diff options
| author |  borenet <borenet@chromium.org> | 2016-09-27 06:52:26 -0700 |
|---|---|---|
| committer |  Commit bot <commit-bot@chromium.org> | 2016-09-27 06:52:26 -0700 |
| commit | 9766f0016669029fcdc49f00ceaaf66b01b03f4a (patch) | |
| tree | 3a7c334621e1cfd2d177b258a473cea398e4a3f2 | |
| parent | 7bfc08b9fb4289381609cee1deb0dc6325f878e1 (diff) | |
Add swarming task for upload_dm_results
BUG=skia:5719
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2360203004
Review-Url: https://codereview.chromium.org/2360203004
19 files changed, 2579 insertions, 861 deletions
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android.json b/infra/bots/recipes/swarm_trigger.expected/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android.json index e2daf6cb5e..1540c7ecff 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android.json @@ -707,28 +707,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -738,9 +716,7 @@ "--json", "{\"base_task_name\": \"test_skia/Android/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\", \"tasks\": {\"test_skia/Android/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Android", "~followup_annotations": [ @@ -791,21 +767,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -813,31 +846,161 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android", - "5", - "", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "buildername=Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android", + "mastername=client.skia", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json index 405be3f18e..a3c0f2d6c0 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json @@ -695,28 +695,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -726,9 +704,7 @@ "--json", "{\"base_task_name\": \"test_skia/Android/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\", \"tasks\": {\"test_skia/Android/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Android", "~followup_annotations": [ @@ -779,21 +755,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -801,31 +834,161 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan", - "5", - "", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "buildername=Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan", + "mastername=client.skia", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json index 780f02a1e9..1c130a4550 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json @@ -695,28 +695,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -726,9 +704,7 @@ "--json", "{\"base_task_name\": \"test_skia/Android/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\", \"tasks\": {\"test_skia/Android/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Android", "~followup_annotations": [ @@ -779,21 +755,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -801,31 +834,161 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release", - "5", - "", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "buildername=Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release", + "mastername=client.skia", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json index f59ad2c300..eb7ffc54ee 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json @@ -674,28 +674,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -705,9 +683,7 @@ "--json", "{\"base_task_name\": \"test_skia/Mac/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\", \"tasks\": {\"test_skia/Mac/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Mac", "~followup_annotations": [ @@ -758,21 +734,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -780,31 +813,161 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release", - "5", - "", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "buildername=Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release", + "mastername=client.skia", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json index 78b61c5690..0c02a22299 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json @@ -443,28 +443,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -474,9 +452,7 @@ "--json", "{\"base_task_name\": \"test_skia/Ubuntu/[dummy has/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5\", \"tasks\": {\"test_skia/Ubuntu/[dummy has/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Ubuntu", "~followup_annotations": [ @@ -524,131 +500,6 @@ ] }, { - "cmd": [ - "git", - "show", - "HEAD", - "--format=%at", - "-s" - ], - "cwd": "[SLAVE_BUILD]/skia", - "name": "git show", - "stdout": "/path/to/tmp/" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[build::gsutil]/resources/gsutil_wrapper.py", - "--", - "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/abc123.cov", - "gs://skia-infra/coverage-raw-v1/trybot/2012/05/14/12/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5/500/abc123.cov" - ], - "env": { - "AWS_CREDENTIAL_FILE": null, - "BOTO_CONFIG": null - }, - "name": "gsutil upload raw coverage data", - "~followup_annotations": [ - "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/skia-infra/coverage-raw-v1/trybot/2012/05/14/12/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5/500/abc123.cov@@@" - ] - }, - { - "cmd": [ - "python", - "-u", - "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/nanobench_abc123.json", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/nanobench_abc123_1408633190.json" - ], - "name": "nanobench JSON" - }, - { - "cmd": [ - "python", - "-u", - "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/nanobench_abc123.json" - ], - "name": "old nanobench JSON", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@import sys@@@", - "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[skia::core]/resources/upload_bench_results.py", - "Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot", - "5", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0", - "abc123", - "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py", - "500" - ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "upload nanobench coverage results" - }, - { - "cmd": [ - "python", - "-u", - "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/coverage_by_line_abc123.json", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/coverage_by_line_abc123_1408633190.json" - ], - "name": "Line-by-line coverage JSON" - }, - { - "cmd": [ - "python", - "-u", - "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/coverage_by_line_abc123.json" - ], - "name": "old line-by-line coverage JSON", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@import sys@@@", - "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[build::gsutil]/resources/gsutil_wrapper.py", - "--", - "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/coverage_by_line_abc123_1408633190.json", - "gs://skia-infra/coverage-json-v1/trybot/2012/05/14/12/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5/500/coverage_by_line_abc123_1408633190.json" - ], - "env": { - "AWS_CREDENTIAL_FILE": null, - "BOTO_CONFIG": null - }, - "name": "gsutil upload line-by-line coverage data", - "~followup_annotations": [ - "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/skia-infra/coverage-json-v1/trybot/2012/05/14/12/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5/500/coverage_by_line_abc123_1408633190.json@@@" - ] - }, - { "name": "$result", "recipe_result": null, "status_code": 0 diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json index 5c39f43d85..2d2e8d3ae9 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json @@ -698,28 +698,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -729,9 +707,7 @@ "--json", "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Ubuntu", "~followup_annotations": [ diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json index c4cad61129..a34a500890 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json @@ -674,28 +674,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -705,9 +683,7 @@ "--json", "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Ubuntu", "~followup_annotations": [ @@ -758,21 +734,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -780,31 +813,161 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug", - "5", - "", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug", + "mastername=client.skia", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared.json index cdf2291ee6..709afa38d3 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared.json @@ -674,28 +674,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -705,9 +683,7 @@ "--json", "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Ubuntu", "~followup_annotations": [ @@ -758,21 +734,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -780,31 +813,161 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared", - "5", - "", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared", + "mastername=client.skia", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json index 3eb69e4864..58c36bc47b 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json @@ -671,28 +671,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -702,9 +680,7 @@ "--json", "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Ubuntu", "~followup_annotations": [ diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release.json index 3524a6ecb7..16526702ed 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release.json @@ -683,28 +683,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -714,9 +692,7 @@ "--json", "{\"base_task_name\": \"test_skia/Windows/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\", \"tasks\": {\"test_skia/Windows/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Windows", "~followup_annotations": [ @@ -767,21 +743,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -789,31 +822,161 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release", - "5", - "", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "buildername=Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release", + "mastername=client.skia", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release.json index 9626d20247..8742862625 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release.json @@ -686,28 +686,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -717,9 +695,7 @@ "--json", "{\"base_task_name\": \"test_skia/Windows-2008ServerR2-SP1/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\", \"tasks\": {\"test_skia/Windows-2008ServerR2-SP1/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Windows-2008ServerR2-SP1", "~followup_annotations": [ @@ -770,21 +746,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -792,31 +825,161 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release", - "5", - "", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "buildername=Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release", + "mastername=client.skia", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release.json index 3c2fa52e21..cebeb0291f 100644 --- a/infra/bots/recipes/swarm_trigger.expected/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release.json +++ b/infra/bots/recipes/swarm_trigger.expected/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release.json @@ -671,28 +671,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -702,9 +680,7 @@ "--json", "{\"base_task_name\": \"test_skia/iOS-9.2/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\", \"tasks\": {\"test_skia/iOS-9.2/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on iOS-9.2", "~followup_annotations": [ @@ -755,21 +731,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -777,31 +810,161 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release", - "5", - "", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "buildername=Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release", + "mastername=client.skia", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json b/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json index 246f91d326..5ac2417d0c 100644 --- a/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json +++ b/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json @@ -682,28 +682,6 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" - ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree test_skia", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", "[SLAVE_BUILD]/swarming.client/swarming.py", "collect", "--swarming", @@ -713,9 +691,7 @@ "--json", "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", "--task-summary-json", - "/path/to/tmp/json", - "--task-output-dir", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + "/path/to/tmp/json" ], "name": "test_skia on Ubuntu", "~followup_annotations": [ @@ -766,21 +742,78 @@ "cmd": [ "python", "-u", - "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", - "[SLAVE_BUILD]/dm" + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" ], - "env": { - "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" - }, - "name": "rmtree dm", + "name": "makedirs swarming tmp dir (3)", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", - "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", - "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Linux\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "Write upload_dm_results_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json" + ], + "name": "isolate tests (3)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated", + "abc123" + ], + "name": "add_isolated_input (2)", + "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", - "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", - "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", "@@@STEP_LOG_END@python.inline@@@" ] }, @@ -788,31 +821,165 @@ "cmd": [ "python", "-u", - "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", - "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", - "[SLAVE_BUILD]/dm", - "0" + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated" ], - "name": "dm_dir" + "name": "upload new .isolated file for upload_dm_results_skia", + "stdout": "/path/to/tmp/" }, { "cmd": [ "python", "-u", - "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", - "[SLAVE_BUILD]/dm", - "abc123", - "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot", - "5", - "2100", - "[SLAVE_BUILD]/skia/common/py/utils" + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Linux", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:upload_dm_results_skia", + "--tag", + "os:Linux", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:upload_dm_results_skia on Linux", + "--idempotent", + "def456", + "--", + "--workdir", + "../../..", + "upload_dm_results", + "event.change.number=2100", + "repository=skia", + "buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot", + "mastername=client.skia", + "patch_storage=gerrit", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "event.patchSet.ref=refs/changes/00/2100/2", + "revision=abc123" ], - "cwd": "[SLAVE_BUILD]/skia", - "env": { - "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", - "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" - }, - "name": "Upload DM Results" + "name": "[trigger] upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "upload_dm_results_skia on Linux", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] }, { "name": "$result", diff --git a/infra/bots/recipes/swarm_trigger.py b/infra/bots/recipes/swarm_trigger.py index fdf81004e8..8a9360cd31 100644 --- a/infra/bots/recipes/swarm_trigger.py +++ b/infra/bots/recipes/swarm_trigger.py @@ -434,26 +434,6 @@ def get_timeouts(builder_cfg): return expiration, hard_timeout, io_timeout -def gsutil_env(api, boto_file): - """Environment variables for gsutil.""" - home_dir = os.path.expanduser('~') - if api.path._test_data.enabled: - home_dir = '[HOME]' - - boto_path = None - if boto_file: - boto_path = api.path.join(home_dir, boto_file) - return {'AWS_CREDENTIAL_FILE': boto_path, - 'BOTO_CONFIG': boto_path} - - -def get_issue_num(api): - if api.properties.get('patch_storage') == 'gerrit': - return str(api.properties['event.change.number']) - else: - return str(api.properties['issue']) - - def perf_steps_trigger(api, builder_cfg, got_revision, infrabots_dir, extra_hashes, cipd_packages): """Trigger perf tests via Swarming.""" @@ -526,110 +506,39 @@ def test_steps_trigger(api, builder_cfg, got_revision, infrabots_dir, expiration=expiration, hard_timeout=hard_timeout, io_timeout=io_timeout, - cipd_packages=cipd_packages) + cipd_packages=cipd_packages, + store_output=False) -def test_steps_collect(api, task, got_revision, is_trybot, builder_cfg): +def test_steps_collect(api, task, builder_cfg, got_revision, infrabots_dir): """Collect the test results from Swarming.""" # Wait for tests to finish, download the results. - api.run.rmtree(task.task_output_dir) - api.swarming.collect_swarming_task(task) + if not api.vars.upload_dm_results: # pragma: nocover + api.swarming.collect_swarming_task(task) + return + + dm_hash = api.swarming.collect_swarming_task_isolate_hash(task) # Upload the results. - if api.vars.upload_dm_results: - dm_dir = api.path['slave_build'].join('dm') - dm_src = task.task_output_dir.join('0', 'dm') - api.run.rmtree(dm_dir) - api.file.copytree('dm_dir', dm_src, dm_dir, infra_step=True) - - # Upload them to Google Storage. - api.python( - 'Upload DM Results', - script=api.core.resource('upload_dm_results.py'), - args=[ - dm_dir, - got_revision, - api.properties['buildername'], - api.properties['buildnumber'], - get_issue_num(api) if is_trybot else '', - api.path['slave_build'].join('skia', 'common', 'py', 'utils'), - ], - cwd=api.path['checkout'], - env=gsutil_env(api, 'chromium-skia-gm.boto'), - infra_step=True) - - if builder_cfg['configuration'] == 'Coverage': - upload_coverage_results(api, task, got_revision, is_trybot) - - -def upload_coverage_results(api, task, got_revision, is_trybot): - results_dir = task.task_output_dir.join('0') - git_timestamp = api.git.get_timestamp(test_data='1408633190', - infra_step=True) - - # Upload raw coverage data. - cov_file_basename = '%s.cov' % got_revision - cov_file = results_dir.join(cov_file_basename) - now = api.time.utcnow() - gs_json_path = '/'.join(( - str(now.year).zfill(4), str(now.month).zfill(2), - str(now.day).zfill(2), str(now.hour).zfill(2), + task = trigger_task( + api, + 'upload_dm_results', api.properties['buildername'], - str(api.properties['buildnumber']))) - if is_trybot: - gs_json_path = '/'.join(('trybot', gs_json_path, get_issue_num(api))) - api.gsutil.upload( - name='upload raw coverage data', - source=cov_file, - bucket='skia-infra', - dest='/'.join(('coverage-raw-v1', gs_json_path, - cov_file_basename)), - env={'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None}, - ) + api.properties['mastername'], + api.properties['slavename'], + api.properties['buildnumber'], + builder_cfg, + got_revision, + infrabots_dir, + idempotent=True, + store_output=False, + cipd_packages=None, + extra_isolate_hashes=[dm_hash], + recipe_name='upload_dm_results', + isolate_file='upload_dm_results.isolate', + dimensions=UPLOAD_DIMENSIONS) - # Transform the nanobench_${git_hash}.json file received from swarming bot - # into the nanobench_${git_hash}_${timestamp}.json file - # upload_bench_results.py expects. - src_nano_file = results_dir.join('nanobench_%s.json' % got_revision) - dst_nano_file = results_dir.join( - 'nanobench_%s_%s.json' % (got_revision, git_timestamp)) - api.file.copy('nanobench JSON', src_nano_file, dst_nano_file, - infra_step=True) - api.file.remove('old nanobench JSON', src_nano_file) - - # Upload nanobench JSON data. - gsutil_path = api.depot_tools.gsutil_py_path - upload_args = [api.properties['buildername'], api.properties['buildnumber'], - results_dir, got_revision, gsutil_path] - if is_trybot: - upload_args.append(get_issue_num(api)) - api.python( - 'upload nanobench coverage results', - script=api.core.resource('upload_bench_results.py'), - args=upload_args, - cwd=api.path['checkout'], - env=gsutil_env(api, 'chromium-skia-gm.boto'), - infra_step=True) - - # Transform the coverage_by_line_${git_hash}.json file received from - # swarming bot into a coverage_by_line_${git_hash}_${timestamp}.json file. - src_lbl_file = results_dir.join('coverage_by_line_%s.json' % got_revision) - dst_lbl_file_basename = 'coverage_by_line_%s_%s.json' % ( - got_revision, git_timestamp) - dst_lbl_file = results_dir.join(dst_lbl_file_basename) - api.file.copy('Line-by-line coverage JSON', src_lbl_file, dst_lbl_file, - infra_step=True) - api.file.remove('old line-by-line coverage JSON', src_lbl_file) - - # Upload line-by-line coverage data. - api.gsutil.upload( - name='upload line-by-line coverage data', - source=dst_lbl_file, - bucket='skia-infra', - dest='/'.join(('coverage-json-v1', gs_json_path, - dst_lbl_file_basename)), - env={'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None}, - ) + return api.swarming.collect_swarming_task(task) def cipd_pkg(api, infrabots_dir, asset_name): @@ -738,13 +647,12 @@ def RunSteps(api): if do_perf_steps: perf_task = perf_steps_trigger(api, builder_cfg, got_revision, infrabots_dir, extra_hashes, cipd_packages) - is_trybot = builder_cfg['is_trybot'] # Wait for results, then upload them if necessary. if test_task: - test_steps_collect(api, test_task, - got_revision, is_trybot, builder_cfg) + test_steps_collect(api, test_task, builder_cfg, + got_revision, infrabots_dir) if perf_task: perf_steps_collect(api, perf_task, builder_cfg, @@ -778,6 +686,10 @@ def test_for_bot(api, builder, mastername, slavename, testname=None): test += api.step_data( 'upload new .isolated file for test_skia', stdout=api.raw_io.output('def456 XYZ.isolated')) + if not 'Valgrind' in builder and not 'MSAN' in builder: + test += api.step_data( + 'upload new .isolated file for upload_dm_results_skia', + stdout=api.raw_io.output('def456 XYZ.isolated')) if 'Perf' in builder and '-CT_' not in builder: test += api.step_data( 'upload new .isolated file for perf_skia', @@ -830,5 +742,8 @@ def GenTests(api): **gerrit_kwargs) + api.step_data( 'upload new .isolated file for test_skia', + stdout=api.raw_io.output('def456 XYZ.isolated')) + + api.step_data( + 'upload new .isolated file for upload_dm_results_skia', stdout=api.raw_io.output('def456 XYZ.isolated')) ) diff --git a/infra/bots/recipes/upload_dm_results.expected/normal_bot.json b/infra/bots/recipes/upload_dm_results.expected/normal_bot.json new file mode 100644 index 0000000000..8b35adc182 --- /dev/null +++ b/infra/bots/recipes/upload_dm_results.expected/normal_bot.json @@ -0,0 +1,125 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/dm.json", + "/path/to/tmp/json" + ], + "name": "validate dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output (invalid)@null@@@", + "@@@STEP_LOG_END@json.output (invalid)@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CWD]/tmp_upload", + "511" + ], + "name": "makedirs tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/dm.json", + "[CWD]/tmp_upload" + ], + "name": "copy dm.json" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/verbose.log", + "[CWD]/tmp_upload" + ], + "name": "copy verbose.log" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CWD]/dm/dm.json" + ], + "name": "rm old dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CWD]/dm/verbose.log" + ], + "name": "rm old verbose.log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[CWD]/dm/*" + ], + "name": "find images" + }, + { + "cmd": [ + "gsutil", + "cp", + "[CWD]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images" + }, + { + "cmd": [ + "gsutil", + "cp", + "-z", + "json,log", + "[CWD]/tmp_upload/*", + "gs://skia-infra-gm/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/1337018001" + ], + "name": "upload JSON and logs" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipes/upload_dm_results.expected/trybot.json b/infra/bots/recipes/upload_dm_results.expected/trybot.json new file mode 100644 index 0000000000..ced83e6861 --- /dev/null +++ b/infra/bots/recipes/upload_dm_results.expected/trybot.json @@ -0,0 +1,125 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/dm.json", + "/path/to/tmp/json" + ], + "name": "validate dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output (invalid)@null@@@", + "@@@STEP_LOG_END@json.output (invalid)@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CWD]/tmp_upload", + "511" + ], + "name": "makedirs tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/dm.json", + "[CWD]/tmp_upload" + ], + "name": "copy dm.json" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/verbose.log", + "[CWD]/tmp_upload" + ], + "name": "copy verbose.log" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CWD]/dm/dm.json" + ], + "name": "rm old dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CWD]/dm/verbose.log" + ], + "name": "rm old verbose.log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[CWD]/dm/*" + ], + "name": "find images" + }, + { + "cmd": [ + "gsutil", + "cp", + "[CWD]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images" + }, + { + "cmd": [ + "gsutil", + "cp", + "-z", + "json,log", + "[CWD]/tmp_upload/*", + "gs://skia-infra-gm/trybot/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/1337018001/12345/1002" + ], + "name": "upload JSON and logs" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py new file mode 100644 index 0000000000..20d2c064c3 --- /dev/null +++ b/infra/bots/recipes/upload_dm_results.py @@ -0,0 +1,106 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# Recipe for uploading DM results. + + +DEPS = [ + 'build/file', + 'recipe_engine/json', + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/shutil', + 'recipe_engine/step', + 'recipe_engine/time', +] + + +import time + + +DM_JSON = 'dm.json' +GS_BUCKET = 'gs://skia-infra-gm' +VERBOSE_LOG = 'verbose.log' + + +def RunSteps(api): + builder_name = api.properties['buildername'] + revision = api.properties['revision'] + issue = str(api.properties.get('issue', '')) + patchset = str(api.properties.get('patchset', '')) + + results_dir = api.path['cwd'].join('dm') + + # Validate the JSON file. + json_file = results_dir.join(DM_JSON) + api.json.read('validate dm.json', json_file) + + # Move dm.json and verbose.log to their own directory. + log_file = results_dir.join(VERBOSE_LOG) + tmp_dir = api.path['cwd'].join('tmp_upload') + api.shutil.makedirs('tmp dir', tmp_dir, infra_step=True) + api.shutil.copy('copy dm.json', json_file, tmp_dir) + api.shutil.copy('copy verbose.log', log_file, tmp_dir) + api.shutil.remove('rm old dm.json', json_file) + api.shutil.remove('rm old verbose.log', log_file) + + # Upload the images. + image_dest_path = '/'.join((GS_BUCKET, 'dm-images-v1')) + files_to_upload = api.file.glob( + 'find images', + results_dir.join('*'), + test_data=['someimage.png'], + infra_step=True) + if len(files_to_upload) > 0: + api.step( + 'upload images', + cmd=['gsutil', 'cp', results_dir.join('*'), image_dest_path], + ) + + # Upload the JSON summary and verbose.log. + now = api.time.utcnow() + summary_dest_path = '/'.join([ + 'dm-json-v1', + str(now.year ).zfill(4), + str(now.month).zfill(2), + str(now.day ).zfill(2), + str(now.hour ).zfill(2), + revision, + builder_name, + str(int(time.mktime(now.utctimetuple())))]) + + # Trybot results are further siloed by issue/patchset. + if builder_name.endswith('-Trybot'): + if not (issue and patchset): # pragma: nocover + raise Exception('issue and patchset properties are required for trybots.') + summary_dest_path = '/'.join(('trybot', summary_dest_path, issue, patchset)) + + summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path)) + + api.step( + 'upload JSON and logs', + cmd=['gsutil', 'cp', '-z', 'json,log', tmp_dir.join('*'), + summary_dest_path], + ) + + +def GenTests(api): + builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug' + yield ( + api.test('normal_bot') + + api.properties(buildername=builder, + revision='abc123', + path_config='kitchen') + ) + + builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot' + yield ( + api.test('trybot') + + api.properties(buildername=builder, + revision='abc123', + path_config='kitchen', + issue='12345', + patchset='1002') + ) diff --git a/infra/bots/recipes/upload_nano_results.py b/infra/bots/recipes/upload_nano_results.py index 8cc39be693..2abe388e85 100644 --- a/infra/bots/recipes/upload_nano_results.py +++ b/infra/bots/recipes/upload_nano_results.py @@ -18,8 +18,8 @@ DEPS = [ def RunSteps(api): # Upload the nanobench resuls. builder_name = api.properties['buildername'] - issue = str(api.properties.get('issue')) - patchset = str(api.properties.get('patchset')) + issue = str(api.properties.get('issue', '')) + patchset = str(api.properties.get('patchset', '')) now = api.time.utcnow() diff --git a/infra/bots/upload_dm_results.isolate b/infra/bots/upload_dm_results.isolate new file mode 100644 index 0000000000..767fa3af9e --- /dev/null +++ b/infra/bots/upload_dm_results.isolate @@ -0,0 +1,10 @@ +{ + 'includes': [ + 'swarm_recipe.isolate', + ], + 'variables': { + 'files': [ + '../../../.gclient', + ], + }, +} |