From 7e97dc0aa1ff5b03593e689fe1cfdcc3550c169e Mon Sep 17 00:00:00 2001 From: Eric Boren Date: Thu, 2 Feb 2017 09:02:37 -0500 Subject: Copy modules back in from skia-recipes repo BUG=skia:6070 Change-Id: I86b3f039c27bd8287516c1db8bd1cae62888171b Reviewed-on: https://skia-review.googlesource.com/7890 Reviewed-by: Ravi Mistry Commit-Queue: Eric Boren --- .../recipe_modules/upload_dm_results/__init__.py | 13 ++ infra/bots/recipe_modules/upload_dm_results/api.py | 91 ++++++++++++ .../example.expected/failed_all.json | 157 +++++++++++++++++++++ .../example.expected/failed_once.json | 124 ++++++++++++++++ .../example.expected/normal_bot.json | 111 +++++++++++++++ .../example.expected/recipe_with_gerrit_patch.json | 111 +++++++++++++++ .../upload_dm_results/example.expected/trybot.json | 111 +++++++++++++++ .../recipe_modules/upload_dm_results/example.py | 70 +++++++++ 8 files changed, 788 insertions(+) create mode 100644 infra/bots/recipe_modules/upload_dm_results/__init__.py create mode 100644 infra/bots/recipe_modules/upload_dm_results/api.py create mode 100644 infra/bots/recipe_modules/upload_dm_results/example.expected/failed_all.json create mode 100644 infra/bots/recipe_modules/upload_dm_results/example.expected/failed_once.json create mode 100644 infra/bots/recipe_modules/upload_dm_results/example.expected/normal_bot.json create mode 100644 infra/bots/recipe_modules/upload_dm_results/example.expected/recipe_with_gerrit_patch.json create mode 100644 infra/bots/recipe_modules/upload_dm_results/example.expected/trybot.json create mode 100644 infra/bots/recipe_modules/upload_dm_results/example.py (limited to 'infra/bots/recipe_modules/upload_dm_results') diff --git a/infra/bots/recipe_modules/upload_dm_results/__init__.py b/infra/bots/recipe_modules/upload_dm_results/__init__.py new file mode 100644 index 0000000000..df2e0054da --- /dev/null +++ b/infra/bots/recipe_modules/upload_dm_results/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'build/file', + 'recipe_engine/json', + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/shutil', + 'recipe_engine/step', + 'recipe_engine/time', +] diff --git a/infra/bots/recipe_modules/upload_dm_results/api.py b/infra/bots/recipe_modules/upload_dm_results/api.py new file mode 100644 index 0000000000..cc395fd722 --- /dev/null +++ b/infra/bots/recipe_modules/upload_dm_results/api.py @@ -0,0 +1,91 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# Recipe for uploading DM results. + + + +import calendar + +from recipe_engine import recipe_api + + +DM_JSON = 'dm.json' +GS_BUCKET = 'gs://skia-infra-gm' +UPLOAD_ATTEMPTS = 5 +VERBOSE_LOG = 'verbose.log' + + +class UploadDmResultsApi(recipe_api.RecipeApi): + def cp(self, name, src, dst, extra_args=None): + cmd = ['gsutil', 'cp'] + if extra_args: + cmd.extend(extra_args) + cmd.extend([src, dst]) + + name = 'upload %s' % name + for i in xrange(UPLOAD_ATTEMPTS): + step_name = name + if i > 0: + step_name += ' (attempt %d)' % (i+1) + try: + self.m.step(step_name, cmd=cmd) + break + except self.m.step.StepFailure: + if i == UPLOAD_ATTEMPTS - 1: + raise + + def run(self): + builder_name = self.m.properties['buildername'] + revision = self.m.properties['revision'] + + results_dir = self.m.path['start_dir'].join('dm') + + # Move dm.json and verbose.log to their own directory. + json_file = results_dir.join(DM_JSON) + log_file = results_dir.join(VERBOSE_LOG) + tmp_dir = self.m.path['start_dir'].join('tmp_upload') + self.m.shutil.makedirs('tmp dir', tmp_dir, infra_step=True) + self.m.shutil.copy('copy dm.json', json_file, tmp_dir) + self.m.shutil.copy('copy verbose.log', log_file, tmp_dir) + self.m.shutil.remove('rm old dm.json', json_file) + self.m.shutil.remove('rm old verbose.log', log_file) + + # Upload the images. + image_dest_path = '/'.join((GS_BUCKET, 'dm-images-v1')) + files_to_upload = self.m.file.glob( + 'find images', + results_dir.join('*'), + test_data=['someimage.png'], + infra_step=True) + if len(files_to_upload) > 0: + self.cp('images', results_dir.join('*'), image_dest_path) + + # Upload the JSON summary and verbose.log. + now = self.m.time.utcnow() + summary_dest_path = '/'.join([ + 'dm-json-v1', + str(now.year ).zfill(4), + str(now.month).zfill(2), + str(now.day ).zfill(2), + str(now.hour ).zfill(2), + revision, + builder_name, + str(int(calendar.timegm(now.utctimetuple())))]) + + # Trybot results are further siloed by issue/patchset. + issue = str(self.m.properties.get('issue', '')) + patchset = str(self.m.properties.get('patchset', '')) + if self.m.properties.get('patch_storage', '') == 'gerrit': + issue = str(self.m.properties['patch_issue']) + patchset = str(self.m.properties['patch_set']) + if issue and patchset: + summary_dest_path = '/'.join(( + 'trybot', summary_dest_path, issue, patchset)) + + summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path)) + + self.cp('JSON and logs', tmp_dir.join('*'), summary_dest_path, + ['-z', 'json,log']) diff --git a/infra/bots/recipe_modules/upload_dm_results/example.expected/failed_all.json b/infra/bots/recipe_modules/upload_dm_results/example.expected/failed_all.json new file mode 100644 index 0000000000..7ec0362e4d --- /dev/null +++ b/infra/bots/recipe_modules/upload_dm_results/example.expected/failed_all.json @@ -0,0 +1,157 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[START_DIR]/tmp_upload", + "511" + ], + "name": "makedirs tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/dm.json", + "[START_DIR]/tmp_upload" + ], + "name": "copy dm.json" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/verbose.log", + "[START_DIR]/tmp_upload" + ], + "name": "copy verbose.log" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/dm.json" + ], + "name": "rm old dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/verbose.log" + ], + "name": "rm old verbose.log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[START_DIR]/dm/*" + ], + "name": "find images" + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 2)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 3)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 4)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 5)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "name": "$result", + "reason": "Step('upload images (attempt 5)') failed with return_code 1", + "recipe_result": null, + "status_code": 1 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/upload_dm_results/example.expected/failed_once.json b/infra/bots/recipe_modules/upload_dm_results/example.expected/failed_once.json new file mode 100644 index 0000000000..b6ed1fe3a9 --- /dev/null +++ b/infra/bots/recipe_modules/upload_dm_results/example.expected/failed_once.json @@ -0,0 +1,124 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[START_DIR]/tmp_upload", + "511" + ], + "name": "makedirs tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/dm.json", + "[START_DIR]/tmp_upload" + ], + "name": "copy dm.json" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/verbose.log", + "[START_DIR]/tmp_upload" + ], + "name": "copy verbose.log" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/dm.json" + ], + "name": "rm old dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/verbose.log" + ], + "name": "rm old verbose.log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[START_DIR]/dm/*" + ], + "name": "find images" + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 2)" + }, + { + "cmd": [ + "gsutil", + "cp", + "-z", + "json,log", + "[START_DIR]/tmp_upload/*", + "gs://skia-infra-gm/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/1337000001" + ], + "name": "upload JSON and logs" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/upload_dm_results/example.expected/normal_bot.json b/infra/bots/recipe_modules/upload_dm_results/example.expected/normal_bot.json new file mode 100644 index 0000000000..9ea0d30397 --- /dev/null +++ b/infra/bots/recipe_modules/upload_dm_results/example.expected/normal_bot.json @@ -0,0 +1,111 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[START_DIR]/tmp_upload", + "511" + ], + "name": "makedirs tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/dm.json", + "[START_DIR]/tmp_upload" + ], + "name": "copy dm.json" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/verbose.log", + "[START_DIR]/tmp_upload" + ], + "name": "copy verbose.log" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/dm.json" + ], + "name": "rm old dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/verbose.log" + ], + "name": "rm old verbose.log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[START_DIR]/dm/*" + ], + "name": "find images" + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images" + }, + { + "cmd": [ + "gsutil", + "cp", + "-z", + "json,log", + "[START_DIR]/tmp_upload/*", + "gs://skia-infra-gm/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/1337000001" + ], + "name": "upload JSON and logs" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/upload_dm_results/example.expected/recipe_with_gerrit_patch.json b/infra/bots/recipe_modules/upload_dm_results/example.expected/recipe_with_gerrit_patch.json new file mode 100644 index 0000000000..03bb50541a --- /dev/null +++ b/infra/bots/recipe_modules/upload_dm_results/example.expected/recipe_with_gerrit_patch.json @@ -0,0 +1,111 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[START_DIR]/tmp_upload", + "511" + ], + "name": "makedirs tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/dm.json", + "[START_DIR]/tmp_upload" + ], + "name": "copy dm.json" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/verbose.log", + "[START_DIR]/tmp_upload" + ], + "name": "copy verbose.log" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/dm.json" + ], + "name": "rm old dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/verbose.log" + ], + "name": "rm old verbose.log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[START_DIR]/dm/*" + ], + "name": "find images" + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images" + }, + { + "cmd": [ + "gsutil", + "cp", + "-z", + "json,log", + "[START_DIR]/tmp_upload/*", + "gs://skia-infra-gm/trybot/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/1337000001/456789/12" + ], + "name": "upload JSON and logs" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/upload_dm_results/example.expected/trybot.json b/infra/bots/recipe_modules/upload_dm_results/example.expected/trybot.json new file mode 100644 index 0000000000..3ebc3fa0af --- /dev/null +++ b/infra/bots/recipe_modules/upload_dm_results/example.expected/trybot.json @@ -0,0 +1,111 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[START_DIR]/tmp_upload", + "511" + ], + "name": "makedirs tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/dm.json", + "[START_DIR]/tmp_upload" + ], + "name": "copy dm.json" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/dm/verbose.log", + "[START_DIR]/tmp_upload" + ], + "name": "copy verbose.log" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/dm.json" + ], + "name": "rm old dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/dm/verbose.log" + ], + "name": "rm old verbose.log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[START_DIR]/dm/*" + ], + "name": "find images" + }, + { + "cmd": [ + "gsutil", + "cp", + "[START_DIR]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images" + }, + { + "cmd": [ + "gsutil", + "cp", + "-z", + "json,log", + "[START_DIR]/tmp_upload/*", + "gs://skia-infra-gm/trybot/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/1337000001/12345/1002" + ], + "name": "upload JSON and logs" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/upload_dm_results/example.py b/infra/bots/recipe_modules/upload_dm_results/example.py new file mode 100644 index 0000000000..54448458ab --- /dev/null +++ b/infra/bots/recipe_modules/upload_dm_results/example.py @@ -0,0 +1,70 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# Example recipe w/ coverage. + + +DEPS = [ + 'upload_dm_results', + 'recipe_engine/properties', +] + + +def RunSteps(api): + api.upload_dm_results.run() + + +def GenTests(api): + builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug' + yield ( + api.test('normal_bot') + + api.properties(buildername=builder, + revision='abc123', + path_config='kitchen') + ) + + yield ( + api.test('failed_once') + + api.properties(buildername=builder, + revision='abc123', + path_config='kitchen') + + api.step_data('upload images', retcode=1) + ) + + yield ( + api.test('failed_all') + + api.properties(buildername=builder, + revision='abc123', + path_config='kitchen') + + api.step_data('upload images', retcode=1) + + api.step_data('upload images (attempt 2)', retcode=1) + + api.step_data('upload images (attempt 3)', retcode=1) + + api.step_data('upload images (attempt 4)', retcode=1) + + api.step_data('upload images (attempt 5)', retcode=1) + ) + + builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot' + yield ( + api.test('trybot') + + api.properties(buildername=builder, + revision='abc123', + path_config='kitchen', + issue='12345', + patchset='1002') + ) + + yield ( + api.test('recipe_with_gerrit_patch') + + api.properties( + buildername=builder, + revision='abc123', + path_config='kitchen', + patch_storage='gerrit') + + api.properties.tryserver( + buildername=builder, + gerrit_project='skia', + gerrit_url='https://skia-review.googlesource.com/', + ) + ) -- cgit v1.2.3