From 4c7754c499d092c8b81fe3c5119aebe4dcfd09f4 Mon Sep 17 00:00:00 2001 From: Eric Boren Date: Mon, 10 Apr 2017 08:19:10 -0400 Subject: Move recipe content from modules back into recipes At one point I moved the contents of the recipes into modules so that they could be shared between repos. It turns out that we don't need that, and it adds complexity. Bug: skia:6473 Change-Id: I75a920b6a8474dcdd8b37ee9edd52aac801d1ab0 Reviewed-on: https://skia-review.googlesource.com/12622 Reviewed-by: Ravi Mistry Commit-Queue: Eric Boren --- infra/bots/recipes/upload_dm_results.py | 140 +++++++++++++++++++++++++++++++- 1 file changed, 136 insertions(+), 4 deletions(-) (limited to 'infra/bots/recipes/upload_dm_results.py') diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py index c161225d1f..099565615b 100644 --- a/infra/bots/recipes/upload_dm_results.py +++ b/infra/bots/recipes/upload_dm_results.py @@ -6,21 +6,153 @@ # Recipe for uploading DM results. +import calendar + + DEPS = [ + 'build/file', + 'recipe_engine/json', + 'recipe_engine/path', 'recipe_engine/properties', - 'upload_dm_results', + 'recipe_engine/shutil', + 'recipe_engine/step', + 'recipe_engine/time', ] +DM_JSON = 'dm.json' +UPLOAD_ATTEMPTS = 5 +VERBOSE_LOG = 'verbose.log' + + +def cp(api, name, src, dst, extra_args=None): + cmd = ['gsutil', 'cp'] + if extra_args: + cmd.extend(extra_args) + cmd.extend([src, dst]) + + name = 'upload %s' % name + for i in xrange(UPLOAD_ATTEMPTS): + step_name = name + if i > 0: + step_name += ' (attempt %d)' % (i+1) + try: + api.step(step_name, cmd=cmd) + break + except api.step.StepFailure: + if i == UPLOAD_ATTEMPTS - 1: + raise + + def RunSteps(api): - api.upload_dm_results.run() + builder_name = api.properties['buildername'] + revision = api.properties['revision'] + + results_dir = api.path['start_dir'].join('dm') + + # Move dm.json and verbose.log to their own directory. + json_file = results_dir.join(DM_JSON) + log_file = results_dir.join(VERBOSE_LOG) + tmp_dir = api.path['start_dir'].join('tmp_upload') + api.shutil.makedirs('tmp dir', tmp_dir, infra_step=True) + api.shutil.copy('copy dm.json', json_file, tmp_dir) + api.shutil.copy('copy verbose.log', log_file, tmp_dir) + api.shutil.remove('rm old dm.json', json_file) + api.shutil.remove('rm old verbose.log', log_file) + + # Upload the images. + image_dest_path = 'gs://%s/dm-images-v1' % api.properties['gs_bucket'] + files_to_upload = api.file.glob( + 'find images', + results_dir.join('*'), + test_data=[results_dir.join('someimage.png')], + infra_step=True) + if len(files_to_upload) > 0: + cp(api, 'images', results_dir.join('*'), image_dest_path) + + # Upload the JSON summary and verbose.log. + now = api.time.utcnow() + summary_dest_path = '/'.join([ + 'dm-json-v1', + str(now.year ).zfill(4), + str(now.month).zfill(2), + str(now.day ).zfill(2), + str(now.hour ).zfill(2), + revision, + builder_name, + str(int(calendar.timegm(now.utctimetuple())))]) + + # Trybot results are further siloed by issue/patchset. + issue = str(api.properties.get('issue', '')) + patchset = str(api.properties.get('patchset', '')) + if api.properties.get('patch_storage', '') == 'gerrit': + issue = str(api.properties['patch_issue']) + patchset = str(api.properties['patch_set']) + if issue and patchset: + summary_dest_path = '/'.join(( + 'trybot', summary_dest_path, issue, patchset)) + + summary_dest_path = 'gs://%s/%s' % (api.properties['gs_bucket'], + summary_dest_path) + + cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path, + ['-z', 'json,log']) def GenTests(api): + builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug' yield ( - api.test('upload') + - api.properties(buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug', + api.test('normal_bot') + + api.properties(buildername=builder, gs_bucket='skia-infra-gm', revision='abc123', path_config='kitchen') ) + + yield ( + api.test('failed_once') + + api.properties(buildername=builder, + gs_bucket='skia-infra-gm', + revision='abc123', + path_config='kitchen') + + api.step_data('upload images', retcode=1) + ) + + yield ( + api.test('failed_all') + + api.properties(buildername=builder, + gs_bucket='skia-infra-gm', + revision='abc123', + path_config='kitchen') + + api.step_data('upload images', retcode=1) + + api.step_data('upload images (attempt 2)', retcode=1) + + api.step_data('upload images (attempt 3)', retcode=1) + + api.step_data('upload images (attempt 4)', retcode=1) + + api.step_data('upload images (attempt 5)', retcode=1) + ) + + builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot' + yield ( + api.test('trybot') + + api.properties(buildername=builder, + gs_bucket='skia-infra-gm', + revision='abc123', + path_config='kitchen', + issue='12345', + patchset='1002') + ) + + yield ( + api.test('recipe_with_gerrit_patch') + + api.properties( + buildername=builder, + gs_bucket='skia-infra-gm', + revision='abc123', + path_config='kitchen', + patch_storage='gerrit') + + api.properties.tryserver( + buildername=builder, + gerrit_project='skia', + gerrit_url='https://skia-review.googlesource.com/', + ) + ) -- cgit v1.2.3