Move recipe content from modules back into recipes

At one point I moved the contents of the recipes into modules so that
they could be shared between repos. It turns out that we don't need
that, and it adds complexity.

Bug: skia:6473
Change-Id: I75a920b6a8474dcdd8b37ee9edd52aac801d1ab0
Reviewed-on: https://skia-review.googlesource.com/12622
Reviewed-by: Ravi Mistry <rmistry@google.com>
Commit-Queue: Eric Boren <borenet@google.com>

1 files changed, 136 insertions, 4 deletions

diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py
index c161225d1f..099565615b 100644
--- a/infra/bots/recipes/upload_dm_results.py
+++ b/infra/bots/recipes/upload_dm_results.py
@@ -6,21 +6,153 @@
 # Recipe for uploading DM results.
 
 
+import calendar
+
+
 DEPS = [
+  'build/file',
+  'recipe_engine/json',
+  'recipe_engine/path',
   'recipe_engine/properties',
-  'upload_dm_results',
+  'recipe_engine/shutil',
+  'recipe_engine/step',
+  'recipe_engine/time',
 ]
 
 
+DM_JSON = 'dm.json'
+UPLOAD_ATTEMPTS = 5
+VERBOSE_LOG = 'verbose.log'
+
+
+def cp(api, name, src, dst, extra_args=None):
+  cmd = ['gsutil', 'cp']
+  if extra_args:
+    cmd.extend(extra_args)
+  cmd.extend([src, dst])
+
+  name = 'upload %s' % name
+  for i in xrange(UPLOAD_ATTEMPTS):
+    step_name = name
+    if i > 0:
+      step_name += ' (attempt %d)' % (i+1)
+    try:
+      api.step(step_name, cmd=cmd)
+      break
+    except api.step.StepFailure:
+      if i == UPLOAD_ATTEMPTS - 1:
+        raise
+
+
 def RunSteps(api):
-  api.upload_dm_results.run()
+  builder_name = api.properties['buildername']
+  revision = api.properties['revision']
+
+  results_dir = api.path['start_dir'].join('dm')
+
+  # Move dm.json and verbose.log to their own directory.
+  json_file = results_dir.join(DM_JSON)
+  log_file = results_dir.join(VERBOSE_LOG)
+  tmp_dir = api.path['start_dir'].join('tmp_upload')
+  api.shutil.makedirs('tmp dir', tmp_dir, infra_step=True)
+  api.shutil.copy('copy dm.json', json_file, tmp_dir)
+  api.shutil.copy('copy verbose.log', log_file, tmp_dir)
+  api.shutil.remove('rm old dm.json', json_file)
+  api.shutil.remove('rm old verbose.log', log_file)
+
+  # Upload the images.
+  image_dest_path = 'gs://%s/dm-images-v1' % api.properties['gs_bucket']
+  files_to_upload = api.file.glob(
+      'find images',
+      results_dir.join('*'),
+      test_data=[results_dir.join('someimage.png')],
+      infra_step=True)
+  if len(files_to_upload) > 0:
+    cp(api, 'images', results_dir.join('*'), image_dest_path)
+
+  # Upload the JSON summary and verbose.log.
+  now = api.time.utcnow()
+  summary_dest_path = '/'.join([
+      'dm-json-v1',
+      str(now.year ).zfill(4),
+      str(now.month).zfill(2),
+      str(now.day  ).zfill(2),
+      str(now.hour ).zfill(2),
+      revision,
+      builder_name,
+      str(int(calendar.timegm(now.utctimetuple())))])
+
+  # Trybot results are further siloed by issue/patchset.
+  issue = str(api.properties.get('issue', ''))
+  patchset = str(api.properties.get('patchset', ''))
+  if api.properties.get('patch_storage', '') == 'gerrit':
+    issue = str(api.properties['patch_issue'])
+    patchset = str(api.properties['patch_set'])
+  if issue and patchset:
+    summary_dest_path = '/'.join((
+        'trybot', summary_dest_path, issue, patchset))
+
+  summary_dest_path = 'gs://%s/%s' % (api.properties['gs_bucket'],
+                                      summary_dest_path)
+
+  cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path,
+     ['-z', 'json,log'])
 
 
 def GenTests(api):
+  builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug'
   yield (
-    api.test('upload') +
-    api.properties(buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug',
+    api.test('normal_bot') +
+    api.properties(buildername=builder,
                    gs_bucket='skia-infra-gm',
                    revision='abc123',
                    path_config='kitchen')
   )
+
+  yield (
+    api.test('failed_once') +
+    api.properties(buildername=builder,
+                   gs_bucket='skia-infra-gm',
+                   revision='abc123',
+                   path_config='kitchen') +
+    api.step_data('upload images', retcode=1)
+  )
+
+  yield (
+    api.test('failed_all') +
+    api.properties(buildername=builder,
+                   gs_bucket='skia-infra-gm',
+                   revision='abc123',
+                   path_config='kitchen') +
+    api.step_data('upload images', retcode=1) +
+    api.step_data('upload images (attempt 2)', retcode=1) +
+    api.step_data('upload images (attempt 3)', retcode=1) +
+    api.step_data('upload images (attempt 4)', retcode=1) +
+    api.step_data('upload images (attempt 5)', retcode=1)
+  )
+
+  builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot'
+  yield (
+    api.test('trybot') +
+    api.properties(buildername=builder,
+                   gs_bucket='skia-infra-gm',
+                   revision='abc123',
+                   path_config='kitchen',
+                   issue='12345',
+                   patchset='1002')
+  )
+
+  yield (
+      api.test('recipe_with_gerrit_patch') +
+      api.properties(
+          buildername=builder,
+          gs_bucket='skia-infra-gm',
+          revision='abc123',
+          path_config='kitchen',
+          patch_storage='gerrit') +
+      api.properties.tryserver(
+          buildername=builder,
+          gerrit_project='skia',
+          gerrit_url='https://skia-review.googlesource.com/',
+      )
+  )