Add retries to upload_dm_results

BUG=skia:5818
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2390723003

Review-Url: https://codereview.chromium.org/2390723003

3 files changed, 324 insertions, 9 deletions

diff --git a/infra/bots/recipes/upload_dm_results.expected/failed_all.json b/infra/bots/recipes/upload_dm_results.expected/failed_all.json
new file mode 100644
index 0000000000..d88927f05e
--- /dev/null
+++ b/infra/bots/recipes/upload_dm_results.expected/failed_all.json
@@ -0,0 +1,157 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[CWD]/tmp_upload",
+      "511"
+    ],
+    "name": "makedirs tmp dir",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/dm.json",
+      "[CWD]/tmp_upload"
+    ],
+    "name": "copy dm.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/verbose.log",
+      "[CWD]/tmp_upload"
+    ],
+    "name": "copy verbose.log"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
+      "[CWD]/dm/dm.json"
+    ],
+    "name": "rm old dm.json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
+      "[CWD]/dm/verbose.log"
+    ],
+    "name": "rm old verbose.log",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n  f.write('\\n'.join(glob.glob(sys.argv[2])))\n",
+      "/path/to/tmp/",
+      "[CWD]/dm/*"
+    ],
+    "name": "find images"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images (attempt 2)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images (attempt 3)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images (attempt 4)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images (attempt 5)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "reason": "Step('upload images (attempt 5)') failed with return_code 1",
+    "recipe_result": null,
+    "status_code": 1
+  }
+]
\ No newline at end of file
diff --git a/infra/bots/recipes/upload_dm_results.expected/failed_once.json b/infra/bots/recipes/upload_dm_results.expected/failed_once.json
new file mode 100644
index 0000000000..48d11423ed
--- /dev/null
+++ b/infra/bots/recipes/upload_dm_results.expected/failed_once.json
@@ -0,0 +1,124 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[CWD]/tmp_upload",
+      "511"
+    ],
+    "name": "makedirs tmp dir",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/dm.json",
+      "[CWD]/tmp_upload"
+    ],
+    "name": "copy dm.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/verbose.log",
+      "[CWD]/tmp_upload"
+    ],
+    "name": "copy verbose.log"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
+      "[CWD]/dm/dm.json"
+    ],
+    "name": "rm old dm.json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
+      "[CWD]/dm/verbose.log"
+    ],
+    "name": "rm old verbose.log",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n  f.write('\\n'.join(glob.glob(sys.argv[2])))\n",
+      "/path/to/tmp/",
+      "[CWD]/dm/*"
+    ],
+    "name": "find images"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images (attempt 2)"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-z",
+      "json,log",
+      "[CWD]/tmp_upload/*",
+      "gs://skia-infra-gm/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/1337018001"
+    ],
+    "name": "upload JSON and logs"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file
diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py
index 6d1b0eacd0..53e68451bd 100644
--- a/infra/bots/recipes/upload_dm_results.py
+++ b/infra/bots/recipes/upload_dm_results.py
@@ -22,9 +22,29 @@ import time
 
 DM_JSON = 'dm.json'
 GS_BUCKET = 'gs://skia-infra-gm'
+UPLOAD_ATTEMPTS = 5
 VERBOSE_LOG = 'verbose.log'
 
 
+def cp(api, name, src, dst, extra_args=None):
+  cmd = ['gsutil', 'cp']
+  if extra_args:
+    cmd.extend(extra_args)
+  cmd.extend([src, dst])
+
+  name = 'upload %s' % name
+  for i in xrange(UPLOAD_ATTEMPTS):
+    step_name = name
+    if i > 0:
+      step_name += ' (attempt %d)' % (i+1)
+    try:
+      api.step(step_name, cmd=cmd)
+      break
+    except api.step.StepFailure:
+      if i == UPLOAD_ATTEMPTS - 1:
+        raise
+
+
 def RunSteps(api):
   builder_name = api.properties['buildername']
   revision = api.properties['revision']
@@ -60,10 +80,7 @@ def RunSteps(api):
       test_data=['someimage.png'],
       infra_step=True)
   if len(files_to_upload) > 0:
-    api.step(
-        'upload images',
-        cmd=['gsutil', 'cp', results_dir.join('*'), image_dest_path],
-     )
+    cp(api, 'images', results_dir.join('*'), image_dest_path)
 
   # Upload the JSON summary and verbose.log.
   now = api.time.utcnow()
@@ -85,11 +102,8 @@ def RunSteps(api):
 
   summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path))
 
-  api.step(
-      'upload JSON and logs',
-      cmd=['gsutil', 'cp', '-z', 'json,log', tmp_dir.join('*'),
-           summary_dest_path],
-  )
+  cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path,
+     ['-z', 'json,log'])
 
 
 def GenTests(api):
@@ -101,6 +115,26 @@ def GenTests(api):
                    path_config='kitchen')
   )
 
+  yield (
+    api.test('failed_once') +
+    api.properties(buildername=builder,
+                   revision='abc123',
+                   path_config='kitchen') +
+    api.step_data('upload images', retcode=1)
+  )
+
+  yield (
+    api.test('failed_all') +
+    api.properties(buildername=builder,
+                   revision='abc123',
+                   path_config='kitchen') +
+    api.step_data('upload images', retcode=1) +
+    api.step_data('upload images (attempt 2)', retcode=1) +
+    api.step_data('upload images (attempt 3)', retcode=1) +
+    api.step_data('upload images (attempt 4)', retcode=1) +
+    api.step_data('upload images (attempt 5)', retcode=1)
+  )
+
   builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot'
   yield (
     api.test('trybot') +