Retry ADB commands on all devices.

Change-Id: Ieb289043bd871ec45c70339787ae45f09d3ac1dc
Reviewed-on: https://skia-review.googlesource.com/130128
Commit-Queue: Kevin Lubick <kjlubick@google.com>
Auto-Submit: Ben Wagner <benjaminwagner@google.com>
Reviewed-by: Kevin Lubick <kjlubick@google.com>

3 files changed, 108 insertions, 7 deletions

diff --git a/infra/bots/recipe_modules/flavor/android.py b/infra/bots/recipe_modules/flavor/android.py
index ed8d9af2d4..ea93cb451c 100644
--- a/infra/bots/recipe_modules/flavor/android.py
+++ b/infra/bots/recipe_modules/flavor/android.py
@@ -85,10 +85,8 @@ class AndroidFlavor(default.DefaultFlavor):
       kwargs['infra_step'] = True
 
     self._ever_ran_adb = True
-    attempts = 1
-    flaky_devices = ['NexusPlayer', 'PixelC']
-    if self.m.vars.builder_cfg.get('model') in flaky_devices:
-      attempts = 3
+    # ADB seems to be occasionally flaky on every device, so always retry.
+    attempts = 3
 
     def wait_for_device(attempt):
       self.m.run(self.m.step,
diff --git a/infra/bots/recipes/test.expected/failed_pull.json b/infra/bots/recipes/test.expected/failed_pull.json
index 45bcd536f1..c033f341de 100644
--- a/infra/bots/recipes/test.expected/failed_pull.json
+++ b/infra/bots/recipes/test.expected/failed_pull.json
@@ -812,6 +812,106 @@
   },
   {
     "cmd": [
+      "/usr/bin/adb.1.0.35",
+      "kill-server"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "ADB_VENDOR_KEYS": "/home/chrome-bot/.android/adbkey",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "infra_step": true,
+    "name": "kill adb server after failure of 'pull /sdcard/revenge_of_the_skiabot/dm_out [START_DIR]/[SWARM_OUT_DIR]/dm' (attempt 1)",
+    "timeout": 30
+  },
+  {
+    "cmd": [
+      "/usr/bin/adb.1.0.35",
+      "wait-for-device"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "ADB_VENDOR_KEYS": "/home/chrome-bot/.android/adbkey",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "infra_step": true,
+    "name": "wait for device after failure of 'pull /sdcard/revenge_of_the_skiabot/dm_out [START_DIR]/[SWARM_OUT_DIR]/dm' (attempt 1)",
+    "timeout": 180
+  },
+  {
+    "cmd": [
+      "/usr/bin/adb.1.0.35",
+      "pull",
+      "/sdcard/revenge_of_the_skiabot/dm_out",
+      "[START_DIR]/[SWARM_OUT_DIR]/dm"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "ADB_VENDOR_KEYS": "/home/chrome-bot/.android/adbkey",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "infra_step": true,
+    "name": "pull /sdcard/revenge_of_the_skiabot/dm_out [START_DIR]/[SWARM_OUT_DIR]/dm (attempt 2)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_EXCEPTION@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "/usr/bin/adb.1.0.35",
+      "kill-server"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "ADB_VENDOR_KEYS": "/home/chrome-bot/.android/adbkey",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "infra_step": true,
+    "name": "kill adb server after failure of 'pull /sdcard/revenge_of_the_skiabot/dm_out [START_DIR]/[SWARM_OUT_DIR]/dm' (attempt 2)",
+    "timeout": 30
+  },
+  {
+    "cmd": [
+      "/usr/bin/adb.1.0.35",
+      "wait-for-device"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "ADB_VENDOR_KEYS": "/home/chrome-bot/.android/adbkey",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "infra_step": true,
+    "name": "wait for device after failure of 'pull /sdcard/revenge_of_the_skiabot/dm_out [START_DIR]/[SWARM_OUT_DIR]/dm' (attempt 2)",
+    "timeout": 180
+  },
+  {
+    "cmd": [
+      "/usr/bin/adb.1.0.35",
+      "pull",
+      "/sdcard/revenge_of_the_skiabot/dm_out",
+      "[START_DIR]/[SWARM_OUT_DIR]/dm"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "ADB_VENDOR_KEYS": "/home/chrome-bot/.android/adbkey",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "infra_step": true,
+    "name": "pull /sdcard/revenge_of_the_skiabot/dm_out [START_DIR]/[SWARM_OUT_DIR]/dm (attempt 3)",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_EXCEPTION@@@"
+    ]
+  },
+  {
+    "cmd": [
       "python",
       "-u",
       "\nimport os\nimport subprocess\nimport sys\nout = sys.argv[1]\nlog = subprocess.check_output(['/usr/bin/adb.1.0.35', 'logcat', '-d'])\nfor line in log.split('\\n'):\n  tokens = line.split()\n  if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':\n    addr, path = tokens[-2:]\n    local = os.path.join(out, os.path.basename(path))\n    if os.path.exists(local):\n      sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])\n      line = line.replace(addr, addr + ' ' + sym.strip())\n  print line\n",
@@ -859,7 +959,7 @@
   },
   {
     "name": "$result",
-    "reason": "Infra Failure: Step('pull /sdcard/revenge_of_the_skiabot/dm_out [START_DIR]/[SWARM_OUT_DIR]/dm') returned 1",
+    "reason": "Infra Failure: Step('pull /sdcard/revenge_of_the_skiabot/dm_out [START_DIR]/[SWARM_OUT_DIR]/dm (attempt 3)') returned 1",
     "recipe_result": null,
     "status_code": 1
   }
diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py
index 8145f368da..62e222a332 100644
--- a/infra/bots/recipes/test.py
+++ b/infra/bots/recipes/test.py
@@ -1182,6 +1182,8 @@ def GenTests(api):
   )
 
   builder = 'Test-Android-Clang-Nexus7-GPU-Tegra3-arm-Debug-All-Android'
+  retry_step_name = ('pull /sdcard/revenge_of_the_skiabot/dm_out '
+                     '[START_DIR]/[SWARM_OUT_DIR]/dm')
   yield (
     api.test('failed_pull') +
     api.properties(buildername=builder,
@@ -1200,8 +1202,9 @@ def GenTests(api):
         api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
     ) +
     api.step_data('dm', retcode=1) +
-    api.step_data('pull /sdcard/revenge_of_the_skiabot/dm_out '+
-                  '[START_DIR]/[SWARM_OUT_DIR]/dm', retcode=1)
+    api.step_data(retry_step_name, retcode=1) +
+    api.step_data(retry_step_name + ' (attempt 2)', retcode=1) +
+    api.step_data(retry_step_name + ' (attempt 3)', retcode=1)
   )
 
   yield (