Make ASAN setup of Android bots more fault tolerant

Bug: skia:7975
Change-Id: I3f123337913870a921d1c09714873eaf2949e88f
Reviewed-on: https://skia-review.googlesource.com/129364
Commit-Queue: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Ben Wagner <benjaminwagner@google.com>

2 files changed, 61 insertions, 29 deletions

diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-Android_ASAN.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-Android_ASAN.json
index 8de4a9ea8f..aff96834e3 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-Android_ASAN.json
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-Android_ASAN.json
@@ -98,7 +98,7 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n  while True:\n    time.sleep(5)\n    print 'Waiting for device'\n    subprocess.check_output([ADB, 'wait-for-device'])\n    bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n                                   'dev.bootcomplete'])\n    bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n                                   'sys.boot_completed'])\n    if '1' in bit1 and '1' in bit2:\n      print 'Device detected'\n      break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n  raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n  print 'Rebooting device'\n  subprocess.check_output([ADB, 'reboot'])\n  wait_for_device()\n\n# ASAN setup script is idempotent, either it installs it or says it's installed\noutput = subprocess.check_output([ADB, 'wait-for-device'])\nprocess = subprocess.Popen([ASAN_SETUP], env={'ADB': ADB},\n                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n# this also blocks until command finishes\n(stdout, stderr) = process.communicate()\nprint stdout\nprint 'Stderr: %s' % stderr\nif process.returncode:\n  raise Exception('setup ASAN returned with non-zero exit code: %d' %\n                  process.returncode)\n\nif 'Please wait until the device restarts' in stdout:\n  # Sleep because device does not reboot instantly\n  time.sleep(30)\nwait_for_device()\n",
+      "\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\nASAN_SETUP = sys.argv[2]\n\ndef wait_for_device():\n  while True:\n    time.sleep(5)\n    print 'Waiting for device'\n    subprocess.check_output([ADB, 'wait-for-device'])\n    bit1 = subprocess.check_output([ADB, 'shell', 'getprop',\n                                   'dev.bootcomplete'])\n    bit2 = subprocess.check_output([ADB, 'shell', 'getprop',\n                                   'sys.boot_completed'])\n    if '1' in bit1 and '1' in bit2:\n      print 'Device detected'\n      break\n\nlog = subprocess.check_output([ADB, 'root'])\n# check for message like 'adbd cannot run as root in production builds'\nprint log\nif 'cannot' in log:\n  raise Exception('adb root failed')\n\noutput = subprocess.check_output([ADB, 'disable-verity'])\nprint output\n\nif 'already disabled' not in output:\n  print 'Rebooting device'\n  subprocess.check_output([ADB, 'reboot'])\n  wait_for_device()\n\ndef installASAN(revert=False):\n  # ASAN setup script is idempotent, either it installs it or\n  # says it's installed.  Returns True on success, false otherwise.\n  out = subprocess.check_output([ADB, 'wait-for-device'])\n  print out\n  cmd = [ASAN_SETUP]\n  if revert:\n    cmd = [ASAN_SETUP, '--revert']\n  process = subprocess.Popen(cmd, env={'ADB': ADB},\n                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n  # this also blocks until command finishes\n  (stdout, stderr) = process.communicate()\n  print stdout\n  print 'Stderr: %s' % stderr\n  return process.returncode == 0\n\nif not installASAN():\n  print 'Trying to revert the ASAN install and then re-install'\n  # ASAN script sometimes has issues if it was interrupted or partially applied\n  # Try reverting it, then re-enabling it\n  if not installASAN(revert=True):\n    raise Exception('reverting ASAN install failed')\n\n  # Sleep because device does not reboot instantly\n  time.sleep(10)\n\n  if not installASAN():\n    raise Exception('Tried twice to setup ASAN and failed.')\n\n# Sleep because device does not reboot instantly\ntime.sleep(10)\nwait_for_device()\n",
       "/opt/infra-android/tools/adb",
       "[START_DIR]/android_ndk_linux/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/6.0.2/bin/asan_device_setup"
     ],
@@ -145,22 +145,38 @@
       "@@@STEP_LOG_LINE@python.inline@  subprocess.check_output([ADB, 'reboot'])@@@",
       "@@@STEP_LOG_LINE@python.inline@  wait_for_device()@@@",
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@# ASAN setup script is idempotent, either it installs it or says it's installed@@@",
-      "@@@STEP_LOG_LINE@python.inline@output = subprocess.check_output([ADB, 'wait-for-device'])@@@",
-      "@@@STEP_LOG_LINE@python.inline@process = subprocess.Popen([ASAN_SETUP], env={'ADB': ADB},@@@",
-      "@@@STEP_LOG_LINE@python.inline@                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)@@@",
+      "@@@STEP_LOG_LINE@python.inline@def installASAN(revert=False):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # ASAN setup script is idempotent, either it installs it or@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # says it's installed.  Returns True on success, false otherwise.@@@",
+      "@@@STEP_LOG_LINE@python.inline@  out = subprocess.check_output([ADB, 'wait-for-device'])@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print out@@@",
+      "@@@STEP_LOG_LINE@python.inline@  cmd = [ASAN_SETUP]@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if revert:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    cmd = [ASAN_SETUP, '--revert']@@@",
+      "@@@STEP_LOG_LINE@python.inline@  process = subprocess.Popen(cmd, env={'ADB': ADB},@@@",
+      "@@@STEP_LOG_LINE@python.inline@                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)@@@",
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@# this also blocks until command finishes@@@",
-      "@@@STEP_LOG_LINE@python.inline@(stdout, stderr) = process.communicate()@@@",
-      "@@@STEP_LOG_LINE@python.inline@print stdout@@@",
-      "@@@STEP_LOG_LINE@python.inline@print 'Stderr: %s' % stderr@@@",
-      "@@@STEP_LOG_LINE@python.inline@if process.returncode:@@@",
-      "@@@STEP_LOG_LINE@python.inline@  raise Exception('setup ASAN returned with non-zero exit code: %d' %@@@",
-      "@@@STEP_LOG_LINE@python.inline@                  process.returncode)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # this also blocks until command finishes@@@",
+      "@@@STEP_LOG_LINE@python.inline@  (stdout, stderr) = process.communicate()@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print stdout@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print 'Stderr: %s' % stderr@@@",
+      "@@@STEP_LOG_LINE@python.inline@  return process.returncode == 0@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not installASAN():@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print 'Trying to revert the ASAN install and then re-install'@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # ASAN script sometimes has issues if it was interrupted or partially applied@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # Try reverting it, then re-enabling it@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if not installASAN(revert=True):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    raise Exception('reverting ASAN install failed')@@@",
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if 'Please wait until the device restarts' in stdout:@@@",
       "@@@STEP_LOG_LINE@python.inline@  # Sleep because device does not reboot instantly@@@",
-      "@@@STEP_LOG_LINE@python.inline@  time.sleep(30)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  time.sleep(10)@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if not installASAN():@@@",
+      "@@@STEP_LOG_LINE@python.inline@    raise Exception('Tried twice to setup ASAN and failed.')@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@# Sleep because device does not reboot instantly@@@",
+      "@@@STEP_LOG_LINE@python.inline@time.sleep(10)@@@",
       "@@@STEP_LOG_LINE@python.inline@wait_for_device()@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
diff --git a/infra/bots/recipe_modules/flavor/gn_android_flavor.py b/infra/bots/recipe_modules/flavor/gn_android_flavor.py
index 8348053407..1c352afc28 100644
--- a/infra/bots/recipe_modules/flavor/gn_android_flavor.py
+++ b/infra/bots/recipe_modules/flavor/gn_android_flavor.py
@@ -389,22 +389,38 @@ if 'already disabled' not in output:
   subprocess.check_output([ADB, 'reboot'])
   wait_for_device()
 
-# ASAN setup script is idempotent, either it installs it or says it's installed
-output = subprocess.check_output([ADB, 'wait-for-device'])
-process = subprocess.Popen([ASAN_SETUP], env={'ADB': ADB},
-                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-# this also blocks until command finishes
-(stdout, stderr) = process.communicate()
-print stdout
-print 'Stderr: %s' % stderr
-if process.returncode:
-  raise Exception('setup ASAN returned with non-zero exit code: %d' %
-                  process.returncode)
-
-if 'Please wait until the device restarts' in stdout:
+def installASAN(revert=False):
+  # ASAN setup script is idempotent, either it installs it or
+  # says it's installed.  Returns True on success, false otherwise.
+  out = subprocess.check_output([ADB, 'wait-for-device'])
+  print out
+  cmd = [ASAN_SETUP]
+  if revert:
+    cmd = [ASAN_SETUP, '--revert']
+  process = subprocess.Popen(cmd, env={'ADB': ADB},
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+  # this also blocks until command finishes
+  (stdout, stderr) = process.communicate()
+  print stdout
+  print 'Stderr: %s' % stderr
+  return process.returncode == 0
+
+if not installASAN():
+  print 'Trying to revert the ASAN install and then re-install'
+  # ASAN script sometimes has issues if it was interrupted or partially applied
+  # Try reverting it, then re-enabling it
+  if not installASAN(revert=True):
+    raise Exception('reverting ASAN install failed')
+
   # Sleep because device does not reboot instantly
-  time.sleep(30)
+  time.sleep(10)
+
+  if not installASAN():
+    raise Exception('Tried twice to setup ASAN and failed.')
+
+# Sleep because device does not reboot instantly
+time.sleep(10)
 wait_for_device()
 """,
         args = [self.ADB_BINARY, asan_setup],