From cf9365a0bc94e2550350dd652a8f2fd2ba80a7f8 Mon Sep 17 00:00:00 2001 From: Ben Wagner Date: Fri, 8 Sep 2017 14:06:38 -0400 Subject: Retry adb commands on flaky devices. No-Try: true Change-Id: I3519131cf064d0a9676abd0fb9d7e946646ce27f Reviewed-on: https://skia-review.googlesource.com/43564 Reviewed-by: Kevin Lubick Commit-Queue: Ben Wagner --- .../retry_adb_command_retries_exhausted.json | 396 +++++++++++++++++++++ 1 file changed, 396 insertions(+) create mode 100644 infra/bots/recipe_modules/flavor/examples/full.expected/retry_adb_command_retries_exhausted.json (limited to 'infra/bots/recipe_modules/flavor/examples/full.expected/retry_adb_command_retries_exhausted.json') diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/retry_adb_command_retries_exhausted.json b/infra/bots/recipe_modules/flavor/examples/full.expected/retry_adb_command_retries_exhausted.json new file mode 100644 index 0000000000..830a4e0423 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/examples/full.expected/retry_adb_command_retries_exhausted.json @@ -0,0 +1,396 @@ +[ + { + "cmd": [ + "python", + "-u", + "[START_DIR]/skia/bin/fetch-gn" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "fetch-gn" + }, + { + "cmd": [ + "[START_DIR]/skia/bin/gn", + "gen", + "[START_DIR]/out/Debug", + "--args=extra_cflags=[\"-O1\"] ndk=\"[START_DIR]/android_ndk_linux\" target_cpu=\"None\"" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "gn gen" + }, + { + "cmd": [ + "ninja", + "-C", + "[START_DIR]/out/Debug" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "ninja" + }, + { + "cmd": [ + "adb", + "push", + "file.txt", + "file.txt" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "push file.txt file.txt" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "results_dir" + ], + "infra_step": true, + "name": "rmtree results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "results_dir" + ], + "infra_step": true, + "name": "makedirs results_dir" + }, + { + "cmd": [ + "adb", + "shell", + "rm", + "-rf", + "device_results_dir" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "rm device_results_dir" + }, + { + "cmd": [ + "adb", + "shell", + "mkdir", + "-p", + "device_results_dir" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "mkdir device_results_dir" + }, + { + "cmd": [ + "adb", + "shell", + "mkdir", + "-p", + "/sdcard/revenge_of_the_skiabot/resources" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "mkdir /sdcard/revenge_of_the_skiabot/resources", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_EXCEPTION@@@" + ] + }, + { + "cmd": [ + "adb", + "kill-server" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "kill adb server after failure of 'mkdir /sdcard/revenge_of_the_skiabot/resources' (attempt 1)", + "timeout": 30 + }, + { + "cmd": [ + "python", + "-u", + "\nimport subprocess\nimport sys\nimport time\nkicks = 0\nwhile True:\n times = 0\n while times < 30:\n print 'Waiting for the device to be connected and ready.'\n try:\n times += 1\n output = subprocess.check_output(['adb', 'shell',\n 'getprop',\n 'sys.boot_completed'])\n if '1' in output:\n print 'Connected'\n sys.exit(0)\n except subprocess.CalledProcessError:\n # no device connected/authorized yet\n pass\n time.sleep(5)\n if kicks >= 3:\n break\n print 'Giving the device a \"kick\" by trying to reboot it.'\n kicks += 1\n print subprocess.check_output(['adb', 'reboot'])\nprint 'Timed out waiting for device'\nsys.exit(1)\n" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "wait for device after failure of 'mkdir /sdcard/revenge_of_the_skiabot/resources' (attempt 1)", + "timeout": 660, + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import subprocess@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@import time@@@", + "@@@STEP_LOG_LINE@python.inline@kicks = 0@@@", + "@@@STEP_LOG_LINE@python.inline@while True:@@@", + "@@@STEP_LOG_LINE@python.inline@ times = 0@@@", + "@@@STEP_LOG_LINE@python.inline@ while times < 30:@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Waiting for the device to be connected and ready.'@@@", + "@@@STEP_LOG_LINE@python.inline@ try:@@@", + "@@@STEP_LOG_LINE@python.inline@ times += 1@@@", + "@@@STEP_LOG_LINE@python.inline@ output = subprocess.check_output(['adb', 'shell',@@@", + "@@@STEP_LOG_LINE@python.inline@ 'getprop',@@@", + "@@@STEP_LOG_LINE@python.inline@ 'sys.boot_completed'])@@@", + "@@@STEP_LOG_LINE@python.inline@ if '1' in output:@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Connected'@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(0)@@@", + "@@@STEP_LOG_LINE@python.inline@ except subprocess.CalledProcessError:@@@", + "@@@STEP_LOG_LINE@python.inline@ # no device connected/authorized yet@@@", + "@@@STEP_LOG_LINE@python.inline@ pass@@@", + "@@@STEP_LOG_LINE@python.inline@ time.sleep(5)@@@", + "@@@STEP_LOG_LINE@python.inline@ if kicks >= 3:@@@", + "@@@STEP_LOG_LINE@python.inline@ break@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Giving the device a \"kick\" by trying to reboot it.'@@@", + "@@@STEP_LOG_LINE@python.inline@ kicks += 1@@@", + "@@@STEP_LOG_LINE@python.inline@ print subprocess.check_output(['adb', 'reboot'])@@@", + "@@@STEP_LOG_LINE@python.inline@print 'Timed out waiting for device'@@@", + "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "adb", + "shell", + "mkdir", + "-p", + "/sdcard/revenge_of_the_skiabot/resources" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "mkdir /sdcard/revenge_of_the_skiabot/resources (attempt 2)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_EXCEPTION@@@" + ] + }, + { + "cmd": [ + "adb", + "kill-server" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "kill adb server after failure of 'mkdir /sdcard/revenge_of_the_skiabot/resources' (attempt 2)", + "timeout": 30 + }, + { + "cmd": [ + "python", + "-u", + "\nimport subprocess\nimport sys\nimport time\nkicks = 0\nwhile True:\n times = 0\n while times < 30:\n print 'Waiting for the device to be connected and ready.'\n try:\n times += 1\n output = subprocess.check_output(['adb', 'shell',\n 'getprop',\n 'sys.boot_completed'])\n if '1' in output:\n print 'Connected'\n sys.exit(0)\n except subprocess.CalledProcessError:\n # no device connected/authorized yet\n pass\n time.sleep(5)\n if kicks >= 3:\n break\n print 'Giving the device a \"kick\" by trying to reboot it.'\n kicks += 1\n print subprocess.check_output(['adb', 'reboot'])\nprint 'Timed out waiting for device'\nsys.exit(1)\n" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "wait for device after failure of 'mkdir /sdcard/revenge_of_the_skiabot/resources' (attempt 2)", + "timeout": 660, + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import subprocess@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@import time@@@", + "@@@STEP_LOG_LINE@python.inline@kicks = 0@@@", + "@@@STEP_LOG_LINE@python.inline@while True:@@@", + "@@@STEP_LOG_LINE@python.inline@ times = 0@@@", + "@@@STEP_LOG_LINE@python.inline@ while times < 30:@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Waiting for the device to be connected and ready.'@@@", + "@@@STEP_LOG_LINE@python.inline@ try:@@@", + "@@@STEP_LOG_LINE@python.inline@ times += 1@@@", + "@@@STEP_LOG_LINE@python.inline@ output = subprocess.check_output(['adb', 'shell',@@@", + "@@@STEP_LOG_LINE@python.inline@ 'getprop',@@@", + "@@@STEP_LOG_LINE@python.inline@ 'sys.boot_completed'])@@@", + "@@@STEP_LOG_LINE@python.inline@ if '1' in output:@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Connected'@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(0)@@@", + "@@@STEP_LOG_LINE@python.inline@ except subprocess.CalledProcessError:@@@", + "@@@STEP_LOG_LINE@python.inline@ # no device connected/authorized yet@@@", + "@@@STEP_LOG_LINE@python.inline@ pass@@@", + "@@@STEP_LOG_LINE@python.inline@ time.sleep(5)@@@", + "@@@STEP_LOG_LINE@python.inline@ if kicks >= 3:@@@", + "@@@STEP_LOG_LINE@python.inline@ break@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Giving the device a \"kick\" by trying to reboot it.'@@@", + "@@@STEP_LOG_LINE@python.inline@ kicks += 1@@@", + "@@@STEP_LOG_LINE@python.inline@ print subprocess.check_output(['adb', 'reboot'])@@@", + "@@@STEP_LOG_LINE@python.inline@print 'Timed out waiting for device'@@@", + "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "adb", + "shell", + "mkdir", + "-p", + "/sdcard/revenge_of_the_skiabot/resources" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "mkdir /sdcard/revenge_of_the_skiabot/resources (attempt 3)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_EXCEPTION@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport subprocess\nimport sys\nout = sys.argv[1]\nlog = subprocess.check_output(['adb', 'logcat', '-d'])\nfor line in log.split('\\n'):\n tokens = line.split()\n if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':\n addr, path = tokens[-2:]\n local = os.path.join(out, os.path.basename(path))\n if os.path.exists(local):\n sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])\n line = line.replace(addr, addr + ' ' + sym.strip())\n print line\n", + "[START_DIR]/out/Debug" + ], + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "dump log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import subprocess@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@out = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@log = subprocess.check_output(['adb', 'logcat', '-d'])@@@", + "@@@STEP_LOG_LINE@python.inline@for line in log.split('\\n'):@@@", + "@@@STEP_LOG_LINE@python.inline@ tokens = line.split()@@@", + "@@@STEP_LOG_LINE@python.inline@ if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':@@@", + "@@@STEP_LOG_LINE@python.inline@ addr, path = tokens[-2:]@@@", + "@@@STEP_LOG_LINE@python.inline@ local = os.path.join(out, os.path.basename(path))@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(local):@@@", + "@@@STEP_LOG_LINE@python.inline@ sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])@@@", + "@@@STEP_LOG_LINE@python.inline@ line = line.replace(addr, addr + ' ' + sym.strip())@@@", + "@@@STEP_LOG_LINE@python.inline@ print line@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "adb", + "shell", + "reboot", + "-p" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "shut down device to quarantine bot" + }, + { + "cmd": [ + "adb", + "kill-server" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "kill adb server" + }, + { + "name": "$result", + "reason": "Infra Failure: Step('mkdir /sdcard/revenge_of_the_skiabot/resources (attempt 3)') returned 1", + "recipe_result": null, + "status_code": 1 + } +] \ No newline at end of file -- cgit v1.2.3