diff options
7 files changed, 720 insertions, 103 deletions
diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All.json index fa51d74d82..f141a1bf2d 100644 --- a/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All.json +++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All.json @@ -208,26 +208,6 @@ }, { "cmd": [ - "ssh", - "-oConnectTimeout=15", - "-oBatchMode=yes", - "-t", - "-t", - "root@192.168.1.2", - "rm", - "/cache/skia/nanobench" - ], - "env": { - "BUILDTYPE": "Release", - "CHROME_HEADLESS": "1", - "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", - "SKIA_OUT": "[START_DIR]/out" - }, - "infra_step": true, - "name": "Delete old nanobench" - }, - { - "cmd": [ "python", "-u", "\nimport os\nimport subprocess\nimport sys\nhost = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n p = os.path.relpath(d, host)\n if p != '.' and p.startswith('.'):\n continue\n for f in fs:\n print os.path.join(p,f)\n hp = os.path.realpath(os.path.join(host, p, f))\n if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n print \"Skipping because it is too big\"\n else:\n subprocess.check_call(['adb', 'push',\n hp, os.path.join(device, p, f)])\n", diff --git a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py b/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py index 771bc8e45b..11ee674ab2 100644 --- a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py @@ -18,6 +18,20 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): self.m.vars.android_bin_dir = self.m.path.join(self.m.vars.android_bin_dir, 'bin') + # Disk space is extremely tight on the Chromecasts (~100M) There is not + # enough space on the android_data_dir (/cache/skia) to fit the images, + # resources, executable and output the dm images. So, we have dm_out be + # on the tempfs (i.e. RAM) /dev/shm. (which is about 140M) + + self.device_dirs = default_flavor.DeviceDirs( + dm_dir = '/dev/shm/skia/dm_out', + perf_data_dir = self.m.vars.android_data_dir + 'perf', + resource_dir = self.m.vars.android_data_dir + 'resources', + images_dir = self.m.vars.android_data_dir + 'images', + skp_dir = self.m.vars.android_data_dir + 'skps', + svg_dir = self.m.vars.android_data_dir + 'svgs', + tmp_dir = self.m.vars.android_data_dir) + @property def user_ip_host(self): if not self._user_ip: @@ -97,11 +111,6 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): super(GNChromecastFlavorUtils, self).install() self._adb('mkdir ' + self.m.vars.android_bin_dir, 'shell', 'mkdir', '-p', self.m.vars.android_bin_dir) - # TODO(kjlubick): Remove this after we are backfilled up and don't need - # to manually delete this - self._ssh('Delete old nanobench', 'rm', '/cache/skia/nanobench', - abort_on_failure=False, fail_build_on_failure=False, - infra_step=True) def _adb(self, title, *cmd, **kwargs): if not self._ever_ran_adb: diff --git a/infra/bots/recipes/perf.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Debug-All.json b/infra/bots/recipes/perf.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Debug-All.json index 77568504b0..088cfc00f2 100644 --- a/infra/bots/recipes/perf.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Debug-All.json +++ b/infra/bots/recipes/perf.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Debug-All.json @@ -76,26 +76,6 @@ }, { "cmd": [ - "ssh", - "-oConnectTimeout=15", - "-oBatchMode=yes", - "-t", - "-t", - "root@192.168.1.2", - "rm", - "/cache/skia/nanobench" - ], - "env": { - "BUILDTYPE": "Debug", - "CHROME_HEADLESS": "1", - "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", - "SKIA_OUT": "[START_DIR]/out" - }, - "infra_step": true, - "name": "Delete old nanobench" - }, - { - "cmd": [ "python", "-u", "\nimport os\nimport subprocess\nimport sys\nhost = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n p = os.path.relpath(d, host)\n if p != '.' and p.startswith('.'):\n continue\n for f in fs:\n print os.path.join(p,f)\n hp = os.path.realpath(os.path.join(host, p, f))\n if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n print \"Skipping because it is too big\"\n else:\n subprocess.check_call(['adb', 'push',\n hp, os.path.join(device, p, f)])\n", diff --git a/infra/bots/recipes/perf.expected/Perf-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All.json b/infra/bots/recipes/perf.expected/Perf-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All.json index 05492aab68..25e8982b43 100644 --- a/infra/bots/recipes/perf.expected/Perf-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All.json +++ b/infra/bots/recipes/perf.expected/Perf-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All.json @@ -76,26 +76,6 @@ }, { "cmd": [ - "ssh", - "-oConnectTimeout=15", - "-oBatchMode=yes", - "-t", - "-t", - "root@192.168.1.2", - "rm", - "/cache/skia/nanobench" - ], - "env": { - "BUILDTYPE": "Release", - "CHROME_HEADLESS": "1", - "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", - "SKIA_OUT": "[START_DIR]/out" - }, - "infra_step": true, - "name": "Delete old nanobench" - }, - { - "cmd": [ "python", "-u", "\nimport os\nimport subprocess\nimport sys\nhost = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n p = os.path.relpath(d, host)\n if p != '.' and p.startswith('.'):\n continue\n for f in fs:\n print os.path.join(p,f)\n hp = os.path.realpath(os.path.join(host, p, f))\n if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n print \"Skipping because it is too big\"\n else:\n subprocess.check_call(['adb', 'push',\n hp, os.path.join(device, p, f)])\n", diff --git a/infra/bots/recipes/test.expected/Test-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All.json b/infra/bots/recipes/test.expected/Test-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All.json new file mode 100644 index 0000000000..d6c807ec0a --- /dev/null +++ b/infra/bots/recipes/test.expected/Test-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All.json @@ -0,0 +1,644 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport os\nCHROMECAST_IP_FILE = os.path.expanduser('~/chromecast.txt')\nwith open(CHROMECAST_IP_FILE, 'r') as f:\n print f.read()\n" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "read chromecast ip", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@CHROMECAST_IP_FILE = os.path.expanduser('~/chromecast.txt')@@@", + "@@@STEP_LOG_LINE@python.inline@with open(CHROMECAST_IP_FILE, 'r') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ print f.read()@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "adb", + "connect", + "192.168.1.2:5555" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "adb connect 192.168.1.2:5555" + }, + { + "cmd": [ + "adb", + "shell", + "mkdir", + "-p", + "/cache/skia/resources" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "mkdir /cache/skia/resources" + }, + { + "cmd": [ + "adb", + "shell", + "mkdir", + "-p", + "/cache/skia/bin" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "mkdir /cache/skia/bin" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport subprocess\nimport sys\nhost = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n p = os.path.relpath(d, host)\n if p != '.' and p.startswith('.'):\n continue\n for f in fs:\n print os.path.join(p,f)\n hp = os.path.realpath(os.path.join(host, p, f))\n if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n print \"Skipping because it is too big\"\n else:\n subprocess.check_call(['adb', 'push',\n hp, os.path.join(device, p, f)])\n", + "[START_DIR]/skia/resources", + "/cache/skia/resources" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "push [START_DIR]/skia/resources/* /cache/skia/resources", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import subprocess@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@host = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@device = sys.argv[2]@@@", + "@@@STEP_LOG_LINE@python.inline@for d, _, fs in os.walk(host):@@@", + "@@@STEP_LOG_LINE@python.inline@ p = os.path.relpath(d, host)@@@", + "@@@STEP_LOG_LINE@python.inline@ if p != '.' and p.startswith('.'):@@@", + "@@@STEP_LOG_LINE@python.inline@ continue@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in fs:@@@", + "@@@STEP_LOG_LINE@python.inline@ print os.path.join(p,f)@@@", + "@@@STEP_LOG_LINE@python.inline@ hp = os.path.realpath(os.path.join(host, p, f))@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.stat(hp).st_size > (1.5 * 1024 * 1024):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"Skipping because it is too big\"@@@", + "@@@STEP_LOG_LINE@python.inline@ else:@@@", + "@@@STEP_LOG_LINE@python.inline@ subprocess.check_call(['adb', 'push',@@@", + "@@@STEP_LOG_LINE@python.inline@ hp, os.path.join(device, p, f)])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/skp/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get downloaded SKP VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SKP_VERSION" + ], + "infra_step": true, + "name": "write SKP_VERSION" + }, + { + "cmd": [ + "adb", + "shell", + "cat", + "/cache/skia/SKP_VERSION" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "read /cache/skia/SKP_VERSION", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "adb", + "shell", + "rm", + "-f", + "/cache/skia/SKP_VERSION" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "rm /cache/skia/SKP_VERSION" + }, + { + "cmd": [ + "adb", + "shell", + "rm", + "-r", + "/cache/skia/skps" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "rm /cache/skia/skps" + }, + { + "cmd": [ + "adb", + "shell", + "mkdir", + "-p", + "/cache/skia/skps" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "mkdir /cache/skia/skps" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport subprocess\nimport sys\nhost = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n p = os.path.relpath(d, host)\n if p != '.' and p.startswith('.'):\n continue\n for f in fs:\n print os.path.join(p,f)\n hp = os.path.realpath(os.path.join(host, p, f))\n if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n print \"Skipping because it is too big\"\n else:\n subprocess.check_call(['adb', 'push',\n hp, os.path.join(device, p, f)])\n", + "[START_DIR]/skp", + "/cache/skia/skps" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "push [START_DIR]/skp/* /cache/skia/skps", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import subprocess@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@host = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@device = sys.argv[2]@@@", + "@@@STEP_LOG_LINE@python.inline@for d, _, fs in os.walk(host):@@@", + "@@@STEP_LOG_LINE@python.inline@ p = os.path.relpath(d, host)@@@", + "@@@STEP_LOG_LINE@python.inline@ if p != '.' and p.startswith('.'):@@@", + "@@@STEP_LOG_LINE@python.inline@ continue@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in fs:@@@", + "@@@STEP_LOG_LINE@python.inline@ print os.path.join(p,f)@@@", + "@@@STEP_LOG_LINE@python.inline@ hp = os.path.realpath(os.path.join(host, p, f))@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.stat(hp).st_size > (1.5 * 1024 * 1024):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"Skipping because it is too big\"@@@", + "@@@STEP_LOG_LINE@python.inline@ else:@@@", + "@@@STEP_LOG_LINE@python.inline@ subprocess.check_call(['adb', 'push',@@@", + "@@@STEP_LOG_LINE@python.inline@ hp, os.path.join(device, p, f)])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "adb", + "push", + "[START_DIR]/tmp/SKP_VERSION", + "/cache/skia/SKP_VERSION" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "push [START_DIR]/tmp/SKP_VERSION /cache/skia/SKP_VERSION" + }, + { + "cmd": [ + "adb", + "shell", + "rm", + "-r", + "/dev/shm/skia/dm_out" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "rm /dev/shm/skia/dm_out" + }, + { + "cmd": [ + "adb", + "shell", + "mkdir", + "-p", + "/dev/shm/skia/dm_out" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "mkdir /dev/shm/skia/dm_out" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "cmd": [ + "python", + "-u", + "\nimport contextlib\nimport math\nimport socket\nimport sys\nimport time\nimport urllib2\n\nHASHES_URL = 'https://storage.googleapis.com/skia-infra-gm/hash_files/gold-prod-hashes.txt'\nRETRIES = 5\nTIMEOUT = 60\nWAIT_BASE = 15\n\nsocket.setdefaulttimeout(TIMEOUT)\nfor retry in range(RETRIES):\n try:\n with contextlib.closing(\n urllib2.urlopen(HASHES_URL, timeout=TIMEOUT)) as w:\n hashes = w.read()\n with open(sys.argv[1], 'w') as f:\n f.write(hashes)\n break\n except Exception as e:\n print 'Failed to get uninteresting hashes from %s:' % HASHES_URL\n print e\n if retry == RETRIES:\n raise\n waittime = WAIT_BASE * math.pow(2, retry)\n print 'Retry in %d seconds.' % waittime\n time.sleep(waittime)\n", + "[START_DIR]/tmp/uninteresting_hashes.txt" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "get uninteresting hashes", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import contextlib@@@", + "@@@STEP_LOG_LINE@python.inline@import math@@@", + "@@@STEP_LOG_LINE@python.inline@import socket@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@import time@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@HASHES_URL = 'https://storage.googleapis.com/skia-infra-gm/hash_files/gold-prod-hashes.txt'@@@", + "@@@STEP_LOG_LINE@python.inline@RETRIES = 5@@@", + "@@@STEP_LOG_LINE@python.inline@TIMEOUT = 60@@@", + "@@@STEP_LOG_LINE@python.inline@WAIT_BASE = 15@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@socket.setdefaulttimeout(TIMEOUT)@@@", + "@@@STEP_LOG_LINE@python.inline@for retry in range(RETRIES):@@@", + "@@@STEP_LOG_LINE@python.inline@ try:@@@", + "@@@STEP_LOG_LINE@python.inline@ with contextlib.closing(@@@", + "@@@STEP_LOG_LINE@python.inline@ urllib2.urlopen(HASHES_URL, timeout=TIMEOUT)) as w:@@@", + "@@@STEP_LOG_LINE@python.inline@ hashes = w.read()@@@", + "@@@STEP_LOG_LINE@python.inline@ with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(hashes)@@@", + "@@@STEP_LOG_LINE@python.inline@ break@@@", + "@@@STEP_LOG_LINE@python.inline@ except Exception as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Failed to get uninteresting hashes from %s:' % HASHES_URL@@@", + "@@@STEP_LOG_LINE@python.inline@ print e@@@", + "@@@STEP_LOG_LINE@python.inline@ if retry == RETRIES:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@ waittime = WAIT_BASE * math.pow(2, retry)@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Retry in %d seconds.' % waittime@@@", + "@@@STEP_LOG_LINE@python.inline@ time.sleep(waittime)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "adb", + "push", + "[START_DIR]/tmp/uninteresting_hashes.txt", + "/cache/skia/uninteresting_hashes.txt" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "push [START_DIR]/tmp/uninteresting_hashes.txt /cache/skia/uninteresting_hashes.txt" + }, + { + "cmd": [ + "python", + "-u", + "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n" + ], + "name": "get swarming bot id", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n" + ], + "name": "get swarming task id", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "adb", + "push", + "[START_DIR]/out/Release/dm", + "/cache/skia/bin" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "push dm" + }, + { + "cmd": [ + "ssh", + "-oConnectTimeout=15", + "-oBatchMode=yes", + "-t", + "-t", + "root@192.168.1.2", + "/cache/skia/bin/dm", + "--resourcePath", + "/cache/skia/resources", + "--skps", + "/cache/skia/skps", + "--images", + "/cache/skia/resources/color_wheel.jpg", + "--nameByHash", + "--properties", + "gitHash", + "abc123", + "builder", + "Test-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All", + "buildbucket_build_id", + "123454321", + "swarming_bot_id", + "skia-bot-123", + "swarming_task_id", + "123456", + "--uninterestingHashesFile", + "/cache/skia/uninteresting_hashes.txt", + "--writePath", + "/dev/shm/skia/dm_out", + "--dont_write", + "pdf", + "--randomProcessorTest", + "--threads", + "0", + "--ignoreSigInt", + "--nogpu", + "--config", + "8888", + "--src", + "tests", + "gm", + "image", + "colorImage", + "--blacklist", + "_", + "test", + "_", + "GrShape", + "--match", + "~matrixconvolution", + "~blur_image_filter", + "~blur_0.01", + "~readpixels", + "~F16Stages", + "~SkRasterPipeline_tail", + "~GM_animated-image-blurs", + "~verylarge", + "~ImageFilterBlurLargeImage", + "~TextBlobCache", + "~bigbitmaprect_", + "~savelayer_clipmask", + "~DrawBitmapRect", + "--nonativeFonts", + "--verbose" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "dm" + }, + { + "cmd": [ + "adb", + "pull", + "/dev/shm/skia/dm_out", + "[CUSTOM_[SWARM_OUT_DIR]]/dm" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "pull /dev/shm/skia/dm_out [CUSTOM_[SWARM_OUT_DIR]]/dm" + }, + { + "cmd": [ + "ssh", + "-oConnectTimeout=15", + "-oBatchMode=yes", + "-t", + "-t", + "root@192.168.1.2", + "rm", + "-r", + "/cache/skia/bin" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "Delete executables" + }, + { + "cmd": [ + "adb", + "disconnect" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "disconnect" + }, + { + "cmd": [ + "adb", + "connect", + "192.168.1.2:5555" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "adb connect 192.168.1.2:5555 (2)" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport subprocess\nimport sys\nout = sys.argv[1]\nlog = subprocess.check_output(['adb', 'logcat', '-d'])\nfor line in log.split('\\n'):\n tokens = line.split()\n if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':\n addr, path = tokens[-2:]\n local = os.path.join(out, os.path.basename(path))\n if os.path.exists(local):\n sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])\n line = line.replace(addr, addr + ' ' + sym.strip())\n print line\n", + "[START_DIR]/out/Release" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "dump log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import subprocess@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@out = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@log = subprocess.check_output(['adb', 'logcat', '-d'])@@@", + "@@@STEP_LOG_LINE@python.inline@for line in log.split('\\n'):@@@", + "@@@STEP_LOG_LINE@python.inline@ tokens = line.split()@@@", + "@@@STEP_LOG_LINE@python.inline@ if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':@@@", + "@@@STEP_LOG_LINE@python.inline@ addr, path = tokens[-2:]@@@", + "@@@STEP_LOG_LINE@python.inline@ local = os.path.join(out, os.path.basename(path))@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(local):@@@", + "@@@STEP_LOG_LINE@python.inline@ sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])@@@", + "@@@STEP_LOG_LINE@python.inline@ line = line.replace(addr, addr + ' ' + sym.strip())@@@", + "@@@STEP_LOG_LINE@python.inline@ print line@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "adb", + "disconnect" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "disconnect (2)" + }, + { + "cmd": [ + "adb", + "kill-server" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "kill adb server" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipes/test.expected/Test-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All.json b/infra/bots/recipes/test.expected/Test-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All.json index 3bd71ebb5f..f790fc6d0e 100644 --- a/infra/bots/recipes/test.expected/Test-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All.json +++ b/infra/bots/recipes/test.expected/Test-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All.json @@ -76,26 +76,6 @@ }, { "cmd": [ - "ssh", - "-oConnectTimeout=15", - "-oBatchMode=yes", - "-t", - "-t", - "root@192.168.1.2", - "rm", - "/cache/skia/nanobench" - ], - "env": { - "BUILDTYPE": "Release", - "CHROME_HEADLESS": "1", - "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]", - "SKIA_OUT": "[START_DIR]/out" - }, - "infra_step": true, - "name": "Delete old nanobench" - }, - { - "cmd": [ "python", "-u", "\nimport os\nimport subprocess\nimport sys\nhost = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n p = os.path.relpath(d, host)\n if p != '.' and p.startswith('.'):\n continue\n for f in fs:\n print os.path.join(p,f)\n hp = os.path.realpath(os.path.join(host, p, f))\n if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n print \"Skipping because it is too big\"\n else:\n subprocess.check_call(['adb', 'push',\n hp, os.path.join(device, p, f)])\n", @@ -293,7 +273,7 @@ "shell", "rm", "-r", - "/cache/skia/dm_out" + "/dev/shm/skia/dm_out" ], "cwd": "[START_DIR]/skia", "env": { @@ -303,7 +283,7 @@ "SKIA_OUT": "[START_DIR]/out" }, "infra_step": true, - "name": "rm /cache/skia/dm_out" + "name": "rm /dev/shm/skia/dm_out" }, { "cmd": [ @@ -311,7 +291,7 @@ "shell", "mkdir", "-p", - "/cache/skia/dm_out" + "/dev/shm/skia/dm_out" ], "cwd": "[START_DIR]/skia", "env": { @@ -321,7 +301,7 @@ "SKIA_OUT": "[START_DIR]/out" }, "infra_step": true, - "name": "mkdir /cache/skia/dm_out" + "name": "mkdir /dev/shm/skia/dm_out" }, { "cmd": [ @@ -458,22 +438,37 @@ "-t", "root@192.168.1.2", "/cache/skia/bin/dm", - "--undefok", "--resourcePath", "/cache/skia/resources", "--skps", "/cache/skia/skps", "--images", "/cache/skia/resources/color_wheel.jpg", + "--nameByHash", + "--properties", + "gitHash", + "abc123", + "builder", + "Test-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All", + "buildbucket_build_id", + "123454321", + "swarming_bot_id", + "skia-bot-123", + "swarming_task_id", + "123456", + "--uninterestingHashesFile", + "/cache/skia/uninteresting_hashes.txt", + "--writePath", + "/dev/shm/skia/dm_out", "--dont_write", "pdf", "--randomProcessorTest", "--threads", "0", + "--ignoreSigInt", "--nocpu", "--config", "gles", - "8888", "--src", "tests", "gm", @@ -585,7 +580,16 @@ "~matrixconvolution", "~blur_image_filter", "~blur_0.01", + "~readpixels", + "~F16Stages", + "~SkRasterPipeline_tail", "~GM_animated-image-blurs", + "~verylarge", + "~ImageFilterBlurLargeImage", + "~TextBlobCache", + "~bigbitmaprect_", + "~savelayer_clipmask", + "~DrawBitmapRect", "--nonativeFonts", "--verbose" ], @@ -601,7 +605,7 @@ "cmd": [ "adb", "pull", - "/cache/skia/dm_out", + "/dev/shm/skia/dm_out", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], "cwd": "[START_DIR]/skia", @@ -612,7 +616,7 @@ "SKIA_OUT": "[START_DIR]/out" }, "infra_step": true, - "name": "pull /cache/skia/dm_out [CUSTOM_[SWARM_OUT_DIR]]/dm" + "name": "pull /dev/shm/skia/dm_out [CUSTOM_[SWARM_OUT_DIR]]/dm" }, { "cmd": [ diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py index 0e6e4e5bda..0b75ef1278 100644 --- a/infra/bots/recipes/test.py +++ b/infra/bots/recipes/test.py @@ -79,7 +79,8 @@ def dm_flags(api, bot): # is ignored and dm will keep attempting to proceed until we actually # exhaust the available resources. if ('NexusPlayer' in bot or - 'PixelC' in bot): + 'PixelC' in bot or + 'Chromecast' in bot): args.append('--ignoreSigInt') if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': @@ -115,6 +116,10 @@ def dm_flags(api, bot): if 'NativeFonts' in bot: configs = ['8888'] + # Just do the basic config on Chromecast to avoid OOM. + if 'Chromecast' in bot: + configs = ['8888'] + elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': args.append('--nocpu') @@ -219,7 +224,7 @@ def dm_flags(api, bot): configs = ['gles'] if 'Chromecast' in bot: - configs = ['gles', '8888'] + configs = ['gles'] # Test coverage counting path renderer. if 'CCPR' in bot: @@ -420,7 +425,7 @@ def dm_flags(api, bot): for test in ['bleed_alpha_image', 'bleed_alpha_image_shader']: blacklist(['serialize-8888', 'gm', '_', test]) # It looks like we skip these only for out-of-memory concerns. - if 'Win' in bot or 'Android' in bot or 'Chromecast' in bot: + if 'Win' in bot or 'Android' in bot: for test in ['verylargebitmap', 'verylarge_picture_image']: blacklist(['serialize-8888', 'gm', '_', test]) if 'Mac' in bot and 'CPU' in bot: @@ -524,11 +529,23 @@ def dm_flags(api, bot): if 'AndroidOne' in bot: # skia:4711 match.append('~WritePixels') - if 'Chromecast' in bot: # skia:6581 + if 'Chromecast' in bot: + # skia:6581 match.append('~matrixconvolution') match.append('~blur_image_filter') match.append('~blur_0.01') + # skia:7497 + match.append('~readpixels') # dies with "Caught signal 7 [Bus error]" + match.append('~F16Stages') + match.append('~SkRasterPipeline_tail') + # Blacklisted to avoid OOM (we see DM just end with "broken pipe") match.append('~GM_animated-image-blurs') + match.append('~verylarge') + match.append('~ImageFilterBlurLargeImage') + match.append('~TextBlobCache') + match.append('~bigbitmaprect_') + match.append('~savelayer_clipmask') + match.append('~DrawBitmapRect') if 'GalaxyS6' in bot: match.append('~SpecialImage') # skia:6338 @@ -838,21 +855,23 @@ def test_steps(api): args.append('--key') args.extend(key_params(api)) - if use_hash_file: - args.extend(['--uninterestingHashesFile', hashes_file]) - if api.vars.upload_dm_results: - args.extend(['--writePath', api.flavor.device_dirs.dm_dir]) if 'Chromecast' in api.vars.builder_cfg.get('os', ''): # Due to limited disk space, we only deal with skps and one image. args = [ 'dm', - '--undefok', # This helps branches that may not know new flags. '--resourcePath', api.flavor.device_dirs.resource_dir, '--skps', api.flavor.device_dirs.skp_dir, '--images', api.flavor.device_path_join( api.flavor.device_dirs.resource_dir, 'color_wheel.jpg'), - ] + '--nameByHash', + '--properties' + ] + properties + + if use_hash_file: + args.extend(['--uninterestingHashesFile', hashes_file]) + if api.vars.upload_dm_results: + args.extend(['--writePath', api.flavor.device_dirs.dm_dir]) args.extend(dm_flags(api, api.vars.builder_name)) @@ -910,6 +929,7 @@ TEST_BUILDERS = [ 'Test-ChromeOS-Clang-ASUSChromebookFlipC100-GPU-MaliT764-arm-Debug-All', ('Test-ChromeOS-Clang-AcerChromebookR13Convertible-GPU-PowerVRGX6250-' 'arm-Debug-All'), + 'Test-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All', 'Test-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All', 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN', 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-shard_00_10-Coverage', |