Add Correctness tests for Chromecast

Bug: skia:
NOTRY=true
Change-Id: Ib604bad8eda6ba4382b850f7c5f318fb0f114401
Reviewed-on: https://skia-review.googlesource.com/20051
Commit-Queue: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Eric Boren <borenet@google.com>

2 files changed, 159 insertions, 15 deletions

diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release.json
index fb0dc13b73..24eef46e3c 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release.json
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release.json
@@ -36,7 +36,8 @@
       "ninja",
       "-C",
       "[START_DIR]/out/Release",
-      "nanobench"
+      "nanobench",
+      "dm"
     ],
     "cwd": "[START_DIR]/skia",
     "env": {
@@ -187,9 +188,46 @@
   },
   {
     "cmd": [
+      "adb",
+      "shell",
+      "mkdir",
+      "-p",
+      "/cache/skia/bin"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "infra_step": true,
+    "name": "mkdir /cache/skia/bin"
+  },
+  {
+    "cmd": [
+      "ssh",
+      "-oConnectTimeout=15",
+      "-oBatchMode=yes",
+      "-t",
+      "-t",
+      "root@192.168.1.2",
+      "rm",
+      "/cache/skia/nanobench"
+    ],
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "name": "Delete old nanobench"
+  },
+  {
+    "cmd": [
       "python",
       "-u",
-      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    hp = os.path.realpath(os.path.join(host, p, f))\n    if os.stat(hp).st_size > (3 * 1024 * 1024):\n      print \"Skipping because it is too big\"\n    else:\n      subprocess.check_call(['adb', 'push',\n                            hp, os.path.join(device, p, f)])\n",
+      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    hp = os.path.realpath(os.path.join(host, p, f))\n    if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n      print \"Skipping because it is too big\"\n    else:\n      subprocess.check_call(['adb', 'push',\n                            hp, os.path.join(device, p, f)])\n",
       "[START_DIR]/skia/resources",
       "/cache/skia/resources"
     ],
@@ -215,7 +253,7 @@
       "@@@STEP_LOG_LINE@python.inline@  for f in fs:@@@",
       "@@@STEP_LOG_LINE@python.inline@    print os.path.join(p,f)@@@",
       "@@@STEP_LOG_LINE@python.inline@    hp = os.path.realpath(os.path.join(host, p, f))@@@",
-      "@@@STEP_LOG_LINE@python.inline@    if os.stat(hp).st_size > (3 * 1024 * 1024):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if os.stat(hp).st_size > (1.5 * 1024 * 1024):@@@",
       "@@@STEP_LOG_LINE@python.inline@      print \"Skipping because it is too big\"@@@",
       "@@@STEP_LOG_LINE@python.inline@    else:@@@",
       "@@@STEP_LOG_LINE@python.inline@      subprocess.check_call(['adb', 'push',@@@",
@@ -327,7 +365,7 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    hp = os.path.realpath(os.path.join(host, p, f))\n    if os.stat(hp).st_size > (3 * 1024 * 1024):\n      print \"Skipping because it is too big\"\n    else:\n      subprocess.check_call(['adb', 'push',\n                            hp, os.path.join(device, p, f)])\n",
+      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    hp = os.path.realpath(os.path.join(host, p, f))\n    if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n      print \"Skipping because it is too big\"\n    else:\n      subprocess.check_call(['adb', 'push',\n                            hp, os.path.join(device, p, f)])\n",
       "[START_DIR]/skp",
       "/cache/skia/skps"
     ],
@@ -353,7 +391,7 @@
       "@@@STEP_LOG_LINE@python.inline@  for f in fs:@@@",
       "@@@STEP_LOG_LINE@python.inline@    print os.path.join(p,f)@@@",
       "@@@STEP_LOG_LINE@python.inline@    hp = os.path.realpath(os.path.join(host, p, f))@@@",
-      "@@@STEP_LOG_LINE@python.inline@    if os.stat(hp).st_size > (3 * 1024 * 1024):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if os.stat(hp).st_size > (1.5 * 1024 * 1024):@@@",
       "@@@STEP_LOG_LINE@python.inline@      print \"Skipping because it is too big\"@@@",
       "@@@STEP_LOG_LINE@python.inline@    else:@@@",
       "@@@STEP_LOG_LINE@python.inline@      subprocess.check_call(['adb', 'push',@@@",
@@ -482,7 +520,7 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    hp = os.path.realpath(os.path.join(host, p, f))\n    if os.stat(hp).st_size > (3 * 1024 * 1024):\n      print \"Skipping because it is too big\"\n    else:\n      subprocess.check_call(['adb', 'push',\n                            hp, os.path.join(device, p, f)])\n",
+      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    hp = os.path.realpath(os.path.join(host, p, f))\n    if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n      print \"Skipping because it is too big\"\n    else:\n      subprocess.check_call(['adb', 'push',\n                            hp, os.path.join(device, p, f)])\n",
       "[START_DIR]/skimage",
       "/cache/skia/images"
     ],
@@ -508,7 +546,7 @@
       "@@@STEP_LOG_LINE@python.inline@  for f in fs:@@@",
       "@@@STEP_LOG_LINE@python.inline@    print os.path.join(p,f)@@@",
       "@@@STEP_LOG_LINE@python.inline@    hp = os.path.realpath(os.path.join(host, p, f))@@@",
-      "@@@STEP_LOG_LINE@python.inline@    if os.stat(hp).st_size > (3 * 1024 * 1024):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if os.stat(hp).st_size > (1.5 * 1024 * 1024):@@@",
       "@@@STEP_LOG_LINE@python.inline@      print \"Skipping because it is too big\"@@@",
       "@@@STEP_LOG_LINE@python.inline@    else:@@@",
       "@@@STEP_LOG_LINE@python.inline@      subprocess.check_call(['adb', 'push',@@@",
@@ -637,7 +675,7 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    hp = os.path.realpath(os.path.join(host, p, f))\n    if os.stat(hp).st_size > (3 * 1024 * 1024):\n      print \"Skipping because it is too big\"\n    else:\n      subprocess.check_call(['adb', 'push',\n                            hp, os.path.join(device, p, f)])\n",
+      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    hp = os.path.realpath(os.path.join(host, p, f))\n    if os.stat(hp).st_size > (1.5 * 1024 * 1024):\n      print \"Skipping because it is too big\"\n    else:\n      subprocess.check_call(['adb', 'push',\n                            hp, os.path.join(device, p, f)])\n",
       "[START_DIR]/svg",
       "/cache/skia/svgs"
     ],
@@ -663,7 +701,7 @@
       "@@@STEP_LOG_LINE@python.inline@  for f in fs:@@@",
       "@@@STEP_LOG_LINE@python.inline@    print os.path.join(p,f)@@@",
       "@@@STEP_LOG_LINE@python.inline@    hp = os.path.realpath(os.path.join(host, p, f))@@@",
-      "@@@STEP_LOG_LINE@python.inline@    if os.stat(hp).st_size > (3 * 1024 * 1024):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if os.stat(hp).st_size > (1.5 * 1024 * 1024):@@@",
       "@@@STEP_LOG_LINE@python.inline@      print \"Skipping because it is too big\"@@@",
       "@@@STEP_LOG_LINE@python.inline@    else:@@@",
       "@@@STEP_LOG_LINE@python.inline@      subprocess.check_call(['adb', 'push',@@@",
@@ -693,7 +731,7 @@
       "adb",
       "push",
       "[START_DIR]/out/Release/dm",
-      "/cache/skia/"
+      "/cache/skia/bin"
     ],
     "cwd": "[START_DIR]/skia",
     "env": {
@@ -713,7 +751,7 @@
       "-t",
       "-t",
       "root@192.168.1.2",
-      "/cache/skia/dm",
+      "/cache/skia/bin/dm",
       "--some-flag"
     ],
     "env": {
@@ -760,6 +798,55 @@
   },
   {
     "cmd": [
+      "ssh",
+      "-oConnectTimeout=15",
+      "-oBatchMode=yes",
+      "-t",
+      "-t",
+      "root@192.168.1.2",
+      "rm",
+      "-r",
+      "/cache/skia/bin"
+    ],
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "name": "Delete executables"
+  },
+  {
+    "cmd": [
+      "adb"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "infra_step": true,
+    "name": "disconnect"
+  },
+  {
+    "cmd": [
+      "adb",
+      "connect",
+      "192.168.1.2:5555"
+    ],
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "infra_step": true,
+    "name": "adb connect 192.168.1.2:5555 (2)"
+  },
+  {
+    "cmd": [
       "python",
       "-u",
       "\nimport os\nimport subprocess\nimport sys\nout = sys.argv[1]\nlog = subprocess.check_output(['adb', 'logcat', '-d'])\nfor line in log.split('\\n'):\n  tokens = line.split()\n  if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':\n    addr, path = tokens[-2:]\n    local = os.path.join(out, os.path.basename(path))\n    if os.path.exists(local):\n      sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])\n      line = line.replace(addr, addr + ' ' + sym.strip())\n  print line\n",
@@ -794,6 +881,20 @@
   },
   {
     "cmd": [
+      "adb"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[START_DIR]/out"
+    },
+    "infra_step": true,
+    "name": "disconnect (2)"
+  },
+  {
+    "cmd": [
       "adb",
       "kill-server"
     ],
diff --git a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py b/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py
index ec48a7dfa9..7f2c167324 100644
--- a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py
+++ b/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py
@@ -15,6 +15,8 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils):
     super(GNChromecastFlavorUtils, self).__init__(m)
     self._ever_ran_adb = False
     self._user_ip = ''
+    self.m.vars.android_bin_dir = self.m.path.join(self.m.vars.android_bin_dir,
+                                                   'bin')
 
   @property
   def user_ip_host(self):
@@ -90,8 +92,16 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils):
 
     self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn'))
     self._run('gn gen', gn, 'gen', self.out_dir, '--args=' + gn_args)
-    # We only build perf for the chromecasts.
-    self._run('ninja', ninja, '-C', self.out_dir, 'nanobench')
+    self._run('ninja', ninja, '-C', self.out_dir, 'nanobench', 'dm')
+
+  def install(self):
+    super(GNChromecastFlavorUtils, self).install()
+    self._adb('mkdir ' + self.m.vars.android_bin_dir,
+              'shell', 'mkdir', '-p', self.m.vars.android_bin_dir)
+    # TODO(kjlubick): Remove this after we are backfilled up and don't need
+    # to manually delete this
+    self._ssh('Delete old nanobench', 'rm', '/cache/skia/nanobench',
+              abort_on_failure=False)
 
   def _adb(self, title, *cmd, **kwargs):
     if not self._ever_ran_adb:
@@ -131,13 +141,45 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils):
       for f in fs:
         print os.path.join(p,f)
         hp = os.path.realpath(os.path.join(host, p, f))
-        if os.stat(hp).st_size > (3 * 1024 * 1024):
+        if os.stat(hp).st_size > (1.5 * 1024 * 1024):
           print "Skipping because it is too big"
         else:
           subprocess.check_call(['adb', 'push',
                                 hp, os.path.join(device, p, f)])
     """, args=[host, device], infra_step=True)
 
+  def cleanup_steps(self):
+    if self._ever_ran_adb:
+      # To clean up disk space for next time
+      self._ssh('Delete executables', 'rm', '-r', self.m.vars.android_bin_dir,
+                abort_on_failure=False)
+      # Reconnect if was disconnected
+      self._adb('disconnect')
+      self._connect_to_remote()
+      self.m.run(self.m.python.inline, 'dump log', program="""
+          import os
+          import subprocess
+          import sys
+          out = sys.argv[1]
+          log = subprocess.check_output(['adb', 'logcat', '-d'])
+          for line in log.split('\\n'):
+            tokens = line.split()
+            if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':
+              addr, path = tokens[-2:]
+              local = os.path.join(out, os.path.basename(path))
+              if os.path.exists(local):
+                sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])
+                line = line.replace(addr, addr + ' ' + sym.strip())
+            print line
+          """,
+          args=[self.m.vars.skia_out.join(self.m.vars.configuration)],
+          infra_step=True,
+          abort_on_failure=False)
+
+      self._adb('disconnect')
+      self._adb('kill adb server', 'kill-server')
+
+
   def _ssh(self, title, *cmd, **kwargs):
     ssh_cmd = ['ssh', '-oConnectTimeout=15', '-oBatchMode=yes',
                '-t', '-t', 'root@%s' % self.user_ip] + list(cmd)
@@ -147,9 +189,10 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils):
 
   def step(self, name, cmd, **kwargs):
     app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0])
+
     self._adb('push %s' % cmd[0],
               'push', app, self.m.vars.android_bin_dir)
 
-    cmd[0] = '%s%s' % (self.m.vars.android_bin_dir, cmd[0])
+    cmd[0] = '%s/%s' % (self.m.vars.android_bin_dir, cmd[0])
     self._ssh(str(name), *cmd)