Add extracting GM comments to the housekeeper bot.

BUG=skia:
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2237033002

Review-Url: https://codereview.chromium.org/2237033002

5 files changed, 103 insertions, 2 deletions

diff --git a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json
index 56837514fb..2fdcad62fe 100644
--- a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json
+++ b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json
@@ -56,6 +56,38 @@
     "name": "generate and upload binary size data"
   },
   {
+    "cmd": [
+      "[SLAVE_BUILD]/go/go/bin/go",
+      "get",
+      "go.skia.org/infra/comments/go/extract_comments"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "GOPATH": "[SLAVE_BUILD]/tmp/golib",
+      "SKIA_OUT": "[SLAVE_BUILD]/out"
+    },
+    "name": "compile extract_comments"
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/tmp/golib/bin/extract_comments",
+      "--dir",
+      "gm",
+      "--dest",
+      "gs://skia-doc/gm/comments.json"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "GOPATH": "[SLAVE_BUILD]/tmp/golib",
+      "SKIA_OUT": "[SLAVE_BUILD]/out"
+    },
+    "name": "run extract_comments"
+  },
+  {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
diff --git a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json
index 3f72a1f0ae..9be48d4af1 100644
--- a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json
+++ b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json
@@ -68,6 +68,38 @@
     "name": "generate and upload binary size data"
   },
   {
+    "cmd": [
+      "[SLAVE_BUILD]/go/go/bin/go",
+      "get",
+      "go.skia.org/infra/comments/go/extract_comments"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "GOPATH": "[SLAVE_BUILD]/tmp/golib",
+      "SKIA_OUT": "[SLAVE_BUILD]/out"
+    },
+    "name": "compile extract_comments"
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/tmp/golib/bin/extract_comments",
+      "--dir",
+      "gm",
+      "--dest",
+      "gs://skia-doc/gm/comments.json"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "GOPATH": "[SLAVE_BUILD]/tmp/golib",
+      "SKIA_OUT": "[SLAVE_BUILD]/out"
+    },
+    "name": "run extract_comments"
+  },
+  {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
diff --git a/infra/bots/recipes/swarm_housekeeper.py b/infra/bots/recipes/swarm_housekeeper.py
index ae31611b1e..43e694c384 100644
--- a/infra/bots/recipes/swarm_housekeeper.py
+++ b/infra/bots/recipes/swarm_housekeeper.py
@@ -5,7 +5,6 @@
 
 # Recipe for the Skia PerCommit Housekeeper.
 
-
 DEPS = [
   'core',
   'recipe_engine/path',
@@ -68,6 +67,30 @@ def RunSteps(api):
     cwd=cwd,
     abort_on_failure=False)
 
+  env = {}
+  env['GOPATH'] = api.vars.tmp_dir.join('golib')
+  extractexe = env['GOPATH'].join('bin', 'extract_comments')
+  goexe = api.vars.slave_dir.join('go', 'go', 'bin', 'go')
+
+  # Compile extract_comments.
+  api.run(
+    api.step,
+    'compile extract_comments',
+    cmd=[goexe, 'get', 'go.skia.org/infra/comments/go/extract_comments'],
+    cwd=cwd,
+    env=env,
+    abort_on_failure=True)
+
+  # Run extract_comments on the gm directory.
+  api.run(
+    api.step,
+    'run extract_comments',
+    cmd=[extractexe, '--dir', 'gm', '--dest', 'gs://skia-doc/gm/comments.json'],
+    cwd=cwd,
+    env=env,
+    abort_on_failure=True)
+
+
 def GenTests(api):
   for mastername, slaves in TEST_BUILDERS.iteritems():
     for slavename, builders_by_slave in slaves.iteritems():
diff --git a/infra/bots/recipes/swarm_trigger.expected/Housekeeper-PerCommit.json b/infra/bots/recipes/swarm_trigger.expected/Housekeeper-PerCommit.json
index 57d3a1637f..9c9eb6f02e 100644
--- a/infra/bots/recipes/swarm_trigger.expected/Housekeeper-PerCommit.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Housekeeper-PerCommit.json
@@ -460,6 +460,16 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/go/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "read go VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
       "[SLAVE_BUILD]/swarming_temp_dir",
       "511"
@@ -601,6 +611,8 @@
       "slavename:skiabot-linux-swarm-000",
       "--tag",
       "stepname:housekeeper_skia on Ubuntu",
+      "--cipd-package",
+      "go:skia/bots/go:version:0",
       "def456",
       "--",
       "--workdir",
diff --git a/infra/bots/recipes/swarm_trigger.py b/infra/bots/recipes/swarm_trigger.py
index 1fdd57174f..c4f1a54937 100644
--- a/infra/bots/recipes/swarm_trigger.py
+++ b/infra/bots/recipes/swarm_trigger.py
@@ -271,7 +271,9 @@ def housekeeper_swarm(api, builder_cfg, got_revision, infrabots_dir,
       infrabots_dir,
       idempotent=False,
       store_output=False,
-      extra_isolate_hashes=extra_isolate_hashes)
+      extra_isolate_hashes=extra_isolate_hashes,
+      cipd_packages=[cipd_pkg(api, infrabots_dir, 'go')],
+      )
   return api.swarming.collect_swarming_task(task)