Download gitcookies from GS for RecreateSKPs bot

NoTry: true
Bug: skia:6797
Change-Id: Id7005ba5252c5f1e3f85a8bff92371e0562ab87b
Reviewed-on: https://skia-review.googlesource.com/22726
Commit-Queue: Ravi Mistry <rmistry@google.com>
Reviewed-by: Eric Boren <borenet@google.com>

3 files changed, 50 insertions, 99 deletions

diff --git a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
index 492804531f..4119fbfc74 100644
--- a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
+++ b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
@@ -154,19 +154,6 @@
       "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
       "--json-output",
       "/path/to/tmp/json",
-      "rmtree",
-      "[START_DIR]/skp_output"
-    ],
-    "infra_step": true,
-    "name": "rmtree skp_output"
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
-      "--json-output",
-      "/path/to/tmp/json",
       "ensure-directory",
       "--mode",
       "0777",
@@ -247,27 +234,14 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n  f.write(contents)\n"
+      "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py",
+      "--",
+      "cp",
+      "gs://skia-buildbots/artifacts/server/.gitcookies_update-skps",
+      "[START_DIR]/update_skps.git_cookies"
     ],
-    "name": "download update_skps.git_cookies",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os@@@",
-      "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_skps.git_cookies'@@@",
-      "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
-      "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
-      "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
-      "@@@STEP_LOG_LINE@python.inline@  f.write(contents)@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
+    "infra_step": true,
+    "name": "gsutil cp"
   },
   {
     "cmd": [
@@ -276,7 +250,7 @@
       "--target_dir",
       "[START_DIR]/skp_output",
       "--gitcookies",
-      "[HOME]/update_skps.git_cookies"
+      "[START_DIR]/update_skps.git_cookies"
     ],
     "cwd": "[CUSTOM_/_B_WORK]/skia",
     "env": {
@@ -293,23 +267,14 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n  os.remove(token_file)\n"
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "remove",
+      "[START_DIR]/update_skps.git_cookies"
     ],
-    "name": "cleanup update_skps.git_cookies",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_skps.git_cookies'@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
-      "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  os.remove(token_file)@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
+    "infra_step": true,
+    "name": "remove [START_DIR]/update_skps.git_cookies"
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/recreate_skps.expected/failed_upload.json b/infra/bots/recipes/recreate_skps.expected/failed_upload.json
index e8ada2c75a..0647f27df0 100644
--- a/infra/bots/recipes/recreate_skps.expected/failed_upload.json
+++ b/infra/bots/recipes/recreate_skps.expected/failed_upload.json
@@ -247,27 +247,14 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n  f.write(contents)\n"
+      "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py",
+      "--",
+      "cp",
+      "gs://skia-buildbots/artifacts/server/.gitcookies_update-skps",
+      "[START_DIR]/update_skps.git_cookies"
     ],
-    "name": "download update_skps.git_cookies",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os@@@",
-      "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_skps.git_cookies'@@@",
-      "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
-      "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
-      "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
-      "@@@STEP_LOG_LINE@python.inline@  f.write(contents)@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
+    "infra_step": true,
+    "name": "gsutil cp"
   },
   {
     "cmd": [
@@ -276,7 +263,7 @@
       "--target_dir",
       "[START_DIR]/skp_output",
       "--gitcookies",
-      "[HOME]/update_skps.git_cookies"
+      "[START_DIR]/update_skps.git_cookies"
     ],
     "cwd": "[CUSTOM_/_B_WORK]/skia",
     "env": {
@@ -294,28 +281,6 @@
     ]
   },
   {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n  os.remove(token_file)\n"
-    ],
-    "name": "cleanup update_skps.git_cookies",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_skps.git_cookies'@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
-      "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  os.remove(token_file)@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
-  {
     "name": "$result",
     "reason": "Step('Upload SKPs') failed with return_code 1",
     "recipe_result": null,
diff --git a/infra/bots/recipes/recreate_skps.py b/infra/bots/recipes/recreate_skps.py
index 8da1ed669d..60b44b69a2 100644
--- a/infra/bots/recipes/recreate_skps.py
+++ b/infra/bots/recipes/recreate_skps.py
@@ -9,6 +9,7 @@
 DEPS = [
   'core',
   'depot_tools/gclient',
+  'depot_tools/gsutil',
   'infra',
   'recipe_engine/context',
   'recipe_engine/file',
@@ -33,7 +34,26 @@ TEST_BUILDERS = {
 
 
 UPDATE_SKPS_GITCOOKIES_FILE = 'update_skps.git_cookies'
-UPDATE_SKPS_KEY = 'update_skps_git_cookies'
+
+UPDATE_SKPS_GITCOOKIES_GS_PATH = (
+    'gs://skia-buildbots/artifacts/server/.gitcookies_update-skps')
+
+
+class DownloadGitCookies(object):
+  """Class to download gitcookies from GS."""
+  def __init__(self, gs_path, local_path, api):
+    self._gs_path = gs_path
+    self._local_path = local_path
+    self._api = api
+
+  def __enter__(self):
+    gsutil_args = ['cp', self._gs_path, self._local_path]
+    self._api.gsutil(gsutil_args, use_retry_wrapper=False)
+
+  def __exit__(self, exc_type, _value, _traceback):
+    if self._api.path.exists(self._local_path):
+      self._api.file.remove('remove %s' % self._local_path, self._local_path)
+
 
 
 def RunSteps(api):
@@ -76,14 +96,14 @@ def RunSteps(api):
   # Upload the SKPs.
   if 'Canary' not in api.properties['buildername']:
     api.infra.update_go_deps()
-    update_skps_gitcookies = api.path.join(api.path.expanduser('~'),
-                                           UPDATE_SKPS_GITCOOKIES_FILE)
+    update_skps_gitcookies = api.path['start_dir'].join(
+        UPDATE_SKPS_GITCOOKIES_FILE)
     cmd = ['python',
            api.vars.skia_dir.join('infra', 'bots', 'upload_skps.py'),
            '--target_dir', output_dir,
            '--gitcookies', str(update_skps_gitcookies)]
-    with api.infra.MetadataFetch(
-        api, UPDATE_SKPS_KEY, UPDATE_SKPS_GITCOOKIES_FILE):
+    with DownloadGitCookies(
+        UPDATE_SKPS_GITCOOKIES_GS_PATH, update_skps_gitcookies, api):
       with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
         api.run(api.step, 'Upload SKPs', cmd=cmd)
 
@@ -108,7 +128,8 @@ def GenTests(api):
                      revision='abc123',
                      path_config='kitchen',
                      swarm_out_dir='[SWARM_OUT_DIR]') +
-      api.path.exists(api.path['start_dir'].join('skp_output'))
+      api.path.exists(api.path['start_dir'].join('skp_output')) +
+      api.path.exists(api.path['start_dir'].join(UPDATE_SKPS_GITCOOKIES_FILE))
   )
 
   yield (