aboutsummaryrefslogtreecommitdiffhomepage
path: root/infra
diff options
context:
space:
mode:
authorGravatar Ravi Mistry <rmistry@google.com>2017-05-17 14:28:06 -0400
committerGravatar Skia Commit-Bot <skia-commit-bot@chromium.org>2017-05-18 15:01:18 +0000
commit01b48e7f13c25af430b6c90aac8cefcd79c0d870 (patch)
treefa34bbf0b19cd947fa31df927fea8c703452a913 /infra
parent081e0e6a32244a7945ddd40c2003fd312673c586 (diff)
Bot to update buildbucket.config daily
Bug: skia:6591 Change-Id: Ic2ec5b033965ca073a00e272693b1c5e1d0ad69f Reviewed-on: https://skia-review.googlesource.com/16541 Commit-Queue: Ravi Mistry <rmistry@google.com> Reviewed-by: Eric Boren <borenet@google.com>
Diffstat (limited to 'infra')
-rw-r--r--infra/bots/gen_tasks.go35
-rw-r--r--infra/bots/jobs.json1
-rw-r--r--infra/bots/meta_config.isolate5
-rw-r--r--infra/bots/recipe_modules/infra/api.py45
-rw-r--r--infra/bots/recipe_modules/infra/example.expected/failed_one_update.json48
-rw-r--r--infra/bots/recipe_modules/infra/example.expected/infra_tests.json48
-rw-r--r--infra/bots/recipe_modules/infra/example.py3
-rw-r--r--infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json4
-rw-r--r--infra/bots/recipes/recreate_skps.expected/failed_upload.json4
-rw-r--r--infra/bots/recipes/recreate_skps.py50
-rw-r--r--infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json98
-rw-r--r--infra/bots/recipes/update_meta_config.expected/failed_update.json103
-rw-r--r--infra/bots/recipes/update_meta_config.expected/trybot_test.json30
-rw-r--r--infra/bots/recipes/update_meta_config.py88
-rw-r--r--infra/bots/tasks.json30
-rw-r--r--infra/bots/update_meta_config.py91
16 files changed, 631 insertions, 52 deletions
diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go
index 917cc5003a..f81b1ac167 100644
--- a/infra/bots/gen_tasks.go
+++ b/infra/bots/gen_tasks.go
@@ -423,6 +423,32 @@ func recreateSKPs(b *specs.TasksCfgBuilder, name string) string {
return name
}
+
+// updateMetaConfig generates a UpdateMetaConfig task. Returns the name of the
+// last task in the generated chain of tasks, which the Job should add as a
+// dependency.
+func updateMetaConfig(b *specs.TasksCfgBuilder, name string) string {
+ b.MustAddTask(name, &specs.TaskSpec{
+ CipdPackages: []*specs.CipdPackage{},
+ Dimensions: linuxGceDimensions(),
+ ExtraArgs: []string{
+ "--workdir", "../../..", "update_meta_config",
+ fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+ fmt.Sprintf("buildername=%s", name),
+ fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+ fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+ fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO),
+ fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+ fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+ fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+ },
+ Isolate: "meta_config.isolate",
+ Priority: 0.8,
+ })
+ return name
+}
+
+
// ctSKPs generates a CT SKPs task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func ctSKPs(b *specs.TasksCfgBuilder, name string) string {
@@ -690,6 +716,11 @@ func process(b *specs.TasksCfgBuilder, name string) {
deps = append(deps, recreateSKPs(b, name))
}
+ // UpdateMetaConfig bot.
+ if strings.Contains(name, "UpdateMetaConfig") {
+ deps = append(deps, updateMetaConfig(b, name))
+ }
+
// CT bots.
if strings.Contains(name, "-CT_") {
deps = append(deps, ctSKPs(b, name))
@@ -716,6 +747,7 @@ func process(b *specs.TasksCfgBuilder, name string) {
name != "Housekeeper-PerCommit-BundleRecipes" &&
name != "Housekeeper-PerCommit-InfraTests" &&
!strings.Contains(name, "RecreateSKPs") &&
+ !strings.Contains(name, "UpdateMetaConfig") &&
!strings.Contains(name, "-CT_") {
compile(b, compileTaskName, compileTaskParts)
}
@@ -769,6 +801,9 @@ func process(b *specs.TasksCfgBuilder, name string) {
if name == "Housekeeper-Nightly-RecreateSKPs_Canary" {
j.Trigger = "nightly"
}
+ if name == "Housekeeper-Nightly-UpdateMetaConfig" {
+ j.Trigger = "nightly"
+ }
if name == "Housekeeper-Weekly-RecreateSKPs" {
j.Trigger = "weekly"
}
diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json
index 5bf6e5207b..45782a2a32 100644
--- a/infra/bots/jobs.json
+++ b/infra/bots/jobs.json
@@ -74,6 +74,7 @@
"Build-Win-MSVC-x86_64-Release-GDI",
"Build-Win-MSVC-x86_64-Release-Vulkan",
"Housekeeper-Nightly-RecreateSKPs_Canary",
+ "Housekeeper-Nightly-UpdateMetaConfig",
"Housekeeper-PerCommit",
"Housekeeper-PerCommit-BundleRecipes",
"Housekeeper-PerCommit-InfraTests",
diff --git a/infra/bots/meta_config.isolate b/infra/bots/meta_config.isolate
new file mode 100644
index 0000000000..d4e1484acd
--- /dev/null
+++ b/infra/bots/meta_config.isolate
@@ -0,0 +1,5 @@
+{
+ 'includes': [
+ 'swarm_recipe.isolate',
+ ],
+}
diff --git a/infra/bots/recipe_modules/infra/api.py b/infra/bots/recipe_modules/infra/api.py
index 0a4784267a..5f37e48b8b 100644
--- a/infra/bots/recipe_modules/infra/api.py
+++ b/infra/bots/recipe_modules/infra/api.py
@@ -63,3 +63,48 @@ class InfraApi(recipe_api.RecipeApi):
'update go pkgs',
UPDATE_GO_ATTEMPTS,
cmd=[self.go_exe, 'get', '-u', '-t', '%s/...' % INFRA_GO_PKG])
+
+ class MetadataFetch():
+ def __init__(self, api, metadata_key, local_file, **kwargs):
+ self.m = api
+ self._key = metadata_key
+ self._local_file = local_file
+
+ def __enter__(self):
+ return self.m.python.inline(
+ 'download ' + self._local_file,
+ """
+import os
+import urllib2
+
+TOKEN_FILE = '%s'
+TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/%s'
+
+req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})
+contents = urllib2.urlopen(req).read()
+
+home = os.path.expanduser('~')
+token_file = os.path.join(home, TOKEN_FILE)
+
+with open(token_file, 'w') as f:
+ f.write(contents)
+ """ % (self._local_file, self._key),
+ )
+
+ def __exit__(self, t, v, tb):
+ self.m.python.inline(
+ 'cleanup ' + self._local_file,
+ """
+import os
+
+
+TOKEN_FILE = '%s'
+
+
+home = os.path.expanduser('~')
+token_file = os.path.join(home, TOKEN_FILE)
+if os.path.isfile(token_file):
+ os.remove(token_file)
+ """ % (self._local_file),
+ )
+ return v is None
diff --git a/infra/bots/recipe_modules/infra/example.expected/failed_one_update.json b/infra/bots/recipe_modules/infra/example.expected/failed_one_update.json
index 3b8a04b0a0..89d7042591 100644
--- a/infra/bots/recipe_modules/infra/example.expected/failed_one_update.json
+++ b/infra/bots/recipe_modules/infra/example.expected/failed_one_update.json
@@ -70,6 +70,54 @@
"name": "update go pkgs (attempt 2)"
},
{
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'file'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ ],
+ "name": "download file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
+ "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\n\n\nTOKEN_FILE = 'file'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
+ ],
+ "name": "cleanup file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
"name": "$result",
"recipe_result": null,
"status_code": 0
diff --git a/infra/bots/recipe_modules/infra/example.expected/infra_tests.json b/infra/bots/recipe_modules/infra/example.expected/infra_tests.json
index 169fc196ae..1e38e2eb95 100644
--- a/infra/bots/recipe_modules/infra/example.expected/infra_tests.json
+++ b/infra/bots/recipe_modules/infra/example.expected/infra_tests.json
@@ -48,6 +48,54 @@
"name": "update go pkgs"
},
{
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'file'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ ],
+ "name": "download file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
+ "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\n\n\nTOKEN_FILE = 'file'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
+ ],
+ "name": "cleanup file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
"name": "$result",
"recipe_result": null,
"status_code": 0
diff --git a/infra/bots/recipe_modules/infra/example.py b/infra/bots/recipe_modules/infra/example.py
index d46a7ee42f..af1c9f2b59 100644
--- a/infra/bots/recipe_modules/infra/example.py
+++ b/infra/bots/recipe_modules/infra/example.py
@@ -11,6 +11,7 @@ DEPS = [
'infra',
'recipe_engine/path',
'recipe_engine/properties',
+ 'recipe_engine/python',
'recipe_engine/step',
'run',
'vars',
@@ -20,6 +21,8 @@ DEPS = [
def RunSteps(api):
api.vars.setup()
api.infra.update_go_deps()
+ with api.infra.MetadataFetch(api, 'key', 'file'):
+ pass
def GenTests(api):
diff --git a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
index 5d461f4b15..b731496138 100644
--- a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
+++ b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
@@ -359,7 +359,7 @@
"-u",
"\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
],
- "name": "download update-skps.gitcookies",
+ "name": "download update_skps.git_cookies",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
@@ -405,7 +405,7 @@
"-u",
"\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
],
- "name": "cleanup update-skps.gitcookies",
+ "name": "cleanup update_skps.git_cookies",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
diff --git a/infra/bots/recipes/recreate_skps.expected/failed_upload.json b/infra/bots/recipes/recreate_skps.expected/failed_upload.json
index 5b1fbca1a8..664077e90e 100644
--- a/infra/bots/recipes/recreate_skps.expected/failed_upload.json
+++ b/infra/bots/recipes/recreate_skps.expected/failed_upload.json
@@ -359,7 +359,7 @@
"-u",
"\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
],
- "name": "download update-skps.gitcookies",
+ "name": "download update_skps.git_cookies",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
@@ -409,7 +409,7 @@
"-u",
"\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
],
- "name": "cleanup update-skps.gitcookies",
+ "name": "cleanup update_skps.git_cookies",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
diff --git a/infra/bots/recipes/recreate_skps.py b/infra/bots/recipes/recreate_skps.py
index b789975cd1..0da6fe2709 100644
--- a/infra/bots/recipes/recreate_skps.py
+++ b/infra/bots/recipes/recreate_skps.py
@@ -36,53 +36,6 @@ UPDATE_SKPS_GITCOOKIES_FILE = 'update_skps.git_cookies'
UPDATE_SKPS_KEY = 'update_skps_git_cookies'
-class gitcookies_auth(object):
- """Download update-skps@skia.org's .gitcookies."""
- def __init__(self, api, metadata_key):
- self.m = api
- self._key = metadata_key
-
- def __enter__(self):
- return self.m.python.inline(
- 'download update-skps.gitcookies',
- """
-import os
-import urllib2
-
-TOKEN_FILE = '%s'
-TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/%s'
-
-req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})
-contents = urllib2.urlopen(req).read()
-
-home = os.path.expanduser('~')
-token_file = os.path.join(home, TOKEN_FILE)
-
-with open(token_file, 'w') as f:
- f.write(contents)
- """ % (UPDATE_SKPS_GITCOOKIES_FILE,
- self._key),
- )
-
- def __exit__(self, t, v, tb):
- self.m.python.inline(
- 'cleanup update-skps.gitcookies',
- """
-import os
-
-
-TOKEN_FILE = '%s'
-
-
-home = os.path.expanduser('~')
-token_file = os.path.join(home, TOKEN_FILE)
-if os.path.isfile(token_file):
- os.remove(token_file)
- """ % (UPDATE_SKPS_GITCOOKIES_FILE),
- )
- return v is None
-
-
def RunSteps(api):
# Check out Chrome.
api.core.setup()
@@ -128,7 +81,8 @@ def RunSteps(api):
api.vars.skia_dir.join('infra', 'bots', 'upload_skps.py'),
'--target_dir', output_dir,
'--gitcookies', str(update_skps_gitcookies)]
- with gitcookies_auth(api, UPDATE_SKPS_KEY):
+ with api.infra.MetadataFetch(
+ api, UPDATE_SKPS_KEY, UPDATE_SKPS_GITCOOKIES_FILE):
with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
api.run(api.step, 'Upload SKPs', cmd=cmd)
diff --git a/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json b/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json
new file mode 100644
index 0000000000..9a1a0aecb7
--- /dev/null
+++ b/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json
@@ -0,0 +1,98 @@
+[
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
+ "[START_DIR]/tmp",
+ "511"
+ ],
+ "infra_step": true,
+ "name": "makedirs tmp_dir",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+ "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+ "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+ "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
+ "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ ],
+ "name": "download update_meta_config.git_cookies",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
+ "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "[START_DIR]/skia/infra/bots/update_meta_config.py",
+ "--repo_name",
+ "skia",
+ "--tasks_json",
+ "[START_DIR]/skia/infra/bots/tasks.json",
+ "--gitcookies",
+ "[HOME]/update_meta_config.git_cookies"
+ ],
+ "cwd": "[START_DIR]/skia",
+ "env": {
+ "BUILDTYPE": "Release",
+ "CHROME_HEADLESS": "1",
+ "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[START_DIR]/out"
+ },
+ "name": "Update meta/config"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\n\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
+ ],
+ "name": "cleanup update_meta_config.git_cookies",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "name": "$result",
+ "recipe_result": null,
+ "status_code": 0
+ }
+] \ No newline at end of file
diff --git a/infra/bots/recipes/update_meta_config.expected/failed_update.json b/infra/bots/recipes/update_meta_config.expected/failed_update.json
new file mode 100644
index 0000000000..1aaf64e1d3
--- /dev/null
+++ b/infra/bots/recipes/update_meta_config.expected/failed_update.json
@@ -0,0 +1,103 @@
+[
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
+ "[START_DIR]/tmp",
+ "511"
+ ],
+ "infra_step": true,
+ "name": "makedirs tmp_dir",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+ "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+ "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+ "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
+ "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ ],
+ "name": "download update_meta_config.git_cookies",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
+ "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "[START_DIR]/skia/infra/bots/update_meta_config.py",
+ "--repo_name",
+ "skia",
+ "--tasks_json",
+ "[START_DIR]/skia/infra/bots/tasks.json",
+ "--gitcookies",
+ "[HOME]/update_meta_config.git_cookies"
+ ],
+ "cwd": "[START_DIR]/skia",
+ "env": {
+ "BUILDTYPE": "Release",
+ "CHROME_HEADLESS": "1",
+ "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[START_DIR]/out"
+ },
+ "name": "Update meta/config",
+ "~followup_annotations": [
+ "step returned non-zero exit code: 1",
+ "@@@STEP_FAILURE@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\n\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
+ ],
+ "name": "cleanup update_meta_config.git_cookies",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "name": "$result",
+ "reason": "Step('Update meta/config') failed with return_code 1",
+ "recipe_result": null,
+ "status_code": 1
+ }
+] \ No newline at end of file
diff --git a/infra/bots/recipes/update_meta_config.expected/trybot_test.json b/infra/bots/recipes/update_meta_config.expected/trybot_test.json
new file mode 100644
index 0000000000..5758c51115
--- /dev/null
+++ b/infra/bots/recipes/update_meta_config.expected/trybot_test.json
@@ -0,0 +1,30 @@
+[
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
+ "[START_DIR]/tmp",
+ "511"
+ ],
+ "infra_step": true,
+ "name": "makedirs tmp_dir",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+ "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+ "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+ "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
+ "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "name": "$result",
+ "reason": "Uncaught Exception: Exception('Cannot run update_meta_config recipe as a trybot',)",
+ "status_code": -1
+ }
+] \ No newline at end of file
diff --git a/infra/bots/recipes/update_meta_config.py b/infra/bots/recipes/update_meta_config.py
new file mode 100644
index 0000000000..b3cbe23b97
--- /dev/null
+++ b/infra/bots/recipes/update_meta_config.py
@@ -0,0 +1,88 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Recipe for the Bot that updates meta config."""
+
+
+DEPS = [
+ 'build/file',
+ 'depot_tools/gclient',
+ 'recipe_engine/context',
+ 'recipe_engine/path',
+ 'recipe_engine/properties',
+ 'recipe_engine/python',
+ 'recipe_engine/raw_io',
+ 'recipe_engine/step',
+ 'core',
+ 'infra',
+ 'run',
+ 'vars',
+]
+
+
+TEST_BUILDERS = {
+ 'client.skia.compile': {
+ 'skiabot-linux-swarm-000': [
+ 'Housekeeper-Nightly-UpdateMetaConfig',
+ ],
+ },
+}
+
+
+UPDATE_META_CONFIG_GITCOOKIES_FILE = 'update_meta_config.git_cookies'
+UPDATE_META_CONFIG_KEY = 'update_meta_config_git_cookies'
+
+
+def RunSteps(api):
+ api.core.setup()
+
+ if api.vars.is_trybot:
+ raise Exception('Cannot run update_meta_config recipe as a trybot')
+ update_meta_config_gitcookies = api.path.join(
+ api.path.expanduser('~'), UPDATE_META_CONFIG_GITCOOKIES_FILE)
+ repo_name = api.properties.get('repository').split('/')[-1].rstrip('.git')
+ cmd = ['python',
+ api.vars.skia_dir.join('infra', 'bots', 'update_meta_config.py'),
+ '--repo_name', repo_name,
+ '--tasks_json', api.vars.skia_dir.join('infra', 'bots', 'tasks.json'),
+ '--gitcookies', str(update_meta_config_gitcookies)]
+ with api.infra.MetadataFetch(
+ api, UPDATE_META_CONFIG_KEY, UPDATE_META_CONFIG_GITCOOKIES_FILE):
+ with api.context(cwd=api.vars.skia_dir):
+ api.run(api.step, 'Update meta/config', cmd=cmd)
+
+
+def GenTests(api):
+ builder = 'Housekeeper-Nightly-UpdateMetaConfig'
+ yield (
+ api.test(builder) +
+ api.properties(buildername=builder,
+ repository='https://skia.googlesource.com/skia.git',
+ revision='abc123',
+ path_config='kitchen',
+ swarm_out_dir='[SWARM_OUT_DIR]')
+ )
+
+ yield (
+ api.test('failed_update') +
+ api.properties(buildername=builder,
+ repository='https://skia.googlesource.com/skia.git',
+ revision='abc123',
+ path_config='kitchen',
+ swarm_out_dir='[SWARM_OUT_DIR]') +
+ api.step_data('Update meta/config', retcode=1)
+ )
+
+ yield (
+ api.test('trybot_test') +
+ api.properties(buildername=builder,
+ repository='https://skia.googlesource.com/skia.git',
+ revision='abc123',
+ path_config='kitchen',
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ patch_issue='123',
+ patch_set='3') +
+ api.expect_exception('Exception')
+ )
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index 6f3dd5fc59..da0c5a576e 100644
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -451,6 +451,13 @@
],
"trigger": "nightly"
},
+ "Housekeeper-Nightly-UpdateMetaConfig": {
+ "priority": 0.8,
+ "tasks": [
+ "Housekeeper-Nightly-UpdateMetaConfig"
+ ],
+ "trigger": "nightly"
+ },
"Housekeeper-PerCommit": {
"priority": 0.8,
"tasks": [
@@ -4379,6 +4386,29 @@
"isolate": "compile_skia.isolate",
"priority": 0.8
},
+ "Housekeeper-Nightly-UpdateMetaConfig": {
+ "dimensions": [
+ "cpu:x86-64-avx2",
+ "gpu:none",
+ "os:Ubuntu-14.04",
+ "pool:Skia"
+ ],
+ "extra_args": [
+ "--workdir",
+ "../../..",
+ "update_meta_config",
+ "repository=<(REPO)",
+ "buildername=Housekeeper-Nightly-UpdateMetaConfig",
+ "swarm_out_dir=${ISOLATED_OUTDIR}",
+ "revision=<(REVISION)",
+ "patch_repo=<(PATCH_REPO)",
+ "patch_storage=<(PATCH_STORAGE)",
+ "patch_issue=<(ISSUE)",
+ "patch_set=<(PATCHSET)"
+ ],
+ "isolate": "meta_config.isolate",
+ "priority": 0.8
+ },
"Housekeeper-PerCommit": {
"cipd_packages": [
{
diff --git a/infra/bots/update_meta_config.py b/infra/bots/update_meta_config.py
new file mode 100644
index 0000000000..663d170c33
--- /dev/null
+++ b/infra/bots/update_meta_config.py
@@ -0,0 +1,91 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Update meta/config of the specified Skia repo."""
+
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import urllib2
+
+import git_utils
+
+
+SKIA_COMMITTER_EMAIL = 'update-meta-config@skia.org'
+SKIA_COMMITTER_NAME = 'Update Meta Config'
+SKIA_REPO_TEMPLATE = 'https://skia.googlesource.com/%s.git'
+
+CQ_INCLUDE_CHROMIUM_TRYBOTS = (
+ 'master.tryserver.blink:linux_trusty_blink_rel,linux_trusty_blink_dbg;'
+ 'master.tryserver.chromium.linux:linux_optional_gpu_tests_rel;'
+ 'master.tryserver.chromium.mac:mac_optional_gpu_tests_rel;'
+ 'master.tryserver.chromium.win:win_optional_gpu_tests_rel;'
+ 'master.tryserver.chromium.android:android_optional_gpu_tests_rel'
+)
+
+
+def addChromiumTrybots(f):
+ for master_section in CQ_INCLUDE_CHROMIUM_TRYBOTS.split(';'):
+ master, bots = master_section.split(':')
+ f.write('[bucket "%s"]\n' % master)
+ for bot in bots.split(','):
+ f.write('\tbuilder = %s\n' % bot)
+
+
+def main(gitcookies, repo_name, tasks_json):
+ skia_repo = SKIA_REPO_TEMPLATE % repo_name
+ with git_utils.NewGitCheckout(repository=skia_repo):
+ # Fetch and checkout the meta/config branch.
+ subprocess.check_call(['git', 'fetch', skia_repo, 'refs/meta/config:cfg'])
+ subprocess.check_call(['git', 'checkout', 'cfg'])
+
+ # Create list of tryjobs from tasks_json.
+ tryjobs = []
+ with open(tasks_json) as tasks_json:
+ data = json.load(tasks_json)
+ for job in data['jobs'].keys():
+ if not job.startswith('Upload-'):
+ tryjobs.append(job)
+ tryjobs.sort()
+
+ # Write to buildbucket.config.
+ buildbucket_config = os.path.join(os.getcwd(), 'buildbucket.config')
+ with open(buildbucket_config, 'w') as f:
+
+ if repo_name == 'skia':
+ addChromiumTrybots(f)
+
+ # Adding all Skia jobs.
+ f.write('[bucket "skia.primary"]\n')
+ for job in tryjobs:
+ f.write('\tbuilder = ' + job + '\n')
+
+ # Push the change as the update-meta-config user.
+ config_dict = {
+ 'user.name': SKIA_COMMITTER_NAME,
+ 'user.email': SKIA_COMMITTER_EMAIL,
+ 'http.cookiefile': gitcookies,
+ }
+ with git_utils.GitLocalConfig(config_dict):
+ subprocess.check_call(['git', 'add', 'buildbucket.config'])
+ try:
+ subprocess.check_call(
+ ['git', 'commit', '-m', 'Update builders in buildbucket.config'])
+ except subprocess.CalledProcessError:
+ print 'No changes to buildbucket.config'
+ return
+
+ subprocess.check_call(['git', 'push', skia_repo, 'cfg:refs/meta/config'])
+
+
+if '__main__' == __name__:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--gitcookies")
+ parser.add_argument("--repo_name")
+ parser.add_argument("--tasks_json")
+ args = parser.parse_args()
+ main(args.gitcookies, args.repo_name, args.tasks_json)