From 22f5ef762343bebc7d72ef84a703ae453616c4e6 Mon Sep 17 00:00:00 2001 From: Eric Boren Date: Fri, 2 Dec 2016 11:01:33 -0500 Subject: [nobuildbot] Housekeeper-PerCommit BUG=skia: Change-Id: I9a6a8970bd70dc52334fc44ed29ea942c29132fc Reviewed-on: https://skia-review.googlesource.com/5507 Reviewed-by: Ravi Mistry Commit-Queue: Eric Boren --- infra/bots/gen_tasks.go | 5 +- infra/bots/housekeeper_skia.isolate | 6 +- infra/bots/recipe_modules/vars/api.py | 3 +- .../Housekeeper-PerCommit-Trybot.json | 96 ++++++++++++++++++++-- .../Housekeeper-PerCommit.json | 93 +++++++++++++++++++-- infra/bots/recipes/swarm_housekeeper.py | 50 ++++++----- infra/bots/tasks.json | 73 ++++++++++++++++ 7 files changed, 290 insertions(+), 36 deletions(-) (limited to 'infra') diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go index d4ad3b208a..1c5dacad38 100644 --- a/infra/bots/gen_tasks.go +++ b/infra/bots/gen_tasks.go @@ -86,6 +86,7 @@ var ( "Build-Win-MSVC-x86_64-Release-GN", "Build-Win-MSVC-x86_64-Release-Vulkan", "Housekeeper-Nightly-RecreateSKPs_Canary", + "Housekeeper-PerCommit", "Housekeeper-PerCommit-InfraTests", "Housekeeper-Weekly-RecreateSKPs", "Perf-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android", @@ -510,7 +511,7 @@ func ctSKPs(b *specs.TasksCfgBuilder, name string) string { // in the generated chain of tasks, which the Job should add as a dependency. func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string { b.MustAddTask(name, &specs.TaskSpec{ - CipdPackages: []*specs.CipdPackage{}, + CipdPackages: []*specs.CipdPackage{b.MustGetCipdPackageFromAsset("go")}, Dependencies: []string{compileTaskName}, Dimensions: LINUX_GCE_DIMENSIONS, ExtraArgs: []string{ @@ -755,7 +756,7 @@ func process(b *specs.TasksCfgBuilder, name string) { } // Housekeeper. - if parts["role"] == "Housekeeper-PerCommit" { + if name == "Housekeeper-PerCommit" { deps = append(deps, housekeeper(b, name, compileTaskName)) } diff --git a/infra/bots/housekeeper_skia.isolate b/infra/bots/housekeeper_skia.isolate index 4172bae45b..767fa3af9e 100644 --- a/infra/bots/housekeeper_skia.isolate +++ b/infra/bots/housekeeper_skia.isolate @@ -1,6 +1,10 @@ { 'includes': [ - 'skia_repo.isolate', 'swarm_recipe.isolate', ], + 'variables': { + 'files': [ + '../../../.gclient', + ], + }, } diff --git a/infra/bots/recipe_modules/vars/api.py b/infra/bots/recipe_modules/vars/api.py index 7a73efdd4a..537a682165 100644 --- a/infra/bots/recipe_modules/vars/api.py +++ b/infra/bots/recipe_modules/vars/api.py @@ -52,7 +52,8 @@ class SkiaVarsApi(recipe_api.RecipeApi): 'RecreateSKPs' in self.builder_name or '-CT_' in self.builder_name or 'Presubmit' in self.builder_name or - 'InfraTests' in self.builder_name) + 'InfraTests' in self.builder_name or + self.builder_name == "Housekeeper-PerCommit") if self.persistent_checkout: if 'Win' in self.builder_name: self.checkout_root = self.make_path('C:\\', 'b', 'work') diff --git a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json index ab5b8ffd1b..8548d767a8 100644 --- a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json +++ b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json @@ -1,4 +1,84 @@ [ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_/_B_WORK]", + "511" + ], + "name": "makedirs checkout_path", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec", + "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"skia\": \"got_revision\"}", + "--git-cache-dir", + "[CUSTOM_/_B_CACHE]", + "--gerrit_repo", + "https://skia.googlesource.com/skia", + "--gerrit_ref", + "refs/changes/89/456789/12", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123", + "--output_manifest" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, { "cmd": [ "python", @@ -24,12 +104,13 @@ { "cmd": [ "python", - "[START_DIR]/skia/platform_tools/android/tests/run_all.py" + "[CUSTOM_/_B_WORK]/skia/platform_tools/android/tests/run_all.py" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", "SKIA_OUT": "[START_DIR]/out" }, "name": "android platform self-tests" @@ -45,12 +126,13 @@ "--gsutil_path", "[DEPOT_TOOLS]/gsutil.py", "--issue_number", - "500" + "456789" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", "SKIA_OUT": "[START_DIR]/out" }, "name": "generate and upload binary size data" @@ -61,11 +143,12 @@ "get", "go.skia.org/infra/comments/go/extract_comments" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", "GOPATH": "[START_DIR]/tmp/golib", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", "SKIA_OUT": "[START_DIR]/out" }, "name": "compile extract_comments" @@ -78,11 +161,12 @@ "--dest", "gs://skia-doc/gm/comments.json" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", "GOPATH": "[START_DIR]/tmp/golib", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", "SKIA_OUT": "[START_DIR]/out" }, "name": "run extract_comments" diff --git a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json index c484a36bb5..9ba24aa18b 100644 --- a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json +++ b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json @@ -1,4 +1,80 @@ [ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_/_B_WORK]", + "511" + ], + "name": "makedirs checkout_path", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec", + "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"skia\": \"got_revision\"}", + "--git-cache-dir", + "[CUSTOM_/_B_CACHE]", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123", + "--output_manifest" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, { "cmd": [ "python", @@ -24,12 +100,13 @@ { "cmd": [ "python", - "[START_DIR]/skia/platform_tools/android/tests/run_all.py" + "[CUSTOM_/_B_WORK]/skia/platform_tools/android/tests/run_all.py" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", "SKIA_OUT": "[START_DIR]/out" }, "name": "android platform self-tests" @@ -39,10 +116,11 @@ "python", "RECIPE_MODULE[skia::core]/resources/generate_and_upload_doxygen.py" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", "SKIA_OUT": "[START_DIR]/out" }, "name": "generate and upload doxygen" @@ -58,10 +136,11 @@ "--gsutil_path", "[DEPOT_TOOLS]/gsutil.py" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", "SKIA_OUT": "[START_DIR]/out" }, "name": "generate and upload binary size data" @@ -72,11 +151,12 @@ "get", "go.skia.org/infra/comments/go/extract_comments" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", "GOPATH": "[START_DIR]/tmp/golib", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", "SKIA_OUT": "[START_DIR]/out" }, "name": "compile extract_comments" @@ -89,11 +169,12 @@ "--dest", "gs://skia-doc/gm/comments.json" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", "GOPATH": "[START_DIR]/tmp/golib", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", "SKIA_OUT": "[START_DIR]/out" }, "name": "run extract_comments" diff --git a/infra/bots/recipes/swarm_housekeeper.py b/infra/bots/recipes/swarm_housekeeper.py index c2e8c6b708..cd9c3b6b58 100644 --- a/infra/bots/recipes/swarm_housekeeper.py +++ b/infra/bots/recipes/swarm_housekeeper.py @@ -57,7 +57,7 @@ def RunSteps(api): '--githash', api.properties['revision'], '--gsutil_path', gsutil_path] if api.vars.is_trybot: - cmd.extend(['--issue_number', str(api.properties['issue'])]) + cmd.extend(['--issue_number', str(api.properties['patch_issue'])]) api.run( api.step, 'generate and upload binary size data', @@ -90,22 +90,32 @@ def RunSteps(api): def GenTests(api): - for mastername, slaves in TEST_BUILDERS.iteritems(): - for slavename, builders_by_slave in slaves.iteritems(): - for buildername in builders_by_slave: - test = ( - api.test(buildername) + - api.properties(buildername=buildername, - mastername=mastername, - slavename=slavename, - buildnumber=5, - revision='abc123', - path_config='kitchen', - swarm_out_dir='[SWARM_OUT_DIR]') + - api.path.exists(api.path['start_dir']) - ) - if 'Trybot' in buildername: - test.properties['issue'] = '500' - test.properties['patchset'] = '1' - test.properties['rietveld'] = 'https://codereview.chromium.org' - yield test + yield ( + api.test('Housekeeper-PerCommit') + + api.properties(buildername='Housekeeper-PerCommit', + mastername='client.skia.fyi', + slavename='skiabot-linux-housekeeper-000', + buildnumber=5, + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.path.exists(api.path['start_dir']) + ) + yield ( + api.test('Housekeeper-PerCommit-Trybot') + + api.properties(buildername='Housekeeper-PerCommit', + mastername='client.skia.fyi', + slavename='skiabot-linux-housekeeper-000', + buildnumber=5, + revision='abc123', + path_config='kitchen', + patch_storage='gerrit', + nobuildbot='True', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.properties.tryserver( + buildername='Housekeeper-PerCommit', + gerrit_project='skia', + gerrit_url='https://skia-review.googlesource.com/', + ) + + api.path.exists(api.path['start_dir']) + ) diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index ab588a835f..bd8c915d9a 100644 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -252,6 +252,12 @@ "Build-Ubuntu-GCC-x86_64-Release-SKNX_NO_SIMD" ] }, + "Build-Ubuntu-GCC-x86_64-Release-Shared": { + "priority": 0.8, + "tasks": [ + "Build-Ubuntu-GCC-x86_64-Release-Shared" + ] + }, "Build-Ubuntu-GCC-x86_64-Release-Valgrind": { "priority": 0.8, "tasks": [ @@ -373,6 +379,12 @@ ], "trigger": "nightly" }, + "Housekeeper-PerCommit": { + "priority": 0.8, + "tasks": [ + "Housekeeper-PerCommit" + ] + }, "Housekeeper-PerCommit-InfraTests": { "priority": 0.8, "tasks": [ @@ -2608,6 +2620,31 @@ "isolate": "compile_skia.isolate", "priority": 0.8 }, + "Build-Ubuntu-GCC-x86_64-Release-Shared": { + "dimensions": [ + "gpu:none", + "os:Ubuntu", + "pool:Skia" + ], + "extra_args": [ + "--workdir", + "../../..", + "swarm_compile", + "repository=<(REPO)", + "buildername=Build-Ubuntu-GCC-x86_64-Release-Shared", + "mastername=fake-master", + "buildnumber=2", + "slavename=fake-buildslave", + "nobuildbot=True", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=<(REVISION)", + "patch_storage=<(PATCH_STORAGE)", + "patch_issue=<(ISSUE)", + "patch_set=<(PATCHSET)" + ], + "isolate": "compile_skia.isolate", + "priority": 0.8 + }, "Build-Ubuntu-GCC-x86_64-Release-Valgrind": { "dimensions": [ "gpu:none", @@ -3247,6 +3284,42 @@ "isolate": "compile_skia.isolate", "priority": 0.8 }, + "Housekeeper-PerCommit": { + "cipd_packages": [ + { + "name": "skia/bots/go", + "path": "go", + "version": "version:0" + } + ], + "dependencies": [ + "Build-Ubuntu-GCC-x86_64-Release-Shared" + ], + "dimensions": [ + "cpu:x86-64-avx2", + "gpu:none", + "os:Ubuntu", + "pool:Skia" + ], + "extra_args": [ + "--workdir", + "../../..", + "swarm_housekeeper", + "repository=<(REPO)", + "buildername=Housekeeper-PerCommit", + "mastername=fake-master", + "buildnumber=2", + "slavename=fake-buildslave", + "nobuildbot=True", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=<(REVISION)", + "patch_storage=<(PATCH_STORAGE)", + "patch_issue=<(ISSUE)", + "patch_set=<(PATCHSET)" + ], + "isolate": "housekeeper_skia.isolate", + "priority": 0.8 + }, "Housekeeper-PerCommit-InfraTests": { "dimensions": [ "cpu:x86-64-avx2", -- cgit v1.2.3