diff options
author | Eric Boren <borenet@google.com> | 2017-05-22 08:35:36 -0400 |
---|---|---|
committer | Skia Commit-Bot <skia-commit-bot@chromium.org> | 2017-05-22 13:08:25 +0000 |
commit | f94514b0ff8eccb2eaef8c77bee8c5f462b83b90 (patch) | |
tree | 208fa3043a2858cfc052c6af8a26a04e68868626 /infra | |
parent | 66f6b1fb48c82505397e8b787f7abc780e8f6029 (diff) |
[recipes] Copy file, isolate, swarming, swarming_client from build.git
Rename swarming -> skia_swarming.
Some required heavy modification to remove other dependencies on modules
in build.git.
Expected changes:
- RECIPE_MODULE[build::<module>] -> RECIPE_MODULE[skia::<module>]
- No more runit; directly run through Python.
Bug: skia:6628
Change-Id: I1b1370ed387966222ce10731771dbde9020cf542
Reviewed-on: https://skia-review.googlesource.com/17448
Commit-Queue: Eric Boren <borenet@google.com>
Reviewed-by: Ravi Mistry <rmistry@google.com>
Diffstat (limited to 'infra')
153 files changed, 9301 insertions, 795 deletions
diff --git a/infra/bots/recipe_modules/core/__init__.py b/infra/bots/recipe_modules/core/__init__.py index 6b541c39aa..1413f2e4dd 100644 --- a/infra/bots/recipe_modules/core/__init__.py +++ b/infra/bots/recipe_modules/core/__init__.py @@ -3,10 +3,10 @@ # found in the LICENSE file. DEPS = [ - 'build/file', 'depot_tools/bot_update', 'depot_tools/gclient', 'depot_tools/tryserver', + 'file', 'flavor', 'recipe_engine/context', 'recipe_engine/path', diff --git a/infra/bots/recipe_modules/ct/__init__.py b/infra/bots/recipe_modules/ct/__init__.py index 3756b3cbb7..792ca353ed 100644 --- a/infra/bots/recipe_modules/ct/__init__.py +++ b/infra/bots/recipe_modules/ct/__init__.py @@ -3,8 +3,8 @@ # found in the LICENSE file. DEPS = [ - 'build/file', 'depot_tools/gsutil', + 'file', 'recipe_engine/path', 'recipe_engine/step', 'run', diff --git a/infra/bots/recipe_modules/ct/example.expected/failed_gsutil.json b/infra/bots/recipe_modules/ct/example.expected/failed_gsutil.json index e51ee004c7..dc6f888199 100644 --- a/infra/bots/recipe_modules/ct/example.expected/failed_gsutil.json +++ b/infra/bots/recipe_modules/ct/example.expected/failed_gsutil.json @@ -3,7 +3,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/skps/slave0" ], diff --git a/infra/bots/recipe_modules/ct/example.expected/test.json b/infra/bots/recipe_modules/ct/example.expected/test.json index dc49be70f0..fa9e44a468 100644 --- a/infra/bots/recipe_modules/ct/example.expected/test.json +++ b/infra/bots/recipe_modules/ct/example.expected/test.json @@ -3,7 +3,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/skps/slave0" ], diff --git a/infra/bots/recipe_modules/file/OWNERS b/infra/bots/recipe_modules/file/OWNERS new file mode 100644 index 0000000000..36f4549a0c --- /dev/null +++ b/infra/bots/recipe_modules/file/OWNERS @@ -0,0 +1,5 @@ +dnj@chromium.org +iannucci@chromium.org +martiniss@chromium.org +nodir@chromium.org +phajdan.jr@chromium.org diff --git a/infra/bots/recipe_modules/file/__init__.py b/infra/bots/recipe_modules/file/__init__.py new file mode 100644 index 0000000000..29b55cb21c --- /dev/null +++ b/infra/bots/recipe_modules/file/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module belongs in the recipe engine. Remove it from this +# repo once it has been moved. + + +DEPS = [ + 'recipe_engine/json', + 'recipe_engine/path', + 'recipe_engine/python', + 'recipe_engine/raw_io', + 'recipe_engine/step', +] diff --git a/infra/bots/recipe_modules/file/api.py b/infra/bots/recipe_modules/file/api.py new file mode 100644 index 0000000000..e2ec61cee6 --- /dev/null +++ b/infra/bots/recipe_modules/file/api.py @@ -0,0 +1,204 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module belongs in the recipe engine. Remove it from this +# repo once it has been moved. + + +from recipe_engine import recipe_api + + +class FileApi(recipe_api.RecipeApi): + """FileApi contains helper functions for reading and writing files.""" + + def __init__(self, **kwargs): + super(FileApi, self).__init__(**kwargs) + + def _run_fileutil(self, name, fileutil_args, **kwargs): + # Failure to perform filesystem operations is considered an infrastructure + # failure. + kwargs = kwargs.copy() + kwargs.setdefault('infra_step', True) + + self.m.python( + name, + self.resource('fileutil.py'), + args=fileutil_args, + **kwargs) + + def copy(self, name, source, dest, step_test_data=None, **kwargs): + """Copy a file.""" + return self.m.python.inline( + name, + """ + import shutil + import sys + shutil.copy(sys.argv[1], sys.argv[2]) + """, + args=[source, dest], + add_python_log=False, + step_test_data=step_test_data, + **kwargs + ) + + def copytree(self, name, source, dest, symlinks=False, **kwargs): + """Run shutil.copytree in a step.""" + return self.m.python.inline( + name, + """ + import shutil + import sys + shutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3])) + """, + args=[source, dest, int(symlinks)], + add_python_log=False, + **kwargs + ) + + def move(self, name, source, dest, **kwargs): + """Run shutil.move in a step.""" + return self.m.python.inline( + name, + """ + import shutil + import sys + shutil.move(sys.argv[1], sys.argv[2]) + """, + args=[source, dest], + add_python_log=False, + **kwargs + ) + + def read(self, name, path, test_data=None, **kwargs): + """Read a file and return its contents.""" + step_test_data = None + if test_data is not None: + step_test_data = lambda: self.m.raw_io.test_api.output_text(test_data) + return self.copy(name, path, self.m.raw_io.output_text(), + step_test_data=step_test_data, **kwargs).raw_io.output_text + + def write(self, name, path, data, **kwargs): + """Write the given data to a file.""" + return self.m.python.inline( + name, + """ + import shutil + import sys + shutil.copy(sys.argv[1], sys.argv[2]) + """, + args=[self.m.raw_io.input_text(data), path], + add_python_log=False, + **kwargs + ) + + def glob(self, name, pattern, test_data=None, **kwargs): + """Performs glob search on a directory. + + Returns list of Path objects for all files found. + """ + step_test_data = None + if test_data is not None: + step_test_data = ( + lambda: self.m.raw_io.test_api.output_text( + '\n'.join(map(str, test_data)))) + step_result = self.m.python.inline( + name, + r""" + import glob + import sys + with open(sys.argv[1], 'w') as f: + f.write('\n'.join(glob.glob(sys.argv[2]))) + """, + args=[self.m.raw_io.output_text(), pattern], + step_test_data=step_test_data, + add_python_log=False, + **kwargs + ) + return map(self.m.path.abs_to_path, + step_result.raw_io.output_text.splitlines()) + + def remove(self, name, path, **kwargs): + """Remove the given file.""" + return self.m.python.inline( + name, + """ + import os + import sys + os.remove(sys.argv[1]) + """, + args=[path], + **kwargs + ) + + def listdir(self, name, path, step_test_data=None, **kwargs): + """Wrapper for os.listdir.""" + return self.m.python.inline('listdir %s' % name, + """ + import json, os, sys + if os.path.exists(sys.argv[1]) and os.path.isdir(sys.argv[1]): + with open(sys.argv[2], 'w') as f: + json.dump(os.listdir(sys.argv[1]), f) + """, + args=[path, self.m.json.output()], + step_test_data=(step_test_data or ( + lambda: self.m.json.test_api.output(['file 1', 'file 2']))), + **kwargs + ).json.output + + def makedirs(self, name, path, mode=0777, **kwargs): + """ + Like os.makedirs, except that if the directory exists, then there is no + error. + """ + self.m.path.assert_absolute(path) + self.m.python.inline( + 'makedirs ' + name, + """ + import sys, os + path = sys.argv[1] + mode = int(sys.argv[2]) + if not os.path.isdir(path): + if os.path.exists(path): + print "%s exists but is not a dir" % path + sys.exit(1) + os.makedirs(path, mode) + """, + args=[path, str(mode)], + **kwargs + ) + self.m.path.mock_add_paths(path) + + def rmtree(self, name, path, **kwargs): + """Wrapper for chromium_utils.RemoveDirectory.""" + self.m.path.assert_absolute(path) + self._run_fileutil( + 'rmtree ' + name, + ['rmtree', path], + **kwargs) + + def rmcontents(self, name, path, **kwargs): + """ + Similar to rmtree, but removes only contents not the directory. + + This is useful e.g. when removing contents of current working directory. + Deleting current working directory makes all further getcwd calls fail + until chdir is called. chdir would be tricky in recipes, so we provide + a call that doesn't delete the directory itself. + """ + self.m.path.assert_absolute(path) + self._run_fileutil( + 'rmcontents ' + name, + ['rmcontents', path], + **kwargs) + + def rmwildcard(self, pattern, path, **kwargs): + """ + Removes all files in the subtree of path matching the glob pattern. + """ + self.m.path.assert_absolute(path) + self._run_fileutil( + 'rmwildcard %s in %s' % (pattern, path), + ['rmwildcard', path, pattern], + **kwargs) diff --git a/infra/bots/recipe_modules/file/example.expected/file_io.json b/infra/bots/recipe_modules/file/example.expected/file_io.json new file mode 100644 index 0000000000..a4e557f9c6 --- /dev/null +++ b/infra/bots/recipe_modules/file/example.expected/file_io.json @@ -0,0 +1,301 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport json, os, sys\nif os.path.exists(sys.argv[1]) and os.path.isdir(sys.argv[1]):\n with open(sys.argv[2], 'w') as f:\n json.dump(os.listdir(sys.argv[1]), f)\n", + "/fake/dir", + "/path/to/tmp/json" + ], + "name": "listdir fake dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"file 1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"file 2\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import json, os, sys@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]) and os.path.isdir(sys.argv[1]):@@@", + "@@@STEP_LOG_LINE@python.inline@ with open(sys.argv[2], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(os.listdir(sys.argv[1]), f)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "some", + "command" + ], + "name": "manipulate file 1" + }, + { + "cmd": [ + "some", + "command" + ], + "name": "manipulate file 2" + }, + { + "cmd": [ + "python", + "-u", + "\nimport json, os, sys\nif os.path.exists(sys.argv[1]) and os.path.isdir(sys.argv[1]):\n with open(sys.argv[2], 'w') as f:\n json.dump(os.listdir(sys.argv[1]), f)\n", + "/faker/dir", + "/path/to/tmp/json" + ], + "name": "listdir other", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"aaa\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import json, os, sys@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]) and os.path.isdir(sys.argv[1]):@@@", + "@@@STEP_LOG_LINE@python.inline@ with open(sys.argv[2], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(os.listdir(sys.argv[1]), f)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "some", + "command" + ], + "name": "manipulate aaa" + }, + { + "cmd": [ + "echo", + "[TMP_BASE]/prefix_a_tmp_1" + ], + "name": "print prefix_a" + }, + { + "cmd": [ + "echo", + "[TMP_BASE]/prefix_b_tmp_2" + ], + "name": "print prefix_b" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.move(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/source", + "[START_DIR]/destination" + ], + "name": "move" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[START_DIR]/some_file" + ], + "name": "remove", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmcontents", + "[START_DIR]/some_dir" + ], + "infra_step": true, + "name": "rmcontents rmcontents" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmwildcard", + "[START_DIR]", + "*.o" + ], + "infra_step": true, + "name": "rmwildcard *.o in [START_DIR]" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "abcde", + "tmp_file.txt" + ], + "name": "write_simple" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "tmp_file.txt", + "/path/to/tmp/" + ], + "name": "read_simple" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "! ~&&", + "tmp_file.txt" + ], + "name": "write_symbols" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "tmp_file.txt", + "/path/to/tmp/" + ], + "name": "read_symbols" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "abcde fgh", + "tmp_file.txt" + ], + "name": "write_spaces" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "tmp_file.txt", + "/path/to/tmp/" + ], + "name": "read_spaces" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "ab\ncd\nefg\n", + "tmp_file.txt" + ], + "name": "write_multiline" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "tmp_file.txt", + "/path/to/tmp/" + ], + "name": "read_multiline" + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[START_DIR]/copytree_example_tmp", + "511" + ], + "name": "makedirs makedirs", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "some file content", + "[START_DIR]/copytree_example_tmp/dummy_file" + ], + "name": "write [START_DIR]/copytree_example_tmp/dummy_file" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", + "[START_DIR]/copytree_example_tmp", + "[START_DIR]/copytree_example_tmp2", + "0" + ], + "name": "copytree" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[START_DIR]/copytree_example_tmp2/dummy_file", + "/path/to/tmp/" + ], + "name": "read [START_DIR]/copytree_example_tmp2/dummy_file" + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[START_DIR]/copytree_example_tmp/*" + ], + "name": "glob" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[START_DIR]/copytree_example_tmp" + ], + "infra_step": true, + "name": "rmtree cleanup" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[START_DIR]/copytree_example_tmp2" + ], + "infra_step": true, + "name": "rmtree cleanup2" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/file/example.py b/infra/bots/recipe_modules/file/example.py new file mode 100644 index 0000000000..b6f00eabfb --- /dev/null +++ b/infra/bots/recipe_modules/file/example.py @@ -0,0 +1,101 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module belongs in the recipe engine. Remove it from this +# repo once it has been moved. + + +from recipe_engine.types import freeze + +DEPS = [ + 'depot_tools/infra_paths', + 'file', + 'recipe_engine/path', + 'recipe_engine/raw_io', + 'recipe_engine/step', +] + + +TEST_CONTENTS = freeze({ + 'simple': 'abcde', + 'spaces': 'abcde fgh', + 'symbols': '! ~&&', + 'multiline': '''ab +cd +efg +''', +}) + + +def RunSteps(api): + # listdir demo. + result = api.file.listdir('fake dir', '/fake/dir') + for element in result: + api.step('manipulate %s' % str(element), ['some', 'command']) + + result = api.file.listdir('other', '/faker/dir') + for element in result: + api.step('manipulate %s' % str(element), ['some', 'command']) + + # mkdtemp demo. + for prefix in ('prefix_a', 'prefix_b'): + # Create temp dir. + temp_dir = api.path.mkdtemp(prefix) + assert api.path.exists(temp_dir) + # Make |temp_dir| surface in expectation files. + api.step('print %s' % prefix, ['echo', temp_dir]) + + # move demo + api.file.move( + 'move', + api.path['start_dir'].join('source'), + api.path['start_dir'].join('destination')) + + # remove demo + api.file.remove('remove', api.path['start_dir'].join('some_file')) + + # rmcontents demo + api.file.rmcontents('rmcontents', api.path['start_dir'].join('some_dir')) + + # rmwildcard demo + api.file.rmwildcard('*.o', api.path['start_dir']) + + for name, content in TEST_CONTENTS.iteritems(): + api.file.write('write_%s' % name, 'tmp_file.txt', content) + actual_content = api.file.read( + 'read_%s' % name, 'tmp_file.txt', + test_data=content + ) + msg = 'expected %s but got %s' % (content, actual_content) + assert actual_content == content, msg + + try: + # copytree + content = 'some file content' + tmp_dir = api.path['start_dir'].join('copytree_example_tmp') + api.file.makedirs('makedirs', tmp_dir) + path = tmp_dir.join('dummy_file') + api.file.write('write %s' % path, path, content) + new_tmp = api.path['start_dir'].join('copytree_example_tmp2') + new_path = new_tmp.join('dummy_file') + api.file.copytree('copytree', tmp_dir, new_tmp) + actual_content = api.file.read('read %s' % new_path, new_path, + test_data=content) + assert actual_content == content + + # glob. + files = api.file.glob( + 'glob', tmp_dir.join('*'), + test_data=[tmp_dir.join('dummy_file')]) + assert files == [tmp_dir.join('dummy_file')], files + + finally: + api.file.rmtree('cleanup', tmp_dir) + api.file.rmtree('cleanup2', new_tmp) + + +def GenTests(api): + yield api.test('file_io') + api.file.listdir('other', ['aaa']) + diff --git a/infra/bots/recipe_modules/file/resources/fileutil.py b/infra/bots/recipe_modules/file/resources/fileutil.py new file mode 100755 index 0000000000..ec73a9b3eb --- /dev/null +++ b/infra/bots/recipe_modules/file/resources/fileutil.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module belongs in the recipe engine. Remove it from this +# repo once it has been moved. + + +"""Utility exporting basic filesystem operations. + +This file was cut from "scripts/common/chromium_utils.py" at: +91310531c31fa645256b4fb5d44b460c42b3e151 +""" + +import argparse +import errno +import fnmatch +import os +import shutil +import subprocess +import sys +import time + + +def _LocateFiles(pattern, root): + """Yeilds files matching pattern found in root and its subdirectories. + + An exception is thrown if root doesn't exist.""" + for path, _, files in os.walk(os.path.abspath(root)): + for filename in fnmatch.filter(files, pattern): + yield os.path.join(path, filename) + + +def _RemoveFilesWildcards(file_wildcard, root): + """Removes files matching 'file_wildcard' in root and its subdirectories, if + any exists. + + An exception is thrown if root doesn't exist.""" + for item in _LocateFiles(file_wildcard, root): + try: + os.remove(item) + except OSError, e: + if e.errno != errno.ENOENT: + raise + + +def _RemoveContents(path): + if os.path.exists(path): + for p in (os.path.join(path, x) for x in os.listdir(path)): + if os.path.isdir(p): + _RemoveDirectory(p) + else: + os.unlink(p) + + +def _RemoveDirectory(*path): + """Recursively removes a directory, even if it's marked read-only. + + Remove the directory located at *path, if it exists. + + shutil.rmtree() doesn't work on Windows if any of the files or directories + are read-only, which svn repositories and some .svn files are. We need to + be able to force the files to be writable (i.e., deletable) as we traverse + the tree. + + Even with all this, Windows still sometimes fails to delete a file, citing + a permission error (maybe something to do with antivirus scans or disk + indexing). The best suggestion any of the user forums had was to wait a + bit and try again, so we do that too. It's hand-waving, but sometimes it + works. :/ + """ + file_path = os.path.join(*path) + if not os.path.exists(file_path): + return + + if sys.platform == 'win32': + # Give up and use cmd.exe's rd command. + file_path = os.path.normcase(file_path) + for _ in xrange(3): + print 'RemoveDirectory running %s' % (' '.join( + ['cmd.exe', '/c', 'rd', '/q', '/s', file_path])) + if not subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', file_path]): + break + print ' Failed' + time.sleep(3) + return + + def RemoveWithRetry_non_win(rmfunc, path): + if os.path.islink(path): + return os.remove(path) + else: + return rmfunc(path) + + remove_with_retry = RemoveWithRetry_non_win + + def RmTreeOnError(function, path, excinfo): + r"""This works around a problem whereby python 2.x on Windows has no ability + to check for symbolic links. os.path.islink always returns False. But + shutil.rmtree will fail if invoked on a symbolic link whose target was + deleted before the link. E.g., reproduce like this: + > mkdir test + > mkdir test\1 + > mklink /D test\current test\1 + > python -c "import chromium_utils; chromium_utils.RemoveDirectory('test')" + To avoid this issue, we pass this error-handling function to rmtree. If + we see the exact sort of failure, we ignore it. All other failures we re- + raise. + """ + + exception_type = excinfo[0] + exception_value = excinfo[1] + # If shutil.rmtree encounters a symbolic link on Windows, os.listdir will + # fail with a WindowsError exception with an ENOENT errno (i.e., file not + # found). We'll ignore that error. Note that WindowsError is not defined + # for non-Windows platforms, so we use OSError (of which it is a subclass) + # to avoid lint complaints about an undefined global on non-Windows + # platforms. + if (function is os.listdir) and issubclass(exception_type, OSError): + if exception_value.errno == errno.ENOENT: + # File does not exist, and we're trying to delete, so we can ignore the + # failure. + print 'WARNING: Failed to list %s during rmtree. Ignoring.\n' % path + else: + raise + else: + raise + + for root, dirs, files in os.walk(file_path, topdown=False): + # For POSIX: making the directory writable guarantees removability. + # Windows will ignore the non-read-only bits in the chmod value. + os.chmod(root, 0770) + for name in files: + remove_with_retry(os.remove, os.path.join(root, name)) + for name in dirs: + remove_with_retry(lambda p: shutil.rmtree(p, onerror=RmTreeOnError), + os.path.join(root, name)) + + remove_with_retry(os.rmdir, file_path) + + +def main(args): + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers() + + # Subcommand: rmtree + subparser = subparsers.add_parser('rmtree', + help='Recursively remove a directory.') + subparser.add_argument('path', nargs='+', help='A path to remove.') + subparser.set_defaults(func=lambda opts: _RemoveDirectory(*opts.path)) + + # Subcommand: rmcontents + subparser = subparsers.add_parser('rmcontents', + help='Recursively remove the contents of a directory.') + subparser.add_argument('path', help='The target directory.') + subparser.set_defaults(func=lambda opts: _RemoveContents(opts.path)) + + # Subcommand: rmwildcard + subparser = subparsers.add_parser('rmwildcard', + help='Recursively remove the contents of a directory.') + subparser.add_argument('root', help='The directory to search through.') + subparser.add_argument('wildcard', help='The wildcard expression to remove.') + subparser.set_defaults(func=lambda opts: + _RemoveFilesWildcards(opts.wildcard, opts.root)) + + # Parse arguments. + opts = parser.parse_args(args) + opts.func(opts) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/infra/bots/recipe_modules/file/test_api.py b/infra/bots/recipe_modules/file/test_api.py new file mode 100644 index 0000000000..8d13cbac1a --- /dev/null +++ b/infra/bots/recipe_modules/file/test_api.py @@ -0,0 +1,18 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module belongs in the recipe engine. Remove it from this +# repo once it has been moved. + + +from recipe_engine import recipe_test_api + + +class FileTestApi(recipe_test_api.RecipeTestApi): + def listdir(self, dirname, files): + return self.step_data( + 'listdir %s' % dirname, + self.m.json.output(files)) + diff --git a/infra/bots/recipe_modules/flavor/__init__.py b/infra/bots/recipe_modules/flavor/__init__.py index a5ef0640e3..beb97ef824 100644 --- a/infra/bots/recipe_modules/flavor/__init__.py +++ b/infra/bots/recipe_modules/flavor/__init__.py @@ -3,10 +3,9 @@ # found in the LICENSE file. DEPS = [ - 'build/adb', - 'build/file', 'builder_name_schema', 'depot_tools/bot_update', + 'file', 'recipe_engine/context', 'recipe_engine/path', 'recipe_engine/platform', diff --git a/infra/bots/recipe_modules/flavor/example.expected/Build-Ubuntu-GCC-x86_64-Release-Flutter_Android.json b/infra/bots/recipe_modules/flavor/example.expected/Build-Ubuntu-GCC-x86_64-Release-Flutter_Android.json index 9d69047d30..7c1bce0c06 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Build-Ubuntu-GCC-x86_64-Release-Flutter_Android.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Build-Ubuntu-GCC-x86_64-Release-Flutter_Android.json @@ -17,7 +17,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/flutter/src/out/android_release" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Debug-Android.json b/infra/bots/recipe_modules/flavor/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Debug-Android.json index 3f53165a17..48084dc6f2 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Debug-Android.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Debug-Android.json @@ -67,7 +67,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/Perf-ChromeOS-Clang-Chromebook_513C24_K01-GPU-MaliT860-arm-Release.json b/infra/bots/recipe_modules/flavor/example.expected/Perf-ChromeOS-Clang-Chromebook_513C24_K01-GPU-MaliT860-arm-Release.json index 30493caf42..41bbac6eaa 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Perf-ChromeOS-Clang-Chromebook_513C24_K01-GPU-MaliT860-arm-Release.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Perf-ChromeOS-Clang-Chromebook_513C24_K01-GPU-MaliT860-arm-Release.json @@ -99,7 +99,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release.json b/infra/bots/recipe_modules/flavor/example.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release.json index 05a3834ef2..8d5bf64def 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release.json @@ -107,7 +107,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-ASAN.json b/infra/bots/recipe_modules/flavor/example.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-ASAN.json index 0b15169c65..56e431a792 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-ASAN.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-ASAN.json @@ -50,7 +50,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], @@ -87,7 +87,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "device_results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-MSAN.json b/infra/bots/recipe_modules/flavor/example.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-MSAN.json index 201a434174..fe214feaed 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-MSAN.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-MSAN.json @@ -50,7 +50,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], @@ -87,7 +87,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "device_results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release.json b/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release.json index e4ecf03ad0..0c4887eee8 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release.json @@ -50,7 +50,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], @@ -87,7 +87,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "device_results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json b/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json index f32eacfa73..235461f8a5 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind_AbandonGpuContext.json @@ -50,7 +50,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], @@ -87,7 +87,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "device_results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug.json b/infra/bots/recipe_modules/flavor/example.expected/Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug.json index c73219efe1..0b67c58f87 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug.json @@ -50,7 +50,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], @@ -87,7 +87,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "device_results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/Test-iOS-Clang-iPadMini4-GPU-GX6450-arm64-Debug.json b/infra/bots/recipe_modules/flavor/example.expected/Test-iOS-Clang-iPadMini4-GPU-GX6450-arm64-Debug.json index 3c7b431b50..0e10c9b049 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/Test-iOS-Clang-iPadMini4-GPU-GX6450-arm64-Debug.json +++ b/infra/bots/recipe_modules/flavor/example.expected/Test-iOS-Clang-iPadMini4-GPU-GX6450-arm64-Debug.json @@ -97,7 +97,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/failed_infra_step.json b/infra/bots/recipe_modules/flavor/example.expected/failed_infra_step.json index 1b760d3c97..3074ffd07c 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/failed_infra_step.json +++ b/infra/bots/recipe_modules/flavor/example.expected/failed_infra_step.json @@ -67,7 +67,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], diff --git a/infra/bots/recipe_modules/flavor/example.expected/failed_read_version.json b/infra/bots/recipe_modules/flavor/example.expected/failed_read_version.json index fd9cdad3f9..66f72bcbc3 100644 --- a/infra/bots/recipe_modules/flavor/example.expected/failed_read_version.json +++ b/infra/bots/recipe_modules/flavor/example.expected/failed_read_version.json @@ -67,7 +67,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "results_dir" ], diff --git a/infra/bots/recipe_modules/isolate/OWNERS b/infra/bots/recipe_modules/isolate/OWNERS new file mode 100644 index 0000000000..a24c7a5e06 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/OWNERS @@ -0,0 +1,3 @@ +dpranke@chromium.org +mcgreevy@chromium.org +tansell@chromium.org diff --git a/infra/bots/recipe_modules/isolate/__init__.py b/infra/bots/recipe_modules/isolate/__init__.py new file mode 100644 index 0000000000..0da966d494 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +DEPS = [ + 'file', + 'depot_tools/git', + 'depot_tools/gsutil', + 'recipe_engine/context', + 'recipe_engine/json', + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/python', + 'recipe_engine/step', + 'recipe_engine/tempfile', + 'swarming_client', +] diff --git a/infra/bots/recipe_modules/isolate/api.py b/infra/bots/recipe_modules/isolate/api.py new file mode 100644 index 0000000000..09aeb99a93 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/api.py @@ -0,0 +1,253 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +import itertools +from recipe_engine import recipe_api + + +class IsolateApi(recipe_api.RecipeApi): + """APIs for interacting with isolates.""" + + def __init__(self, **kwargs): + super(IsolateApi, self).__init__(**kwargs) + self._isolate_server = 'https://isolateserver.appspot.com' + self._isolated_tests = {} + + @property + def isolate_server(self): + """URL of Isolate server to use, default is a production one.""" + return self._isolate_server + + @isolate_server.setter + def isolate_server(self, value): + """Changes URL of Isolate server to use.""" + self._isolate_server = value + + def clean_isolated_files(self, build_dir): + """Cleans out all *.isolated files from the build directory in + preparation for the compile. Needed in order to ensure isolates + are rebuilt properly because their dependencies are currently not + completely described to gyp. + """ + self.m.python( + 'clean isolated files', + self.resource('find_isolated_tests.py'), + [ + '--build-dir', build_dir, + '--clean-isolated-files' + ]) + + def find_isolated_tests(self, build_dir, targets=None, **kwargs): + """Returns a step which finds all *.isolated files in a build directory. + + Useful only with 'archive' isolation mode. + In 'prepare' mode use 'isolate_tests' instead. + + Assigns the dict {target name -> *.isolated file hash} to the swarm_hashes + build property. This implies this step can currently only be run once + per recipe. + + If |targets| is None, the step will use all *.isolated files it finds. + Otherwise, it will verify that all |targets| are found and will use only + them. If some expected targets are missing, will abort the build. + """ + step_result = self.m.python( + 'find isolated tests', + self.resource('find_isolated_tests.py'), + [ + '--build-dir', build_dir, + '--output-json', self.m.json.output(), + ], + step_test_data=lambda: (self.test_api.output_json(targets)), + **kwargs) + + assert isinstance(step_result.json.output, dict) + self._isolated_tests = step_result.json.output + if targets is not None and ( + step_result.presentation.status != self.m.step.FAILURE): + found = set(step_result.json.output) + expected = set(targets) + if found >= expected: # pragma: no cover + # Limit result only to |expected|. + self._isolated_tests = { + target: step_result.json.output[target] for target in expected + } + else: + # Some expected targets are missing? Fail the step. + step_result.presentation.status = self.m.step.FAILURE + step_result.presentation.logs['missing.isolates'] = ( + ['Failed to find *.isolated files:'] + list(expected - found)) + step_result.presentation.properties['swarm_hashes'] = self._isolated_tests + # No isolated files found? That looks suspicious, emit warning. + if (not self._isolated_tests and + step_result.presentation.status != self.m.step.FAILURE): + step_result.presentation.status = self.m.step.WARNING + + def isolate_tests(self, build_dir, targets=None, verbose=False, + set_swarm_hashes=True, always_use_exparchive=False, + **kwargs): + """Archives prepared tests in |build_dir| to isolate server. + + src/tools/isolate_driver.py is invoked by ninja during compilation + to produce *.isolated.gen.json files that describe how to archive tests. + + This step then uses *.isolated.gen.json files to actually performs the + archival. By archiving all tests at once it is able to reduce the total + amount of work. Tests share many common files, and such files are processed + only once. + + Assigns the dict {target name -> *.isolated file hash} to the swarm_hashes + build property (also accessible as 'isolated_tests' property). This implies + this step can currently only be run once per recipe. + """ + # TODO(tansell): Make all steps in this function nested under one overall + # 'isolate tests' master step. + + # TODO(vadimsh): Always require |targets| to be passed explicitly. Currently + # chromium_trybot, blink_trybot and swarming/canary recipes rely on targets + # autodiscovery. The code path in chromium_trybot that needs it is being + # deprecated in favor of to *_ng builders, that pass targets explicitly. + if targets is None: + # Ninja builds <target>.isolated.gen.json files via isolate_driver.py. + paths = self.m.file.glob( + 'find isolated targets', + build_dir.join('*.isolated.gen.json'), + test_data=[ + build_dir.join('dummy_target_%d.isolated.gen.json' % i) + for i in (1, 2) + ]) + targets = [] + for p in paths: + name = self.m.path.basename(p) + assert name.endswith('.isolated.gen.json'), name + targets.append(name[:-len('.isolated.gen.json')]) + + # No isolated tests found. + if not targets: # pragma: no cover + return + + batch_targets = [] + exparchive_targets = [] + for t in targets: + if t.endswith('_exparchive') or always_use_exparchive: + exparchive_targets.append(t) + else: + batch_targets.append(t) + + isolate_steps = [] + try: + args = [ + self.m.swarming_client.path, + 'exparchive', + '--dump-json', self.m.json.output(), + '--isolate-server', self._isolate_server, + '--eventlog-endpoint', 'prod', + ] + (['--verbose'] if verbose else []) + + for target in exparchive_targets: + isolate_steps.append( + self.m.python( + 'isolate %s' % target, + self.resource('isolate.py'), + args + [ + '--isolate', build_dir.join('%s.isolate' % target), + '--isolated', build_dir.join('%s.isolated' % target), + ], + step_test_data=lambda: self.test_api.output_json([target]), + **kwargs)) + + if batch_targets: + # TODO(vadimsh): Differentiate between bad *.isolate and upload errors. + # Raise InfraFailure on upload errors. + args = [ + self.m.swarming_client.path, + 'batcharchive', + '--dump-json', self.m.json.output(), + '--isolate-server', self._isolate_server, + ] + (['--verbose'] if verbose else []) + [ + build_dir.join('%s.isolated.gen.json' % t) for t in batch_targets + ] + isolate_steps.append( + self.m.python( + 'isolate tests', self.resource('isolate.py'), args, + step_test_data=lambda: self.test_api.output_json(batch_targets), + **kwargs)) + + # TODO(tansell): Change this to return a dummy "isolate results" or the + # top level master step. + return isolate_steps[-1] + finally: + step_result = self.m.step.active_result + swarm_hashes = {} + for step in isolate_steps: + if not step.json.output: + continue # pragma: no cover + + for k, v in step.json.output.iteritems(): + # TODO(tansell): Raise an error here when it can't clobber an + # existing error. This code is currently inside a finally block, + # meaning it could be executed when an existing error is occurring. + # See https://chromium-review.googlesource.com/c/437024/ + #assert k not in swarm_hashes or swarm_hashes[k] == v, ( + # "Duplicate hash for target %s was found at step %s." + # "Existing hash: %s, New hash: %s") % ( + # k, step, swarm_hashes[k], v) + swarm_hashes[k] = v + + if swarm_hashes: + self._isolated_tests = swarm_hashes + + if set_swarm_hashes: + step_result.presentation.properties['swarm_hashes'] = swarm_hashes + + missing = sorted( + t for t, h in self._isolated_tests.iteritems() if not h) + if missing: + step_result.presentation.logs['failed to isolate'] = ( + ['Failed to isolate following targets:'] + + missing + + ['', 'See logs for more information.'] + ) + for k in missing: + self._isolated_tests.pop(k) + + @property + def isolated_tests(self): + """The dictionary of 'target name -> isolated hash' for this run. + + These come either from the incoming swarm_hashes build property, + or from calling find_isolated_tests, above, at some point during the run. + """ + hashes = self.m.properties.get('swarm_hashes', self._isolated_tests) + # Be robust in the case where swarm_hashes is an empty string + # instead of an empty dictionary, or similar. + if not hashes: + return {} # pragma: no covergae + return { + k.encode('ascii'): v.encode('ascii') + for k, v in hashes.iteritems() + } + + @property + def _run_isolated_path(self): + """Returns the path to run_isolated.py.""" + return self.m.swarming_client.path.join('run_isolated.py') + + def run_isolated(self, name, isolate_hash, args=None, **kwargs): + """Runs an isolated test.""" + cmd = [ + '--isolated', isolate_hash, + '-I', self.isolate_server, + '--verbose', + ] + if args: + cmd.append('--') + cmd.extend(args) + self.m.python(name, self._run_isolated_path, cmd, **kwargs) diff --git a/infra/bots/recipe_modules/isolate/example.expected/always-use-exparchive.json b/infra/bots/recipe_modules/isolate/example.expected/always-use-exparchive.json new file mode 100644 index 0000000000..f5cbb23b79 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/always-use-exparchive.json @@ -0,0 +1,208 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": \"[dummy hash for test_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\", \"test_exparchive\": \"[dummy hash for test_exparchive]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolated" + ], + "name": "isolate test_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": \"[dummy hash for test_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test1.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test1.isolated" + ], + "name": "isolate test1", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test2.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test2.isolated" + ], + "name": "isolate test2", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\", \"test_exparchive\": \"[dummy hash for test_exparchive]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/basic.json b/infra/bots/recipe_modules/isolate/example.expected/basic.json new file mode 100644 index 0000000000..e5afc9204b --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/basic.json @@ -0,0 +1,151 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "RECIPE_PACKAGE_REPO[skia]/test1.isolated.gen.json", + "RECIPE_PACKAGE_REPO[skia]/test2.isolated.gen.json" + ], + "name": "isolate tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/discover.json b/infra/bots/recipe_modules/isolate/example.expected/discover.json new file mode 100644 index 0000000000..7ec650f308 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/discover.json @@ -0,0 +1,124 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@null@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"dummy_target_1\": \"[dummy hash for dummy_target_1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"dummy_target_2\": \"[dummy hash for dummy_target_2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"dummy_target_1\": \"[dummy hash for dummy_target_1]\", \"dummy_target_2\": \"[dummy hash for dummy_target_2]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/exparchive-batch-bmiss.json b/infra/bots/recipe_modules/isolate/example.expected/exparchive-batch-bmiss.json new file mode 100644 index 0000000000..330bd4b029 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/exparchive-batch-bmiss.json @@ -0,0 +1,187 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": \"[dummy hash for test_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@missing.isolates@Failed to find *.isolated files:@@@", + "@@@STEP_LOG_LINE@missing.isolates@test2@@@", + "@@@STEP_LOG_END@missing.isolates@@@", + "@@@STEP_FAILURE@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test_exparchive\": \"[dummy hash for test_exparchive]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolated" + ], + "name": "isolate test_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": \"[dummy hash for test_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "RECIPE_PACKAGE_REPO[skia]/test1.isolated.gen.json", + "RECIPE_PACKAGE_REPO[skia]/test2.isolated.gen.json" + ], + "name": "isolate tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": null@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@failed to isolate@Failed to isolate following targets:@@@", + "@@@STEP_LOG_LINE@failed to isolate@test2@@@", + "@@@STEP_LOG_LINE@failed to isolate@@@@", + "@@@STEP_LOG_LINE@failed to isolate@See logs for more information.@@@", + "@@@STEP_LOG_END@failed to isolate@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test_exparchive\": \"[dummy hash for test_exparchive]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/exparchive-batch-emiss.json b/infra/bots/recipe_modules/isolate/example.expected/exparchive-batch-emiss.json new file mode 100644 index 0000000000..fe24c12d82 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/exparchive-batch-emiss.json @@ -0,0 +1,187 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@missing.isolates@Failed to find *.isolated files:@@@", + "@@@STEP_LOG_LINE@missing.isolates@test_exparchive@@@", + "@@@STEP_LOG_END@missing.isolates@@@", + "@@@STEP_FAILURE@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolated" + ], + "name": "isolate test_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": null@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "RECIPE_PACKAGE_REPO[skia]/test1.isolated.gen.json", + "RECIPE_PACKAGE_REPO[skia]/test2.isolated.gen.json" + ], + "name": "isolate tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@failed to isolate@Failed to isolate following targets:@@@", + "@@@STEP_LOG_LINE@failed to isolate@test_exparchive@@@", + "@@@STEP_LOG_LINE@failed to isolate@@@@", + "@@@STEP_LOG_LINE@failed to isolate@See logs for more information.@@@", + "@@@STEP_LOG_END@failed to isolate@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/exparchive-batch.json b/infra/bots/recipe_modules/isolate/example.expected/exparchive-batch.json new file mode 100644 index 0000000000..41d070c477 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/exparchive-batch.json @@ -0,0 +1,179 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": \"[dummy hash for test_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\", \"test_exparchive\": \"[dummy hash for test_exparchive]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolated" + ], + "name": "isolate test_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": \"[dummy hash for test_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "RECIPE_PACKAGE_REPO[skia]/test1.isolated.gen.json", + "RECIPE_PACKAGE_REPO[skia]/test2.isolated.gen.json" + ], + "name": "isolate tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\", \"test_exparchive\": \"[dummy hash for test_exparchive]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/exparchive-miss.json b/infra/bots/recipe_modules/isolate/example.expected/exparchive-miss.json new file mode 100644 index 0000000000..fd3a86d7fb --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/exparchive-miss.json @@ -0,0 +1,159 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@missing.isolates@Failed to find *.isolated files:@@@", + "@@@STEP_LOG_LINE@missing.isolates@test_exparchive@@@", + "@@@STEP_LOG_END@missing.isolates@@@", + "@@@STEP_FAILURE@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolated" + ], + "name": "isolate test_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": null@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@failed to isolate@Failed to isolate following targets:@@@", + "@@@STEP_LOG_LINE@failed to isolate@test_exparchive@@@", + "@@@STEP_LOG_LINE@failed to isolate@@@@", + "@@@STEP_LOG_LINE@failed to isolate@See logs for more information.@@@", + "@@@STEP_LOG_END@failed to isolate@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/exparchive-multi-miss.json b/infra/bots/recipe_modules/isolate/example.expected/exparchive-multi-miss.json new file mode 100644 index 0000000000..cd8e4c3531 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/exparchive-multi-miss.json @@ -0,0 +1,188 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1_exparchive\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2_exparchive\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1_exparchive\": \"[dummy hash for test1_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@missing.isolates@Failed to find *.isolated files:@@@", + "@@@STEP_LOG_LINE@missing.isolates@test2_exparchive@@@", + "@@@STEP_LOG_END@missing.isolates@@@", + "@@@STEP_FAILURE@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1_exparchive\": \"[dummy hash for test1_exparchive]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test1_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test1_exparchive.isolated" + ], + "name": "isolate test1_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1_exparchive\": \"[dummy hash for test1_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test2_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test2_exparchive.isolated" + ], + "name": "isolate test2_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test2_exparchive\": null@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@failed to isolate@Failed to isolate following targets:@@@", + "@@@STEP_LOG_LINE@failed to isolate@test2_exparchive@@@", + "@@@STEP_LOG_LINE@failed to isolate@@@@", + "@@@STEP_LOG_LINE@failed to isolate@See logs for more information.@@@", + "@@@STEP_LOG_END@failed to isolate@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1_exparchive\": \"[dummy hash for test1_exparchive]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/exparchive-multi.json b/infra/bots/recipe_modules/isolate/example.expected/exparchive-multi.json new file mode 100644 index 0000000000..589722b387 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/exparchive-multi.json @@ -0,0 +1,180 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1_exparchive\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2_exparchive\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1_exparchive\": \"[dummy hash for test1_exparchive]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2_exparchive\": \"[dummy hash for test2_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1_exparchive\": \"[dummy hash for test1_exparchive]\", \"test2_exparchive\": \"[dummy hash for test2_exparchive]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test1_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test1_exparchive.isolated" + ], + "name": "isolate test1_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1_exparchive\": \"[dummy hash for test1_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test2_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test2_exparchive.isolated" + ], + "name": "isolate test2_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test2_exparchive\": \"[dummy hash for test2_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1_exparchive\": \"[dummy hash for test1_exparchive]\", \"test2_exparchive\": \"[dummy hash for test2_exparchive]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/exparchive.json b/infra/bots/recipe_modules/isolate/example.expected/exparchive.json new file mode 100644 index 0000000000..c29775ae62 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/exparchive.json @@ -0,0 +1,152 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": \"[dummy hash for test_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test_exparchive\": \"[dummy hash for test_exparchive]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "exparchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--eventlog-endpoint", + "prod", + "--isolate", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolate", + "--isolated", + "RECIPE_PACKAGE_REPO[skia]/test_exparchive.isolated" + ], + "name": "isolate test_exparchive", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_exparchive\": \"[dummy hash for test_exparchive]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test_exparchive\": \"[dummy hash for test_exparchive]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/extra.json b/infra/bots/recipe_modules/isolate/example.expected/extra.json new file mode 100644 index 0000000000..3bbbbcd115 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/extra.json @@ -0,0 +1,152 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"extra_test\": \"[dummy hash for extra_test]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "RECIPE_PACKAGE_REPO[skia]/test1.isolated.gen.json", + "RECIPE_PACKAGE_REPO[skia]/test2.isolated.gen.json" + ], + "name": "isolate tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": \"[dummy hash for test2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\", \"test2\": \"[dummy hash for test2]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/missing.json b/infra/bots/recipe_modules/isolate/example.expected/missing.json new file mode 100644 index 0000000000..9dd8532d3c --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/missing.json @@ -0,0 +1,159 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@[@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\"@@@", + "@@@STEP_LOG_LINE@json.output@]@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@missing.isolates@Failed to find *.isolated files:@@@", + "@@@STEP_LOG_LINE@missing.isolates@test2@@@", + "@@@STEP_LOG_END@missing.isolates@@@", + "@@@STEP_FAILURE@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "[START_DIR]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "RECIPE_PACKAGE_REPO[skia]/test1.isolated.gen.json", + "RECIPE_PACKAGE_REPO[skia]/test2.isolated.gen.json" + ], + "name": "isolate tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test1\": \"[dummy hash for test1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"test2\": null@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@failed to isolate@Failed to isolate following targets:@@@", + "@@@STEP_LOG_LINE@failed to isolate@test2@@@", + "@@@STEP_LOG_LINE@failed to isolate@@@@", + "@@@STEP_LOG_LINE@failed to isolate@See logs for more information.@@@", + "@@@STEP_LOG_END@failed to isolate@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test1\": \"[dummy hash for test1]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.expected/none.json b/infra/bots/recipe_modules/isolate/example.expected/none.json new file mode 100644 index 0000000000..fc961f24ad --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.expected/none.json @@ -0,0 +1,122 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "cat" + ], + "name": "read test spec", + "stdout": "/path/to/tmp/json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@null@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "RECIPE_PACKAGE_REPO[skia]", + "--output-json", + "/path/to/tmp/json" + ], + "name": "find isolated tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_WARNINGS@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/example.py b/infra/bots/recipe_modules/isolate/example.py new file mode 100644 index 0000000000..7dfa009f65 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/example.py @@ -0,0 +1,147 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +from recipe_engine.recipe_api import Property + +DEPS = [ + 'isolate', + 'recipe_engine/json', + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/step', + 'swarming_client', +] + +PROPERTIES = { + 'always_use_exparchive': Property( + kind=bool, help="Force usage of exparchive.", default=False), +} + + +def RunSteps(api, always_use_exparchive): + # 'isolate_tests' step needs swarming checkout. + api.swarming_client.checkout('master') + + # Code coverage for isolate_server property. + api.isolate.isolate_server = 'https://isolateserver-dev.appspot.com' + assert api.isolate.isolate_server == 'https://isolateserver-dev.appspot.com' + + # That would read a list of files to search for, generated in GenTests. + step_result = api.step('read test spec', ['cat'], stdout=api.json.output()) + expected_targets = step_result.stdout + + build_path = api.isolate.package_repo_resource() + # Generates code coverage for find_isolated_tests corner cases. + # TODO(vadimsh): This step doesn't actually make any sense when the recipe + # is running for real via run_recipe.py. + api.isolate.find_isolated_tests(build_path, expected_targets) + + # Code coverage for 'isolate_tests'. 'isolated_test' doesn't support discovery + # of isolated targets in build directory, so skip if 'expected_targets' is + # None. + if expected_targets is not None: + api.isolate.isolate_tests( + build_path, expected_targets, + always_use_exparchive=always_use_exparchive) + + +def GenTests(api): + def make_test( + name, + expected_batcharchive_targets, + expected_exparchive_targets, + discovered_targets): + + if expected_batcharchive_targets or expected_exparchive_targets: + all_expected_targets = ( + (expected_batcharchive_targets or []) + + (expected_exparchive_targets or [])) + else: + all_expected_targets = None + + missing = set(all_expected_targets or []) - set(discovered_targets or []) + output = ( + api.test(name) + + api.step_data( + 'read test spec', + stdout=api.json.output(all_expected_targets)) + + api.override_step_data( + 'find isolated tests', + api.isolate.output_json(discovered_targets)) + ) + + # See comment around 'if expected_targets is not None' above. + if all_expected_targets: + for target in sorted(expected_exparchive_targets): + output += api.override_step_data( + 'isolate %s' % target, + api.isolate.output_json([target], missing)) + + if expected_batcharchive_targets: + output += api.override_step_data( + 'isolate tests', + api.isolate.output_json(expected_batcharchive_targets, missing)) + + return output + + # Expected targets == found targets. + yield make_test( + 'basic', ['test1', 'test2'], [], ['test1', 'test2']) + # No expectations, just discovering what's there returned by default mock. + yield make_test( + 'discover', None, None, None) + # Found more than expected. + yield make_test( + 'extra', ['test1', 'test2'], [], ['test1', 'test2', 'extra_test']) + # Didn't find something. + yield ( + make_test('missing', ['test1', 'test2'], [], ['test1']) + + api.properties.generic(buildername='Windows Swarm Test')) + # No expectations, and nothing has been found, produces warning. + yield make_test('none', None, None, []) + # Test the `exparchive` cases + # Only exparchive + yield make_test( + 'exparchive', [], ['test_exparchive'], ['test_exparchive']) + yield make_test( + 'exparchive-miss', [], ['test_exparchive'], []) + yield make_test( + 'exparchive-multi', + [], + ['test1_exparchive', 'test2_exparchive'], + ['test1_exparchive', 'test2_exparchive']) + yield make_test( + 'exparchive-multi-miss', + [], + ['test1_exparchive', 'test2_exparchive'], + ['test1_exparchive']) + # Mixed + yield make_test( + 'exparchive-batch', + ['test1', 'test2'], + ['test_exparchive'], + ['test1', 'test2', 'test_exparchive']) + yield make_test( + 'exparchive-batch-bmiss', + ['test1', 'test2'], + ['test_exparchive'], + ['test1', 'test_exparchive']) + yield make_test( + 'exparchive-batch-emiss', + ['test1', 'test2'], + ['test_exparchive'], + ['test1', 'test2']) + # Use force-exparchive + yield make_test( + 'always-use-exparchive', + [], + ['test_exparchive', 'test1', 'test2'], + ['test_exparchive', 'test1', 'test2']) + api.properties( + always_use_exparchive=True) diff --git a/infra/bots/recipe_modules/isolate/resources/find_isolated_tests.py b/infra/bots/recipe_modules/isolate/resources/find_isolated_tests.py new file mode 100755 index 0000000000..9214f2988a --- /dev/null +++ b/infra/bots/recipe_modules/isolate/resources/find_isolated_tests.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Scans build output directory for .isolated files, calculates their SHA1 +hashes and stores final list in JSON document. + +Used to figure out what tests were build in isolated mode to trigger these +tests to run on swarming. + +For more info see: +https://sites.google.com/a/chromium.org/dev/developers/testing/isolated-testing +""" + +import glob +import hashlib +import json +import optparse +import os +import re +import sys + + +def hash_file(filepath): + """Calculates the hash of a file without reading it all in memory at once.""" + digest = hashlib.sha1() + with open(filepath, 'rb') as f: + while True: + chunk = f.read(1024*1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def main(): + parser = optparse.OptionParser( + usage='%prog --build-dir <path> ' + '[--output-json <path> | --clean-isolated-files]', + description=sys.modules[__name__].__doc__) + parser.add_option( + '--build-dir', + help='Path to a directory to search for *.isolated files.') + parser.add_option( + '--output-json', + help='File to dump JSON results into. ' + 'Mutually exclusive with --clean-isolated-files.') + parser.add_option( + '--clean-isolated-files', + action='store_true', + help='Whether to clean out all .isolated and .isolated.gen.json files. ' + 'Mutually exclusive with --output-json.') + + options, _ = parser.parse_args() + if not options.build_dir: + parser.error('--build-dir option is required') + if not (options.output_json or options.clean_isolated_files): + parser.error('either --output-json or --clean-isolated-files is required') + if options.output_json and options.clean_isolated_files: + parser.error('only one of --output-json and ' + '--clean-isolated-files is allowed') + + result = {} + + # Clean up generated *.isolated.gen.json files, produced by isolate_driver.py + # in test_isolation_mode='prepare'. + if options.clean_isolated_files: + pattern = os.path.join(options.build_dir, '*.isolated.gen.json') + for filepath in sorted(glob.glob(pattern)): + os.remove(filepath) + + # Get the file hash values and output the pair. + pattern = os.path.join(options.build_dir, '*.isolated') + for filepath in sorted(glob.glob(pattern)): + test_name = os.path.splitext(os.path.basename(filepath))[0] + if re.match(r'^.+?\.\d$', test_name): + # It's a split .isolated file, e.g. foo.0.isolated. Ignore these. + continue + + if options.clean_isolated_files: + # TODO(csharp): Remove deletion entirely once the isolate + # tracked dependencies are inputs for the isolated files. + # http://crbug.com/419031 + os.remove(filepath) + else: + sha1_hash = hash_file(filepath) + result[test_name] = sha1_hash + + if options.output_json: + with open(options.output_json, 'wb') as f: + json.dump(result, f) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/infra/bots/recipe_modules/isolate/resources/isolate.py b/infra/bots/recipe_modules/isolate/resources/isolate.py new file mode 100755 index 0000000000..9414df518d --- /dev/null +++ b/infra/bots/recipe_modules/isolate/resources/isolate.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Calls the isolate Go executable in the checkout, failing if it cannot run. +""" + +# TODO(djd): make the caller invoke the Go binary directly, kill this script. + +import os +import subprocess +import sys + + +def try_go(path, args): + """Tries to run the Go implementation of isolate. + """ + luci_go = os.path.join(os.path.dirname(path), 'luci-go') + if sys.platform == 'win32': + exe = os.path.join(luci_go, 'win64', 'isolate.exe') + elif sys.platform == 'darwin': + exe = os.path.join(luci_go, 'mac64', 'isolate') + else: + exe = os.path.join(luci_go, 'linux64', 'isolate') + + return subprocess.call([exe] + args) + + +def main(): + path = sys.argv[1] + args = sys.argv[2:] + return try_go(path, args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/infra/bots/recipe_modules/isolate/test_api.py b/infra/bots/recipe_modules/isolate/test_api.py new file mode 100644 index 0000000000..e56e60b71a --- /dev/null +++ b/infra/bots/recipe_modules/isolate/test_api.py @@ -0,0 +1,30 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +from recipe_engine import recipe_test_api + +class IsolateTestApi(recipe_test_api.RecipeTestApi): + def output_json(self, targets=None, missing=None): + """Mocked output of 'find_isolated_tests' and 'isolate_tests' steps. + + Deterministically synthesizes json.output test data for the given targets. + If |targets| is None, will emit test data with some dummy targets instead, + emulating find_isolated_tests.py finding some files. + + If |missing| is given it's a subset of |targets| that wasn't isolated in + 'isolate_tests' due to some error. + """ + missing = missing or () + if targets is None: + targets = ['dummy_target_1', 'dummy_target_2'] + return self.m.json.output({ + target: None if target in missing else '[dummy hash for %s]' % target + for target in targets + }) diff --git a/infra/bots/recipe_modules/isolate/tests/clean_isolated_files.expected/basic.json b/infra/bots/recipe_modules/isolate/tests/clean_isolated_files.expected/basic.json new file mode 100644 index 0000000000..afe369aac1 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/tests/clean_isolated_files.expected/basic.json @@ -0,0 +1,18 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/find_isolated_tests.py", + "--build-dir", + "None/out/Release", + "--clean-isolated-files" + ], + "name": "clean isolated files" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/tests/clean_isolated_files.py b/infra/bots/recipe_modules/isolate/tests/clean_isolated_files.py new file mode 100644 index 0000000000..075776456f --- /dev/null +++ b/infra/bots/recipe_modules/isolate/tests/clean_isolated_files.py @@ -0,0 +1,16 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'isolate', + 'recipe_engine/path', +] + + +def RunSteps(api): + api.isolate.clean_isolated_files(api.path['checkout'].join('out', 'Release')) + + +def GenTests(api): + yield api.test('basic') diff --git a/infra/bots/recipe_modules/isolate/tests/isolate_tests.expected/basic.json b/infra/bots/recipe_modules/isolate/tests/isolate_tests.expected/basic.json new file mode 100644 index 0000000000..d2fcf462df --- /dev/null +++ b/infra/bots/recipe_modules/isolate/tests/isolate_tests.expected/basic.json @@ -0,0 +1,41 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "None/out/Release/*.isolated.gen.json" + ], + "name": "find isolated targets" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", + "None/tools/swarming_client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "None/out/Release/dummy_target_1.isolated.gen.json", + "None/out/Release/dummy_target_2.isolated.gen.json" + ], + "name": "isolate tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"dummy_target_1\": \"[dummy hash for dummy_target_1]\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"dummy_target_2\": \"[dummy hash for dummy_target_2]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"dummy_target_1\": \"[dummy hash for dummy_target_1]\", \"dummy_target_2\": \"[dummy hash for dummy_target_2]\"}@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/tests/isolate_tests.py b/infra/bots/recipe_modules/isolate/tests/isolate_tests.py new file mode 100644 index 0000000000..9d0ba0f8a4 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/tests/isolate_tests.py @@ -0,0 +1,16 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'isolate', + 'recipe_engine/path', +] + + +def RunSteps(api): + api.isolate.isolate_tests(api.path['checkout'].join('out', 'Release')) + + +def GenTests(api): + yield api.test('basic') diff --git a/infra/bots/recipe_modules/isolate/tests/isolated_tests.expected/basic.json b/infra/bots/recipe_modules/isolate/tests/isolated_tests.expected/basic.json new file mode 100644 index 0000000000..e11a23699b --- /dev/null +++ b/infra/bots/recipe_modules/isolate/tests/isolated_tests.expected/basic.json @@ -0,0 +1,15 @@ +[ + { + "cmd": [], + "name": "isolated_tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@details@isolated_tests: {'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff'}@@@", + "@@@STEP_LOG_END@details@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/tests/isolated_tests.py b/infra/bots/recipe_modules/isolate/tests/isolated_tests.py new file mode 100644 index 0000000000..2101bd7035 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/tests/isolated_tests.py @@ -0,0 +1,27 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'isolate', + 'recipe_engine/properties', + 'recipe_engine/step', +] + + +def RunSteps(api): + api.step('isolated_tests', []) + api.step.active_result.presentation.logs['details'] = [ + 'isolated_tests: %r' % api.isolate.isolated_tests + ] + + +def GenTests(api): + yield ( + api.test('basic') + + api.properties( + swarm_hashes={ + 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', + } + ) + ) diff --git a/infra/bots/recipe_modules/isolate/tests/run_isolated.expected/basic.json b/infra/bots/recipe_modules/isolate/tests/run_isolated.expected/basic.json new file mode 100644 index 0000000000..2ed58f1e94 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/tests/run_isolated.expected/basic.json @@ -0,0 +1,23 @@ +[ + { + "cmd": [ + "python", + "-u", + "None/tools/swarming_client/run_isolated.py", + "--isolated", + "isolate_hash", + "-I", + "https://isolateserver.appspot.com", + "--verbose", + "--", + "some", + "args" + ], + "name": "run_isolated" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/isolate/tests/run_isolated.py b/infra/bots/recipe_modules/isolate/tests/run_isolated.py new file mode 100644 index 0000000000..df83887386 --- /dev/null +++ b/infra/bots/recipe_modules/isolate/tests/run_isolated.py @@ -0,0 +1,15 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'isolate', +] + + +def RunSteps(api): + api.isolate.run_isolated('run_isolated', 'isolate_hash', ['some', 'args']) + + +def GenTests(api): + yield api.test('basic') diff --git a/infra/bots/recipe_modules/run/__init__.py b/infra/bots/recipe_modules/run/__init__.py index 25e4c0dad5..7c13f4f8ee 100644 --- a/infra/bots/recipe_modules/run/__init__.py +++ b/infra/bots/recipe_modules/run/__init__.py @@ -3,8 +3,8 @@ # found in the LICENSE file. DEPS = [ - 'build/file', 'env', + 'file', 'recipe_engine/json', 'recipe_engine/path', 'recipe_engine/platform', diff --git a/infra/bots/recipe_modules/run/example.expected/test.json b/infra/bots/recipe_modules/run/example.expected/test.json index b3b40db128..072c39c913 100644 --- a/infra/bots/recipe_modules/run/example.expected/test.json +++ b/infra/bots/recipe_modules/run/example.expected/test.json @@ -83,7 +83,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "mydir" ], diff --git a/infra/bots/recipe_modules/skia_swarming/__init__.py b/infra/bots/recipe_modules/skia_swarming/__init__.py new file mode 100644 index 0000000000..435ad0be88 --- /dev/null +++ b/infra/bots/recipe_modules/skia_swarming/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'depot_tools/depot_tools', + 'file', + 'isolate', + 'recipe_engine/context', + 'recipe_engine/json', + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/python', + 'recipe_engine/raw_io', + 'recipe_engine/step', + 'run', + 'swarming', + 'swarming_client', +] diff --git a/infra/bots/recipe_modules/skia_swarming/api.py b/infra/bots/recipe_modules/skia_swarming/api.py new file mode 100644 index 0000000000..35dddcf507 --- /dev/null +++ b/infra/bots/recipe_modules/skia_swarming/api.py @@ -0,0 +1,220 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +from recipe_engine import recipe_api +import shlex + + +DEFAULT_TASK_EXPIRATION = 20*60*60 +DEFAULT_TASK_TIMEOUT = 4*60*60 +DEFAULT_IO_TIMEOUT = 40*60 + +MILO_LOG_LINK = 'https://luci-milo.appspot.com/swarming/task/%s' + + +class SkiaSwarmingApi(recipe_api.RecipeApi): + """Provides steps to run Skia tasks on swarming bots.""" + + @property + def swarming_temp_dir(self): + """Path where artifacts like isolate file and json output will be stored.""" + return self.m.path['start_dir'].join('swarming_temp_dir') + + @property + def tasks_output_dir(self): + """Directory where the outputs of the swarming tasks will be stored.""" + return self.swarming_temp_dir.join('outputs') + + def isolated_file_path(self, task_name): + """Get the path to the given task's .isolated file.""" + return self.swarming_temp_dir.join('skia-task-%s.isolated' % task_name) + + def setup(self, luci_go_dir, swarming_rev=None): + """Performs setup steps for swarming.""" + self.m.swarming_client.checkout(revision=swarming_rev) + self.m.swarming.check_client_version(step_test_data=(0, 8, 6)) + self.setup_go_isolate(luci_go_dir) + self.m.swarming.add_default_tag('allow_milo:1') + + # TODO(rmistry): Remove once the Go binaries are moved to recipes or buildbot. + def setup_go_isolate(self, luci_go_dir): + """Generates and puts in place the isolate Go binary.""" + depot_tools_path = self.m.depot_tools.package_repo_resource() + env = {'PATH': self.m.path.pathsep.join([ + str(depot_tools_path), '%(PATH)s'])} + with self.m.context(env=env): + self.m.step('download luci-go linux', + ['download_from_google_storage', '--no_resume', + '--platform=linux*', '--no_auth', + '--bucket', 'chromium-luci', + '-d', luci_go_dir.join('linux64')]) + self.m.step('download luci-go mac', + ['download_from_google_storage', '--no_resume', + '--platform=darwin', '--no_auth', + '--bucket', 'chromium-luci', + '-d', luci_go_dir.join('mac64')]) + self.m.step('download luci-go win', + ['download_from_google_storage', '--no_resume', + '--platform=win32', '--no_auth', '--bucket', + 'chromium-luci', + '-d', luci_go_dir.join('win64')]) + # Copy binaries to the expected location. + dest = self.m.path['start_dir'].join('luci-go') + self.m.run.rmtree(dest) + self.m.file.copytree('Copy Go binary', + source=luci_go_dir, + dest=dest) + + def create_isolated_gen_json(self, isolate_path, base_dir, os_type, + task_name, extra_variables, blacklist=None): + """Creates an isolated.gen.json file (used by the isolate recipe module). + + Args: + isolate_path: path obj. Path to the isolate file. + base_dir: path obj. Dir that is the base of all paths in the isolate file. + os_type: str. The OS type to use when archiving the isolate file. + Eg: linux. + task_name: str. The isolated.gen.json file will be suffixed by this str. + extra_variables: dict of str to str. The extra vars to pass to isolate. + Eg: {'SLAVE_NUM': '1', 'MASTER': 'ChromiumPerfFYI'} + blacklist: list of regular expressions indicating which files/directories + not to archive. + """ + self.m.file.makedirs('swarming tmp dir', self.swarming_temp_dir) + isolated_path = self.isolated_file_path(task_name) + isolate_args = [ + '--isolate', isolate_path, + '--isolated', isolated_path, + '--config-variable', 'OS', os_type, + ] + if blacklist: + for b in blacklist: + isolate_args.extend(['--blacklist', b]) + for k, v in extra_variables.iteritems(): + isolate_args.extend(['--extra-variable', k, v]) + isolated_gen_dict = { + 'version': 1, + 'dir': base_dir, + 'args': isolate_args, + } + isolated_gen_json = self.swarming_temp_dir.join( + '%s.isolated.gen.json' % task_name) + self.m.file.write( + 'Write %s.isolated.gen.json' % task_name, + isolated_gen_json, + self.m.json.dumps(isolated_gen_dict, indent=4), + ) + + def batcharchive(self, targets): + """Calls batcharchive on the skia.isolated.gen.json file. + + Args: + targets: list of str. The suffixes of the isolated.gen.json files to + archive. + + Returns: + list of tuples containing (task_name, swarming_hash). + """ + return self.m.isolate.isolate_tests( + verbose=True, # To avoid no output timeouts. + build_dir=self.swarming_temp_dir, + targets=targets).presentation.properties['swarm_hashes'].items() + + def trigger_swarming_tasks( + self, swarm_hashes, dimensions, idempotent=False, store_output=True, + extra_args=None, expiration=None, hard_timeout=None, io_timeout=None, + cipd_packages=None): + """Triggers swarming tasks using swarm hashes. + + Args: + swarm_hashes: list of str. List of swarm hashes from the isolate server. + dimensions: dict of str to str. The dimensions to run the task on. + Eg: {'os': 'Ubuntu', 'gpu': '10de', 'pool': 'Skia'} + idempotent: bool. Whether or not to de-duplicate tasks. + store_output: bool. Whether task output should be stored. + extra_args: list of str. Extra arguments to pass to the task. + expiration: int. Task will expire if not picked up within this time. + DEFAULT_TASK_EXPIRATION is used if this argument is None. + hard_timeout: int. Task will timeout if not completed within this time. + DEFAULT_TASK_TIMEOUT is used if this argument is None. + io_timeout: int. Task will timeout if there is no output within this time. + DEFAULT_IO_TIMEOUT is used if this argument is None. + cipd_packages: CIPD packages which these tasks depend on. + + Returns: + List of swarming.SwarmingTask instances. + """ + swarming_tasks = [] + for task_name, swarm_hash in swarm_hashes: + swarming_task = self.m.swarming.task( + title=task_name, + cipd_packages=cipd_packages, + isolated_hash=swarm_hash) + if store_output: + swarming_task.task_output_dir = self.tasks_output_dir.join(task_name) + swarming_task.dimensions = dimensions + swarming_task.idempotent = idempotent + swarming_task.priority = 90 + swarming_task.expiration = ( + expiration if expiration else DEFAULT_TASK_EXPIRATION) + swarming_task.hard_timeout = ( + hard_timeout if hard_timeout else DEFAULT_TASK_TIMEOUT) + swarming_task.io_timeout = ( + io_timeout if io_timeout else DEFAULT_IO_TIMEOUT) + if extra_args: + swarming_task.extra_args = extra_args + revision = self.m.properties.get('revision') + if revision: + swarming_task.tags.add('revision:%s' % revision) + swarming_tasks.append(swarming_task) + step_results = self.m.swarming.trigger(swarming_tasks) + for step_result in step_results: + self._add_log_links(step_result, step_result.json.output) + return swarming_tasks + + def collect_swarming_task(self, swarming_task): + """Collects the specified swarming task. + + Args: + swarming_task: An instance of swarming.SwarmingTask. + """ + try: + rv = self.m.swarming.collect_task(swarming_task) + except self.m.step.StepFailure as e: # pragma: no cover + step_result = self.m.step.active_result + # Change step result to Infra failure if the swarming task failed due to + # expiration, time outs, bot crashes or task cancelations. + # Infra failures have step.EXCEPTION. + states_infra_failure = ( + self.m.swarming.State.EXPIRED, self.m.swarming.State.TIMED_OUT, + self.m.swarming.State.BOT_DIED, self.m.swarming.State.CANCELED) + summary = step_result.swarming.summary + if summary['shards'][0]['state'] in states_infra_failure: + step_result.presentation.status = self.m.step.EXCEPTION + raise self.m.step.InfraFailure(e.name, step_result) + raise + finally: + step_result = self.m.step.active_result + # Add log link. + self._add_log_links(step_result, step_result.swarming.summary) + return rv + + def _add_log_links(self, step_result, summary): + """Add Milo log links to all shards in the step.""" + ids = [] + shards = summary.get('shards') + if shards: + for shard in shards: + ids.append(shard['id']) + else: + for _, task in summary.get('tasks', {}).iteritems(): + ids.append(task['task_id']) + for idx, task_id in enumerate(ids): + link = MILO_LOG_LINK % task_id + k = 'view steps on Milo' + if len(ids) > 1: # pragma: nocover + k += ' (shard index %d, %d total)' % (idx, len(ids)) + step_result.presentation.links[k] = link + diff --git a/infra/bots/recipe_modules/swarming/example.expected/test.json b/infra/bots/recipe_modules/skia_swarming/example.expected/test.json index f59d6fa45f..f425f90499 100644 --- a/infra/bots/recipe_modules/swarming/example.expected/test.json +++ b/infra/bots/recipe_modules/skia_swarming/example.expected/test.json @@ -150,7 +150,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -207,7 +207,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -549,17 +549,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/task-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -630,17 +626,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/task-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -711,17 +703,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/task-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -792,17 +780,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/task-0", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -873,17 +857,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/task-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipe_modules/skia_swarming/example.py b/infra/bots/recipe_modules/skia_swarming/example.py new file mode 100644 index 0000000000..bf928e2308 --- /dev/null +++ b/infra/bots/recipe_modules/skia_swarming/example.py @@ -0,0 +1,31 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +DEPS = [ + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/step', + 'skia_swarming', +] + + +def RunSteps(api): + api.skia_swarming.setup('mydir', swarming_rev='abc123') + api.skia_swarming.create_isolated_gen_json( + 'isolate_path', 'isolate_dir', 'linux', 'task', {'myvar': 'myval'}, + blacklist=['*.pyc']) + tasks_to_hashes = api.skia_swarming.batcharchive(targets=[ + 'task-%s' % num for num in range(5)]) + tasks = api.skia_swarming.trigger_swarming_tasks( + tasks_to_hashes, dimensions={'os': 'Linux'}, extra_args=['--extra']) + for t in tasks: + api.skia_swarming.collect_swarming_task(t) + + +def GenTests(api): + yield ( + api.test('test') + + api.properties(revision='abc123') + ) diff --git a/infra/bots/recipe_modules/swarming/OWNERS b/infra/bots/recipe_modules/swarming/OWNERS new file mode 100644 index 0000000000..c14640d2a4 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/OWNERS @@ -0,0 +1,3 @@ +maruel@chromium.org +tansell@chromium.org +vadimsh@chromium.org diff --git a/infra/bots/recipe_modules/swarming/__init__.py b/infra/bots/recipe_modules/swarming/__init__.py index fbd0c006cf..98b20ffd81 100644 --- a/infra/bots/recipe_modules/swarming/__init__.py +++ b/infra/bots/recipe_modules/swarming/__init__.py @@ -1,19 +1,33 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + DEPS = [ - 'build/file', - 'build/isolate', - 'build/swarming', - 'build/swarming_client', - 'depot_tools/depot_tools', + 'isolate', 'recipe_engine/context', 'recipe_engine/json', 'recipe_engine/path', + 'recipe_engine/platform', 'recipe_engine/properties', 'recipe_engine/python', 'recipe_engine/raw_io', 'recipe_engine/step', - 'run', + 'swarming_client', ] + +from recipe_engine.recipe_api import Property + +PROPERTIES = { + 'show_shards_in_collect_step': Property(default=False, kind=bool), + 'show_isolated_out_in_collect_step': Property(default=True, kind=bool), +} + + +# TODO(phajdan.jr): provide coverage (http://crbug.com/693058). +DISABLE_STRICT_COVERAGE = True diff --git a/infra/bots/recipe_modules/swarming/api.py b/infra/bots/recipe_modules/swarming/api.py index 35dddcf507..47e2c84158 100644 --- a/infra/bots/recipe_modules/swarming/api.py +++ b/infra/bots/recipe_modules/swarming/api.py @@ -1,220 +1,905 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import datetime +import functools +import hashlib +import logging +import os.path +from recipe_engine import config_types from recipe_engine import recipe_api -import shlex +from recipe_engine import util as recipe_util +import state -DEFAULT_TASK_EXPIRATION = 20*60*60 -DEFAULT_TASK_TIMEOUT = 4*60*60 -DEFAULT_IO_TIMEOUT = 40*60 -MILO_LOG_LINK = 'https://luci-milo.appspot.com/swarming/task/%s' +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. -class SkiaSwarmingApi(recipe_api.RecipeApi): - """Provides steps to run Skia tasks on swarming bots.""" +# Minimally supported version of swarming.py script (reported by --version). +MINIMAL_SWARMING_VERSION = (0, 8, 6) + + +def text_for_task(task): + lines = [] + + if task.dimensions.get('id'): # pragma: no cover + lines.append('Bot id: %r' % task.dimensions['id']) + if task.dimensions.get('os'): + lines.append('Run on OS: %r' % task.dimensions['os']) + + return '<br/>'.join(lines) + + +def parse_time(value): + """Converts serialized time from the API to datetime.datetime.""" + # When microseconds are 0, the '.123456' suffix is elided. This means the + # serialized format is not consistent, which confuses the hell out of python. + # TODO(maruel): Remove third format once we enforce version >=0.8.2. + for fmt in ('%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S'): + try: + return datetime.datetime.strptime(value, fmt) + except ValueError: # pragma: no cover + pass + raise ValueError('Failed to parse %s' % value) # pragma: no cover + + +class ReadOnlyDict(dict): + def __setitem__(self, key, value): + raise TypeError('ReadOnlyDict is immutable') + + +class SwarmingApi(recipe_api.RecipeApi): + """Recipe module to use swarming.py tool to run tasks on Swarming. + + General usage: + 1. Tweak default task parameters applied to all swarming tasks (such as + default_dimensions and default_priority). + 2. Isolate some test using 'isolate' recipe module. Get isolated hash as + a result of that process. + 3. Create a task configuration using 'task(...)' method, providing + isolated hash obtained previously. + 4. Tweak the task parameters. This step is optional. + 5. Launch the task on swarming by calling 'trigger_task(...)'. + 6. Continue doing useful work locally while the task is running concurrently + on swarming. + 7. Wait for task to finish and collect its result (exit code, logs) + by calling 'collect_task(...)'. + + See also example.py for concrete code. + """ + + State = state.State + + ############################################################################# + # The below are helper functions to help transition between the old and new # + # swarming result formats. TODO(martiniss): remove these # + ############################################################################# + + def _is_expired(self, shard): + # FIXME: We really should only have one format for enums. We want to move to + # strings, currently have numbers. + return ( + shard.get('state') == self.State.EXPIRED or + shard.get('state') == 'EXPIRED') + + def _is_timed_out(self, shard): + # FIXME: We really should only have one format for enums. We want to move to + # strings, currently have numbers. + return ( + shard.get('state') == self.State.TIMED_OUT or + shard.get('state') == 'TIMED_OUT') + + def _get_exit_code(self, shard): + if shard.get('exit_code'): + return shard.get('exit_code') # pragma: no cover + lst = shard.get('exit_codes', []) + return str(lst[0]) if lst else None + + def __init__(self, **kwargs): + super(SwarmingApi, self).__init__(**kwargs) + # All tests default to a x86-64 bot running with no GPU. This simplifies + # management so that new tests are not executed on exotic bots by accidents + # even if misconfigured. + self._default_dimensions = { + 'cpu': 'x86-64', + 'gpu': 'none', + } + # Expirations are set to mildly good values and will be tightened soon. + self._default_expiration = 60*60 + self._default_env = {} + self._default_hard_timeout = 60*60 + self._default_idempotent = False + self._default_io_timeout = 20*60 + # The default priority is extremely low and should be increased dependending + # on the type of task. + self._default_priority = 200 + self._default_tags = set() + self._default_user = None + self._pending_tasks = set() + self._show_isolated_out_in_collect_step = True + self._show_shards_in_collect_step = False + self._swarming_server = 'https://chromium-swarm.appspot.com' + self._verbose = False + + @recipe_util.returns_placeholder + def summary(self): + return self.m.json.output() + + @property + def swarming_server(self): + """URL of Swarming server to use, default is a production one.""" + return self._swarming_server + + @swarming_server.setter + def swarming_server(self, value): + """Changes URL of Swarming server to use.""" + self._swarming_server = value @property - def swarming_temp_dir(self): - """Path where artifacts like isolate file and json output will be stored.""" - return self.m.path['start_dir'].join('swarming_temp_dir') + def verbose(self): + """True to run swarming scripts with verbose output.""" + return self._verbose + + @verbose.setter + def verbose(self, value): + """Enables or disables verbose output in swarming scripts.""" + assert isinstance(value, bool), value + self._verbose = value @property - def tasks_output_dir(self): - """Directory where the outputs of the swarming tasks will be stored.""" - return self.swarming_temp_dir.join('outputs') - - def isolated_file_path(self, task_name): - """Get the path to the given task's .isolated file.""" - return self.swarming_temp_dir.join('skia-task-%s.isolated' % task_name) - - def setup(self, luci_go_dir, swarming_rev=None): - """Performs setup steps for swarming.""" - self.m.swarming_client.checkout(revision=swarming_rev) - self.m.swarming.check_client_version(step_test_data=(0, 8, 6)) - self.setup_go_isolate(luci_go_dir) - self.m.swarming.add_default_tag('allow_milo:1') - - # TODO(rmistry): Remove once the Go binaries are moved to recipes or buildbot. - def setup_go_isolate(self, luci_go_dir): - """Generates and puts in place the isolate Go binary.""" - depot_tools_path = self.m.depot_tools.package_repo_resource() - env = {'PATH': self.m.path.pathsep.join([ - str(depot_tools_path), '%(PATH)s'])} - with self.m.context(env=env): - self.m.step('download luci-go linux', - ['download_from_google_storage', '--no_resume', - '--platform=linux*', '--no_auth', - '--bucket', 'chromium-luci', - '-d', luci_go_dir.join('linux64')]) - self.m.step('download luci-go mac', - ['download_from_google_storage', '--no_resume', - '--platform=darwin', '--no_auth', - '--bucket', 'chromium-luci', - '-d', luci_go_dir.join('mac64')]) - self.m.step('download luci-go win', - ['download_from_google_storage', '--no_resume', - '--platform=win32', '--no_auth', '--bucket', - 'chromium-luci', - '-d', luci_go_dir.join('win64')]) - # Copy binaries to the expected location. - dest = self.m.path['start_dir'].join('luci-go') - self.m.run.rmtree(dest) - self.m.file.copytree('Copy Go binary', - source=luci_go_dir, - dest=dest) - - def create_isolated_gen_json(self, isolate_path, base_dir, os_type, - task_name, extra_variables, blacklist=None): - """Creates an isolated.gen.json file (used by the isolate recipe module). + def default_expiration(self): + """Number of seconds that the server will wait to find a bot able to run the + task. - Args: - isolate_path: path obj. Path to the isolate file. - base_dir: path obj. Dir that is the base of all paths in the isolate file. - os_type: str. The OS type to use when archiving the isolate file. - Eg: linux. - task_name: str. The isolated.gen.json file will be suffixed by this str. - extra_variables: dict of str to str. The extra vars to pass to isolate. - Eg: {'SLAVE_NUM': '1', 'MASTER': 'ChromiumPerfFYI'} - blacklist: list of regular expressions indicating which files/directories - not to archive. - """ - self.m.file.makedirs('swarming tmp dir', self.swarming_temp_dir) - isolated_path = self.isolated_file_path(task_name) - isolate_args = [ - '--isolate', isolate_path, - '--isolated', isolated_path, - '--config-variable', 'OS', os_type, - ] - if blacklist: - for b in blacklist: - isolate_args.extend(['--blacklist', b]) - for k, v in extra_variables.iteritems(): - isolate_args.extend(['--extra-variable', k, v]) - isolated_gen_dict = { - 'version': 1, - 'dir': base_dir, - 'args': isolate_args, - } - isolated_gen_json = self.swarming_temp_dir.join( - '%s.isolated.gen.json' % task_name) - self.m.file.write( - 'Write %s.isolated.gen.json' % task_name, - isolated_gen_json, - self.m.json.dumps(isolated_gen_dict, indent=4), - ) + If not bot runs the task by this number of seconds, the task is canceled as + EXPIRED. - def batcharchive(self, targets): - """Calls batcharchive on the skia.isolated.gen.json file. + This value can be changed per individual task. + """ + return self._default_expiration - Args: - targets: list of str. The suffixes of the isolated.gen.json files to - archive. + @default_expiration.setter + def default_expiration(self, value): + assert 30 <= value <= 24*60*60, value + self._default_expiration = value - Returns: - list of tuples containing (task_name, swarming_hash). + @property + def default_hard_timeout(self): + """Number of seconds in which the task must complete. + + If the task takes more than this amount of time, the process is assumed to + be hung. It forcibly killed via SIGTERM then SIGKILL after a grace period + (default: 30s). Then the task is marked as TIMED_OUT. + + This value can be changed per individual task. + """ + return self._default_hard_timeout + + @default_hard_timeout.setter + def default_hard_timeout(self, value): + assert 30 <= value <= 6*60*60, value + self._default_hard_timeout = value + + @property + def default_io_timeout(self): + """Number of seconds at which interval the task must write to stdout or + stderr. + + If the task takes more than this amount of time between writes to stdout or + stderr, the process is assumed to be hung. It forcibly killed via SIGTERM + then SIGKILL after a grace period (default: 30s). Then the task is marked as + TIMED_OUT. + + This value can be changed per individual task. + """ + return self._default_io_timeout + + @default_io_timeout.setter + def default_io_timeout(self, value): + assert 30 <= value <= 6*60*60, value + self._default_io_timeout = value + + @property + def default_idempotent(self): + """Bool to specify if task deduplication can be done. + + When set, the server will search for another task that ran in the last days + that had the exact same properties. If it finds one, the task will not be + run at all, the previous results will be returned as-is. + + For more infos, see: + https://github.com/luci/luci-py/blob/master/appengine/swarming/doc/User-Guide.md#task-idempotency + + This value can be changed per individual task. + """ + return self._default_idempotent + + @default_idempotent.setter + def default_idempotent(self, value): + assert isinstance(value, bool), value + self._default_idempotent = value + + @property + def default_user(self): + """String to represent who triggered the task. + + The user should be an email address when someone requested testing via + pre-commit or manual testing. + + This value can be changed per individual task. + """ + return self._default_user + + @default_user.setter + def default_user(self, value): + assert value is None or isinstance(value, basestring), value + self._default_user = value + + @property + def default_dimensions(self): + """Returns a copy of the default Swarming dimensions to run task on. + + The dimensions are what is used to filter which bots are able to run the + task successfully. This is particularly useful to discern between OS + versions, type of CPU, GPU card or VM, or preallocated pool. + + Example: + {'cpu': 'x86-64', 'os': 'Windows-XP-SP3'} + + This value can be changed per individual task. + """ + return ReadOnlyDict(self._default_dimensions) + + def set_default_dimension(self, key, value): + assert isinstance(key, basestring), key + assert isinstance(value, basestring) or value is None, value + if value is None: + self._default_dimensions.pop(key, None) + else: + self._default_dimensions[key] = value # pragma: no cover + + @property + def default_env(self): + """Returns a copy of the default environment variable to run tasks with. + + By default the environment variable is not modified. Additional environment + variables can be specified for each task. + + This value can be changed per individual task. + """ + return ReadOnlyDict(self._default_env) + + def set_default_env(self, key, value): + assert isinstance(key, basestring), key + assert isinstance(value, basestring), value + self._default_env[key] = value + + @property + def default_priority(self): + """Swarming task priority for tasks triggered from the recipe. + + Priority ranges from 1 to 255. The lower the value, the most important the + task is and will preempty any task with a lower priority. + + This value can be changed per individual task. + """ + return self._default_priority + + @default_priority.setter + def default_priority(self, value): + assert 1 <= value <= 255 + self._default_priority = value + + def add_default_tag(self, tag): + """Adds a tag to the Swarming tasks triggered. + + Tags are used for maintenance, they can be used to calculate the number of + tasks run for a day to calculate the cost of a type of type (CQ, ASAN, etc). + + Tags can be added per individual task. + """ + assert ':' in tag, tag + self._default_tags.add(tag) + + @property + def show_isolated_out_in_collect_step(self): + """Show the shard's isolated out link in each collect step.""" + return self._show_isolated_out_in_collect_step + + @show_isolated_out_in_collect_step.setter + def show_isolated_out_in_collect_step(self, value): + self._show_isolated_out_in_collect_step = value + + @property + def show_shards_in_collect_step(self): + """Show the shard link in each collect step.""" + return self._show_shards_in_collect_step + + @show_shards_in_collect_step.setter + def show_shards_in_collect_step(self, value): + self._show_shards_in_collect_step = value + + @staticmethod + def prefered_os_dimension(platform): + """Given a platform name returns the prefered Swarming OS dimension. + + Platform name is usually provided by 'platform' recipe module, it's one + of 'win', 'linux', 'mac'. This function returns more concrete Swarming OS + dimension that represent this platform on Swarming by default. + + Recipes are free to use other OS dimension if there's a need for it. For + example WinXP try bot recipe may explicitly specify 'Windows-XP-SP3' + dimension. """ - return self.m.isolate.isolate_tests( - verbose=True, # To avoid no output timeouts. - build_dir=self.swarming_temp_dir, - targets=targets).presentation.properties['swarm_hashes'].items() + return { + 'linux': 'Ubuntu-14.04', + 'mac': 'Mac-10.9', + 'win': 'Windows-7-SP1', + }[platform] + + def task(self, title, isolated_hash, ignore_task_failure=False, shards=1, + task_output_dir=None, extra_args=None, idempotent=None, + cipd_packages=None, build_properties=None, merge=None): + """Returns a new SwarmingTask instance to run an isolated executable on + Swarming. + + For google test executables, use gtest_task() instead. - def trigger_swarming_tasks( - self, swarm_hashes, dimensions, idempotent=False, store_output=True, - extra_args=None, expiration=None, hard_timeout=None, io_timeout=None, - cipd_packages=None): - """Triggers swarming tasks using swarm hashes. + At the time of this writting, this code is used by V8, Skia and iOS. + + The return value can be customized if necessary (see SwarmingTask class + below). Pass it to 'trigger_task' to launch it on swarming. Later pass the + same instance to 'collect_task' to wait for the task to finish and fetch its + results. Args: - swarm_hashes: list of str. List of swarm hashes from the isolate server. - dimensions: dict of str to str. The dimensions to run the task on. - Eg: {'os': 'Ubuntu', 'gpu': '10de', 'pool': 'Skia'} - idempotent: bool. Whether or not to de-duplicate tasks. - store_output: bool. Whether task output should be stored. - extra_args: list of str. Extra arguments to pass to the task. - expiration: int. Task will expire if not picked up within this time. - DEFAULT_TASK_EXPIRATION is used if this argument is None. - hard_timeout: int. Task will timeout if not completed within this time. - DEFAULT_TASK_TIMEOUT is used if this argument is None. - io_timeout: int. Task will timeout if there is no output within this time. - DEFAULT_IO_TIMEOUT is used if this argument is None. - cipd_packages: CIPD packages which these tasks depend on. + title: name of the test, used as part of a task ID. + isolated_hash: hash of isolated test on isolate server, the test should + be already isolated there, see 'isolate' recipe module. + ignore_task_failure: whether to ignore the test failure of swarming + tasks. By default, this is set to False. + shards: if defined, the number of shards to use for the task. By default + this value is either 1 or based on the title. + task_output_dir: if defined, the directory where task results are placed. + The caller is responsible for removing this folder when finished. + extra_args: list of command line arguments to pass to isolated tasks. + idempotent: whether this task is considered idempotent. Defaults + to self.default_idempotent if not specified. + cipd_packages: list of 3-tuples corresponding to CIPD packages needed for + the task: ('path', 'package_name', 'version'), defined as follows: + path: Path relative to the Swarming root dir in which to install + the package. + package_name: Name of the package to install, + eg. "infra/tools/authutil/${platform}" + version: Version of the package, either a package instance ID, + ref, or tag key/value pair. + build_properties: An optional dict containing various build properties. + These are typically but not necessarily the properties emitted by + bot_update. + merge: An optional dict containing: + "script": path to a script to call to post process and merge the + collected outputs from the tasks. The script should take one + named (but required) parameter, '-o' (for output), that represents + the path that the merged results should be written to, and accept + N additional paths to result files to merge. The merged results + should be in the JSON Results File Format + (https://www.chromium.org/developers/the-json-test-results-format) + and may optionally contain a top level "links" field that + may contain a dict mapping link text to URLs, for a set of + links that will be included in the buildbot output. + "args": an optional list of additional arguments to pass to the + above script. + """ + if idempotent is None: + idempotent = self.default_idempotent + return SwarmingTask( + title=title, + isolated_hash=isolated_hash, + dimensions=self._default_dimensions, + env=self._default_env, + priority=self.default_priority, + shards=shards, + buildername=self.m.properties.get('buildername'), + buildnumber=self.m.properties.get('buildnumber'), + user=self.default_user, + expiration=self.default_expiration, + io_timeout=self.default_io_timeout, + hard_timeout=self.default_hard_timeout, + idempotent=idempotent, + ignore_task_failure=ignore_task_failure, + extra_args=extra_args, + collect_step=self._default_collect_step, + task_output_dir=task_output_dir, + cipd_packages=cipd_packages, + build_properties=build_properties, + merge=merge) - Returns: - List of swarming.SwarmingTask instances. - """ - swarming_tasks = [] - for task_name, swarm_hash in swarm_hashes: - swarming_task = self.m.swarming.task( - title=task_name, - cipd_packages=cipd_packages, - isolated_hash=swarm_hash) - if store_output: - swarming_task.task_output_dir = self.tasks_output_dir.join(task_name) - swarming_task.dimensions = dimensions - swarming_task.idempotent = idempotent - swarming_task.priority = 90 - swarming_task.expiration = ( - expiration if expiration else DEFAULT_TASK_EXPIRATION) - swarming_task.hard_timeout = ( - hard_timeout if hard_timeout else DEFAULT_TASK_TIMEOUT) - swarming_task.io_timeout = ( - io_timeout if io_timeout else DEFAULT_IO_TIMEOUT) - if extra_args: - swarming_task.extra_args = extra_args - revision = self.m.properties.get('revision') - if revision: - swarming_task.tags.add('revision:%s' % revision) - swarming_tasks.append(swarming_task) - step_results = self.m.swarming.trigger(swarming_tasks) - for step_result in step_results: - self._add_log_links(step_result, step_result.json.output) - return swarming_tasks - - def collect_swarming_task(self, swarming_task): - """Collects the specified swarming task. + def check_client_version(self, step_test_data=None): + """Yields steps to verify compatibility with swarming_client version.""" + return self.m.swarming_client.ensure_script_version( + 'swarming.py', MINIMAL_SWARMING_VERSION, step_test_data) + + def trigger_task(self, task, **kwargs): + """Triggers one task. + + It the task is sharded, will trigger all shards. This steps justs posts + the task and immediately returns. Use 'collect_task' to wait for a task to + finish and grab its result. + + Behaves as a regular recipe step: returns StepData with step results + on success or raises StepFailure if step fails. Args: - swarming_task: An instance of swarming.SwarmingTask. + task: SwarmingTask instance. + kwargs: passed to recipe step constructor as-is. """ + assert isinstance(task, SwarmingTask) + assert task.task_name not in self._pending_tasks, ( + 'Triggered same task twice: %s' % task.task_name) + assert 'os' in task.dimensions, task.dimensions + self._pending_tasks.add(task.task_name) + + # Trigger parameters. + args = [ + 'trigger', + '--swarming', self.swarming_server, + '--isolate-server', self.m.isolate.isolate_server, + '--priority', str(task.priority), + '--shards', str(task.shards), + '--task-name', task.task_name, + '--dump-json', self.m.json.output(), + '--expiration', str(task.expiration), + '--io-timeout', str(task.io_timeout), + '--hard-timeout', str(task.hard_timeout), + ] + for name, value in sorted(task.dimensions.iteritems()): + assert isinstance(value, basestring), value + args.extend(['--dimension', name, value]) + for name, value in sorted(task.env.iteritems()): + assert isinstance(value, basestring), value + args.extend(['--env', name, value]) + + # Default tags. + tags = set(task.tags) + tags.update(self._default_tags) + tags.add('data:' + task.isolated_hash) + tags.add('name:' + task.title.split(' ')[0]) + mastername = self.m.properties.get('mastername') + if mastername: # pragma: no cover + tags.add('master:' + mastername) + if task.buildername: # pragma: no cover + tags.add('buildername:' + task.buildername) + if task.buildnumber: # pragma: no cover + tags.add('buildnumber:%s' % task.buildnumber) + if task.dimensions.get('os'): + tags.add('os:' + task.dimensions['os']) + if self.m.properties.get('bot_id'): # pragma: no cover + tags.add('slavename:%s' % self.m.properties['bot_id']) + tags.add('stepname:%s' % self.get_step_name('', task)) + rietveld = self.m.properties.get('rietveld') + issue = self.m.properties.get('issue') + patchset = self.m.properties.get('patchset') + if rietveld and issue and patchset: + # The expected format is strict to the usage of buildbot properties on the + # Chromium Try Server. Fix if necessary. + tags.add('rietveld:%s/%s/#ps%s' % (rietveld, issue, patchset)) + for tag in sorted(tags): + assert ':' in tag, tag + args.extend(['--tag', tag]) + + if self.verbose: + args.append('--verbose') + if task.idempotent: + args.append('--idempotent') + if task.user: + args.extend(['--user', task.user]) + + if task.cipd_packages: + for path, pkg, version in task.cipd_packages: + args.extend(['--cipd-package', '%s:%s:%s' % (path, pkg, version)]) + + # What isolated command to trigger. + args.extend(('--isolated', task.isolated_hash)) + + # Additional command line args for isolated command. + if task.extra_args: # pragma: no cover + args.append('--') + args.extend(task.extra_args) + + # The step can fail only on infra failures, so mark it as 'infra_step'. try: - rv = self.m.swarming.collect_task(swarming_task) - except self.m.step.StepFailure as e: # pragma: no cover - step_result = self.m.step.active_result - # Change step result to Infra failure if the swarming task failed due to - # expiration, time outs, bot crashes or task cancelations. - # Infra failures have step.EXCEPTION. - states_infra_failure = ( - self.m.swarming.State.EXPIRED, self.m.swarming.State.TIMED_OUT, - self.m.swarming.State.BOT_DIED, self.m.swarming.State.CANCELED) - summary = step_result.swarming.summary - if summary['shards'][0]['state'] in states_infra_failure: - step_result.presentation.status = self.m.step.EXCEPTION - raise self.m.step.InfraFailure(e.name, step_result) - raise + return self.m.python( + name=self.get_step_name('trigger', task), + script=self.m.swarming_client.path.join('swarming.py'), + args=args, + step_test_data=functools.partial( + self._gen_trigger_step_test_data, task), + infra_step=True, + **kwargs) finally: + # Store trigger output with the |task|, print links to triggered shards. step_result = self.m.step.active_result - # Add log link. - self._add_log_links(step_result, step_result.swarming.summary) - return rv - - def _add_log_links(self, step_result, summary): - """Add Milo log links to all shards in the step.""" - ids = [] - shards = summary.get('shards') - if shards: - for shard in shards: - ids.append(shard['id']) + step_result.presentation.step_text += text_for_task(task) + + if step_result.presentation != self.m.step.FAILURE: + task._trigger_output = step_result.json.output + links = step_result.presentation.links + for index in xrange(task.shards): + url = task.get_shard_view_url(index) + if url: + links['shard #%d' % index] = url + assert not hasattr(step_result, 'swarming_task') + step_result.swarming_task = task + + def collect_task(self, task, **kwargs): + """Waits for a single triggered task to finish. + + If the task is sharded, will wait for all shards to finish. Behaves as + a regular recipe step: returns StepData with step results on success or + raises StepFailure if task fails. + + Args: + task: SwarmingTask instance, previously triggered with 'trigger' method. + kwargs: passed to recipe step constructor as-is. + """ + # TODO(vadimsh): Raise InfraFailure on Swarming failures. + assert isinstance(task, SwarmingTask) + assert task.task_name in self._pending_tasks, ( + 'Trying to collect a task that was not triggered: %s' % + task.task_name) + self._pending_tasks.remove(task.task_name) + + try: + return task.collect_step(task, **kwargs) + finally: + try: + self.m.step.active_result.swarming_task = task + except Exception: # pragma: no cover + # If we don't have an active_result, something failed very early, + # so we eat this exception and let that one propagate. + pass + + def trigger(self, tasks, **kwargs): # pragma: no cover + """Batch version of 'trigger_task'. + + Deprecated, to be removed soon. Use 'trigger_task' in a loop instead, + properly handling exceptions. This method doesn't handle trigger failures + well (it aborts on a first failure). + """ + return [self.trigger_task(t, **kwargs) for t in tasks] + + def collect(self, tasks, **kwargs): # pragma: no cover + """Batch version of 'collect_task'. + + Deprecated, to be removed soon. Use 'collect_task' in a loop instead, + properly handling exceptions. This method doesn't handle collect failures + well (it aborts on a first failure). + """ + return [self.collect_task(t, **kwargs) for t in tasks] + + # To keep compatibility with some build_internal code. To be removed as well. + collect_each = collect + + @staticmethod + def _display_pending(summary_json, step_presentation): + """Shows max pending time in seconds across all shards if it exceeds 10s.""" + pending_times = [ + (parse_time(shard['started_ts']) - + parse_time(shard['created_ts'])).total_seconds() + for shard in summary_json.get('shards', []) if shard.get('started_ts') + ] + max_pending = max(pending_times) if pending_times else 0 + + # Only display annotation when pending more than 10 seconds to reduce noise. + if max_pending > 10: + step_presentation.step_text += '<br>swarming pending %ds' % max_pending + + def _default_collect_step( + self, task, merged_test_output=None, + step_test_data=None, + **kwargs): + """Produces a step that collects a result of an arbitrary task.""" + task_output_dir = task.task_output_dir or self.m.raw_io.output_dir() + + # If we don't already have a Placeholder, wrap the task_output_dir in one + # so we can read out of it later w/ step_result.raw_io.output_dir. + if not isinstance(task_output_dir, recipe_util.Placeholder): + task_output_dir = self.m.raw_io.output_dir(leak_to=task_output_dir) + + task_args = [ + '-o', merged_test_output or self.m.json.output(), + '--task-output-dir', task_output_dir, + ] + + merge_script = (task.merge.get('script') + or self.resource('noop_merge.py')) + merge_args = (task.merge.get('args') or []) + + task_args.extend([ + '--merge-script', merge_script, + '--merge-additional-args', self.m.json.dumps(merge_args), + ]) + + if task.build_properties: # pragma: no cover + properties = dict(task.build_properties) + properties.update(self.m.properties) + task_args.extend([ + '--build-properties', self.m.json.dumps(properties), + ]) + + task_args.append('--') + # Arguments for the actual 'collect' command. + collect_cmd = [ + 'python', + '-u', + self.m.swarming_client.path.join('swarming.py'), + ] + collect_cmd.extend(self.get_collect_cmd_args(task)) + collect_cmd.extend([ + '--task-summary-json', self.summary(), + ]) + + task_args.extend(collect_cmd) + + allowed_return_codes = {0} + if task.ignore_task_failure: # pragma: no cover + allowed_return_codes = 'any' + + # The call to collect_task emits two JSON files: + # 1) a task summary JSON emitted by swarming + # 2) a gtest results JSON emitted by the task + # This builds an instance of StepTestData that covers both. + step_test_data = step_test_data or ( + self.test_api.canned_summary_output(task.shards) + + self.m.json.test_api.output({})) + + try: + with self.m.context(cwd=self.m.path['start_dir']): + return self.m.python( + name=self.get_step_name('', task), + script=self.resource('collect_task.py'), + args=task_args, + ok_ret=allowed_return_codes, + step_test_data=lambda: step_test_data, + **kwargs) + finally: + step_result = None + try: + step_result = self.m.step.active_result + step_result.presentation.step_text = text_for_task(task) + summary_json = step_result.swarming.summary + self._handle_summary_json(task, summary_json, step_result) + + links = {} + if hasattr(step_result, 'json') and hasattr(step_result.json, 'output'): + links = step_result.json.output.get('links', {}) + for k, v in links.iteritems(): # pragma: no cover + step_result.presentation.links[k] = v + except Exception as e: + if step_result: + step_result.presentation.logs['no_results_exc'] = [str(e)] + + def get_step_name(self, prefix, task): + """SwarmingTask -> name of a step of a waterfall. + + Will take a task title (+ step name prefix) and append OS dimension to it. + + Args: + prefix: prefix to append to task title, like 'trigger'. + task: SwarmingTask instance. + + Returns: + '[<prefix>] <task title> on <OS>' + """ + prefix = '[%s] ' % prefix if prefix else '' + task_os = task.dimensions['os'] + + bot_os = self.prefered_os_dimension(self.m.platform.name) + suffix = ('' if ( + task_os == bot_os or task_os.lower() == self.m.platform.name.lower()) + else ' on %s' % task_os) + # Note: properly detecting dimensions of the bot the recipe is running + # on is somewhat non-trivial. It is not safe to assume it uses default + # or preferred dimensions for its OS. For example, the version of the OS + # can differ. + return ''.join((prefix, task.title, suffix)) + + def _handle_summary_json(self, task, summary, step_result): + # We store this now, and add links to all shards first, before failing the + # build. Format is tuple of (error message, shard that failed) + infra_failures = [] + links = step_result.presentation.links + for index, shard in enumerate(summary['shards']): + url = task.get_shard_view_url(index) + display_text = 'shard #%d' % index + + if not shard or shard.get('internal_failure'): # pragma: no cover + display_text = ( + 'shard #%d had an internal swarming failure' % index) + infra_failures.append((index, 'Internal swarming failure')) + elif self._is_expired(shard): + display_text = ( + 'shard #%d expired, not enough capacity' % index) + infra_failures.append(( + index, 'There isn\'t enough capacity to run your test')) + elif self._is_timed_out(shard): + display_text = ( + 'shard #%d timed out, took too much time to complete' % index) + elif self._get_exit_code(shard) != '0': # pragma: no cover + display_text = 'shard #%d (failed)' % index + + if self.show_isolated_out_in_collect_step: + isolated_out = shard.get('isolated_out') + if isolated_out: + link_name = 'shard #%d isolated out' % index + links[link_name] = isolated_out['view_url'] + + if url and self.show_shards_in_collect_step: + links[display_text] = url + + self._display_pending(summary, step_result.presentation) + + if infra_failures: + template = 'Shard #%s failed: %s' + + # Done so that raising an InfraFailure doesn't cause an error. + # TODO(martiniss): Remove this hack. Requires recipe engine change + step_result._retcode = 2 + step_result.presentation.status = self.m.step.EXCEPTION + raise recipe_api.InfraFailure( + '\n'.join(template % f for f in infra_failures), result=step_result) + + def get_collect_cmd_args(self, task): + """SwarmingTask -> argument list for 'swarming.py' command.""" + args = [ + 'collect', + '--swarming', self.swarming_server, + '--decorate', + '--print-status-updates', + ] + if self.verbose: + args.append('--verbose') + args.extend(('--json', self.m.json.input(task.trigger_output))) + return args + + def _gen_trigger_step_test_data(self, task): + """Generates an expected value of --dump-json in 'trigger' step. + + Used when running recipes to generate test expectations. + """ + # Suffixes of shard subtask names. + subtasks = [] + if task.shards == 1: + subtasks = [''] else: - for _, task in summary.get('tasks', {}).iteritems(): - ids.append(task['task_id']) - for idx, task_id in enumerate(ids): - link = MILO_LOG_LINK % task_id - k = 'view steps on Milo' - if len(ids) > 1: # pragma: nocover - k += ' (shard index %d, %d total)' % (idx, len(ids)) - step_result.presentation.links[k] = link + subtasks = [':%d:%d' % (task.shards, i) for i in range(task.shards)] + return self.m.json.test_api.output({ + 'base_task_name': task.task_name, + 'tasks': { + '%s%s' % (task.task_name, suffix): { + 'task_id': '1%02d00' % i, + 'shard_index': i, + 'view_url': '%s/user/task/1%02d00' % (self.swarming_server, i), + } for i, suffix in enumerate(subtasks) + }, + }) + + +class SwarmingTask(object): + """Definition of a task to run on swarming.""" + def __init__(self, title, isolated_hash, ignore_task_failure, dimensions, + env, priority, shards, buildername, buildnumber, expiration, + user, io_timeout, hard_timeout, idempotent, extra_args, + collect_step, task_output_dir, cipd_packages=None, + build_properties=None, merge=None): + """Configuration of a swarming task. + + Args: + title: display name of the task, hints to what task is doing. Usually + corresponds to a name of a test executable. Doesn't have to be unique. + isolated_hash: hash of isolated file that describes all files needed to + run the task as well as command line to launch. See 'isolate' recipe + module. + ignore_task_failure: whether to ignore the test failure of swarming + tasks. + cipd_packages: list of 3-tuples corresponding to CIPD packages needed for + the task: ('path', 'package_name', 'version'), defined as follows: + path: Path relative to the Swarming root dir in which to install + the package. + package_name: Name of the package to install, + eg. "infra/tools/authutil/${platform}" + version: Version of the package, either a package instance ID, + ref, or tag key/value pair. + collect_step: callback that will be called to collect and processes + results of task execution, signature is collect_step(task, **kwargs). + dimensions: key-value mapping with swarming dimensions that specify + on what Swarming slaves task can run. One important dimension is 'os', + which defines platform flavor to run the task on. See Swarming doc. + env: key-value mapping with additional environment variables to add to + environment before launching the task executable. + priority: integer [0, 255] that defines how urgent the task is. + Lower value corresponds to higher priority. Swarming service executes + tasks with higher priority first. + shards: how many concurrent shards to run, makes sense only for + isolated tests based on gtest. Swarming uses GTEST_SHARD_INDEX + and GTEST_TOTAL_SHARDS environment variables to tell the executable + what shard to run. + buildername: buildbot builder this task was triggered from. + buildnumber: build number of a build this task was triggered from. + expiration: number of schedule until the task shouldn't even be run if it + hadn't started yet. + user: user that requested this task, if applicable. + io_timeout: number of seconds that the task is allowed to not emit any + stdout bytes, after which it is forcibly killed. + hard_timeout: number of seconds for which the task is allowed to run, + after which it is forcibly killed. + idempotent: True if the results from a previous task can be reused. E.g. + this task has no side-effects. + extra_args: list of command line arguments to pass to isolated tasks. + task_output_dir: if defined, the directory where task results are placed + during the collect step. + build_properties: An optional dict containing various build properties. + These are typically but not necessarily the properties emitted by + bot_update. + merge: An optional dict containing: + "script": path to a script to call to post process and merge the + collected outputs from the tasks. + "args": an optional list of additional arguments to pass to the + above script. + """ + self._trigger_output = None + self.build_properties = build_properties + self.buildername = buildername + self.buildnumber = buildnumber + self.cipd_packages = cipd_packages + self.collect_step = collect_step + self.dimensions = dimensions.copy() + self.env = env.copy() + self.expiration = expiration + self.extra_args = tuple(extra_args or []) + self.hard_timeout = hard_timeout + self.idempotent = idempotent + self.ignore_task_failure = ignore_task_failure + self.io_timeout = io_timeout + self.isolated_hash = isolated_hash + self.merge = merge or {} + self.priority = priority + self.shards = shards + self.tags = set() + self.task_output_dir = task_output_dir + self.title = title + self.user = user + + @property + def task_name(self): + """Name of this task, derived from its other properties. + + The task name is purely to make sense of the task and is not used in any + other way. + """ + out = '%s/%s/%s' % ( + self.title, self.dimensions['os'], self.isolated_hash[:10]) + if self.buildername: # pragma: no cover + out += '/%s/%s' % (self.buildername, self.buildnumber or -1) + return out + + @property + def trigger_output(self): + """JSON results of 'trigger' step or None if not triggered.""" + return self._trigger_output + + def get_shard_view_url(self, index): + """Returns URL of HTML page with shard details or None if not available. + + Works only after the task has been successfully triggered. + """ + if self._trigger_output and self._trigger_output.get('tasks'): + for shard_dict in self._trigger_output['tasks'].itervalues(): + if shard_dict['shard_index'] == index: + return shard_dict['view_url'] diff --git a/infra/bots/recipe_modules/swarming/example.expected/basic.json b/infra/bots/recipe_modules/swarming/example.expected/basic.json new file mode 100644 index 0000000000..7a2448dd56 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/example.expected/basic.json @@ -0,0 +1,683 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.8.6@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "win", + "--verbose" + ], + "name": "archive for win", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "linux", + "--verbose" + ], + "name": "archive for linux", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "mac", + "--verbose" + ], + "name": "archive for mac", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "1", + "--task-name", + "hello_world/Windows-7-SP1/hash_for_w", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Windows-7-SP1", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_win", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Windows-7-SP1", + "--tag", + "os:win", + "--tag", + "stepname:hello_world on Windows-7-SP1", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_win" + ], + "infra_step": true, + "name": "[trigger] hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Windows-7-SP1/hash_for_w\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "2", + "--task-name", + "hello_world/Ubuntu-14.04/hash_for_l", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Ubuntu-14.04", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_linux", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Ubuntu-14.04", + "--tag", + "os:linux", + "--tag", + "stepname:hello_world", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_linux" + ], + "infra_step": true, + "name": "[trigger] hello_world", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Ubuntu-14.04'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Ubuntu-14.04/hash_for_l\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Ubuntu-14.04/hash_for_l:2:0\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Ubuntu-14.04/hash_for_l:2:1\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10100\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10100\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@shard #1@https://chromium-swarm-dev.appspot.com/user/task/10100@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "1", + "--task-name", + "hello_world/Mac-10.9/hash_for_m", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Mac-10.9", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_mac", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Mac-10.9", + "--tag", + "os:mac", + "--tag", + "stepname:hello_world on Mac-10.9", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_mac" + ], + "infra_step": true, + "name": "[trigger] hello_world on Mac-10.9", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Mac-10.9'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Mac-10.9/hash_for_m\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Mac-10.9/hash_for_m\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "echo", + "running something locally" + ], + "name": "local step" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", \"tasks\": {\"hello_world/Windows-7-SP1/hash_for_w\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 5.7, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 31.5@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs_ref\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Ubuntu-14.04/hash_for_l\", \"tasks\": {\"hello_world/Ubuntu-14.04/hash_for_l:2:0\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}, \"hello_world/Ubuntu-14.04/hash_for_l:2:1\": {\"shard_index\": 1, \"task_id\": \"10100\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10100\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Ubuntu-14.04'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 5.7, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 31.5@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs_ref\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 5.7, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 31.5@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0100\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs_ref\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@shard #1 isolated out@blah@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Mac-10.9/hash_for_m\", \"tasks\": {\"hello_world/Mac-10.9/hash_for_m\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world on Mac-10.9", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Mac-10.9'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 5.7, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 31.5@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs_ref\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[TMP_BASE]/hello_isolated_world_tmp_1" + ], + "infra_step": true, + "name": "rmtree remove temp dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/swarming/example.expected/show_isolated_out_in_collect_step.json b/infra/bots/recipe_modules/swarming/example.expected/show_isolated_out_in_collect_step.json new file mode 100644 index 0000000000..d356498c03 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/example.expected/show_isolated_out_in_collect_step.json @@ -0,0 +1,297 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.8.6@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "win", + "--verbose" + ], + "name": "archive for win", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "1", + "--task-name", + "hello_world/Windows-7-SP1/hash_for_w", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Windows-7-SP1", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_win", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Windows-7-SP1", + "--tag", + "os:win", + "--tag", + "rietveld:https://codereview.chromium.org/123/#ps1001", + "--tag", + "stepname:hello_world on Windows-7-SP1", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_win" + ], + "infra_step": true, + "name": "[trigger] hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Windows-7-SP1/hash_for_w\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "echo", + "running something locally" + ], + "name": "local step" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", \"tasks\": {\"hello_world/Windows-7-SP1/hash_for_w\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 5.7, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 31.5@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs_ref\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[TMP_BASE]/hello_isolated_world_tmp_1" + ], + "infra_step": true, + "name": "rmtree remove temp dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/swarming/example.expected/show_shards_in_collect_step.json b/infra/bots/recipe_modules/swarming/example.expected/show_shards_in_collect_step.json new file mode 100644 index 0000000000..ddd8dd029d --- /dev/null +++ b/infra/bots/recipe_modules/swarming/example.expected/show_shards_in_collect_step.json @@ -0,0 +1,299 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.8.6@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "win", + "--verbose" + ], + "name": "archive for win", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "1", + "--task-name", + "hello_world/Windows-7-SP1/hash_for_w", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Windows-7-SP1", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_win", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Windows-7-SP1", + "--tag", + "os:win", + "--tag", + "rietveld:https://codereview.chromium.org/123/#ps1001", + "--tag", + "stepname:hello_world on Windows-7-SP1", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_win" + ], + "infra_step": true, + "name": "[trigger] hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Windows-7-SP1/hash_for_w\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "echo", + "running something locally" + ], + "name": "local step" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", \"tasks\": {\"hello_world/Windows-7-SP1/hash_for_w\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 5.7, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 31.5@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs_ref\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[TMP_BASE]/hello_isolated_world_tmp_1" + ], + "infra_step": true, + "name": "rmtree remove temp dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/swarming/example.expected/swarming_expired_new.json b/infra/bots/recipe_modules/swarming/example.expected/swarming_expired_new.json new file mode 100644 index 0000000000..132e5b23a2 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/example.expected/swarming_expired_new.json @@ -0,0 +1,281 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.8.6@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "win", + "--verbose" + ], + "name": "archive for win", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "1", + "--task-name", + "hello_world/Windows-7-SP1/hash_for_w", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Windows-7-SP1", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_win", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Windows-7-SP1", + "--tag", + "os:win", + "--tag", + "stepname:hello_world on Windows-7-SP1", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_win" + ], + "infra_step": true, + "name": "[trigger] hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Windows-7-SP1/hash_for_w\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "echo", + "running something locally" + ], + "name": "local step" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", \"tasks\": {\"hello_world/Windows-7-SP1/hash_for_w\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0100\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": \"EXPIRED\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@", + "@@@STEP_LOG_LINE@no_results_exc@Infra Failure in Shard #0 failed: There isn't enough capacity to run your test@@@", + "@@@STEP_LOG_END@no_results_exc@@@", + "@@@STEP_EXCEPTION@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[TMP_BASE]/hello_isolated_world_tmp_1" + ], + "infra_step": true, + "name": "rmtree remove temp dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/swarming/example.expected/swarming_expired_old.json b/infra/bots/recipe_modules/swarming/example.expected/swarming_expired_old.json new file mode 100644 index 0000000000..190094d84d --- /dev/null +++ b/infra/bots/recipe_modules/swarming/example.expected/swarming_expired_old.json @@ -0,0 +1,281 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.8.6@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "win", + "--verbose" + ], + "name": "archive for win", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "1", + "--task-name", + "hello_world/Windows-7-SP1/hash_for_w", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Windows-7-SP1", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_win", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Windows-7-SP1", + "--tag", + "os:win", + "--tag", + "stepname:hello_world on Windows-7-SP1", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_win" + ], + "infra_step": true, + "name": "[trigger] hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Windows-7-SP1/hash_for_w\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "echo", + "running something locally" + ], + "name": "local step" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", \"tasks\": {\"hello_world/Windows-7-SP1/hash_for_w\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0100\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": 48, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@", + "@@@STEP_LOG_LINE@no_results_exc@Infra Failure in Shard #0 failed: There isn't enough capacity to run your test@@@", + "@@@STEP_LOG_END@no_results_exc@@@", + "@@@STEP_EXCEPTION@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[TMP_BASE]/hello_isolated_world_tmp_1" + ], + "infra_step": true, + "name": "rmtree remove temp dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/swarming/example.expected/swarming_timeout_new.json b/infra/bots/recipe_modules/swarming/example.expected/swarming_timeout_new.json new file mode 100644 index 0000000000..469aa3cb1c --- /dev/null +++ b/infra/bots/recipe_modules/swarming/example.expected/swarming_timeout_new.json @@ -0,0 +1,278 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.8.6@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "win", + "--verbose" + ], + "name": "archive for win", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "1", + "--task-name", + "hello_world/Windows-7-SP1/hash_for_w", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Windows-7-SP1", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_win", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Windows-7-SP1", + "--tag", + "os:win", + "--tag", + "stepname:hello_world on Windows-7-SP1", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_win" + ], + "infra_step": true, + "name": "[trigger] hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Windows-7-SP1/hash_for_w\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "echo", + "running something locally" + ], + "name": "local step" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", \"tasks\": {\"hello_world/Windows-7-SP1/hash_for_w\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0100\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": \"TIMED_OUT\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[TMP_BASE]/hello_isolated_world_tmp_1" + ], + "infra_step": true, + "name": "rmtree remove temp dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/swarming/example.expected/swarming_timeout_old.json b/infra/bots/recipe_modules/swarming/example.expected/swarming_timeout_old.json new file mode 100644 index 0000000000..e8fc09ee47 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/example.expected/swarming_timeout_old.json @@ -0,0 +1,278 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.8.6@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "win", + "--verbose" + ], + "name": "archive for win", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "1", + "--task-name", + "hello_world/Windows-7-SP1/hash_for_w", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Windows-7-SP1", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_win", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Windows-7-SP1", + "--tag", + "os:win", + "--tag", + "stepname:hello_world on Windows-7-SP1", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_win" + ], + "infra_step": true, + "name": "[trigger] hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Windows-7-SP1/hash_for_w\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "echo", + "running something locally" + ], + "name": "local step" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", \"tasks\": {\"hello_world/Windows-7-SP1/hash_for_w\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0100\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": 64, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[TMP_BASE]/hello_isolated_world_tmp_1" + ], + "infra_step": true, + "name": "rmtree remove temp dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/swarming/example.expected/trybot.json b/infra/bots/recipe_modules/swarming/example.expected/trybot.json new file mode 100644 index 0000000000..741efcc340 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/example.expected/trybot.json @@ -0,0 +1,298 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.8.6@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "archive", + "--isolate", + "[START_DIR]/swarming.client/example/payload/hello_world.isolate", + "--isolated", + "[TMP_BASE]/hello_isolated_world_tmp_1/hello_world.isolated", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--config-variable", + "OS", + "win", + "--verbose" + ], + "name": "archive for win", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--isolate-server", + "https://isolateserver-dev.appspot.com", + "--priority", + "30", + "--shards", + "1", + "--task-name", + "hello_world/Windows-7-SP1/hash_for_w", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "3600", + "--io-timeout", + "1200", + "--hard-timeout", + "3600", + "--dimension", + "cpu", + "x86-64", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Windows-7-SP1", + "--env", + "TESTING", + "1", + "--tag", + "data:hash_for_win", + "--tag", + "master:tryserver", + "--tag", + "name:hello_world", + "--tag", + "os:Windows-7-SP1", + "--tag", + "os:win", + "--tag", + "rietveld:https://codereview.chromium.org/123/#ps1001", + "--tag", + "stepname:hello_world on Windows-7-SP1", + "--verbose", + "--idempotent", + "--user", + "joe", + "--cipd-package", + "bin:super/awesome/pkg:git_revision:deadbeef", + "--isolated", + "hash_for_win" + ], + "infra_step": true, + "name": "[trigger] hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"hello_world/Windows-7-SP1/hash_for_w\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm-dev.appspot.com/user/task/10000@@@" + ] + }, + { + "cmd": [ + "echo", + "running something locally" + ], + "name": "local step" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", + "-o", + "/path/to/tmp/json", + "--task-output-dir", + "[TMP_BASE]/hello_isolated_world_tmp_1/task_output_dir", + "--merge-script", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", + "--merge-additional-args", + "[]", + "--", + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm-dev.appspot.com", + "--decorate", + "--print-status-updates", + "--verbose", + "--json", + "{\"base_task_name\": \"hello_world/Windows-7-SP1/hash_for_w\", \"tasks\": {\"hello_world/Windows-7-SP1/hash_for_w\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm-dev.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "hello_world on Windows-7-SP1", + "~followup_annotations": [ + "@@@STEP_TEXT@Run on OS: 'Windows-7-SP1'<br>swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@swarming.summary@{@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 5.7, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 31.5@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ 0@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ], @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"outputs_ref\": {@@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@swarming.summary@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@swarming.summary@ }@@@", + "@@@STEP_LOG_LINE@swarming.summary@ ]@@@", + "@@@STEP_LOG_LINE@swarming.summary@}@@@", + "@@@STEP_LOG_END@swarming.summary@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", + "rmtree", + "[TMP_BASE]/hello_isolated_world_tmp_1" + ], + "infra_step": true, + "name": "rmtree remove temp dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/swarming/example.py b/infra/bots/recipe_modules/swarming/example.py index b248410c5b..f61576a24d 100644 --- a/infra/bots/recipe_modules/swarming/example.py +++ b/infra/bots/recipe_modules/swarming/example.py @@ -1,31 +1,242 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +import json + DEPS = [ + 'file', + 'isolate', + 'recipe_engine/json', 'recipe_engine/path', 'recipe_engine/properties', + 'recipe_engine/python', + 'recipe_engine/raw_io', 'recipe_engine/step', 'swarming', + 'swarming_client', ] +from recipe_engine.recipe_api import Property + +PROPERTIES = { + 'platforms': Property(default=('win',)), + 'show_isolated_out_in_collect_step': Property(default=True), + 'show_shards_in_collect_step': Property(default=False), + 'gtest_task': Property(default=False), + #'isolated_script_task': Property(default=False), + 'merge': Property(default=None), +} + +def RunSteps(api, platforms, show_isolated_out_in_collect_step, + show_shards_in_collect_step, gtest_task, merge): + # Checkout swarming client. + api.swarming_client.checkout('master') + + # Ensure swarming_client version is fresh enough. + api.swarming.check_client_version(step_test_data=(0, 8, 6)) + + # Configure isolate & swarming modules (this is optional). + api.isolate.isolate_server = 'https://isolateserver-dev.appspot.com' + api.swarming.swarming_server = 'https://chromium-swarm-dev.appspot.com' + api.swarming.add_default_tag('master:tryserver') + api.swarming.default_expiration = 60*60 + api.swarming.default_hard_timeout = 60*60 + api.swarming.default_io_timeout = 20*60 + api.swarming.default_idempotent = True + api.swarming.default_priority = 30 + api.swarming.default_user = 'joe' + api.swarming.set_default_env('TESTING', '1') + api.swarming.verbose = True + + api.swarming.set_default_dimension('inexistent', None) + + api.swarming.show_shards_in_collect_step = show_shards_in_collect_step + api.swarming.show_isolated_out_in_collect_step = ( + show_isolated_out_in_collect_step) + + try: + # Testing ReadOnlyDict.__setattr__() coverage. + api.swarming.default_dimensions['invalid'] = 'foo' + except TypeError: + pass + try: + api.swarming.default_env['invalid'] = 'foo' + except TypeError: + pass + + # Create a temp dir to put *.isolated files into. + temp_dir = api.path.mkdtemp('hello_isolated_world') + + # Prepare a bunch of swarming tasks to run hello_world on multiple platforms. + tasks = [] + for platform in platforms: + # Isolate example hello_world.isolate from swarming client repo. + # TODO(vadimsh): Add a thin wrapper around isolate.py to 'isolate' module? + step_result = api.python( + 'archive for %s' % platform, + api.swarming_client.path.join('isolate.py'), + [ + 'archive', + '--isolate', api.swarming_client.path.join( + 'example', 'payload', 'hello_world.isolate'), + '--isolated', temp_dir.join('hello_world.isolated'), + '--isolate-server', api.isolate.isolate_server, + '--config-variable', 'OS', platform, + '--verbose', + ], stdout=api.raw_io.output_text()) + # TODO(vadimsh): Pass result from isolate.py though --output-json option. + isolated_hash = step_result.stdout.split()[0].strip() + + # Create a task to run the isolated file on swarming, set OS dimension. + # Also generate code coverage for multi-shard case by triggering multiple + # shards on Linux. + task = api.swarming.task('hello_world', isolated_hash, + task_output_dir=temp_dir.join('task_output_dir')) + task.dimensions['os'] = api.swarming.prefered_os_dimension(platform) + task.shards = 2 if platform == 'linux' else 1 + task.tags.add('os:' + platform) + if api.swarming_client.get_script_version('swarming.py') >= (0, 8, 6): + task.cipd_packages = [ + ('bin', 'super/awesome/pkg', 'git_revision:deadbeef')] + tasks.append(task) + + # Launch all tasks. + for task in tasks: + step_result = api.swarming.trigger_task(task) + assert step_result.swarming_task in tasks + + # Recipe can do something useful here locally while tasks are + # running on swarming. + api.step('local step', ['echo', 'running something locally']) + + # Wait for all tasks to complete. + for task in tasks: + step_result = api.swarming.collect_task(task) + data = step_result.swarming.summary + + state = data['shards'][0]['state'] + if api.swarming.State.COMPLETED == state: + state_name = api.swarming.State.to_string(state) + assert 'Completed' == state_name, state_name + assert step_result.swarming_task in tasks -def RunSteps(api): - api.swarming.setup('mydir', swarming_rev='abc123') - api.swarming.create_isolated_gen_json( - 'isolate_path', 'isolate_dir', 'linux', 'task', {'myvar': 'myval'}, - blacklist=['*.pyc']) - tasks_to_hashes = api.swarming.batcharchive(targets=[ - 'task-%s' % num for num in range(5)]) - tasks = api.swarming.trigger_swarming_tasks( - tasks_to_hashes, dimensions={'os': 'Linux'}, extra_args=['--extra']) - for t in tasks: - api.swarming.collect_swarming_task(t) + # Cleanup. + api.file.rmtree('remove temp dir', temp_dir) def GenTests(api): yield ( - api.test('test') + - api.properties(revision='abc123') - ) + api.test('basic') + + api.step_data( + 'archive for win', + stdout=api.raw_io.output_text('hash_for_win hello_world.isolated')) + + api.step_data( + 'archive for linux', + stdout=api.raw_io.output_text( + 'hash_for_linux hello_world.isolated')) + + api.step_data( + 'archive for mac', + stdout=api.raw_io.output_text('hash_for_mac hello_world.isolated')) + + api.properties(platforms=('win', 'linux', 'mac'))) + + yield ( + api.test('trybot') + + api.step_data( + 'archive for win', + stdout=api.raw_io.output_text('hash_for_win hello_world.isolated')) + + api.properties( + rietveld='https://codereview.chromium.org', + issue='123', + patchset='1001')) + + yield ( + api.test('show_shards_in_collect_step') + + api.step_data( + 'archive for win', + stdout=api.raw_io.output_text('hash_for_win hello_world.isolated')) + + api.properties( + rietveld='https://codereview.chromium.org', + issue='123', + patchset='1001', + show_shards_in_collect_step=True)) + + yield ( + api.test('show_isolated_out_in_collect_step') + + api.step_data( + 'archive for win', + stdout=api.raw_io.output_text('hash_for_win hello_world.isolated')) + + api.properties( + rietveld='https://codereview.chromium.org', + issue='123', + patchset='1001', + show_isolated_out_in_collect_step=False)) + + data = { + 'shards': [ + { + '': '', + } + ] + } + + data = { + 'shards': [ + { + 'abandoned_ts': '2014-09-25T01:41:00.123', + 'bot_id': 'vm30', + 'completed_ts': None, + 'created_ts': '2014-09-25T01:41:00.123', + 'durations': None, + 'exit_codes': [], + 'failure': False, + 'id': '148aa78d7aa0100', + 'internal_failure': False, + 'isolated_out': None, + 'modified_ts': '2014-09-25 01:42:00', + 'name': 'heartbeat-canary-2014-09-25_01:41:55-os=Windows', + 'outputs': [], + 'started_ts': '2014-09-25T01:42:11.123', + 'state': 0x30, # EXPIRED (old) + 'try_number': None, + 'user': 'unknown', + } + ], + } + + yield ( + api.test('swarming_expired_old') + + api.step_data( + 'archive for win', + stdout=api.raw_io.output_text('hash_for_win hello_world.isolated')) + + api.step_data('hello_world on Windows-7-SP1', api.swarming.summary(data))) + + data['shards'][0]['state'] = 'EXPIRED' + yield ( + api.test('swarming_expired_new') + + api.step_data( + 'archive for win', + stdout=api.raw_io.output_text('hash_for_win hello_world.isolated')) + + api.step_data('hello_world on Windows-7-SP1', api.swarming.summary(data))) + + data['shards'][0]['state'] = 0x40 # TIMED_OUT (old) + yield ( + api.test('swarming_timeout_old') + + api.step_data( + 'archive for win', + stdout=api.raw_io.output_text('hash_for_win hello_world.isolated')) + + api.step_data('hello_world on Windows-7-SP1', api.swarming.summary(data))) + + data['shards'][0]['state'] = 'TIMED_OUT' # TIMED_OUT (old) + yield ( + api.test('swarming_timeout_new') + + api.step_data( + 'archive for win', + stdout=api.raw_io.output_text('hash_for_win hello_world.isolated')) + + api.step_data('hello_world on Windows-7-SP1', api.swarming.summary(data))) diff --git a/infra/bots/recipe_modules/swarming/resources/collect_task.py b/infra/bots/recipe_modules/swarming/resources/collect_task.py new file mode 100755 index 0000000000..cb4f15e96a --- /dev/null +++ b/infra/bots/recipe_modules/swarming/resources/collect_task.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import json +import logging +import os +import subprocess +import sys + + +def collect_task( + collect_cmd, merge_script, build_properties, merge_arguments, + task_output_dir, output_json): + """Collect and merge the results of a task. + + This is a relatively thin wrapper script around a `swarming.py collect` + command and a subsequent results merge to ensure that the recipe system + treats them as a single step. The results merge can either be the default + one provided by results_merger or a python script provided as merge_script. + + Args: + collect_cmd: The `swarming.py collect` command to run. Should not contain + a --task-output-dir argument. + merge_script: A merge/postprocessing script that should be run to + merge the results. This script will be invoked as + + <merge_script> \ + [--build-properties <string JSON>] \ + [merge arguments...] \ + --summary-json <summary json> \ + -o <merged json path> \ + <shard json>... + + where the merge arguments are the contents of merge_arguments_json. + build_properties: A string containing build information to + pass to the merge script in JSON form. + merge_arguments: A string containing additional arguments to pass to + the merge script in JSON form. + task_output_dir: A path to a directory in which swarming will write the + output of the task, including a summary JSON and all of the individual + shard results. + output_json: A path to a JSON file to which the merged results should be + written. The merged results should be in the JSON Results File Format + (https://www.chromium.org/developers/the-json-test-results-format) + and may optionally contain a top level "links" field that may contain a + dict mapping link text to URLs, for a set of links that will be included + in the buildbot output. + Returns: + The exit code of collect_cmd or merge_cmd. + """ + logging.debug('Using task_output_dir: %r', task_output_dir) + if os.path.exists(task_output_dir): + logging.warn('task_output_dir %r already exists!', task_output_dir) + existing_contents = [] + try: + for p in os.listdir(task_output_dir): + existing_contents.append(os.path.join(task_output_dir, p)) + except (OSError, IOError) as e: + logging.error('Error while examining existing task_output_dir: %s', e) + + logging.warn('task_output_dir existing content: %r', existing_contents) + + collect_cmd.extend(['--task-output-dir', task_output_dir]) + + logging.info('collect_cmd: %s', ' '.join(collect_cmd)) + collect_result = subprocess.call(collect_cmd) + if collect_result != 0: + logging.warn('collect_cmd had non-zero return code: %s', collect_result) + + task_output_dir_contents = [] + try: + task_output_dir_contents.extend( + os.path.join(task_output_dir, p) + for p in os.listdir(task_output_dir)) + except (OSError, IOError) as e: + logging.error('Error while processing task_output_dir: %s', e) + + logging.debug('Contents of task_output_dir: %r', task_output_dir_contents) + if not task_output_dir_contents: + logging.warn( + 'No files found in task_output_dir: %r', + task_output_dir) + + task_output_subdirs = ( + p for p in task_output_dir_contents + if os.path.isdir(p)) + shard_json_files = [ + os.path.join(subdir, 'output.json') + for subdir in task_output_subdirs] + extant_shard_json_files = [ + f for f in shard_json_files if os.path.exists(f)] + + if shard_json_files != extant_shard_json_files: + logging.warn( + 'Expected output.json file missing: %r\nFound: %r\nExpected: %r\n', + set(shard_json_files) - set(extant_shard_json_files), + extant_shard_json_files, + shard_json_files) + + if not extant_shard_json_files: + logging.warn( + 'No shard json files found in task_output_dir: %r\nFound %r', + task_output_dir, task_output_dir_contents) + + logging.debug('Found shard_json_files: %r', shard_json_files) + + summary_json_file = os.path.join(task_output_dir, 'summary.json') + + merge_result = 0 + + merge_cmd = [sys.executable, merge_script] + if build_properties: + merge_cmd.extend(('--build-properties', build_properties)) + if os.path.exists(summary_json_file): + merge_cmd.extend(('--summary-json', summary_json_file)) + else: + logging.warn('Summary json file missing: %r', summary_json_file) + if merge_arguments: + merge_cmd.extend(json.loads(merge_arguments)) + merge_cmd.extend(('-o', output_json)) + merge_cmd.extend(extant_shard_json_files) + + logging.info('merge_cmd: %s', ' '.join(merge_cmd)) + merge_result = subprocess.call(merge_cmd) + if merge_result != 0: + logging.warn('merge_cmd had non-zero return code: %s', merge_result) + + if not os.path.exists(output_json): + logging.warn( + 'merge_cmd did not create output_json file: %r', output_json) + + return collect_result or merge_result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--build-properties') + parser.add_argument('--merge-additional-args') + parser.add_argument('--merge-script', required=True) + parser.add_argument('--task-output-dir', required=True) + parser.add_argument('-o', '--output-json', required=True) + parser.add_argument('--verbose', action='store_true') + parser.add_argument('collect_cmd', nargs='+') + + args = parser.parse_args() + if args.verbose: + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + + return collect_task( + args.collect_cmd, + args.merge_script, args.build_properties, args.merge_additional_args, + args.task_output_dir, args.output_json) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/infra/bots/recipe_modules/swarming/resources/noop_merge.py b/infra/bots/recipe_modules/swarming/resources/noop_merge.py new file mode 100755 index 0000000000..740e0d3b32 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/resources/noop_merge.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import json +import shutil +import sys + + +def noop_merge(output_json, jsons_to_merge): + """Use the first supplied JSON as the output JSON. + + Primarily intended for unsharded tasks. + + Args: + output_json: A path to a JSON file to which the results should be written. + jsons_to_merge: A list of paths to JSON files. + """ + if len(jsons_to_merge) > 1: + print >> sys.stderr, ( + 'Multiple JSONs provided: %s' % ','.join(jsons_to_merge)) + return 1 + if jsons_to_merge: + shutil.copyfile(jsons_to_merge[0], output_json) + else: + with open(output_json, 'w') as f: + json.dump({}, f) + return 0 + + +def main(raw_args): + parser = argparse.ArgumentParser() + parser.add_argument('--build-properties', help=argparse.SUPPRESS) + parser.add_argument('--summary-json', help=argparse.SUPPRESS) + parser.add_argument('-o', '--output-json', required=True) + parser.add_argument('jsons_to_merge', nargs='*') + + args = parser.parse_args(raw_args) + + return noop_merge(args.output_json, args.jsons_to_merge) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/infra/bots/recipe_modules/swarming/resources/results_merger.py b/infra/bots/recipe_modules/swarming/resources/results_merger.py new file mode 100755 index 0000000000..3ea045345e --- /dev/null +++ b/infra/bots/recipe_modules/swarming/resources/results_merger.py @@ -0,0 +1,278 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import copy +import json +import sys + +# These fields must appear in the test result output +REQUIRED = { + 'interrupted', + 'num_failures_by_type', + 'seconds_since_epoch', + 'tests', + } + +# These fields are optional, but must have the same value on all shards +OPTIONAL_MATCHING = ( + 'builder_name', + 'build_number', + 'chromium_revision', + 'has_pretty_patch', + 'has_wdiff', + 'path_delimiter', + 'pixel_tests_enabled', + 'random_order_seed', + ) + +OPTIONAL_IGNORED = ( + 'layout_tests_dir', + ) + +# These fields are optional and will be summed together +OPTIONAL_COUNTS = ( + 'fixable', + 'num_flaky', + 'num_passes', + 'num_regressions', + 'skipped', + 'skips', + ) + + +class MergeException(Exception): + pass + + +def merge_test_results(shard_results_list): + """ Merge list of results. + + Args: + shard_results_list: list of results to merge. All the results must have the + same format. Supported format are simplified JSON format & Chromium JSON + test results format version 3 (see + https://www.chromium.org/developers/the-json-test-results-format) + + Returns: + a dictionary that represent the merged results. Its format follow the same + format of all results in |shard_results_list|. + """ + if not shard_results_list: + return {} + + if 'seconds_since_epoch' in shard_results_list[0]: + return _merge_json_test_result_format(shard_results_list) + else: + return _merge_simplified_json_format(shard_results_list) + + +def _merge_simplified_json_format(shard_results_list): + # This code is specialized to the "simplified" JSON format that used to be + # the standard for recipes. + + # These are the only keys we pay attention to in the output JSON. + merged_results = { + 'successes': [], + 'failures': [], + 'valid': True, + } + + for result_json in shard_results_list: + successes = result_json.get('successes', []) + failures = result_json.get('failures', []) + valid = result_json.get('valid', True) + + if (not isinstance(successes, list) or not isinstance(failures, list) or + not isinstance(valid, bool)): + raise MergeException( + 'Unexpected value type in %s' % result_json) # pragma: no cover + + merged_results['successes'].extend(successes) + merged_results['failures'].extend(failures) + merged_results['valid'] = merged_results['valid'] and valid + return merged_results + + +def _merge_json_test_result_format(shard_results_list): + # This code is specialized to the Chromium JSON test results format version 3: + # https://www.chromium.org/developers/the-json-test-results-format + + # These are required fields for the JSON test result format version 3. + merged_results = { + 'tests': {}, + 'interrupted': False, + 'version': 3, + 'seconds_since_epoch': float('inf'), + 'num_failures_by_type': { + } + } + + # To make sure that we don't mutate existing shard_results_list. + shard_results_list = copy.deepcopy(shard_results_list) + for result_json in shard_results_list: + # TODO(tansell): check whether this deepcopy is actually neccessary. + result_json = copy.deepcopy(result_json) + + # Check the version first + version = result_json.pop('version', -1) + if version != 3: + raise MergeException( # pragma: no cover (covered by + # results_merger_unittest). + 'Unsupported version %s. Only version 3 is supported' % version) + + # Check the results for each shard have the required keys + missing = REQUIRED - set(result_json) + if missing: + raise MergeException( # pragma: no cover (covered by + # results_merger_unittest). + 'Invalid json test results (missing %s)' % missing) + + # Curry merge_values for this result_json. + merge = lambda key, merge_func: merge_value( + result_json, merged_results, key, merge_func) + + # Traverse the result_json's test trie & merged_results's test tries in + # DFS order & add the n to merged['tests']. + merge('tests', merge_tries) + + # If any were interrupted, we are interrupted. + merge('interrupted', lambda x,y: x|y) + + # Use the earliest seconds_since_epoch value + merge('seconds_since_epoch', min) + + # Sum the number of failure types + merge('num_failures_by_type', sum_dicts) + + # Optional values must match + for optional_key in OPTIONAL_MATCHING: + if optional_key not in result_json: + continue + + if optional_key not in merged_results: + # Set this value to None, then blindly copy over it. + merged_results[optional_key] = None + merge(optional_key, lambda src, dst: src) + else: + merge(optional_key, ensure_match) + + # Optional values ignored + for optional_key in OPTIONAL_IGNORED: + if optional_key in result_json: + merged_results[optional_key] = result_json.pop( + # pragma: no cover (covered by + # results_merger_unittest). + optional_key) + + # Sum optional value counts + for count_key in OPTIONAL_COUNTS: + if count_key in result_json: # pragma: no cover + # TODO(mcgreevy): add coverage. + merged_results.setdefault(count_key, 0) + merge(count_key, lambda a, b: a+b) + + if result_json: + raise MergeException( # pragma: no cover (covered by + # results_merger_unittest). + 'Unmergable values %s' % result_json.keys()) + + return merged_results + + +def merge_tries(source, dest): + """ Merges test tries. + + This is intended for use as a merge_func parameter to merge_value. + + Args: + source: A result json test trie. + dest: A json test trie merge destination. + """ + # merge_tries merges source into dest by performing a lock-step depth-first + # traversal of dest and source. + # pending_nodes contains a list of all sub-tries which have been reached but + # need further merging. + # Each element consists of a trie prefix, and a sub-trie from each of dest + # and source which is reached via that prefix. + pending_nodes = [('', dest, source)] + while pending_nodes: + prefix, dest_node, curr_node = pending_nodes.pop() + for k, v in curr_node.iteritems(): + if k in dest_node: + if not isinstance(v, dict): + raise MergeException( + "%s:%s: %r not mergable, curr_node: %r\ndest_node: %r" % ( + prefix, k, v, curr_node, dest_node)) + pending_nodes.append(("%s:%s" % (prefix, k), dest_node[k], v)) + else: + dest_node[k] = v + return dest + + +def ensure_match(source, dest): + """ Returns source if it matches dest. + + This is intended for use as a merge_func parameter to merge_value. + + Raises: + MergeException if source != dest + """ + if source != dest: + raise MergeException( # pragma: no cover (covered by + # results_merger_unittest). + "Values don't match: %s, %s" % (source, dest)) + return source + + +def sum_dicts(source, dest): + """ Adds values from source to corresponding values in dest. + + This is intended for use as a merge_func parameter to merge_value. + """ + for k, v in source.iteritems(): + dest.setdefault(k, 0) + dest[k] += v + + return dest + + +def merge_value(source, dest, key, merge_func): + """ Merges a value from source to dest. + + The value is deleted from source. + + Args: + source: A dictionary from which to pull a value, identified by key. + dest: The dictionary into to which the value is to be merged. + key: The key which identifies the value to be merged. + merge_func(src, dst): A function which merges its src into dst, + and returns the result. May modify dst. May raise a MergeException. + + Raises: + MergeException if the values can not be merged. + """ + try: + dest[key] = merge_func(source[key], dest[key]) + except MergeException as e: + e.message = "MergeFailure for %s\n%s" % (key, e.message) + e.args = tuple([e.message] + list(e.args[1:])) + raise + del source[key] + + +def main(files): + if len(files) < 2: + sys.stderr.write("Not enough JSON files to merge.\n") + return 1 + sys.stderr.write('Starting with %s\n' % files[0]) + result = json.load(open(files[0])) + for f in files[1:]: + sys.stderr.write('Merging %s\n' % f) + result = merge_test_results([result, json.load(open(f))]) + print json.dumps(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/infra/bots/recipe_modules/swarming/resources/standard_gtest_merge.py b/infra/bots/recipe_modules/swarming/resources/standard_gtest_merge.py new file mode 100755 index 0000000000..ca3abcf004 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/resources/standard_gtest_merge.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import json +import os +import shutil +import sys +import tempfile +import traceback + +from common import gtest_utils +from slave import annotation_utils +from slave import slave_utils + + +MISSING_SHARDS_MSG = r"""Missing results from the following shard(s): %s + +This can happen in following cases: + * Test failed to start (missing *.dll/*.so dependency for example) + * Test crashed or hung + * Task expired because there are not enough bots available and are all used + * Swarming service experienced problems + +Please examine logs to figure out what happened. +""" + + +def emit_warning(title, log=None): + print '@@@STEP_WARNINGS@@@' + print title + if log: + slave_utils.WriteLogLines(title, log.split('\n')) + + +def merge_shard_results(summary_json, jsons_to_merge): + """Reads JSON test output from all shards and combines them into one. + + Returns dict with merged test output on success or None on failure. Emits + annotations. + """ + # summary.json is produced by swarming.py itself. We are mostly interested + # in the number of shards. + try: + with open(summary_json) as f: + summary = json.load(f) + except (IOError, ValueError): + emit_warning( + 'summary.json is missing or can not be read', + 'Something is seriously wrong with swarming_client/ or the bot.') + return None + + # Merge all JSON files together. Keep track of missing shards. + merged = { + 'all_tests': set(), + 'disabled_tests': set(), + 'global_tags': set(), + 'missing_shards': [], + 'per_iteration_data': [], + 'swarming_summary': summary, + } + for index, result in enumerate(summary['shards']): + if result is not None: + # Author note: this code path doesn't trigger convert_to_old_format() in + # client/swarming.py, which means the state enum is saved in its string + # name form, not in the number form. + state = result.get('state') + if state == u'BOT_DIED': + emit_warning('Shard #%d had a Swarming internal failure' % index) + elif state == u'EXPIRED': + emit_warning('There wasn\'t enough capacity to run your test') + elif state == u'TIMED_OUT': + emit_warning( + 'Test runtime exceeded allocated time', + 'Either it ran for too long (hard timeout) or it didn\'t produce ' + 'I/O for an extended period of time (I/O timeout)') + elif state == u'COMPLETED': + json_data, err_msg = load_shard_json(index, jsons_to_merge) + if json_data: + # Set-like fields. + for key in ('all_tests', 'disabled_tests', 'global_tags'): + merged[key].update(json_data.get(key), []) + + # 'per_iteration_data' is a list of dicts. Dicts should be merged + # together, not the 'per_iteration_data' list itself. + merged['per_iteration_data'] = merge_list_of_dicts( + merged['per_iteration_data'], + json_data.get('per_iteration_data', [])) + continue + else: + emit_warning('Task ran but no result was found: %s' % err_msg) + else: + emit_warning('Invalid Swarming task state: %s' % state) + merged['missing_shards'].append(index) + + # If some shards are missing, make it known. Continue parsing anyway. Step + # should be red anyway, since swarming.py return non-zero exit code in that + # case. + if merged['missing_shards']: + as_str = ', '.join(map(str, merged['missing_shards'])) + emit_warning( + 'some shards did not complete: %s' % as_str, + MISSING_SHARDS_MSG % as_str) + # Not all tests run, combined JSON summary can not be trusted. + merged['global_tags'].add('UNRELIABLE_RESULTS') + + # Convert to jsonish dict. + for key in ('all_tests', 'disabled_tests', 'global_tags'): + merged[key] = sorted(merged[key]) + return merged + + +OUTPUT_JSON_SIZE_LIMIT = 100 * 1024 * 1024 # 100 MB + + +def load_shard_json(index, jsons_to_merge): + """Reads JSON output of the specified shard. + + Args: + output_dir: The directory in which to look for the JSON output to load. + index: The index of the shard to load data for. + + Returns: A tuple containing: + * The contents of path, deserialized into a python object. + * An error string. + (exactly one of the tuple elements will be non-None). + """ + # 'output.json' is set in swarming/api.py, gtest_task method. + matching_json_files = [ + j for j in jsons_to_merge + if (os.path.basename(j) == 'output.json' + and os.path.basename(os.path.dirname(j)) == str(index))] + + if not matching_json_files: + print >> sys.stderr, 'shard %s test output missing' % index + return (None, 'shard %s test output was missing' % index) + elif len(matching_json_files) > 1: + print >> sys.stderr, 'duplicate test output for shard %s' % index + return (None, 'shard %s test output was duplicated' % index) + + path = matching_json_files[0] + + try: + filesize = os.stat(path).st_size + if filesize > OUTPUT_JSON_SIZE_LIMIT: + print >> sys.stderr, 'output.json is %d bytes. Max size is %d' % ( + filesize, OUTPUT_JSON_SIZE_LIMIT) + return (None, 'shard %s test output exceeded the size limit' % index) + + with open(path) as f: + return (json.load(f), None) + except (IOError, ValueError, OSError) as e: + print >> sys.stderr, 'Missing or invalid gtest JSON file: %s' % path + print >> sys.stderr, '%s: %s' % (type(e).__name__, e) + + return (None, 'shard %s test output was missing or invalid' % index) + + +def merge_list_of_dicts(left, right): + """Merges dicts left[0] with right[0], left[1] with right[1], etc.""" + output = [] + for i in xrange(max(len(left), len(right))): + left_dict = left[i] if i < len(left) else {} + right_dict = right[i] if i < len(right) else {} + merged_dict = left_dict.copy() + merged_dict.update(right_dict) + output.append(merged_dict) + return output + + +def standard_gtest_merge( + output_json, summary_json, jsons_to_merge): + + output = merge_shard_results(summary_json, jsons_to_merge) + with open(output_json, 'wb') as f: + json.dump(output, f) + + return 0 + + +def main(raw_args): + + parser = argparse.ArgumentParser() + parser.add_argument('--build-properties') + parser.add_argument('--summary-json') + parser.add_argument('-o', '--output-json', required=True) + parser.add_argument('jsons_to_merge', nargs='*') + + args = parser.parse_args(raw_args) + + return standard_gtest_merge( + args.output_json, args.summary_json, args.jsons_to_merge) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/infra/bots/recipe_modules/swarming/resources/standard_isolated_script_merge.py b/infra/bots/recipe_modules/swarming/resources/standard_isolated_script_merge.py new file mode 100755 index 0000000000..e3c860f433 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/resources/standard_isolated_script_merge.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import json +import sys + +import results_merger + + +def StandardIsolatedScriptMerge(output_json, jsons_to_merge): + """Merge the contents of one or more results JSONs into a single JSON. + + Args: + output_json: A path to a JSON file to which the merged results should be + written. + jsons_to_merge: A list of paths to JSON files that should be merged. + """ + shard_results_list = [] + for j in jsons_to_merge: + with open(j) as f: + shard_results_list.append(json.load(f)) + merged_results = results_merger.merge_test_results(shard_results_list) + + with open(output_json, 'w') as f: + json.dump(merged_results, f) + + return 0 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--output-json', required=True) + parser.add_argument('--build-properties', help=argparse.SUPPRESS) + parser.add_argument('--summary-json', help=argparse.SUPPRESS) + parser.add_argument('jsons_to_merge', nargs='*') + + args = parser.parse_args() + return StandardIsolatedScriptMerge(args.output_json, args.jsons_to_merge) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/infra/bots/recipe_modules/swarming/state.py b/infra/bots/recipe_modules/swarming/state.py new file mode 100644 index 0000000000..3834b901c0 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/state.py @@ -0,0 +1,46 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +class State(object): + """Copied from appengine/swarming/server/task_result.py. + + KEEP IN SYNC. + + Used to parse the 'state' value in task result. + """ + RUNNING = 0x10 # 16 + PENDING = 0x20 # 32 + EXPIRED = 0x30 # 48 + TIMED_OUT = 0x40 # 64 + BOT_DIED = 0x50 # 80 + CANCELED = 0x60 # 96 + COMPLETED = 0x70 # 112 + + STATES = ( + RUNNING, PENDING, EXPIRED, TIMED_OUT, BOT_DIED, CANCELED, COMPLETED) + STATES_RUNNING = (RUNNING, PENDING) + STATES_NOT_RUNNING = (EXPIRED, TIMED_OUT, BOT_DIED, CANCELED, COMPLETED) + STATES_DONE = (TIMED_OUT, COMPLETED) + STATES_ABANDONED = (EXPIRED, BOT_DIED, CANCELED) + + _NAMES = { + RUNNING: 'Running', + PENDING: 'Pending', + EXPIRED: 'Expired', + TIMED_OUT: 'Execution timed out', + BOT_DIED: 'Bot died', + CANCELED: 'User canceled', + COMPLETED: 'Completed', + } + + @classmethod + def to_string(cls, state): + """Returns a user-readable string representing a State.""" + return cls._NAMES[state] diff --git a/infra/bots/recipe_modules/swarming/test_api.py b/infra/bots/recipe_modules/swarming/test_api.py new file mode 100644 index 0000000000..3066fd6189 --- /dev/null +++ b/infra/bots/recipe_modules/swarming/test_api.py @@ -0,0 +1,56 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +from recipe_engine import recipe_test_api + +import state + +class SwarmingTestApi(recipe_test_api.RecipeTestApi): + + @recipe_test_api.placeholder_step_data + def summary(self, data): + return self.m.json.output(data) + + def canned_summary_output( + self, shards=1, failure=False, internal_failure=False): + return self.summary({ + 'shards': [ + { + 'abandoned_ts': None, + 'bot_id': 'vm30', + 'completed_ts': '2014-09-25T01:42:00.123', + 'created_ts': '2014-09-25T01:41:00.123', + 'durations': [5.7, 31.5], + 'exit_codes': [0, 0], + 'failure': failure, + 'id': '148aa78d7aa%02d00' % i, + 'internal_failure': internal_failure, + 'isolated_out': { + 'isolated': 'abc123', + 'isolatedserver': 'https://isolateserver.appspot.com', + 'namespace': 'default-gzip', + 'view_url': 'blah', + }, + 'modified_ts': '2014-09-25 01:42:00', + 'name': 'heartbeat-canary-2014-09-25_01:41:55-os=Windows', + 'outputs': [ + 'Heart beat succeeded on win32.\n', + 'Foo', + ], + 'outputs_ref': { + 'view_url': 'blah', + }, + 'started_ts': '2014-09-25T01:42:11.123', + 'state': state.State.COMPLETED, + 'try_number': 1, + 'user': 'unknown', + } for i in xrange(shards) + ], + }) diff --git a/infra/bots/recipe_modules/swarming_client/OWNERS b/infra/bots/recipe_modules/swarming_client/OWNERS new file mode 100644 index 0000000000..8da4472230 --- /dev/null +++ b/infra/bots/recipe_modules/swarming_client/OWNERS @@ -0,0 +1,2 @@ +martiniss@chromium.org +vadimsh@chromium.org diff --git a/infra/bots/recipe_modules/swarming_client/__init__.py b/infra/bots/recipe_modules/swarming_client/__init__.py new file mode 100644 index 0000000000..76ce2748f2 --- /dev/null +++ b/infra/bots/recipe_modules/swarming_client/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +DEPS = [ + 'depot_tools/git', + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/python', + 'recipe_engine/raw_io', + 'recipe_engine/step', +] diff --git a/infra/bots/recipe_modules/swarming_client/api.py b/infra/bots/recipe_modules/swarming_client/api.py new file mode 100644 index 0000000000..13e6c01dca --- /dev/null +++ b/infra/bots/recipe_modules/swarming_client/api.py @@ -0,0 +1,135 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +from recipe_engine import recipe_api + + +class SwarmingClientApi(recipe_api.RecipeApi): + """Code that both isolate and swarming recipe modules depend on. + + Both swarming and isolate scripts live in a single repository called + 'swarming client'. This module include common functionality like finding + existing swarming client checkout, fetching a new one, getting version of + a swarming script, etc. + """ + + def __init__(self, **kwargs): + super(SwarmingClientApi, self).__init__(**kwargs) + self._client_path = None + self._script_version = {} + + def checkout(self, revision=None, curl_trace_file=None, can_fail_build=True): + """Returns a step to checkout swarming client into a separate directory. + + Ordinarily swarming client is checked out via Chromium DEPS into + src/tools/swarming_client. This step configures recipe module to use + a separate checkout. + + If |revision| is None, this requires the build property + 'parent_got_swarming_client_revision' to be present, and raises an exception + otherwise. Fail-fast behavior is used because if machines silently fell back + to checking out the entire workspace, that would cause dramatic increases + in cycle time if a misconfiguration were made and it were no longer possible + for the bot to check out swarming_client separately. + """ + # If the following line throws an exception, it either means the + # bot is misconfigured, or, if you're testing locally, that you + # need to pass in some recent legal revision for this property. + if revision is None: + revision = self.m.properties['parent_got_swarming_client_revision'] + self._client_path = self.m.path['start_dir'].join('swarming.client') + self.m.git.checkout( + url='https://chromium.googlesource.com/external/swarming.client.git', + ref=revision, + dir_path=self._client_path, + step_suffix='swarming_client', + curl_trace_file=curl_trace_file, + can_fail_build=can_fail_build) + + @property + def path(self): + """Returns path to a swarming client checkout. + + It's subdirectory of Chromium src/ checkout or a separate directory if + 'checkout_swarming_client' step was used. + """ + if self._client_path: + return self._client_path + # Default is swarming client path in chromium src/ checkout. + # TODO(vadimsh): This line assumes the recipe is working with + # Chromium checkout. + return self.m.path['checkout'].join('tools', 'swarming_client') + + def query_script_version(self, script, step_test_data=None): + """Yields a step to query a swarming script for its version. + + Version tuple is later accessible via 'get_script_version' method. If + |step_test_data| is given, it is a tuple with version to use in expectation + tests by default. + + Does nothing if script's version is already known. + """ + # Convert |step_test_data| from tuple of ints back to a version string. + if step_test_data: + assert isinstance(step_test_data, tuple) + assert all(isinstance(x, int) for x in step_test_data) + as_text = '.'.join(map(str, step_test_data)) + step_test_data_cb = lambda: self.m.raw_io.test_api.stream_output(as_text) + else: + step_test_data_cb = None + + if script not in self._script_version: + try: + self.m.python( + name='%s --version' % script, + script=self.path.join(script), + args=['--version'], + stdout=self.m.raw_io.output_text(), + step_test_data=step_test_data_cb) + finally: + step_result = self.m.step.active_result + version = step_result.stdout.strip() + step_result.presentation.step_text = version + self._script_version[script] = tuple(map(int, version.split('.'))) + + return step_result + + def get_script_version(self, script): + """Returns a version of some swarming script as a tuple (Major, Minor, Rev). + + It should have been queried by 'query_script_version' step before. Raises + AssertionError if it wasn't. + """ + assert script in self._script_version, script + return self._script_version[script] + + def ensure_script_version(self, script, min_version, step_test_data=None): + """Yields steps to ensure a script version is not older than |min_version|. + + Will abort recipe execution if it is. + """ + step_result = self.query_script_version( + script, step_test_data=step_test_data or min_version) + version = self.get_script_version(script) + if version < min_version: + expecting = '.'.join(map(str, min_version)) + got = '.'.join(map(str, version)) + abort_reason = 'Expecting at least v%s, got v%s' % (expecting, got) + + # TODO(martiniss) remove once recipe 1.5 migration done + step_result = self.m.python.inline( + '%s is too old' % script, + 'import sys; sys.exit(1)', + add_python_log=False) + # TODO(martiniss) get rid of this bare string. + step_result.presentation.status = self.m.step.FAILURE + step_result.presentation.step_text = abort_reason + + raise self.m.step.StepFailure(abort_reason) diff --git a/infra/bots/recipe_modules/swarming_client/example.expected/basic.json b/infra/bots/recipe_modules/swarming_client/example.expected/basic.json new file mode 100644 index 0000000000..86bc9a7e42 --- /dev/null +++ b/infra/bots/recipe_modules/swarming_client/example.expected/basic.json @@ -0,0 +1,136 @@ +[ + { + "cmd": [], + "name": "client path" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "sample_sha" + ], + "cwd": "[START_DIR]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "infra_step": true, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/swarming.client", + "infra_step": true, + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.4.4@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/swarming.client/isolate.py", + "--version" + ], + "name": "isolate.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.3.1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "import sys; sys.exit(1)" + ], + "name": "swarming.py is too old", + "~followup_annotations": [ + "@@@STEP_TEXT@Expecting at least v20.0.0, got v0.4.4@@@", + "@@@STEP_FAILURE@@@" + ] + }, + { + "name": "$result", + "reason": "Expecting at least v20.0.0, got v0.4.4", + "recipe_result": null, + "status_code": 1 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/swarming_client/example.py b/infra/bots/recipe_modules/swarming_client/example.py new file mode 100644 index 0000000000..223c29d526 --- /dev/null +++ b/infra/bots/recipe_modules/swarming_client/example.py @@ -0,0 +1,46 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# TODO(borenet): This module was copied from build.git and heavily modified to +# remove dependencies on other modules in build.git. It belongs in a different +# repo. Remove this once it has been moved. + + +DEPS = [ + 'recipe_engine/properties', + 'recipe_engine/raw_io', + 'recipe_engine/step', + 'swarming_client', +] + + +def RunSteps(api): + # Code coverage for these methods. + api.step('client path', []) + api.step.active_result.step_text = api.swarming_client.path + api.swarming_client.checkout() + #api.swarming_client.checkout('master') + api.swarming_client.query_script_version('swarming.py') + api.swarming_client.ensure_script_version('swarming.py', (0, 4, 4)) + + # Coverage for |step_test_data| argument. + api.swarming_client.query_script_version( + 'isolate.py', step_test_data=(0, 3, 1)) + + # 'master' had swarming.py at v0.4.4 at the moment of writing this example. + assert api.swarming_client.get_script_version('swarming.py') >= (0, 4, 4) + + # Coverage for 'fail' path of ensure_script_version. + api.swarming_client.ensure_script_version('swarming.py', (20, 0, 0)) + + +def GenTests(api): + yield ( + api.test('basic') + + api.properties(parent_got_swarming_client_revision='sample_sha') + + api.step_data( + 'swarming.py --version', + stdout=api.raw_io.output_text('0.4.4')) + ) diff --git a/infra/bots/recipes/compile.expected/Build-Ubuntu-GCC-x86_64-Release-Flutter_Android.json b/infra/bots/recipes/compile.expected/Build-Ubuntu-GCC-x86_64-Release-Flutter_Android.json index a318c9643b..a3b167b1e2 100644 --- a/infra/bots/recipes/compile.expected/Build-Ubuntu-GCC-x86_64-Release-Flutter_Android.json +++ b/infra/bots/recipes/compile.expected/Build-Ubuntu-GCC-x86_64-Release-Flutter_Android.json @@ -164,7 +164,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/flutter/src/out/android_release" ], diff --git a/infra/bots/recipes/compile.expected/flutter_trybot.json b/infra/bots/recipes/compile.expected/flutter_trybot.json index 4cec7e6ef9..c6d1556ea8 100644 --- a/infra/bots/recipes/compile.expected/flutter_trybot.json +++ b/infra/bots/recipes/compile.expected/flutter_trybot.json @@ -168,7 +168,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/flutter/src/out/android_release" ], diff --git a/infra/bots/recipes/ct_skps.expected/CT_CPU_BENCH_10k_SKPs.json b/infra/bots/recipes/ct_skps.expected/CT_CPU_BENCH_10k_SKPs.json index 744e7c77a6..e02579acd9 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_CPU_BENCH_10k_SKPs.json +++ b/infra/bots/recipes/ct_skps.expected/CT_CPU_BENCH_10k_SKPs.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_10k_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_10k_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_10k_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_10k_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_10k_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1275,17 +1275,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1425,17 +1421,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1575,17 +1567,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1725,17 +1713,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1875,17 +1859,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_DM_100k_SKPs.json b/infra/bots/recipes/ct_skps.expected/CT_DM_100k_SKPs.json index 4f76beded3..6fd8305db0 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_DM_100k_SKPs.json +++ b/infra/bots/recipes/ct_skps.expected/CT_DM_100k_SKPs.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1275,17 +1275,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1356,17 +1352,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1437,17 +1429,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1518,17 +1506,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1599,17 +1583,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_DM_10k_SKPs.json b/infra/bots/recipes/ct_skps.expected/CT_DM_10k_SKPs.json index 36a82c279e..083745d09c 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_DM_10k_SKPs.json +++ b/infra/bots/recipes/ct_skps.expected/CT_DM_10k_SKPs.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1275,17 +1275,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1356,17 +1352,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1437,17 +1429,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1518,17 +1506,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1599,17 +1583,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_DM_10k_SKPs_Trybot.json b/infra/bots/recipes/ct_skps.expected/CT_DM_10k_SKPs_Trybot.json index 9c0a020a8f..0e893d2a7a 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_DM_10k_SKPs_Trybot.json +++ b/infra/bots/recipes/ct_skps.expected/CT_DM_10k_SKPs_Trybot.json @@ -398,7 +398,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -427,7 +427,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -441,7 +441,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave1" ], @@ -532,7 +532,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave2" ], @@ -623,7 +623,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave3" ], @@ -714,7 +714,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave4" ], @@ -805,7 +805,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs/slave5" ], @@ -907,7 +907,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1299,17 +1299,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1380,17 +1376,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1461,17 +1453,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1542,17 +1530,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1623,17 +1607,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs.json b/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs.json index 4cadd20dfb..7ad7faacb7 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs.json +++ b/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1275,17 +1275,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1356,17 +1352,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1437,17 +1429,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1518,17 +1506,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1599,17 +1583,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs_2slaves_failure.json b/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs_2slaves_failure.json index a51eb49921..0f1a7f5fd8 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs_2slaves_failure.json +++ b/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs_2slaves_failure.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1275,17 +1275,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1360,17 +1356,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1441,17 +1433,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1526,17 +1514,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1607,17 +1591,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs_slave3_failure.json b/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs_slave3_failure.json index db51003f30..ee33430822 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs_slave3_failure.json +++ b/infra/bots/recipes/ct_skps.expected/CT_DM_1m_SKPs_slave3_failure.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1275,17 +1275,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1356,17 +1352,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1437,17 +1429,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1522,17 +1510,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1603,17 +1587,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-dm-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_GPU_BENCH_10k_SKPs.json b/infra/bots/recipes/ct_skps.expected/CT_GPU_BENCH_10k_SKPs.json index 0bba222009..d74bb38805 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_GPU_BENCH_10k_SKPs.json +++ b/infra/bots/recipes/ct_skps.expected/CT_GPU_BENCH_10k_SKPs.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_10k_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_10k_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_10k_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_10k_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_10k_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1290,17 +1290,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1440,17 +1436,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1590,17 +1582,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1740,17 +1728,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1890,17 +1874,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_GPU_BENCH_1k_SKPs.json b/infra/bots/recipes/ct_skps.expected/CT_GPU_BENCH_1k_SKPs.json index 16f2e8550b..6f03a47caf 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_GPU_BENCH_1k_SKPs.json +++ b/infra/bots/recipes/ct_skps.expected/CT_GPU_BENCH_1k_SKPs.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1290,17 +1290,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1440,17 +1436,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1590,17 +1582,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1740,17 +1728,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1890,17 +1874,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-nanobench-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_100k_SKPs.json b/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_100k_SKPs.json index 4cbbd25fa2..22f0f39770 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_100k_SKPs.json +++ b/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_100k_SKPs.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_100k_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_100k_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_100k_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_100k_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_100k_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1275,17 +1275,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1356,17 +1352,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1437,17 +1429,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1518,17 +1506,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1599,17 +1583,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_10k_SKPs.json b/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_10k_SKPs.json index 3815dd3465..98f23bf6fb 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_10k_SKPs.json +++ b/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_10k_SKPs.json @@ -394,7 +394,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -423,7 +423,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -437,7 +437,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave1" ], @@ -528,7 +528,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave2" ], @@ -619,7 +619,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave3" ], @@ -710,7 +710,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave4" ], @@ -801,7 +801,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave5" ], @@ -903,7 +903,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1275,17 +1275,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1356,17 +1352,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1437,17 +1429,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1518,17 +1506,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1599,17 +1583,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_10k_SKPs_Trybot.json b/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_10k_SKPs_Trybot.json index a7c3f10d76..7e5ed894f1 100644 --- a/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_10k_SKPs_Trybot.json +++ b/infra/bots/recipes/ct_skps.expected/CT_IMG_DECODE_10k_SKPs_Trybot.json @@ -398,7 +398,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/luci-go" ], @@ -427,7 +427,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/swarming_temp_dir" ], @@ -441,7 +441,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave1" ], @@ -532,7 +532,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave2" ], @@ -623,7 +623,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave3" ], @@ -714,7 +714,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave4" ], @@ -805,7 +805,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_/_B_WORK]/skps/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_10k_SKPs/slave5" ], @@ -907,7 +907,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "RECIPE_MODULE[skia::isolate]/resources/isolate.py", "[START_DIR]/swarming.client", "batcharchive", "--dump-json", @@ -1309,17 +1309,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-1", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1390,17 +1386,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-2", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1471,17 +1463,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-3", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1552,17 +1540,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-4", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", @@ -1633,17 +1617,13 @@ "cmd": [ "python", "-u", - "RECIPE_PACKAGE_REPO[build]/scripts/tools/runit.py", - "--show-path", - "--", - "python", - "RECIPE_MODULE[build::swarming]/resources/collect_task.py", + "RECIPE_MODULE[skia::swarming]/resources/collect_task.py", "-o", "/path/to/tmp/json", "--task-output-dir", "[START_DIR]/swarming_temp_dir/outputs/ct-get_images_from_skps-5", "--merge-script", - "RECIPE_MODULE[build::swarming]/resources/noop_merge.py", + "RECIPE_MODULE[skia::swarming]/resources/noop_merge.py", "--merge-additional-args", "[]", "--", diff --git a/infra/bots/recipes/ct_skps.py b/infra/bots/recipes/ct_skps.py index 0abdd8f7fd..86982330db 100644 --- a/infra/bots/recipes/ct_skps.py +++ b/infra/bots/recipes/ct_skps.py @@ -7,8 +7,8 @@ import math DEPS = [ - 'build/file', 'depot_tools/gsutil', + 'file', 'recipe_engine/context', 'recipe_engine/json', 'recipe_engine/path', @@ -19,7 +19,7 @@ DEPS = [ 'ct', 'flavor', 'run', - 'swarming', + 'skia_swarming', 'vars', ] @@ -83,7 +83,7 @@ def RunSteps(api): api.run.copy_build_products( api.flavor.out_dir, isolate_dir) - api.swarming.setup( + api.skia_swarming.setup( infrabots_dir.join('tools', 'luci-go'), swarming_rev='') @@ -103,7 +103,7 @@ def RunSteps(api): download_skps_link) # Delete swarming_temp_dir to ensure it starts from a clean slate. - api.run.rmtree(api.swarming.swarming_temp_dir) + api.run.rmtree(api.skia_swarming.swarming_temp_dir) num_per_slave = api.properties.get( 'num_per_slave', @@ -166,7 +166,7 @@ def RunSteps(api): 'CONFIGURATION': api.vars.configuration, 'BUILDER': buildername, } - api.swarming.create_isolated_gen_json( + api.skia_swarming.create_isolated_gen_json( isolate_path, isolate_dir, 'linux', 'ct-%s-%s' % (skia_tool, slave_num), extra_variables, blacklist=blacklist_skps) @@ -186,7 +186,7 @@ def RunSteps(api): tasks_to_swarm_hashes = [] for slave_start_num in xrange(1, ct_num_slaves+1, max_slaves_to_batcharchive): m = min(max_slaves_to_batcharchive, ct_num_slaves) - batcharchive_output = api.swarming.batcharchive( + batcharchive_output = api.skia_swarming.batcharchive( targets=['ct-' + skia_tool + '-%s' % num for num in range( slave_start_num, slave_start_num + m)]) tasks_to_swarm_hashes.extend(batcharchive_output) @@ -197,7 +197,7 @@ def RunSteps(api): dimensions={'os': 'Ubuntu-14.04', 'cpu': 'x86-64', 'pool': 'Chrome'} if 'GPU' in buildername: dimensions['gpu'] = '10de:104a' - tasks = api.swarming.trigger_swarming_tasks( + tasks = api.skia_swarming.trigger_swarming_tasks( tasks_to_swarm_hashes, dimensions=dimensions, io_timeout=40*60) # Now collect all tasks. @@ -205,10 +205,10 @@ def RunSteps(api): failed_tasks = [] for task in tasks: try: - api.swarming.collect_swarming_task(task) + api.skia_swarming.collect_swarming_task(task) if skia_tool == 'nanobench': - output_dir = api.swarming.tasks_output_dir.join( + output_dir = api.skia_swarming.tasks_output_dir.join( task.title).join('0') utc = api.time.utcnow() gs_dest_dir = 'ct/%s/%d/%02d/%02d/%02d/' % ( diff --git a/infra/bots/recipes/perf.expected/Perf-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json b/infra/bots/recipes/perf.expected/Perf-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json index bbb4f59d1e..cab4a88f85 100644 --- a/infra/bots/recipes/perf.expected/Perf-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json +++ b/infra/bots/recipes/perf.expected/Perf-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/data" ], diff --git a/infra/bots/recipes/perf.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release.json b/infra/bots/recipes/perf.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release.json index ece01be4de..7048b33b4b 100644 --- a/infra/bots/recipes/perf.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release.json +++ b/infra/bots/recipes/perf.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release/data" ], diff --git a/infra/bots/recipes/perf.expected/Perf-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json b/infra/bots/recipes/perf.expected/Perf-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json index ee5118b671..69bb9ce39c 100644 --- a/infra/bots/recipes/perf.expected/Perf-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json +++ b/infra/bots/recipes/perf.expected/Perf-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release/data" ], diff --git a/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-AlphaR2-GPU-RadeonR9M470X-x86_64-Release-ANGLE.json b/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-AlphaR2-GPU-RadeonR9M470X-x86_64-Release-ANGLE.json index 71168c09b3..a866cc57b9 100644 --- a/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-AlphaR2-GPU-RadeonR9M470X-x86_64-Release-ANGLE.json +++ b/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-AlphaR2-GPU-RadeonR9M470X-x86_64-Release-ANGLE.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\perfdata\\Perf-Win10-MSVC-AlphaR2-GPU-RadeonR9M470X-x86_64-Release-ANGLE\\data" ], diff --git a/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-ANGLE.json b/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-ANGLE.json index 1819167514..2221c2c526 100644 --- a/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-ANGLE.json +++ b/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-ANGLE.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\perfdata\\Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-ANGLE\\data" ], diff --git a/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-Vulkan.json b/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-Vulkan.json index f7cfd0e94c..f60c0e424a 100644 --- a/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-Vulkan.json +++ b/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-Vulkan.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\perfdata\\Perf-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-Vulkan\\data" ], diff --git a/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Release-ANGLE.json b/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Release-ANGLE.json index 9a18e4b63c..a85f320356 100644 --- a/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Release-ANGLE.json +++ b/infra/bots/recipes/perf.expected/Perf-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Release-ANGLE.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\perfdata\\Perf-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Release-ANGLE\\data" ], diff --git a/infra/bots/recipes/perf.expected/Perf-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Release.json b/infra/bots/recipes/perf.expected/Perf-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Release.json index d52213b21c..68372a3883 100644 --- a/infra/bots/recipes/perf.expected/Perf-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Release.json +++ b/infra/bots/recipes/perf.expected/Perf-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Release.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\perfdata\\Perf-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Release\\data" ], diff --git a/infra/bots/recipes/perf.expected/trybot.json b/infra/bots/recipes/perf.expected/trybot.json index daa2d7dcd6..352da826d2 100644 --- a/infra/bots/recipes/perf.expected/trybot.json +++ b/infra/bots/recipes/perf.expected/trybot.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Win10-MSVC-ShuttleB-GPU-IntelHD4600-x86_64-Release/data" ], diff --git a/infra/bots/recipes/perf.py b/infra/bots/recipes/perf.py index a85f75aa4e..318ba2ec68 100644 --- a/infra/bots/recipes/perf.py +++ b/infra/bots/recipes/perf.py @@ -10,9 +10,9 @@ import calendar DEPS = [ - 'build/file', 'core', 'env', + 'file', 'flavor', 'recipe_engine/json', 'recipe_engine/path', diff --git a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Nightly-RecreateSKPs_Canary.json b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Nightly-RecreateSKPs_Canary.json index 819fbb4b6d..340f802ab7 100644 --- a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Nightly-RecreateSKPs_Canary.json +++ b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Nightly-RecreateSKPs_Canary.json @@ -255,7 +255,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/skp_output" ], diff --git a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json index b731496138..f5246a03a1 100644 --- a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json +++ b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json @@ -255,7 +255,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/skp_output" ], diff --git a/infra/bots/recipes/recreate_skps.expected/failed_upload.json b/infra/bots/recipes/recreate_skps.expected/failed_upload.json index 664077e90e..f6328542ea 100644 --- a/infra/bots/recipes/recreate_skps.expected/failed_upload.json +++ b/infra/bots/recipes/recreate_skps.expected/failed_upload.json @@ -255,7 +255,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[START_DIR]/skp_output" ], diff --git a/infra/bots/recipes/recreate_skps.py b/infra/bots/recipes/recreate_skps.py index 0da6fe2709..8ca604c44d 100644 --- a/infra/bots/recipes/recreate_skps.py +++ b/infra/bots/recipes/recreate_skps.py @@ -7,8 +7,8 @@ DEPS = [ - 'build/file', 'depot_tools/gclient', + 'file', 'recipe_engine/context', 'recipe_engine/path', 'recipe_engine/properties', diff --git a/infra/bots/recipes/skpbench.py b/infra/bots/recipes/skpbench.py index 08b60d9b23..1d416f7897 100644 --- a/infra/bots/recipes/skpbench.py +++ b/infra/bots/recipes/skpbench.py @@ -10,8 +10,8 @@ import calendar DEPS = [ - 'build/file', 'core', + 'file', 'recipe_engine/context', 'recipe_engine/path', 'recipe_engine/properties', diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Release-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Release-Android.json index 62e34e745a..c6fe8f7f62 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Release-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Release-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-Ci20-CPU-IngenicJZ4780-mipsel-Release-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-Ci20-CPU-IngenicJZ4780-mipsel-Release-Android.json index 7604b5de9b..9a3ce7d4a6 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-Ci20-CPU-IngenicJZ4780-mipsel-Release-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-Ci20-CPU-IngenicJZ4780-mipsel-Release-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-GalaxyS6-GPU-MaliT760-arm64-Debug-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-GalaxyS6-GPU-MaliT760-arm64-Debug-Android.json index 07dd273620..fbbaf0b996 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-GalaxyS6-GPU-MaliT760-arm64-Debug-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-GalaxyS6-GPU-MaliT760-arm64-Debug-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-GalaxyS7_G930A-GPU-Adreno530-arm64-Debug-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-GalaxyS7_G930A-GPU-Adreno530-arm64-Debug-Android.json index 484e9a81bf..a13d632c2b 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-GalaxyS7_G930A-GPU-Adreno530-arm64-Debug-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-GalaxyS7_G930A-GPU-Adreno530-arm64-Debug-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Debug-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Debug-Android.json index 8cc5cc6477..ef02107e12 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Debug-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Debug-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus10-GPU-MaliT604-arm-Release-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus10-GPU-MaliT604-arm-Release-Android.json index 8640ee0065..5728d9496c 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus10-GPU-MaliT604-arm-Release-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus10-GPU-MaliT604-arm-Release-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus5-GPU-Adreno330-arm-Release-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus5-GPU-Adreno330-arm-Release-Android.json index fa779b7685..2f5a159f77 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus5-GPU-Adreno330-arm-Release-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus5-GPU-Adreno330-arm-Release-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Debug-Android_Vulkan.json b/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Debug-Android_Vulkan.json index 22febe0cc1..1425278e88 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Debug-Android_Vulkan.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Debug-Android_Vulkan.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus7-GPU-Tegra3-arm-Debug-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus7-GPU-Tegra3-arm-Debug-Android.json index 87d7d614c9..c779d41c4f 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus7-GPU-Tegra3-arm-Debug-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-Nexus7-GPU-Tegra3-arm-Debug-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-NexusPlayer-CPU-SSE4-x86-Release-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-NexusPlayer-CPU-SSE4-x86-Release-Android.json index f459f27eb8..d85d7dd5b5 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-NexusPlayer-CPU-SSE4-x86-Release-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-NexusPlayer-CPU-SSE4-x86-Release-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-Android_Vulkan.json b/infra/bots/recipes/test.expected/Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-Android_Vulkan.json index 19ad113816..b198dbe189 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-Android_Vulkan.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-Android_Vulkan.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Android-Clang-PixelC-CPU-TegraX1-arm64-Debug-Android.json b/infra/bots/recipes/test.expected/Test-Android-Clang-PixelC-CPU-TegraX1-arm64-Debug-Android.json index abdd739e4c..45c304a414 100644 --- a/infra/bots/recipes/test.expected/Test-Android-Clang-PixelC-CPU-TegraX1-arm64-Debug-Android.json +++ b/infra/bots/recipes/test.expected/Test-Android-Clang-PixelC-CPU-TegraX1-arm64-Debug-Android.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-ChromeOS-Clang-Chromebook_C100p-GPU-MaliT764-arm-Debug.json b/infra/bots/recipes/test.expected/Test-ChromeOS-Clang-Chromebook_C100p-GPU-MaliT764-arm-Debug.json index 460bc89e87..cd2bcecc4f 100644 --- a/infra/bots/recipes/test.expected/Test-ChromeOS-Clang-Chromebook_C100p-GPU-MaliT764-arm-Debug.json +++ b/infra/bots/recipes/test.expected/Test-ChromeOS-Clang-Chromebook_C100p-GPU-MaliT764-arm-Debug.json @@ -572,7 +572,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Debug.json b/infra/bots/recipes/test.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Debug.json index 8b585dc4f7..9ae4b9f7be 100644 --- a/infra/bots/recipes/test.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Debug.json +++ b/infra/bots/recipes/test.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Debug.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Debug-CommandBuffer.json b/infra/bots/recipes/test.expected/Test-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Debug-CommandBuffer.json index 3066a8a377..ad6ce5cdd6 100644 --- a/infra/bots/recipes/test.expected/Test-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Debug-CommandBuffer.json +++ b/infra/bots/recipes/test.expected/Test-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Debug-CommandBuffer.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-SK_FORCE_RASTER_PIPELINE_BLITTER.json b/infra/bots/recipes/test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-SK_FORCE_RASTER_PIPELINE_BLITTER.json index e47e0af364..ba9ffccdc1 100644 --- a/infra/bots/recipes/test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-SK_FORCE_RASTER_PIPELINE_BLITTER.json +++ b/infra/bots/recipes/test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-SK_FORCE_RASTER_PIPELINE_BLITTER.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug.json b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug.json index 7f567069bc..4d46a13b52 100644 --- a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug.json +++ b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE.json b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE.json index c8a5941dda..05fb89bfe7 100644 --- a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE.json +++ b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json index 566da63f28..45fc09d226 100644 --- a/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json +++ b/infra/bots/recipes/test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug.json b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug.json index 796207bdfa..e38cc6a566 100644 --- a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug.json +++ b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-Vulkan.json b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-Vulkan.json index db2006b362..20d3612714 100644 --- a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-Vulkan.json +++ b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-Vulkan.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json index 1db3d85ff4..276ab89db8 100644 --- a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json +++ b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUCDE3815TYKHE-GPU-IntelBayTrail-x86_64-Debug.json b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUCDE3815TYKHE-GPU-IntelBayTrail-x86_64-Debug.json index 7e5a3f0c52..1855a39fe0 100644 --- a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUCDE3815TYKHE-GPU-IntelBayTrail-x86_64-Debug.json +++ b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUCDE3815TYKHE-GPU-IntelBayTrail-x86_64-Debug.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Win10-MSVC-AlphaR2-GPU-RadeonR9M470X-x86_64-Debug-Vulkan.json b/infra/bots/recipes/test.expected/Test-Win10-MSVC-AlphaR2-GPU-RadeonR9M470X-x86_64-Debug-Vulkan.json index 317e2fc99b..82c124742c 100644 --- a/infra/bots/recipes/test.expected/Test-Win10-MSVC-AlphaR2-GPU-RadeonR9M470X-x86_64-Debug-Vulkan.json +++ b/infra/bots/recipes/test.expected/Test-Win10-MSVC-AlphaR2-GPU-RadeonR9M470X-x86_64-Debug-Vulkan.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC5i7RYH-GPU-IntelIris6100-x86_64-Release-ReleaseAndAbandonGpuContext.json b/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC5i7RYH-GPU-IntelIris6100-x86_64-Release-ReleaseAndAbandonGpuContext.json index fddeed76c2..c1a3a32888 100644 --- a/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC5i7RYH-GPU-IntelIris6100-x86_64-Release-ReleaseAndAbandonGpuContext.json +++ b/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC5i7RYH-GPU-IntelIris6100-x86_64-Release-ReleaseAndAbandonGpuContext.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-ANGLE.json b/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-ANGLE.json index a9c8984913..f662a99886 100644 --- a/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-ANGLE.json +++ b/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-ANGLE.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-Vulkan.json b/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-Vulkan.json index e455cf87ef..37f73cd3fb 100644 --- a/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-Vulkan.json +++ b/infra/bots/recipes/test.expected/Test-Win10-MSVC-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-Vulkan.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug-Vulkan.json b/infra/bots/recipes/test.expected/Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug-Vulkan.json index 04ca84c21a..62152a1c09 100644 --- a/infra/bots/recipes/test.expected/Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug-Vulkan.json +++ b/infra/bots/recipes/test.expected/Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug-Vulkan.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Debug-ANGLE.json b/infra/bots/recipes/test.expected/Test-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Debug-ANGLE.json index a5b70c566e..d9c4e3c5de 100644 --- a/infra/bots/recipes/test.expected/Test-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Debug-ANGLE.json +++ b/infra/bots/recipes/test.expected/Test-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Debug-ANGLE.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Win10-MSVC-ZBOX-GPU-GTX1070-x86_64-Debug-Vulkan.json b/infra/bots/recipes/test.expected/Test-Win10-MSVC-ZBOX-GPU-GTX1070-x86_64-Debug-Vulkan.json index 3a147913ee..ff8b9075fa 100644 --- a/infra/bots/recipes/test.expected/Test-Win10-MSVC-ZBOX-GPU-GTX1070-x86_64-Debug-Vulkan.json +++ b/infra/bots/recipes/test.expected/Test-Win10-MSVC-ZBOX-GPU-GTX1070-x86_64-Debug-Vulkan.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\dm" ], diff --git a/infra/bots/recipes/test.expected/Test-Win8-MSVC-Golo-CPU-AVX-x86-Debug.json b/infra/bots/recipes/test.expected/Test-Win8-MSVC-Golo-CPU-AVX-x86-Debug.json index 5d6c05d583..de3127c560 100644 --- a/infra/bots/recipes/test.expected/Test-Win8-MSVC-Golo-CPU-AVX-x86-Debug.json +++ b/infra/bots/recipes/test.expected/Test-Win8-MSVC-Golo-CPU-AVX-x86-Debug.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]\\resources\\fileutil.py", + "RECIPE_MODULE[skia::file]\\resources\\fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]\\dm" ], diff --git a/infra/bots/recipes/test.expected/Test-iOS-Clang-iPadMini4-GPU-GX6450-arm-Release.json b/infra/bots/recipes/test.expected/Test-iOS-Clang-iPadMini4-GPU-GX6450-arm-Release.json index 126ea2489a..62182a1f39 100644 --- a/infra/bots/recipes/test.expected/Test-iOS-Clang-iPadMini4-GPU-GX6450-arm-Release.json +++ b/infra/bots/recipes/test.expected/Test-iOS-Clang-iPadMini4-GPU-GX6450-arm-Release.json @@ -456,7 +456,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/failed_dm.json b/infra/bots/recipes/test.expected/failed_dm.json index 9d907de7cb..9e51944319 100644 --- a/infra/bots/recipes/test.expected/failed_dm.json +++ b/infra/bots/recipes/test.expected/failed_dm.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/failed_get_hashes.json b/infra/bots/recipes/test.expected/failed_get_hashes.json index 0a5aec9612..2610d7a8a2 100644 --- a/infra/bots/recipes/test.expected/failed_get_hashes.json +++ b/infra/bots/recipes/test.expected/failed_get_hashes.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/failed_pull.json b/infra/bots/recipes/test.expected/failed_pull.json index edf16fe6a9..3727d80d96 100644 --- a/infra/bots/recipes/test.expected/failed_pull.json +++ b/infra/bots/recipes/test.expected/failed_pull.json @@ -494,7 +494,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.expected/trybot.json b/infra/bots/recipes/test.expected/trybot.json index 4b2e35683c..a2654935b7 100644 --- a/infra/bots/recipes/test.expected/trybot.json +++ b/infra/bots/recipes/test.expected/trybot.json @@ -69,7 +69,7 @@ "cmd": [ "python", "-u", - "RECIPE_MODULE[build::file]/resources/fileutil.py", + "RECIPE_MODULE[skia::file]/resources/fileutil.py", "rmtree", "[CUSTOM_[SWARM_OUT_DIR]]/dm" ], diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py index c4eaff160a..ac53e89d95 100644 --- a/infra/bots/recipes/test.py +++ b/infra/bots/recipes/test.py @@ -7,9 +7,9 @@ DEPS = [ - 'build/file', 'core', 'env', + 'file', 'flavor', 'recipe_engine/context', 'recipe_engine/json', diff --git a/infra/bots/recipes/update_meta_config.py b/infra/bots/recipes/update_meta_config.py index 3c6086c2f5..a3ca4f6ddd 100644 --- a/infra/bots/recipes/update_meta_config.py +++ b/infra/bots/recipes/update_meta_config.py @@ -7,8 +7,8 @@ DEPS = [ - 'build/file', 'depot_tools/gclient', + 'file', 'recipe_engine/context', 'recipe_engine/path', 'recipe_engine/properties', diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py index 562d6e589d..d919f59077 100644 --- a/infra/bots/recipes/upload_dm_results.py +++ b/infra/bots/recipes/upload_dm_results.py @@ -10,7 +10,7 @@ import calendar DEPS = [ - 'build/file', + 'file', 'recipe_engine/json', 'recipe_engine/path', 'recipe_engine/properties', diff --git a/infra/bots/recipes/upload_nano_results.py b/infra/bots/recipes/upload_nano_results.py index 20a723b093..1bf4ef1d54 100644 --- a/infra/bots/recipes/upload_nano_results.py +++ b/infra/bots/recipes/upload_nano_results.py @@ -7,7 +7,7 @@ DEPS = [ - 'build/file', + 'file', 'recipe_engine/context', 'recipe_engine/path', 'recipe_engine/properties', diff --git a/infra/config/recipes.cfg b/infra/config/recipes.cfg index 78d19ade6b..d278be24e5 100644 --- a/infra/config/recipes.cfg +++ b/infra/config/recipes.cfg @@ -12,11 +12,6 @@ } }, "deps": { - "build": { - "branch": "master", - "revision": "bc9dcd6ed4e53b32dcb134a634fcba2b3556f60f", - "url": "https://chromium.googlesource.com/chromium/tools/build.git" - }, "depot_tools": { "branch": "master", "revision": "063814f82efdda25a0d1737395172faa88a5fd0f", |