diff options
Diffstat (limited to 'infra/bots/recipe_modules/gsutil')
6 files changed, 229 insertions, 0 deletions
diff --git a/infra/bots/recipe_modules/gsutil/__init__.py b/infra/bots/recipe_modules/gsutil/__init__.py new file mode 100644 index 0000000000..94595a0214 --- /dev/null +++ b/infra/bots/recipe_modules/gsutil/__init__.py @@ -0,0 +1,10 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'recipe_engine/context', + 'recipe_engine/step', + 'run', + 'vars', +] diff --git a/infra/bots/recipe_modules/gsutil/api.py b/infra/bots/recipe_modules/gsutil/api.py new file mode 100644 index 0000000000..babf85270b --- /dev/null +++ b/infra/bots/recipe_modules/gsutil/api.py @@ -0,0 +1,38 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +from recipe_engine import recipe_api + +UPLOAD_ATTEMPTS = 5 + +class GSUtilApi(recipe_api.RecipeApi): + def cp(self, name, src, dst, extra_args=None): + """Attempt to upload or download files to/from Google Cloud Storage (GCS). + + Args: + name: string. Will be used to fill out the step name. + src: string. Absolute path for a local file or gcs file (e.g. gs://...) + dst: string. Same as src. + extra_args: optional list of args to be passed to gsutil. e.g. [-Z] asks + all files be compressed with gzip after upload and before download. + + If the operation fails, it will be retried multiple times. + """ + cmd = ['gsutil', 'cp'] + if extra_args: + cmd.extend(extra_args) + cmd.extend([src, dst]) + + name = 'upload %s' % name + for i in xrange(UPLOAD_ATTEMPTS): + step_name = name + if i > 0: + step_name += ' (attempt %d)' % (i+1) + try: + self.m.step(step_name, cmd=cmd) + break + except self.m.step.StepFailure: + if i == UPLOAD_ATTEMPTS - 1: + raise diff --git a/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json new file mode 100644 index 0000000000..2307ef7d90 --- /dev/null +++ b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json @@ -0,0 +1,78 @@ +[ + { + "cmd": [ + "gsutil", + "cp", + "-Z", + "/foo/file", + "gs://bar-bucket/file" + ], + "name": "upload test file", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "-Z", + "/foo/file", + "gs://bar-bucket/file" + ], + "name": "upload test file (attempt 2)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "-Z", + "/foo/file", + "gs://bar-bucket/file" + ], + "name": "upload test file (attempt 3)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "-Z", + "/foo/file", + "gs://bar-bucket/file" + ], + "name": "upload test file (attempt 4)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "-Z", + "/foo/file", + "gs://bar-bucket/file" + ], + "name": "upload test file (attempt 5)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "name": "$result", + "reason": "Step('upload test file (attempt 5)') failed with return_code 1", + "recipe_result": null, + "status_code": 1 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json new file mode 100644 index 0000000000..ab20ee6052 --- /dev/null +++ b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json @@ -0,0 +1,31 @@ +[ + { + "cmd": [ + "gsutil", + "cp", + "-Z", + "/foo/file", + "gs://bar-bucket/file" + ], + "name": "upload test file", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "-Z", + "/foo/file", + "gs://bar-bucket/file" + ], + "name": "upload test file (attempt 2)" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json b/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json new file mode 100644 index 0000000000..c29e1c4612 --- /dev/null +++ b/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json @@ -0,0 +1,17 @@ +[ + { + "cmd": [ + "gsutil", + "cp", + "-Z", + "/foo/file", + "gs://bar-bucket/file" + ], + "name": "upload test file" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/gsutil/examples/full.py b/infra/bots/recipe_modules/gsutil/examples/full.py new file mode 100644 index 0000000000..8ed9745dcc --- /dev/null +++ b/infra/bots/recipe_modules/gsutil/examples/full.py @@ -0,0 +1,55 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# Recipe which runs the Skia gsutils tests. + + +DEPS = [ + 'gsutil', + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/python', + 'recipe_engine/step', + 'run', + 'vars', +] + + +def RunSteps(api): + api.gsutil.cp('test file', '/foo/file', 'gs://bar-bucket/file', ['-Z']) + +def GenTests(api): + yield ( + api.test('gsutil_tests') + + api.properties(buildername='Housekeeper-PerCommit-InfraTests', + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + ) + + yield ( + api.test('failed_one_upload') + + api.properties(buildername='Housekeeper-PerCommit-InfraTests', + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.step_data('upload test file', retcode=1) + ) + + yield ( + api.test('failed_all_uploads') + + api.properties(buildername='Housekeeper-PerCommit-InfraTests', + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.step_data('upload test file', retcode=1) + + api.step_data('upload test file (attempt 2)', retcode=1) + + api.step_data('upload test file (attempt 3)', retcode=1) + + api.step_data('upload test file (attempt 4)', retcode=1) + + api.step_data('upload test file (attempt 5)', retcode=1) + ) |