diff options
12 files changed, 31 insertions, 8 deletions
diff --git a/infra/bots/recipe_modules/gsutil/api.py b/infra/bots/recipe_modules/gsutil/api.py index 8467354a75..2ddcb265b0 100644 --- a/infra/bots/recipe_modules/gsutil/api.py +++ b/infra/bots/recipe_modules/gsutil/api.py @@ -12,7 +12,7 @@ class GSUtilApi(recipe_api.RecipeApi): """Run gsutil with the given args.""" return self.m.step(step_name, cmd=['gsutil'] + list(args)) - def cp(self, name, src, dst, extra_args=None): + def cp(self, name, src, dst, extra_args=None, multithread=False): """Attempt to upload or download files to/from Google Cloud Storage (GCS). Args: @@ -21,10 +21,14 @@ class GSUtilApi(recipe_api.RecipeApi): dst: string. Same as src. extra_args: optional list of args to be passed to gsutil. e.g. [-Z] asks all files be compressed with gzip after upload and before download. + multi_thread: if the -m argument should be used to copy multiple items + at once (e.g. gsutil -m cp foo* gs://bar/dir) If the operation fails, it will be retried multiple times. """ cmd = ['cp'] + if multithread: + cmd = ['-m'] + cmd if extra_args: cmd.extend(extra_args) cmd.extend([src, dst]) diff --git a/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json index 2307ef7d90..ac1e7e4de7 100644 --- a/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json +++ b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json @@ -2,6 +2,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "-Z", "/foo/file", @@ -16,6 +17,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "-Z", "/foo/file", @@ -30,6 +32,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "-Z", "/foo/file", @@ -44,6 +47,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "-Z", "/foo/file", @@ -58,6 +62,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "-Z", "/foo/file", diff --git a/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json index ab20ee6052..cdc36662b4 100644 --- a/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json +++ b/infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json @@ -2,6 +2,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "-Z", "/foo/file", @@ -16,6 +17,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "-Z", "/foo/file", diff --git a/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json b/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json index c29e1c4612..2f33635946 100644 --- a/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json +++ b/infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json @@ -2,6 +2,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "-Z", "/foo/file", diff --git a/infra/bots/recipe_modules/gsutil/examples/full.py b/infra/bots/recipe_modules/gsutil/examples/full.py index 8ed9745dcc..1ed09066dd 100644 --- a/infra/bots/recipe_modules/gsutil/examples/full.py +++ b/infra/bots/recipe_modules/gsutil/examples/full.py @@ -18,7 +18,8 @@ DEPS = [ def RunSteps(api): - api.gsutil.cp('test file', '/foo/file', 'gs://bar-bucket/file', ['-Z']) + api.gsutil.cp('test file', '/foo/file', 'gs://bar-bucket/file', + extra_args=['-Z'], multithread=True) def GenTests(api): yield ( diff --git a/infra/bots/recipes/upload_coverage_results.py b/infra/bots/recipes/upload_coverage_results.py index fd9a24ec8e..91fa79c2dd 100644 --- a/infra/bots/recipes/upload_coverage_results.py +++ b/infra/bots/recipes/upload_coverage_results.py @@ -91,7 +91,7 @@ def RunSteps(api): gcs_file = PARSED_FILE % builder_name api.gsutil.cp('parsed data', indexed_data, - 'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z']) + 'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z']) # Create text coverage output output_data = api.path['start_dir'].join('coverage_text') @@ -108,7 +108,7 @@ def RunSteps(api): # downloading and untarring all the coverage data. gcs_file = SUMMARY_FILE % builder_name api.gsutil.cp('coverage summary', output_data.join('index.txt'), - 'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z']) + 'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z']) tar_file = api.path['start_dir'].join('coverage.text.tar') @@ -119,7 +119,7 @@ def RunSteps(api): gcs_file = COVERAGE_TEXT_FILE % builder_name api.gsutil.cp('text coverage data', tar_file, - 'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z']) + 'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z']) # Create html coverage output output_data = api.path['start_dir'].join('coverage_html') @@ -141,7 +141,7 @@ def RunSteps(api): gcs_file = COVERAGE_HTML_FILE % builder_name api.gsutil.cp('html coverage data', tar_file, - 'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z']) + 'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z']) def GenTests(api): diff --git a/infra/bots/recipes/upload_dm_results.expected/alternate_bucket.json b/infra/bots/recipes/upload_dm_results.expected/alternate_bucket.json index 27319c8dbc..fca5954dd0 100644 --- a/infra/bots/recipes/upload_dm_results.expected/alternate_bucket.json +++ b/infra/bots/recipes/upload_dm_results.expected/alternate_bucket.json @@ -90,6 +90,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" diff --git a/infra/bots/recipes/upload_dm_results.expected/failed_all.json b/infra/bots/recipes/upload_dm_results.expected/failed_all.json index 563b37d1ae..3d643abb6f 100644 --- a/infra/bots/recipes/upload_dm_results.expected/failed_all.json +++ b/infra/bots/recipes/upload_dm_results.expected/failed_all.json @@ -90,6 +90,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" @@ -103,6 +104,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" @@ -116,6 +118,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" @@ -129,6 +132,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" @@ -142,6 +146,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" diff --git a/infra/bots/recipes/upload_dm_results.expected/failed_once.json b/infra/bots/recipes/upload_dm_results.expected/failed_once.json index 1e6afe841c..c2531d9e24 100644 --- a/infra/bots/recipes/upload_dm_results.expected/failed_once.json +++ b/infra/bots/recipes/upload_dm_results.expected/failed_once.json @@ -90,6 +90,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" @@ -103,6 +104,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" diff --git a/infra/bots/recipes/upload_dm_results.expected/normal_bot.json b/infra/bots/recipes/upload_dm_results.expected/normal_bot.json index 247a612e02..4a8fd11bf4 100644 --- a/infra/bots/recipes/upload_dm_results.expected/normal_bot.json +++ b/infra/bots/recipes/upload_dm_results.expected/normal_bot.json @@ -90,6 +90,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" diff --git a/infra/bots/recipes/upload_dm_results.expected/trybot.json b/infra/bots/recipes/upload_dm_results.expected/trybot.json index 3ea86e0033..5a10528d1d 100644 --- a/infra/bots/recipes/upload_dm_results.expected/trybot.json +++ b/infra/bots/recipes/upload_dm_results.expected/trybot.json @@ -90,6 +90,7 @@ { "cmd": [ "gsutil", + "-m", "cp", "[START_DIR]/dm/*.png", "gs://skia-infra-gm/dm-images-v1" diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py index a46c787d26..7d08089a1d 100644 --- a/infra/bots/recipes/upload_dm_results.py +++ b/infra/bots/recipes/upload_dm_results.py @@ -53,7 +53,7 @@ def RunSteps(api): files_to_upload = [f for f in files_to_upload if str(f).endswith(ext)] if len(files_to_upload) > 0: api.gsutil.cp('images', results_dir.join('*%s' % ext), - image_dest_path) + image_dest_path, multithread=True) # Upload the JSON summary and verbose.log. now = api.time.utcnow() @@ -78,7 +78,7 @@ def RunSteps(api): summary_dest_path) api.gsutil.cp('JSON and logs', tmp_dir.join('*'), summary_dest_path, - ['-z', 'json,log']) + extra_args=['-z', 'json,log']) def GenTests(api): |