Couple small changes to speed up Goma compiles

 - Use more threads
 - Avoid failures due to Goma compiler_proxy running after Swarming
   task terminates, e.g.
   https://chromium-swarm.appspot.com/task?id=3b309dfecdaa9b10

Change-Id: I8521ad0dbaead6f22a2d3a3401dfe0b2695742b9
Reviewed-on: https://skia-review.googlesource.com/99020
Commit-Queue: Ben Wagner <benjaminwagner@google.com>
Reviewed-by: Eric Boren <borenet@google.com>

4 files changed, 78 insertions, 4 deletions

diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Debug-GomaNoFallback.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Debug-GomaNoFallback.json
index 3cdb5c2565..2611037a3d 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Debug-GomaNoFallback.json
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Debug-GomaNoFallback.json
@@ -131,7 +131,7 @@
       "-C",
       "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Debug-GomaNoFallback/Debug_x64",
       "-j",
-      "100"
+      "2000"
     ],
     "cwd": "[CUSTOM_C:\\_B_WORK]/skia",
     "env": {
@@ -189,6 +189,30 @@
     "name": "stop goma"
   },
   {
+    "cmd": [
+      "python",
+      "-u",
+      "import time; time.sleep(2)"
+    ],
+    "cwd": "[CACHE]/goma",
+    "env": {
+      "BUILDTYPE": "Debug_x64",
+      "CHROME_HEADLESS": "1",
+      "GOMA_FALLBACK": "0",
+      "GOMA_HERMETIC": "error",
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
+      "GOMA_USE_LOCAL": "0",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Debug-GomaNoFallback"
+    },
+    "infra_step": true,
+    "name": "wait for goma_ctl stop",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import time; time.sleep(2)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-ANGLE_Goma.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-ANGLE_Goma.json
index c5e57e03f8..51e8c81440 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-ANGLE_Goma.json
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-ANGLE_Goma.json
@@ -125,7 +125,7 @@
       "-C",
       "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-ANGLE_Goma/Release_x64",
       "-j",
-      "100"
+      "2000"
     ],
     "cwd": "[CUSTOM_C:\\_B_WORK]/skia",
     "env": {
@@ -174,6 +174,27 @@
     "name": "stop goma"
   },
   {
+    "cmd": [
+      "python",
+      "-u",
+      "import time; time.sleep(2)"
+    ],
+    "cwd": "[CACHE]/goma",
+    "env": {
+      "BUILDTYPE": "Release_x64",
+      "CHROME_HEADLESS": "1",
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-ANGLE_Goma"
+    },
+    "infra_step": true,
+    "name": "wait for goma_ctl stop",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import time; time.sleep(2)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json
index 33bbb1c21e..0999cc537f 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json
@@ -125,7 +125,7 @@
       "-C",
       "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma/Release_x64",
       "-j",
-      "100"
+      "2000"
     ],
     "cwd": "[CUSTOM_C:\\_B_WORK]/skia",
     "env": {
@@ -174,6 +174,27 @@
     "name": "stop goma"
   },
   {
+    "cmd": [
+      "python",
+      "-u",
+      "import time; time.sleep(2)"
+    ],
+    "cwd": "[CACHE]/goma",
+    "env": {
+      "BUILDTYPE": "Release_x64",
+      "CHROME_HEADLESS": "1",
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
+      "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
+    },
+    "infra_step": true,
+    "name": "wait for goma_ctl stop",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import time; time.sleep(2)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py
index 63465517c9..22733cbe59 100644
--- a/infra/bots/recipe_modules/flavor/gn_flavor.py
+++ b/infra/bots/recipe_modules/flavor/gn_flavor.py
@@ -192,7 +192,7 @@ with open(sys.argv[1], 'w') as f:
         # ANGLE uses case-insensitive include paths in D3D code. Not sure why
         # only Goma warns about this.
         extra_cflags.append('-Wno-nonportable-include-path')
-      ninja_args.extend(['-j', '100'])
+      ninja_args.extend(['-j', '2000'])
 
     sanitize = ''
     for t in extra_tokens:
@@ -248,6 +248,14 @@ with open(sys.argv[1], 'w') as f:
           self.m.run(self.m.python, 'stop goma',
                      script='goma_ctl.py', args=['stop'], infra_step=True,
                      abort_on_failure=False, fail_build_on_failure=False)
+          # Hack: goma_ctl stop is asynchronous, so the process often does not
+          # stop before the recipe exits, which causes Swarming to freak out.
+          # Wait a couple seconds for it to exit normally.
+          # TODO(dogben): Remove after internal b/72128121 is resolved.
+          self.m.run(self.m.python.inline, 'wait for goma_ctl stop',
+                     program="""import time; time.sleep(2)""",
+                     infra_step=True,
+                     abort_on_failure=False, fail_build_on_failure=False)
 
   def copy_extra_build_products(self, swarming_out_dir):
     configuration = self.m.vars.builder_cfg.get('configuration', '')