update bazel's embedded jdk to jdk9

- Updates the embedded JDK to Azul Zulu 9.0.7
- All integration tests use Bazel with the embedded JDK

Also updated: http://storage.googleapis.com/bazel-mirror/openjdk/index.html

Closes #5312, #5314, #5315

PiperOrigin-RevId: 200055008

14 files changed, 79 insertions, 45 deletions

diff --git a/WORKSPACE b/WORKSPACE
index a0669072cb..e6136610d4 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -117,31 +117,31 @@ distdir_tar(
 # OpenJDK distributions used to create a version of Bazel bundled with the OpenJDK.
 http_file(
     name = "openjdk_linux",
-    sha256 = "85b81652b3fe8cfb0a2cfb835988672bde7844f613dae5b9487b5b44921a1afd",
+    sha256 = "45f2dfbee93b91b1468cf81d843fc6d9a47fef1f831c0b7ceff4f1eb6e6851c8",
     urls = [
-        "https://mirror.bazel.build/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-linux_x64.tar.gz",
-        "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-linux_x64.tar.gz",
-        "https://cdn.azul.com/zulu/bin/zulu8.28.0.1-jdk8.0.163-linux_x64.tar.gz",
+        "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-linux_x64.tar.gz",
+        "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-linux_x64.tar.gz",
+        "https://cdn.azul.com/zulu/bin/zulu9.0.7.1-jdk9.0.7-linux_x64.tar.gz",
     ],
 )
 
 http_file(
     name = "openjdk_macos",
-    sha256 = "292a44695e708a8822b3d0a462faf32887ddb80b56aa91e7598fd4b6f6341f69",
+    sha256 = "5a5b3225b86d3fdb51e9add5335f43cc19c6b2d9b8b5558e72b52d7b2ce9162e",
     urls = [
-        "https://mirror.bazel.build/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-macosx_x64.tar.gz",
-        "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-macosx_x64.tar.gz",
-        "https://cdn.azul.com/zulu/bin/zulu8.28.0.1-jdk8.0.163-macosx_x64.tar.gz",
+        "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-macosx_x64.tar.gz",
+        "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-macosx_x64.tar.gz",
+        "https://cdn.azul.com/zulu/bin/zulu9.0.7.1-jdk9.0.7-macosx_x64.tar.gz",
     ],
 )
 
 http_file(
     name = "openjdk_win",
-    sha256 = "0c5ea3634ae7d7851630cf61fdd6344cce110cf9246593345e49fc430bb39442",
+    sha256 = "75f76c53c6a1f12b1a571b86bd9708ab75adf582d689dddc94fdd77dcc0f3f5c",
     urls = [
-        "https://mirror.bazel.build/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-win_x64.zip",
-        "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-win_x64.zip",
-        "https://cdn.azul.com/zulu/bin/zulu8.28.0.1-jdk8.0.163-win_x64.zip",
+        "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-win_x64.zip",
+        "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-win_x64.zip",
+        "https://cdn.azul.com/zulu/bin/zulu9.0.7.1-jdk9.0.7-win_x64.zip",
     ],
 )
 
diff --git a/scripts/BUILD b/scripts/BUILD
index ee6ef86c37..3ff56c4200 100644
--- a/scripts/BUILD
+++ b/scripts/BUILD
@@ -10,19 +10,14 @@ genrule(
     outs = ["bazel-complete.bash"],
     cmd = " ".join([
         "$(location :generate_bash_completion.sh)",
-        "--bazel=$(location //src:bazel)",
-        "--javabase=$(JAVABASE)",
+        "--bazel=$(location //src:bazel_with_jdk)",
         "--output=$@",
         "--prepend=$(location bazel-complete-template.bash)",
     ]),
     output_to_bindir = 1,
-    # Bazel 0.8.0 doesn't have this target under @bazel_tools, so we have to
-    # use it from the main repository
-    toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"],
     tools = [
         ":generate_bash_completion.sh",
-        "//src:bazel",
-        "//tools/defaults:jdk",
+        "//src:bazel_with_jdk",
     ],
     visibility = ["//scripts/packages:__subpackages__"],
 )
diff --git a/src/BUILD b/src/BUILD
index d1ef9366fd..d2431844fb 100644
--- a/src/BUILD
+++ b/src/BUILD
@@ -173,7 +173,18 @@ py_binary(
         "//conditions:default": [
             "//src/tools/singlejar:singlejar",
         ],
-    }) + (select({
+    }) + ([":embedded_jdk"] if (suffix == "_with_jdk") else []),
+    visibility = [
+        "//src/test:__subpackages__",  # For integration tests
+    ],
+) for suffix in [
+    "",
+    "_with_jdk",
+]]
+
+filegroup(
+    name = "embedded_jdk",
+    srcs = select({
         "//src/conditions:darwin": [
             "@openjdk_macos//file",
         ],
@@ -186,14 +197,9 @@ py_binary(
         "//conditions:default": [
             "@openjdk_linux//file",
         ],
-    }) if (suffix == "_with_jdk") else []),
-    visibility = [
-        "//src/test:__subpackages__",  # For integration tests
-    ],
-) for suffix in [
-    "",
-    "_with_jdk",
-]]
+    }),
+    visibility = ["//src/test/shell/bazel:__pkg__"],
+)
 
 [srcsfile(
     name = "embedded_tools" + suffix + "_params",
diff --git a/src/main/cpp/blaze.cc b/src/main/cpp/blaze.cc
index 7487c76c90..d6ede0c7b0 100644
--- a/src/main/cpp/blaze.cc
+++ b/src/main/cpp/blaze.cc
@@ -417,6 +417,13 @@ static vector<string> GetArgumentArray(
   result.push_back("-XX:HeapDumpPath=" +
                    blaze_util::PathAsJvmFlag(heap_crash_path));
 
+  // TODO(b/109998449): only assume JDK >= 9 for embedded JDKs
+  if (!globals->options->GetEmbeddedJavabase().empty()) {
+    // quiet warnings from com.google.protobuf.UnsafeUtil,
+    // see: https://github.com/google/protobuf/issues/3781
+    result.push_back("--add-opens=java.base/java.nio=ALL-UNNAMED");
+  }
+
   result.push_back("-Xverify:none");
 
   vector<string> user_options;
diff --git a/src/main/cpp/startup_options.cc b/src/main/cpp/startup_options.cc
index 8f028f55c6..7b5625c2ef 100644
--- a/src/main/cpp/startup_options.cc
+++ b/src/main/cpp/startup_options.cc
@@ -414,16 +414,24 @@ string StartupOptions::GetSystemJavabase() const {
   return blaze::GetSystemJavabase();
 }
 
+string StartupOptions::GetEmbeddedJavabase() {
+  string bundled_jre_path = blaze_util::JoinPath(
+      install_base, "_embedded_binaries/embedded_tools/jdk");
+  if (blaze_util::CanExecuteFile(blaze_util::JoinPath(
+          bundled_jre_path, GetJavaBinaryUnderJavabase()))) {
+    return bundled_jre_path;
+  }
+  return "";
+}
+
 string StartupOptions::GetHostJavabase() {
   // 1) Allow overriding the host_javabase via --host_javabase.
   if (!host_javabase.empty()) {
     return host_javabase;
   }
   if (default_host_javabase.empty()) {
-    string bundled_jre_path = blaze_util::JoinPath(
-        install_base, "_embedded_binaries/embedded_tools/jdk");
-    if (blaze_util::CanExecuteFile(blaze_util::JoinPath(
-            bundled_jre_path, GetJavaBinaryUnderJavabase()))) {
+    string bundled_jre_path = GetEmbeddedJavabase();
+    if (!bundled_jre_path.empty()) {
       // 2) Use a bundled JVM if we have one.
       default_host_javabase = bundled_jre_path;
     } else {
diff --git a/src/main/cpp/startup_options.h b/src/main/cpp/startup_options.h
index 84079f1b3f..3998195545 100644
--- a/src/main/cpp/startup_options.h
+++ b/src/main/cpp/startup_options.h
@@ -266,6 +266,9 @@ class StartupOptions {
   // from a blazerc file, if a key is not present, it is the default.
   std::map<std::string, std::string> option_sources;
 
+  // Returns the embedded JDK, or an empty string.
+  std::string GetEmbeddedJavabase();
+
   // Returns the GetHostJavabase. This should be called after parsing
   // the --host_javabase option.
   std::string GetHostJavabase();
diff --git a/src/test/py/bazel/test_base.py b/src/test/py/bazel/test_base.py
index 4947341aec..7f0f705bce 100644
--- a/src/test/py/bazel/test_base.py
+++ b/src/test/py/bazel/test_base.py
@@ -357,6 +357,8 @@ class TestBase(unittest.TestCase):
           'BAZEL_SH':
               TestBase.GetEnv('BAZEL_SH',
                               'c:\\tools\\msys64\\usr\\bin\\bash.exe'),
+          'JAVA_HOME':
+              TestBase.GetEnv('JAVA_HOME'),
       }
     else:
       env = {'HOME': os.path.join(self._temp, 'home')}
diff --git a/src/test/py/bazel/windows_remote_test.py b/src/test/py/bazel/windows_remote_test.py
index fc9a03a139..e962ce590d 100644
--- a/src/test/py/bazel/windows_remote_test.py
+++ b/src/test/py/bazel/windows_remote_test.py
@@ -203,8 +203,7 @@ class WindowsRemoteTest(test_base.TestBase):
 
     # Test.
     exit_code, stdout, stderr = self._RunRemoteBazel([
-        'test', '--test_output=all', '--host_javabase=//foo:jdk8',
-        '--javabase=//foo:jdk8', '//foo:foo_test'
+        'test', '--test_output=all', '--javabase=//foo:jdk8', '//foo:foo_test'
     ])
     self.AssertExitCode(exit_code, 0, stderr, stdout)
 
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index 3e3eea92ef..b3039d424c 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -27,7 +27,7 @@ filegroup(
         "testing_server.py",
         ":langtools-copy",
         "//examples:srcs",
-        "//src:bazel",
+        "//src:bazel_with_jdk",
         "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar",
         "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar",
         "//src/main/tools:linux-sandbox",
@@ -497,10 +497,14 @@ sh_test(
     name = "bazel_bootstrap_distfile_test",
     timeout = "eternal",
     srcs = ["bazel_bootstrap_distfile_test.sh"],
-    args = ["$(location //:bazel-distfile)"],
+    args = [
+        "$(location //:bazel-distfile)",
+        "$(location //src:embedded_jdk)",
+    ],
     data = [
         ":test-deps",
         "//:bazel-distfile",
+        "//src:embedded_jdk",
     ],
     tags = ["jdk8"],
 )
diff --git a/src/test/shell/bazel/bazel_bootstrap_distfile_test.sh b/src/test/shell/bazel/bazel_bootstrap_distfile_test.sh
index 3f4dd06efa..273fe8d5cd 100755
--- a/src/test/shell/bazel/bazel_bootstrap_distfile_test.sh
+++ b/src/test/shell/bazel/bazel_bootstrap_distfile_test.sh
@@ -19,6 +19,7 @@
 
 set -u
 DISTFILE=$(rlocation io_bazel/${1#./})
+EMBEDDED_JDK=$(rlocation io_bazel/${2#./})
 shift 1
 
 # Load the test setup defined in the parent directory
@@ -41,12 +42,23 @@ function test_bootstrap()  {
     export SOURCE_DATE_EPOCH=1501234567
     _log_progress "unzip"
     unzip -q "${DISTFILE}"
+    if [[ $EMBEDDED_JDK == *.tar.gz ]]; then
+      tar xf $EMBEDDED_JDK
+    elif [[ $EMBEDDED_JDK == *.zip ]]; then
+      unzip -q $EMBEDDED_JDK
+    fi
+    JAVABASE=$(echo zulu*)
+
     _log_progress "bootstrap"
     env EXTRA_BAZEL_ARGS="--curses=no --strategy=Javac=standalone" ./compile.sh \
         || fail "Expected to be able to bootstrap bazel"
     _log_progress "run"
-    ./output/bazel version > "${TEST_log}" || fail "Generated bazel not working"
-    ./output/bazel shutdown
+    ./output/bazel \
+      --host_javabase=$JAVABASE --host_jvm_args=--add-opens=java.base/java.nio=ALL-UNNAMED \
+      version > "${TEST_log}" || fail "Generated bazel not working"
+    ./output/bazel \
+      --host_javabase=$JAVABASE --host_jvm_args=--add-opens=java.base/java.nio=ALL-UNNAMED \
+      shutdown
     _log_progress "assert"
     expect_log "${SOURCE_DATE_EPOCH}"
     cd "${olddir}"
diff --git a/src/test/shell/bazel/bazel_determinism_test.sh b/src/test/shell/bazel/bazel_determinism_test.sh
index 8abc11f4f5..a3f64ff7c5 100755
--- a/src/test/shell/bazel/bazel_determinism_test.sh
+++ b/src/test/shell/bazel/bazel_determinism_test.sh
@@ -56,11 +56,11 @@ function test_determinism()  {
     unzip -q "${DISTFILE}"
 
     # Build Bazel once.
-    bazel --output_base="${TEST_TMPDIR}/out1" build --nostamp //src:bazel
+    bazel --output_base="${TEST_TMPDIR}/out1" build --nostamp //src:bazel_with_jdk
     hash_outputs >"${TEST_TMPDIR}/sum1"
 
     # Build Bazel twice.
-    bazel-bin/src/bazel --output_base="${TEST_TMPDIR}/out2" build --nostamp //src:bazel
+    bazel-bin/src/bazel_with_jdk --output_base="${TEST_TMPDIR}/out2" build --nostamp //src:bazel_with_jdk
     hash_outputs >"${TEST_TMPDIR}/sum2"
 
     if ! diff -U0 "${TEST_TMPDIR}/sum1" "${TEST_TMPDIR}/sum2" >$TEST_log; then
diff --git a/src/test/shell/bin/bazel b/src/test/shell/bin/bazel
index 27975618ee..975d4df609 100755
--- a/src/test/shell/bin/bazel
+++ b/src/test/shell/bin/bazel
@@ -17,4 +17,4 @@
 # Wrapper script to run bazel in the tests. Any change to this file will
 # affect all our integration tests.
 #
-exec $(rlocation io_bazel/src/bazel) --bazelrc=$TEST_TMPDIR/bazelrc "$@"
\ No newline at end of file
+exec $(rlocation io_bazel/src/bazel_with_jdk) --bazelrc=$TEST_TMPDIR/bazelrc "$@"
diff --git a/src/test/shell/integration/test_test.sh b/src/test/shell/integration/test_test.sh
index c2e599749b..a6d70dde04 100755
--- a/src/test/shell/integration/test_test.sh
+++ b/src/test/shell/integration/test_test.sh
@@ -64,7 +64,7 @@ EOF
 
   bazel test --nocache_test_results //tests:fail &>$TEST_log \
       && fail "expected failure" || true
-  expect_log "^//tests:fail[[:space:]]\+FAILED in [[:digit:]]\+\.[[:digit:]]\+s"
+  expect_log "^//tests:fail[[:space:]]\+FAILED in [[:digit:]]\+[\.,][[:digit:]]\+s"
   expect_log "^Executed 1 out of 1 test: 1 fails"
 }
 
diff --git a/src/test/shell/testenv.sh b/src/test/shell/testenv.sh
index 9de4521fea..805304a333 100755
--- a/src/test/shell/testenv.sh
+++ b/src/test/shell/testenv.sh
@@ -60,7 +60,7 @@ if is_windows; then
 fi
 
 # Make the command "bazel" available for tests.
-PATH_TO_BAZEL_BIN=$(rlocation io_bazel/src/bazel)
+PATH_TO_BAZEL_BIN=$(rlocation io_bazel/src/bazel_with_jdk)
 PATH_TO_BAZEL_WRAPPER="$(dirname $(rlocation io_bazel/src/test/shell/bin/bazel))"
 # Convert PATH_TO_BAZEL_WRAPPER to Unix path style on Windows, because it will be
 # added into PATH. There's problem if PATH=C:/msys64/usr/bin:/usr/local,
@@ -266,8 +266,6 @@ function setup_bazelrc() {
   cat >$TEST_TMPDIR/bazelrc <<EOF
 # Set the user root properly for this test invocation.
 startup --output_user_root=${bazel_root}
-# Set the correct javabase from the outer bazel invocation.
-startup --host_javabase=${bazel_javabase}
 
 # Print all progress messages because we regularly grep the output in tests.
 common --show_progress_rate_limit=-1