From 72141a1330aaf88d4a44e5fda8541c2cdf915c77 Mon Sep 17 00:00:00 2001 From: cushon Date: Mon, 11 Jun 2018 08:30:15 -0700 Subject: update bazel's embedded jdk to jdk9 - Updates the embedded JDK to Azul Zulu 9.0.7 - All integration tests use Bazel with the embedded JDK Also updated: http://storage.googleapis.com/bazel-mirror/openjdk/index.html Closes #5312, #5314, #5315 PiperOrigin-RevId: 200055008 --- WORKSPACE | 24 +++++++++++----------- scripts/BUILD | 9 ++------ src/BUILD | 24 ++++++++++++++-------- src/main/cpp/blaze.cc | 7 +++++++ src/main/cpp/startup_options.cc | 16 +++++++++++---- src/main/cpp/startup_options.h | 3 +++ src/test/py/bazel/test_base.py | 2 ++ src/test/py/bazel/windows_remote_test.py | 3 +-- src/test/shell/bazel/BUILD | 8 ++++++-- .../shell/bazel/bazel_bootstrap_distfile_test.sh | 16 +++++++++++++-- src/test/shell/bazel/bazel_determinism_test.sh | 4 ++-- src/test/shell/bin/bazel | 2 +- src/test/shell/integration/test_test.sh | 2 +- src/test/shell/testenv.sh | 4 +--- 14 files changed, 79 insertions(+), 45 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index a0669072cb..e6136610d4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -117,31 +117,31 @@ distdir_tar( # OpenJDK distributions used to create a version of Bazel bundled with the OpenJDK. http_file( name = "openjdk_linux", - sha256 = "85b81652b3fe8cfb0a2cfb835988672bde7844f613dae5b9487b5b44921a1afd", + sha256 = "45f2dfbee93b91b1468cf81d843fc6d9a47fef1f831c0b7ceff4f1eb6e6851c8", urls = [ - "https://mirror.bazel.build/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-linux_x64.tar.gz", - "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-linux_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu8.28.0.1-jdk8.0.163-linux_x64.tar.gz", + "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-linux_x64.tar.gz", + "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-linux_x64.tar.gz", + "https://cdn.azul.com/zulu/bin/zulu9.0.7.1-jdk9.0.7-linux_x64.tar.gz", ], ) http_file( name = "openjdk_macos", - sha256 = "292a44695e708a8822b3d0a462faf32887ddb80b56aa91e7598fd4b6f6341f69", + sha256 = "5a5b3225b86d3fdb51e9add5335f43cc19c6b2d9b8b5558e72b52d7b2ce9162e", urls = [ - "https://mirror.bazel.build/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-macosx_x64.tar.gz", - "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-macosx_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu8.28.0.1-jdk8.0.163-macosx_x64.tar.gz", + "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-macosx_x64.tar.gz", + "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-macosx_x64.tar.gz", + "https://cdn.azul.com/zulu/bin/zulu9.0.7.1-jdk9.0.7-macosx_x64.tar.gz", ], ) http_file( name = "openjdk_win", - sha256 = "0c5ea3634ae7d7851630cf61fdd6344cce110cf9246593345e49fc430bb39442", + sha256 = "75f76c53c6a1f12b1a571b86bd9708ab75adf582d689dddc94fdd77dcc0f3f5c", urls = [ - "https://mirror.bazel.build/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-win_x64.zip", - "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-8.28.0.1-jdk8.0.163/zulu8.28.0.1-jdk8.0.163-win_x64.zip", - "https://cdn.azul.com/zulu/bin/zulu8.28.0.1-jdk8.0.163-win_x64.zip", + "https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-win_x64.zip", + "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-win_x64.zip", + "https://cdn.azul.com/zulu/bin/zulu9.0.7.1-jdk9.0.7-win_x64.zip", ], ) diff --git a/scripts/BUILD b/scripts/BUILD index ee6ef86c37..3ff56c4200 100644 --- a/scripts/BUILD +++ b/scripts/BUILD @@ -10,19 +10,14 @@ genrule( outs = ["bazel-complete.bash"], cmd = " ".join([ "$(location :generate_bash_completion.sh)", - "--bazel=$(location //src:bazel)", - "--javabase=$(JAVABASE)", + "--bazel=$(location //src:bazel_with_jdk)", "--output=$@", "--prepend=$(location bazel-complete-template.bash)", ]), output_to_bindir = 1, - # Bazel 0.8.0 doesn't have this target under @bazel_tools, so we have to - # use it from the main repository - toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"], tools = [ ":generate_bash_completion.sh", - "//src:bazel", - "//tools/defaults:jdk", + "//src:bazel_with_jdk", ], visibility = ["//scripts/packages:__subpackages__"], ) diff --git a/src/BUILD b/src/BUILD index d1ef9366fd..d2431844fb 100644 --- a/src/BUILD +++ b/src/BUILD @@ -173,7 +173,18 @@ py_binary( "//conditions:default": [ "//src/tools/singlejar:singlejar", ], - }) + (select({ + }) + ([":embedded_jdk"] if (suffix == "_with_jdk") else []), + visibility = [ + "//src/test:__subpackages__", # For integration tests + ], +) for suffix in [ + "", + "_with_jdk", +]] + +filegroup( + name = "embedded_jdk", + srcs = select({ "//src/conditions:darwin": [ "@openjdk_macos//file", ], @@ -186,14 +197,9 @@ py_binary( "//conditions:default": [ "@openjdk_linux//file", ], - }) if (suffix == "_with_jdk") else []), - visibility = [ - "//src/test:__subpackages__", # For integration tests - ], -) for suffix in [ - "", - "_with_jdk", -]] + }), + visibility = ["//src/test/shell/bazel:__pkg__"], +) [srcsfile( name = "embedded_tools" + suffix + "_params", diff --git a/src/main/cpp/blaze.cc b/src/main/cpp/blaze.cc index 7487c76c90..d6ede0c7b0 100644 --- a/src/main/cpp/blaze.cc +++ b/src/main/cpp/blaze.cc @@ -417,6 +417,13 @@ static vector GetArgumentArray( result.push_back("-XX:HeapDumpPath=" + blaze_util::PathAsJvmFlag(heap_crash_path)); + // TODO(b/109998449): only assume JDK >= 9 for embedded JDKs + if (!globals->options->GetEmbeddedJavabase().empty()) { + // quiet warnings from com.google.protobuf.UnsafeUtil, + // see: https://github.com/google/protobuf/issues/3781 + result.push_back("--add-opens=java.base/java.nio=ALL-UNNAMED"); + } + result.push_back("-Xverify:none"); vector user_options; diff --git a/src/main/cpp/startup_options.cc b/src/main/cpp/startup_options.cc index 8f028f55c6..7b5625c2ef 100644 --- a/src/main/cpp/startup_options.cc +++ b/src/main/cpp/startup_options.cc @@ -414,16 +414,24 @@ string StartupOptions::GetSystemJavabase() const { return blaze::GetSystemJavabase(); } +string StartupOptions::GetEmbeddedJavabase() { + string bundled_jre_path = blaze_util::JoinPath( + install_base, "_embedded_binaries/embedded_tools/jdk"); + if (blaze_util::CanExecuteFile(blaze_util::JoinPath( + bundled_jre_path, GetJavaBinaryUnderJavabase()))) { + return bundled_jre_path; + } + return ""; +} + string StartupOptions::GetHostJavabase() { // 1) Allow overriding the host_javabase via --host_javabase. if (!host_javabase.empty()) { return host_javabase; } if (default_host_javabase.empty()) { - string bundled_jre_path = blaze_util::JoinPath( - install_base, "_embedded_binaries/embedded_tools/jdk"); - if (blaze_util::CanExecuteFile(blaze_util::JoinPath( - bundled_jre_path, GetJavaBinaryUnderJavabase()))) { + string bundled_jre_path = GetEmbeddedJavabase(); + if (!bundled_jre_path.empty()) { // 2) Use a bundled JVM if we have one. default_host_javabase = bundled_jre_path; } else { diff --git a/src/main/cpp/startup_options.h b/src/main/cpp/startup_options.h index 84079f1b3f..3998195545 100644 --- a/src/main/cpp/startup_options.h +++ b/src/main/cpp/startup_options.h @@ -266,6 +266,9 @@ class StartupOptions { // from a blazerc file, if a key is not present, it is the default. std::map option_sources; + // Returns the embedded JDK, or an empty string. + std::string GetEmbeddedJavabase(); + // Returns the GetHostJavabase. This should be called after parsing // the --host_javabase option. std::string GetHostJavabase(); diff --git a/src/test/py/bazel/test_base.py b/src/test/py/bazel/test_base.py index 4947341aec..7f0f705bce 100644 --- a/src/test/py/bazel/test_base.py +++ b/src/test/py/bazel/test_base.py @@ -357,6 +357,8 @@ class TestBase(unittest.TestCase): 'BAZEL_SH': TestBase.GetEnv('BAZEL_SH', 'c:\\tools\\msys64\\usr\\bin\\bash.exe'), + 'JAVA_HOME': + TestBase.GetEnv('JAVA_HOME'), } else: env = {'HOME': os.path.join(self._temp, 'home')} diff --git a/src/test/py/bazel/windows_remote_test.py b/src/test/py/bazel/windows_remote_test.py index fc9a03a139..e962ce590d 100644 --- a/src/test/py/bazel/windows_remote_test.py +++ b/src/test/py/bazel/windows_remote_test.py @@ -203,8 +203,7 @@ class WindowsRemoteTest(test_base.TestBase): # Test. exit_code, stdout, stderr = self._RunRemoteBazel([ - 'test', '--test_output=all', '--host_javabase=//foo:jdk8', - '--javabase=//foo:jdk8', '//foo:foo_test' + 'test', '--test_output=all', '--javabase=//foo:jdk8', '//foo:foo_test' ]) self.AssertExitCode(exit_code, 0, stderr, stdout) diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 3e3eea92ef..b3039d424c 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -27,7 +27,7 @@ filegroup( "testing_server.py", ":langtools-copy", "//examples:srcs", - "//src:bazel", + "//src:bazel_with_jdk", "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar", "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar", "//src/main/tools:linux-sandbox", @@ -497,10 +497,14 @@ sh_test( name = "bazel_bootstrap_distfile_test", timeout = "eternal", srcs = ["bazel_bootstrap_distfile_test.sh"], - args = ["$(location //:bazel-distfile)"], + args = [ + "$(location //:bazel-distfile)", + "$(location //src:embedded_jdk)", + ], data = [ ":test-deps", "//:bazel-distfile", + "//src:embedded_jdk", ], tags = ["jdk8"], ) diff --git a/src/test/shell/bazel/bazel_bootstrap_distfile_test.sh b/src/test/shell/bazel/bazel_bootstrap_distfile_test.sh index 3f4dd06efa..273fe8d5cd 100755 --- a/src/test/shell/bazel/bazel_bootstrap_distfile_test.sh +++ b/src/test/shell/bazel/bazel_bootstrap_distfile_test.sh @@ -19,6 +19,7 @@ set -u DISTFILE=$(rlocation io_bazel/${1#./}) +EMBEDDED_JDK=$(rlocation io_bazel/${2#./}) shift 1 # Load the test setup defined in the parent directory @@ -41,12 +42,23 @@ function test_bootstrap() { export SOURCE_DATE_EPOCH=1501234567 _log_progress "unzip" unzip -q "${DISTFILE}" + if [[ $EMBEDDED_JDK == *.tar.gz ]]; then + tar xf $EMBEDDED_JDK + elif [[ $EMBEDDED_JDK == *.zip ]]; then + unzip -q $EMBEDDED_JDK + fi + JAVABASE=$(echo zulu*) + _log_progress "bootstrap" env EXTRA_BAZEL_ARGS="--curses=no --strategy=Javac=standalone" ./compile.sh \ || fail "Expected to be able to bootstrap bazel" _log_progress "run" - ./output/bazel version > "${TEST_log}" || fail "Generated bazel not working" - ./output/bazel shutdown + ./output/bazel \ + --host_javabase=$JAVABASE --host_jvm_args=--add-opens=java.base/java.nio=ALL-UNNAMED \ + version > "${TEST_log}" || fail "Generated bazel not working" + ./output/bazel \ + --host_javabase=$JAVABASE --host_jvm_args=--add-opens=java.base/java.nio=ALL-UNNAMED \ + shutdown _log_progress "assert" expect_log "${SOURCE_DATE_EPOCH}" cd "${olddir}" diff --git a/src/test/shell/bazel/bazel_determinism_test.sh b/src/test/shell/bazel/bazel_determinism_test.sh index 8abc11f4f5..a3f64ff7c5 100755 --- a/src/test/shell/bazel/bazel_determinism_test.sh +++ b/src/test/shell/bazel/bazel_determinism_test.sh @@ -56,11 +56,11 @@ function test_determinism() { unzip -q "${DISTFILE}" # Build Bazel once. - bazel --output_base="${TEST_TMPDIR}/out1" build --nostamp //src:bazel + bazel --output_base="${TEST_TMPDIR}/out1" build --nostamp //src:bazel_with_jdk hash_outputs >"${TEST_TMPDIR}/sum1" # Build Bazel twice. - bazel-bin/src/bazel --output_base="${TEST_TMPDIR}/out2" build --nostamp //src:bazel + bazel-bin/src/bazel_with_jdk --output_base="${TEST_TMPDIR}/out2" build --nostamp //src:bazel_with_jdk hash_outputs >"${TEST_TMPDIR}/sum2" if ! diff -U0 "${TEST_TMPDIR}/sum1" "${TEST_TMPDIR}/sum2" >$TEST_log; then diff --git a/src/test/shell/bin/bazel b/src/test/shell/bin/bazel index 27975618ee..975d4df609 100755 --- a/src/test/shell/bin/bazel +++ b/src/test/shell/bin/bazel @@ -17,4 +17,4 @@ # Wrapper script to run bazel in the tests. Any change to this file will # affect all our integration tests. # -exec $(rlocation io_bazel/src/bazel) --bazelrc=$TEST_TMPDIR/bazelrc "$@" \ No newline at end of file +exec $(rlocation io_bazel/src/bazel_with_jdk) --bazelrc=$TEST_TMPDIR/bazelrc "$@" diff --git a/src/test/shell/integration/test_test.sh b/src/test/shell/integration/test_test.sh index c2e599749b..a6d70dde04 100755 --- a/src/test/shell/integration/test_test.sh +++ b/src/test/shell/integration/test_test.sh @@ -64,7 +64,7 @@ EOF bazel test --nocache_test_results //tests:fail &>$TEST_log \ && fail "expected failure" || true - expect_log "^//tests:fail[[:space:]]\+FAILED in [[:digit:]]\+\.[[:digit:]]\+s" + expect_log "^//tests:fail[[:space:]]\+FAILED in [[:digit:]]\+[\.,][[:digit:]]\+s" expect_log "^Executed 1 out of 1 test: 1 fails" } diff --git a/src/test/shell/testenv.sh b/src/test/shell/testenv.sh index 9de4521fea..805304a333 100755 --- a/src/test/shell/testenv.sh +++ b/src/test/shell/testenv.sh @@ -60,7 +60,7 @@ if is_windows; then fi # Make the command "bazel" available for tests. -PATH_TO_BAZEL_BIN=$(rlocation io_bazel/src/bazel) +PATH_TO_BAZEL_BIN=$(rlocation io_bazel/src/bazel_with_jdk) PATH_TO_BAZEL_WRAPPER="$(dirname $(rlocation io_bazel/src/test/shell/bin/bazel))" # Convert PATH_TO_BAZEL_WRAPPER to Unix path style on Windows, because it will be # added into PATH. There's problem if PATH=C:/msys64/usr/bin:/usr/local, @@ -266,8 +266,6 @@ function setup_bazelrc() { cat >$TEST_TMPDIR/bazelrc <