Refactor JDK 7 build to use the vendored version of it

Along the path, fix the build for JDK 7 and get rid of
most ugliness in the JDK 7 build. Now simply setting
JAVA_VERSION to 1.7 will build a JDK 7 compatible version.

Fixes #1159.

--
Change-Id: I9599283844a57d9e053f12d37445907f22a9232e
Reviewed-on: https://bazel-review.googlesource.com/#/c/3452
MOS_MIGRATED_REVID=120332747

13 files changed, 140 insertions, 67 deletions

diff --git a/compile.sh b/compile.sh
index 56299b8f1b..59ec71c72e 100755
--- a/compile.sh
+++ b/compile.sh
@@ -141,23 +141,24 @@ if [ $DO_TESTS ]; then
   fi
 
   [ -n "$JAVAC_VERSION" ] || get_java_version
-  if [[ ! "${BAZEL_TEST_FILTERS-}" =~ "-jdk8" ]] \
-      && [ "8" -gt ${JAVAC_VERSION#*.} ]; then
-    display "$WARNING Your version of Java is lower than 1.8!"
-    display "$WARNING Deactivating Java 8 tests, please use a JDK 8 to fully"
-    display "$WARNING test Bazel."
-    if [ -n "${BAZEL_TEST_FILTERS-}" ]; then
-      BAZEL_TEST_FILTERS="${BAZEL_TEST_FILTERS},-jdk8"
-    else
-      BAZEL_TEST_FILTERS="-jdk8"
+  if [[ ! "${BAZEL_TEST_FILTERS-}" =~ "-jdk8" ]]; then
+    if [ "8" -gt ${JAVAC_VERSION#*.} ] || [ "${JAVA_VERSION}" = "1.7" ]; then
+      display "$WARNING Your version of Java is lower than 1.8!"
+      display "$WARNING Deactivating Java 8 tests, please use a JDK 8 to fully"
+      display "$WARNING test Bazel."
+      if [ -n "${BAZEL_TEST_FILTERS-}" ]; then
+        BAZEL_TEST_FILTERS="${BAZEL_TEST_FILTERS},-jdk8"
+      else
+        BAZEL_TEST_FILTERS="-jdk8"
+      fi
     fi
   fi
   $BAZEL --bazelrc=${BAZELRC} --nomaster_bazelrc test \
       --test_tag_filters="${BAZEL_TEST_FILTERS-}" \
       --build_tests_only \
       --nolegacy_bazel_java_test \
+      --define JAVA_VERSION=${JAVA_VERSION} \
       ${EXTRA_BAZEL_ARGS} \
-      --javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \
       -k --test_output=errors //src/... //third_party/ijar/... //scripts/... \
       || fail "Tests failed"
 fi
diff --git a/scripts/bootstrap/bootstrap.sh b/scripts/bootstrap/bootstrap.sh
index 1394b62acb..8732d98aca 100755
--- a/scripts/bootstrap/bootstrap.sh
+++ b/scripts/bootstrap/bootstrap.sh
@@ -30,10 +30,17 @@ fi
 
 : ${JAVA_VERSION:="1.8"}
 
-: ${BAZEL_ARGS:=--java_toolchain=//src/java_tools/buildjar:bootstrap_toolchain \
-      --strategy=Javac=worker --worker_quit_after_build \
-      --genrule_strategy=standalone --spawn_strategy=standalone \
-      "${EXTRA_BAZEL_ARGS:-}"}
+if [ "${JAVA_VERSION}" = "1.7" ]; then
+  : ${BAZEL_ARGS:=--java_toolchain=//src/java_tools/buildjar:bootstrap_toolchain_jdk7 \
+        --define JAVA_VERSION=1.7 \
+        --genrule_strategy=standalone --spawn_strategy=standalone \
+        "${EXTRA_BAZEL_ARGS:-}"}
+else
+  : ${BAZEL_ARGS:=--java_toolchain=//src/java_tools/buildjar:bootstrap_toolchain \
+        --strategy=Javac=worker --worker_quit_after_build \
+        --genrule_strategy=standalone --spawn_strategy=standalone \
+        "${EXTRA_BAZEL_ARGS:-}"}
+fi
 
 if [ -z "${BAZEL-}" ]; then
   function bazel_build() {
@@ -82,6 +89,7 @@ function bootstrap_test() {
       ${EXTRA_BAZEL_ARGS-} \
       --strategy=Javac=worker --worker_quit_after_build \
       --fetch --nostamp \
+      --define "JAVA_VERSION=${JAVA_VERSION}" \
       --javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \
       ${BAZEL_TARGET} || return $?
   if [ -n "${BAZEL_SUM}" ]; then
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
index df7e794b39..3d63948f11 100755
--- a/scripts/ci/build.sh
+++ b/scripts/ci/build.sh
@@ -98,45 +98,6 @@ EOF
   fi
 }
 
-# Set the various arguments when JDK 7 is required (deprecated).
-# This method is here to continue to build binary release of Bazel
-# for JDK 7. We will drop this method and JDK 7 support when our
-# ci system turn red on this one.
-function setup_jdk7() {
-  # This is a JDK 7 JavaBuilder from release 0.1.0.
-  local javabuilder_url="https://storage.googleapis.com/bazel/0.1.0/JavaBuilder_deploy.jar"
-  local javac_url="https://github.com/bazelbuild/bazel/blob/0.1.0/third_party/java/jdk/langtools/javac.jar?raw=true"
-  sed -i.bak 's/_version = "8"/_version = "7"/' tools/jdk/BUILD
-  rm -f tools/jdk/BUILD.bak
-  rm -f third_party/java/jdk/langtools/javac.jar
-  curl -Ls -o tools/jdk/JavaBuilder_deploy.jar "${javabuilder_url}"
-  curl -Ls -o third_party/java/jdk/langtools/javac.jar "${javac_url}"
-  # Do not use the skylark bootstrapped version of JavaBuilder
-  export BAZEL_ARGS="--singlejar_top=//src/java_tools/singlejar:bootstrap_deploy.jar \
-      --genclass_top=//src/java_tools/buildjar:bootstrap_genclass_deploy.jar \
-      --ijar_top=//third_party/ijar"
-  # Skip building JavaBuilder
-  export BAZEL_SKIP_TOOL_COMPILATION=tools/jdk/JavaBuilder_deploy.jar
-  # Ignore JDK8 tests
-  export BAZEL_TEST_FILTERS="-jdk8"
-  if ! grep -Fq 'RealJavaBuilder' src/java_tools/buildjar/BUILD; then
-    # And more ugly hack. Overwrite the BUILD file of JavaBuilder
-    # so we use the pre-built version in integration tests.
-    sed -i.bak 's/name = \"JavaBuilder\"/name = \"RealJavaBuilder\"/' \
-        src/java_tools/buildjar/BUILD
-    rm -f src/java_tools/buildjar/BUILD.bak
-    cat >>src/java_tools/buildjar/BUILD <<'EOF'
-genrule(
-    name = "JavaBuilder",
-    outs = ["JavaBuilder_deploy.jar"],
-    srcs = ["//tools/jdk:JavaBuilder_deploy.jar"],
-    cmd = "cp $< $@",
-    visibility = ["//visibility:public"],
-)
-EOF
-  fi
-}
-
 # Main entry point for building bazel.
 # It sets the embed label to the release name if any, calls the whole
 # test suite, compile the various packages, then copy the artifacts
@@ -151,7 +112,6 @@ function bazel_build() {
 
   if [[ "${JAVA_VERSION-}" =~ ^(1\.)?7$ ]]; then
     JAVA_VERSION=1.7
-    setup_jdk7
     release_label="${release_label}-jdk7"
   else
     JAVA_VERSION=1.8
diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD
index 2fba80fed2..e2f5efef92 100644
--- a/scripts/packages/BUILD
+++ b/scripts/packages/BUILD
@@ -116,18 +116,13 @@ genrule(
     stamp = 1,
 )
 
-config_setting(
-    name = "jdk7",
-    values = {"define": "JAVA_VERSION=1.7"},
-)
-
 pkg_deb(
     name = "bazel-debian",
     architecture = "amd64",
     built_using = "bazel (HEAD)",
     data = ":debian-data",
     depends = select({
-        ":jdk7": [
+        "//tools/jdk:jdk7": [
             "java7-jdk | java7-sdk",
         ],
         "//conditions:default": [
diff --git a/src/BUILD b/src/BUILD
index 98460b1868..c4fa2a7457 100644
--- a/src/BUILD
+++ b/src/BUILD
@@ -100,7 +100,7 @@ genrule(
         "//src/tools/android/java/com/google/devtools/build/android/incrementaldeployment:srcs",
         "//src/tools/android/java/com/google/devtools/build/android/ziputils:embedded_tools",
         "//src/main/protobuf:srcs",
-        "//src/java_tools/buildjar:JavaBuilder_deploy.jar",
+        "//src/java_tools/buildjar:JavaBuilderDeploy",
         "//src/java_tools/singlejar:SingleJar_deploy.jar",
         "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar",
         "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar",
diff --git a/src/create_embedded_tools.sh b/src/create_embedded_tools.sh
index e3b751c3b0..8de48375d4 100755
--- a/src/create_embedded_tools.sh
+++ b/src/create_embedded_tools.sh
@@ -28,7 +28,9 @@ trap "rm -fr \"${PACKAGE_DIR}\"" EXIT
 
 for i in $*; do
   case "$i" in
-    *JavaBuilder_deploy.jar) OUTPUT_PATH=tools/jdk/JavaBuilder_deploy.jar ;;
+    *tools/jdk/BUILD*) OUTPUT_PATH=tools/jdk/BUILD ;;
+    *JavaBuilder*_deploy.jar) OUTPUT_PATH=tools/jdk/JavaBuilder_deploy.jar ;;
+    *javac*.jar) OUTPUT_PATH=third_party/java/jdk/langtools/javac.jar ;;
     *SingleJar_deploy.jar) OUTPUT_PATH=tools/jdk/SingleJar_deploy.jar ;;
     *GenClass_deploy.jar) OUTPUT_PATH=tools/jdk/GenClass_deploy.jar ;;
     *Runner_deploy.jar) OUTPUT_PATH=tools/jdk/TestRunner_deploy.jar ;;
diff --git a/src/java_tools/buildjar/BUILD b/src/java_tools/buildjar/BUILD
index f7f53a7ae6..56298dfd1a 100644
--- a/src/java_tools/buildjar/BUILD
+++ b/src/java_tools/buildjar/BUILD
@@ -8,6 +8,14 @@ java_binary(
 )
 
 filegroup(
+    name = "JavaBuilderDeploy",
+    srcs = select({
+        "//tools/jdk:jdk7": ["//third_party/java/jdk/javabuilder:JavaBuilder_0.1.0"],
+        "//conditions:default": [":JavaBuilder_deploy.jar"],
+    }),
+)
+
+filegroup(
     name = "bootstrap_deploy.jar",
     srcs = ["//src/java_tools/buildjar/java/com/google/devtools/build/buildjar:bootstrap_deploy.jar"],
     visibility = ["//visibility:public"],
@@ -52,3 +60,19 @@ java_toolchain(
     target_version = "8",
     visibility = ["//visibility:public"],
 )
+
+java_toolchain(
+    name = "bootstrap_toolchain_jdk7",
+    bootclasspath = ["//tools/jdk:bootclasspath"],
+    encoding = "UTF-8",
+    extclasspath = ["//tools/jdk:extdir"],
+    genclass = ["bootstrap_genclass_deploy.jar"],
+    ijar = ["//third_party/ijar"],
+    javabuilder = ["//third_party/java/jdk/javabuilder:JavaBuilder_0.1.0"],
+    javac = ["//third_party/java/jdk/langtools:javac7_jar"],
+    jvm_opts = ["-client"],
+    singlejar = ["//src/java_tools/singlejar:bootstrap_deploy.jar"],
+    source_version = "7",
+    target_version = "7",
+    visibility = ["//visibility:public"],
+)
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index 349db3738b..3c45a3f6a8 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -50,7 +50,7 @@ filegroup(
         "//external:extdir",
         "//external:jdk-default",
         "//src:bazel",
-        "//src/java_tools/buildjar:JavaBuilder_deploy.jar",
+        "//src/java_tools/buildjar:JavaBuilderDeploy",
         "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar",
         "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar",
         "//src/java_tools/singlejar:SingleJar_deploy.jar",
diff --git a/src/test/shell/bazel/testenv.sh b/src/test/shell/bazel/testenv.sh
index 8820f93a6e..4db59d358e 100755
--- a/src/test/shell/bazel/testenv.sh
+++ b/src/test/shell/bazel/testenv.sh
@@ -39,7 +39,7 @@ langtools_dir="${TEST_SRCDIR}/third_party/java/jdk/langtools"
 EXTRA_BAZELRC="build --ios_sdk_version=8.4"
 
 # Java tooling
-javabuilder_path="${TEST_SRCDIR}/src/java_tools/buildjar/JavaBuilder_deploy.jar"
+javabuilder_path="$(find ${TEST_SRCDIR} -name JavaBuilder_*.jar)"
 langtools_path="${TEST_SRCDIR}/third_party/java/jdk/langtools/javac.jar"
 singlejar_path="${TEST_SRCDIR}/src/java_tools/singlejar/SingleJar_deploy.jar"
 genclass_path="${TEST_SRCDIR}/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass/GenClass_deploy.jar"
@@ -106,6 +106,11 @@ hamcrest_jar="${TEST_SRCDIR}/third_party/hamcrest/hamcrest-*.jar"
 # This function copies the tools directory from Bazel.
 function copy_tools_directory() {
   cp -RL ${tools_dir}/* tools
+  # tools/jdk/BUILD file for JDK 7 is generated.
+  if [ -f tools/jdk/BUILD.* ]; then
+    cp tools/jdk/BUILD.* tools/jdk/BUILD
+    chmod +w tools/jdk/BUILD
+  fi
   # To support custom langtools
   cp ${langtools} tools/jdk/langtools.jar
   cat >>tools/jdk/BUILD <<'EOF'
diff --git a/third_party/java/jdk/README.md b/third_party/java/jdk/README.md
new file mode 100644
index 0000000000..02c794ae20
--- /dev/null
+++ b/third_party/java/jdk/README.md
@@ -0,0 +1,23 @@
+# Java compilers in Bazel
+
+Bazel compiles Java code using a custom builder. This builder is called
+JavaBuilder and its code lies in //src/java_tools/buildjar. To build Java
+code, JavaBuilder use the Java compiler from the JDK. To support
+[ErrorProne](http://errorprone.info) checks, we vendor a custom build
+of the Java compiler code. This is the raw version of the Java compiler
+from [OpenJDK](https://openjdk.java.net) but compiled for a lower
+version of the JRE. Those builds are vendored in
+//third_party/java/jdk/langtools.
+
+Current Bazel supports running on a JRE 8 only because the default Java
+compiler used (//third_party/java/jdk/langtools/javac.jar) is the
+Java compiler of OpenJDK 9 compiled to run on a JRE 8. This cannot
+be built to run on a JRE 7 because of code incompatiblity. Bazel's
+JavaBuilder at HEAD cannot be linked with earlier version of the
+Java compiler (it depends on some internals of the Java compiler).
+
+To build a version of Bazel that can run on a JRE 7, we need to rely
+on the version of JavaBuilder provided with Bazel 0.1.0
+(//third_party/java/jdk/javabuilder/JavaBuilder_0.1.0_deploy.jar) which works
+with a Java compiler of OpenJDK 8 compiled to run on a JRE 7
+(//third_party/java/jdk/langtools/javac7.jar).
diff --git a/third_party/java/jdk/javabuilder/BUILD b/third_party/java/jdk/javabuilder/BUILD
new file mode 100644
index 0000000000..f670c45bcd
--- /dev/null
+++ b/third_party/java/jdk/javabuilder/BUILD
@@ -0,0 +1,13 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0 license
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+)
+
+filegroup(
+    name = "JavaBuilder_0.1.0",
+    srcs = ["JavaBuilder_0.1.0_deploy.jar"],
+)
diff --git a/third_party/java/jdk/langtools/BUILD b/third_party/java/jdk/langtools/BUILD
index a542ae28cb..e87fa27c1d 100644
--- a/third_party/java/jdk/langtools/BUILD
+++ b/third_party/java/jdk/langtools/BUILD
@@ -4,7 +4,16 @@ licenses(["restricted"])  # GNU GPL v2 with Classpath exception
 
 filegroup(
     name = "srcs",
-    srcs = glob(["**"]),
+    srcs = select({
+        "//tools/jdk:jdk7": [
+            "BUILD",
+            "javac7.jar",
+        ],
+        "//conditions:default": [
+            "BUILD",
+            "javac.jar",
+        ],
+    }),
 )
 
 java_import(
@@ -14,5 +23,18 @@ java_import(
 
 filegroup(
     name = "javac_jar",
-    srcs = ["javac.jar"],
+    srcs = select({
+        "//tools/jdk:jdk7": ["javac7.jar"],
+        "//conditions:default": ["javac.jar"],
+    }),
+)
+
+java_import(
+    name = "javac7",
+    jars = ["javac7.jar"],
+)
+
+filegroup(
+    name = "javac7_jar",
+    srcs = ["javac7.jar"],
 )
diff --git a/tools/jdk/BUILD b/tools/jdk/BUILD
index 5e5fc19a05..220815f51d 100644
--- a/tools/jdk/BUILD
+++ b/tools/jdk/BUILD
@@ -1,5 +1,25 @@
 package(default_visibility = ["//visibility:public"])
 
+config_setting(
+    name = "jdk7",
+    values = {"define": "JAVA_VERSION=1.7"},
+)
+
+genrule(
+    name = "BUILD-jdk7",
+    srcs = [":BUILD"],
+    outs = ["BUILD.jdk7"],
+    cmd = "sed 's/_version = \"8\"/_version = \"7\"/' $< > $@",
+)
+
+filegroup(
+    name = "BUILD-jdk",
+    srcs = select({
+        ":jdk7": [":BUILD-jdk7"],
+        "//conditions:default": [":BUILD"],
+    }),
+)
+
 filegroup(
     name = "jni_header",
     srcs = ["//external:jni_header"],
@@ -112,7 +132,7 @@ java_toolchain(
 filegroup(
     name = "srcs",
     srcs = [
-        "BUILD",  # Tools are build from the workspace for tests.
+        "BUILD-jdk",  # Tools are build from the workspace for tests.
         "proguard_whitelister.py",
         "proguard_whitelister_test.py",
         "proguard_whitelister_test_input.cfg",