From 6cb8d820689ad029a9d0dc4ee1100db9b2d96515 Mon Sep 17 00:00:00 2001 From: Damien Martin-Guillerez Date: Tue, 12 Apr 2016 13:59:41 +0000 Subject: Refactor build for JDK 7 Now the JDK 7 tuning happens all in Bazel, removing logic from the CI script. It uses remote repositories to access JDK 7 dependencies. -- Change-Id: Iff590c6642ca5b2343aa15096f8fd837d1c80787 Reviewed-on: https://bazel-review.googlesource.com/#/c/3327 MOS_MIGRATED_REVID=119634530 --- WORKSPACE | 16 ++++ compile.sh | 5 +- scripts/bootstrap/bootstrap.sh | 32 +++++--- scripts/bootstrap/compile.sh | 2 +- scripts/ci/build.sh | 49 ++---------- scripts/packages/BUILD | 172 +++++++++++++++++++++++++---------------- src/BUILD | 68 ++++++++++++---- src/create_embedded_tools.sh | 2 + tools/jdk/BUILD | 8 ++ 9 files changed, 217 insertions(+), 137 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 05c5117286..ed50f775fe 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,6 +23,22 @@ bind( actual = "//:dummy", ) +# For building for JDK 7 +# This is a JDK 7 JavaBuilder from release 0.1.0. +# Do not use the skylark bootstrapped version of JavaBuilder +http_file( + name = "io_bazel_javabuilder_jdk7", + # This was built from Bazel 0.1.0 (after ./compile.sh) and JDK 7. + url = "https://storage.googleapis.com/bazel/0.1.0/JavaBuilder_deploy.jar", + sha256 = "374f07be2f73ec170ef6bdd8057530e6842cb0cbc7a240caccbd7b80be239366", +) + +http_file( + name = "io_bazel_javac_jdk7", + url = "https://github.com/bazelbuild/bazel/blob/0.1.0/third_party/java/jdk/langtools/javac.jar?raw=true", + sha256 = "e063cf60cf9a1d24acd043f55dc605da559b913793c6a4e51c5ddbc4f5d494df", +) + # For tools/cpp/test/... load("//tools/cpp/test:docker_repository.bzl", "docker_repository") docker_repository() diff --git a/compile.sh b/compile.sh index 56299b8f1b..da75ec02d3 100755 --- a/compile.sh +++ b/compile.sh @@ -96,8 +96,8 @@ if [ $DO_COMPILE ]; then new_step 'Building Bazel with Bazel' display "." log "Building output/bazel" - bazel_build "src:bazel${EXE_EXT}" - cp -f "bazel-bin/src/bazel${EXE_EXT}" "output/bazel${EXE_EXT}" + bazel_build "src:bazel${BAZEL_FLAVOUR}${EXE_EXT}" + cp -f "bazel-bin/src/bazel${BAZEL_FLAVOUR}${EXE_EXT}" "output/bazel${EXE_EXT}" chmod 0755 "output/bazel${EXE_EXT}" BAZEL="$(pwd)/output/bazel${EXE_EXT}" fi @@ -155,7 +155,6 @@ if [ $DO_TESTS ]; then $BAZEL --bazelrc=${BAZELRC} --nomaster_bazelrc test \ --test_tag_filters="${BAZEL_TEST_FILTERS-}" \ --build_tests_only \ - --nolegacy_bazel_java_test \ ${EXTRA_BAZEL_ARGS} \ --javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \ -k --test_output=errors //src/... //third_party/ijar/... //scripts/... \ diff --git a/scripts/bootstrap/bootstrap.sh b/scripts/bootstrap/bootstrap.sh index 2fea800680..3fae3027c4 100755 --- a/scripts/bootstrap/bootstrap.sh +++ b/scripts/bootstrap/bootstrap.sh @@ -28,21 +28,33 @@ if [ -n "${EMBED_LABEL}" ]; then EMBED_LABEL_ARG=(--stamp --embed_label "${EMBED_LABEL}") fi -: ${JAVA_VERSION:="1.8"} - -: ${BAZEL_ARGS:=--singlejar_top=//src/java_tools/singlejar:bootstrap_deploy.jar \ - --javabuilder_top=//src/java_tools/buildjar:bootstrap_deploy.jar \ - --genclass_top=//src/java_tools/buildjar:bootstrap_genclass_deploy.jar \ - --ijar_top=//third_party/ijar \ +COMMON_BAZEL_ARGS="--singlejar_top=//src/java_tools/singlejar:bootstrap_deploy.jar \ + --genclass_top=//src/java_tools/buildjar:bootstrap_genclass_deploy.jar \ + --ijar_top=//third_party/ijar" +if [[ "${JAVA_VERSION-}" == 1.7 ]]; then + # Set the various arguments when JDK 7 is required (deprecated). + # This setting is here to continue to build binary release of Bazel + # for JDK 7. We will drop this method and JDK 7 support when our + # ci system turn red on this one. + : ${BAZEL_ARGS:=--java_toolchain=//tools/jdk:toolchain7 \ + --javabuilder_top=@io_bazel_javabuilder_jdk7//file \ + --java_langtools=@io_bazel_javac_jdk7//file \ + ${COMMON_BAZEL_ARGS} \ + "${EXTRA_BAZEL_ARGS:-}"} + : ${BAZEL_FLAVOUR:="-jdk7"} +else + : ${BAZEL_ARGS:=--javabuilder_top=//src/java_tools/buildjar:bootstrap_deploy.jar \ --strategy=Javac=worker --worker_quit_after_build \ --genrule_strategy=standalone --spawn_strategy=standalone \ + ${COMMON_BAZEL_ARGS} \ "${EXTRA_BAZEL_ARGS:-}"} + : ${BAZEL_FLAVOUR:=""} +fi if [ -z "${BAZEL-}" ]; then function bazel_build() { bootstrap_build ${BAZEL_ARGS-} \ --verbose_failures \ - --javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \ "${EMBED_LABEL_ARG[@]}" \ "${@}" } @@ -51,7 +63,6 @@ else ${BAZEL} --bazelrc=${BAZELRC} build \ ${BAZEL_ARGS-} \ --verbose_failures \ - --javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \ "${EMBED_LABEL_ARG[@]}" \ "${@}" } @@ -77,7 +88,7 @@ function get_outputs_sum() { function bootstrap_test() { local BAZEL_BIN=$1 local BAZEL_SUM=$2 - local BAZEL_TARGET=${3:-src:bazel} + local BAZEL_TARGET=${3:-src:bazel${BAZEL_FLAVOUR}} [ -x "${BAZEL_BIN}" ] || fail "syntax: bootstrap bazel-binary" run ${BAZEL_BIN} --nomaster_bazelrc --bazelrc=${BAZELRC} clean \ --expunge || return $? @@ -85,7 +96,6 @@ function bootstrap_test() { ${EXTRA_BAZEL_ARGS-} \ --strategy=Javac=worker --worker_quit_after_build \ --fetch --nostamp \ - --javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \ ${BAZEL_TARGET} || return $? if [ -n "${BAZEL_SUM}" ]; then cat bazel-genfiles/src/java.version >${BAZEL_SUM} @@ -93,7 +103,7 @@ function bootstrap_test() { fi if [ -z "${BOOTSTRAP:-}" ]; then tempdir - BOOTSTRAP=${NEW_TMPDIR}/bazel + BOOTSTRAP=${NEW_TMPDIR}/bazel${BAZEL_FLAVOUR} local FILE=bazel-bin/${BAZEL_TARGET##//} cp -f ${FILE/:/\/} $BOOTSTRAP chmod +x $BOOTSTRAP diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh index abb7e1e899..f949dbd8b4 100755 --- a/scripts/bootstrap/compile.sh +++ b/scripts/bootstrap/compile.sh @@ -17,7 +17,7 @@ # Script for building bazel from scratch without bazel PROTO_FILES=$(ls src/main/protobuf/*.proto) -LIBRARY_JARS=$(find third_party -name '*.jar' | tr "\n" " ") +LIBRARY_JARS=$(find third_party -name '*.jar' | grep -v "javac.jar" | tr "\n" " ") DIRS=$(echo src/{java_tools/singlejar/java/com/google/devtools/build/zip,main/java,tools/xcode-common/java/com/google/devtools/build/xcode/{common,util}} ${OUTPUT_DIR}/src) EXCLUDE_FILES=src/main/java/com/google/devtools/build/lib/server/GrpcServer.java diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index df7e794b39..63f7de705f 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -98,45 +98,6 @@ EOF fi } -# Set the various arguments when JDK 7 is required (deprecated). -# This method is here to continue to build binary release of Bazel -# for JDK 7. We will drop this method and JDK 7 support when our -# ci system turn red on this one. -function setup_jdk7() { - # This is a JDK 7 JavaBuilder from release 0.1.0. - local javabuilder_url="https://storage.googleapis.com/bazel/0.1.0/JavaBuilder_deploy.jar" - local javac_url="https://github.com/bazelbuild/bazel/blob/0.1.0/third_party/java/jdk/langtools/javac.jar?raw=true" - sed -i.bak 's/_version = "8"/_version = "7"/' tools/jdk/BUILD - rm -f tools/jdk/BUILD.bak - rm -f third_party/java/jdk/langtools/javac.jar - curl -Ls -o tools/jdk/JavaBuilder_deploy.jar "${javabuilder_url}" - curl -Ls -o third_party/java/jdk/langtools/javac.jar "${javac_url}" - # Do not use the skylark bootstrapped version of JavaBuilder - export BAZEL_ARGS="--singlejar_top=//src/java_tools/singlejar:bootstrap_deploy.jar \ - --genclass_top=//src/java_tools/buildjar:bootstrap_genclass_deploy.jar \ - --ijar_top=//third_party/ijar" - # Skip building JavaBuilder - export BAZEL_SKIP_TOOL_COMPILATION=tools/jdk/JavaBuilder_deploy.jar - # Ignore JDK8 tests - export BAZEL_TEST_FILTERS="-jdk8" - if ! grep -Fq 'RealJavaBuilder' src/java_tools/buildjar/BUILD; then - # And more ugly hack. Overwrite the BUILD file of JavaBuilder - # so we use the pre-built version in integration tests. - sed -i.bak 's/name = \"JavaBuilder\"/name = \"RealJavaBuilder\"/' \ - src/java_tools/buildjar/BUILD - rm -f src/java_tools/buildjar/BUILD.bak - cat >>src/java_tools/buildjar/BUILD <<'EOF' -genrule( - name = "JavaBuilder", - outs = ["JavaBuilder_deploy.jar"], - srcs = ["//tools/jdk:JavaBuilder_deploy.jar"], - cmd = "cp $< $@", - visibility = ["//visibility:public"], -) -EOF - fi -} - # Main entry point for building bazel. # It sets the embed label to the release name if any, calls the whole # test suite, compile the various packages, then copy the artifacts @@ -151,9 +112,10 @@ function bazel_build() { if [[ "${JAVA_VERSION-}" =~ ^(1\.)?7$ ]]; then JAVA_VERSION=1.7 - setup_jdk7 + FLAVOUR="-jdk7" release_label="${release_label}-jdk7" else + FLAVOUR="" JAVA_VERSION=1.8 fi @@ -177,15 +139,16 @@ function bazel_build() { --workspace_status_command=scripts/ci/build_status_command.sh \ --define JAVA_VERSION=${JAVA_VERSION} \ ${ARGS} \ - //scripts/packages/... || exit $? + //scripts/packages:install${FLAVOUR}.sh \ + //scripts/bazel-debian${FLAVOUR}.deb} || exit $? if [ -n "${1-}" ]; then # Copy the results to the output directory mkdir -p $1/packages cp output/bazel $1/bazel - cp bazel-bin/scripts/packages/install.sh $1/bazel-${release_label}-installer.sh + cp bazel-bin/scripts/packages/install${FLAVOUR}.sh $1/bazel-${release_label}-installer.sh if [ "$PLATFORM" = "linux" ]; then - cp bazel-bin/scripts/packages/bazel-debian.deb $1/bazel_${release_label}.deb + cp bazel-bin/scripts/packages/bazel-debian${FLAVOUR}.deb $1/bazel_${release_label}.deb fi cp bazel-genfiles/scripts/packages/README.md $1/README.md fi diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD index 2fba80fed2..86ca34dfa6 100644 --- a/scripts/packages/BUILD +++ b/scripts/packages/BUILD @@ -32,6 +32,13 @@ genrule( cmd = "cp $< $@", ) +genrule( + name = "rename-bazel-bin-jdk7", + srcs = ["//src:bazel_jdk7"], + outs = ["jdk7/bazel-real"], + cmd = "cp $< $@", +) + genrule( name = "rename-bazel-sh", srcs = ["bazel.sh"], @@ -41,24 +48,49 @@ genrule( load(":self_extract_binary.bzl", "self_extract_binary") -self_extract_binary( - name = "install.sh", - empty_files = ["tools/defaults/BUILD"], - flatten_resources = [ +[ + self_extract_binary( + name = "install%s.sh" % flavour, + empty_files = ["tools/defaults/BUILD"], + flatten_resources = [ + ":bazel-real" + flavour, + ":bazel", + "//scripts:bash_completion", + ], + launcher = ":launcher_bin.sh", + ) + for flavour in [ + "", + "-jdk7", + ] +] + +load("//tools/build_defs/pkg:pkg.bzl", "pkg_tar", "pkg_deb") + +pkg_tar( + name = "bazel-real-bin", + files = [ ":bazel-real", - ":bazel", - "//scripts:bash_completion", ], - launcher = ":launcher_bin.sh", + mode = "0755", + package_dir = "/usr/bin", + strip_prefix = ".", ) -load("//tools/build_defs/pkg:pkg.bzl", "pkg_tar", "pkg_deb") +pkg_tar( + name = "bazel-real-bin-jdk7", + files = [ + ":bazel-real-jdk7", + ], + mode = "0755", + package_dir = "/usr/bin", + strip_prefix = "scripts/packages/jdk7/", +) pkg_tar( name = "bazel-bin", files = [ ":bazel", - ":bazel-real", ], mode = "0755", package_dir = "/usr/bin", @@ -87,63 +119,71 @@ pkg_tar( strip_prefix = ".", ) -pkg_tar( - name = "debian-data", - extension = "tar.gz", - deps = [ - ":bazel-bin", - ":bazel-completion", - ":bazel-rc", - ], -) - -genrule( - name = "embedded_label", - srcs = [":debian-data"], - outs = [ - "version.txt", - ], - cmd = - "find . -name stable-status.txt -exec cat '{}' ';'" + - " | grep EMBED_LABEL | cut -d ' ' -f 2-" + - # Debian package manager does not order our rcs correctly, insert a tilde - # Non RC version number are not matched by the regex, example translations: - # 0.1.2 0.1.2 - # 0.1.2rc2 0.1.2~rc2 - # 0.1.2rc2-jdk7 0.1.2~rc2-jdk7 - " | sed -E 's/^([0-9\\.]+)([^0-9\\.].*)$$/\\1~\\2/'" + - " > $@\n", - stamp = 1, -) - -config_setting( - name = "jdk7", - values = {"define": "JAVA_VERSION=1.7"}, -) +[ + pkg_tar( + name = "debian-data" + flavour, + extension = "tar.gz", + deps = [ + ":bazel-real-bin" + flavour, + ":bazel-bin", + ":bazel-completion", + ":bazel-rc", + ], + ) + for flavour in [ + "", + "-jdk7", + ] +] -pkg_deb( - name = "bazel-debian", - architecture = "amd64", - built_using = "bazel (HEAD)", - data = ":debian-data", - depends = select({ - ":jdk7": [ - "java7-jdk | java7-sdk", +[ + genrule( + name = "embedded_label" + flavour, + srcs = [":debian-data"], + outs = [ + "version%s.txt" % flavour, ], - "//conditions:default": [ - "java8-jdk | java8-sdk", + cmd = + "find . -name stable-status.txt -exec cat '{}' ';'" + + " | grep EMBED_LABEL | cut -d ' ' -f 2-" + + # Debian package manager does not order our rcs correctly, insert a tilde + # Non RC version number are not matched by the regex, example translations: + # 0.1.2 0.1.2 + # 0.1.2rc2 0.1.2~rc2 + # 0.1.2rc2-jdk7 0.1.2~rc2-jdk7 + " | sed -E 's/^([0-9\\.]+)([^0-9\\.].*)$$/\\1~\\2/'" + + " > $@\n", + stamp = 1, + ) + for flavour in [ + "", + "-jdk7", + ] +] + +[ + pkg_deb( + name = "bazel-debian" + flavour, + architecture = "amd64", + built_using = "bazel (HEAD)", + data = ":debian-data" + flavour, + depends = [ + "java7-jdk | java7-sdk" if (flavour == "-jdk7") else "java8-jdk | java8-sdk", + "pkg-config", + "zip", + "g++", + "zlib1g-dev", + "unzip", + "bash-completion", ], - }) + [ - "pkg-config", - "zip", - "g++", - "zlib1g-dev", - "unzip", - "bash-completion", - ], - description_file = "debian/description", - homepage = "http://bazel.io", - maintainer = "The Bazel Authors ", - package = "bazel", - version_file = ":version.txt", -) + description_file = "debian/description", + homepage = "http://bazel.io", + maintainer = "The Bazel Authors ", + package = "bazel", + version_file = ":version%s.txt" % flavour, + ) + for flavour in [ + "", + "-jdk7", + ] +] diff --git a/src/BUILD b/src/BUILD index 7ff67d458f..07cd4fd63f 100644 --- a/src/BUILD +++ b/src/BUILD @@ -38,6 +38,7 @@ filegroup( }), ) for suffix, embedded_tools in { "": [":embedded_tools"], + "-jdk7": [":embedded_tools-jdk7"], "_notools": [], }.items()] @@ -87,25 +88,32 @@ filegroup( }), ) -genrule( - name = "embedded_tools", +filegroup( + name = "common_embedded_tools", srcs = [ - ":create_embedded_tools.sh", - "//tools:embedded_tools_srcs", - "//third_party:embedded_tools_srcs", + "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar", "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/jarhelper:srcs", + "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar", + "//src/java_tools/singlejar:SingleJar_deploy.jar", + "//src/main/protobuf:srcs", "//src/tools/android/java/com/google/devtools/build/android:embedded_tools", "//src/tools/android/java/com/google/devtools/build/android/ideinfo:embedded_tools", "//src/tools/android/java/com/google/devtools/build/android/idlclass:embedded_tools", "//src/tools/android/java/com/google/devtools/build/android/incrementaldeployment:srcs", "//src/tools/android/java/com/google/devtools/build/android/ziputils:embedded_tools", - "//src/main/protobuf:srcs", - "//src/java_tools/buildjar:JavaBuilder_deploy.jar", - "//src/java_tools/singlejar:SingleJar_deploy.jar", - "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar", - "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar", + "//third_party:embedded_tools_srcs", "//third_party/ijar", "//third_party/java/apkbuilder:embedded_tools", + "//tools:embedded_tools_srcs", + ], +) + +genrule( + name = "embedded_tools", + srcs = [ + ":create_embedded_tools.sh", + ":common_embedded_tools", + "//src/java_tools/buildjar:JavaBuilder_deploy.jar", ] + select({ ":darwin": [ ":darwin_tools", @@ -122,9 +130,40 @@ genrule( cmd = "$(location :create_embedded_tools.sh) $@ $(SRCS)", ) +genrule( + name = "tools_jdk_jdk7_build", + srcs = ["//tools/jdk:BUILD"], + outs = ["tools_jdk_jdk7.build"], + cmd = "cat $< | sed 's|\"toolchain\"|\"toolchain8\"|' | sed 's|\"toolchain7\"|\"toolchain\"|' > $@", +) + +genrule( + name = "embedded_tools-jdk7", + srcs = [ + ":create_embedded_tools.sh", + ":common_embedded_tools", + "@io_bazel_javabuilder_jdk7//file", + "@io_bazel_javac_jdk7//file", + ":tools_jdk_jdk7_build", + ] + select({ + ":darwin": [ + ":darwin_tools", + "//third_party/ijar:zipper", + ], + ":darwin_x86_64": [ + ":darwin_tools", + "//third_party/ijar:zipper", + ], + ":windows": [], # Windows specifically exclude zipper for now. + "//conditions:default": ["//third_party/ijar:zipper"], + }), + outs = ["embedded_tools-jdk7.zip"], + cmd = "$(location :create_embedded_tools.sh) $@ $(SRCS)", +) + [genrule( name = "package-zip" + suffix, - srcs = ([":embedded_tools.zip"] if embed else []) + [ + srcs = ([":embedded_%s.zip" % embed] if embed else []) + [ # The script assumes that the embedded tools zip (if exists) is the # first item here, the deploy jar the second and install base key is the # third @@ -144,8 +183,9 @@ genrule( cmd = "$(location :package-bazel.sh) $@ " + ("" if embed else "''") + " $(SRCS)", tools = ["package-bazel.sh"], ) for suffix, embed in [ - ("", True), - ("_notools", False), + ("", "tools"), + ("_notools", None), + ("-jdk7", "tools-jdk7"), ]] [genrule( @@ -166,6 +206,7 @@ genrule( ) for suffix in [ "", "_notools", + "-jdk7", ]] # Build an executable named `bazel.exe`. @@ -186,6 +227,7 @@ genrule( ) for suffix in [ "", "_notools", + "-jdk7", ]] config_setting( diff --git a/src/create_embedded_tools.sh b/src/create_embedded_tools.sh index e3b751c3b0..78bd9e4afb 100755 --- a/src/create_embedded_tools.sh +++ b/src/create_embedded_tools.sh @@ -28,6 +28,8 @@ trap "rm -fr \"${PACKAGE_DIR}\"" EXIT for i in $*; do case "$i" in + *javac.jar) OUTPUT_PATH=third_party/java/jdk/langtools/javac.jar ;; + *tools_jdk_jdk7.build) OUTPUT_PATH=tools/jdk/BUILD ;; *JavaBuilder_deploy.jar) OUTPUT_PATH=tools/jdk/JavaBuilder_deploy.jar ;; *SingleJar_deploy.jar) OUTPUT_PATH=tools/jdk/SingleJar_deploy.jar ;; *GenClass_deploy.jar) OUTPUT_PATH=tools/jdk/GenClass_deploy.jar ;; diff --git a/tools/jdk/BUILD b/tools/jdk/BUILD index 50feeb0c2e..2793124902 100644 --- a/tools/jdk/BUILD +++ b/tools/jdk/BUILD @@ -102,6 +102,14 @@ java_toolchain( target_version = "8", ) +java_toolchain( + name = "toolchain7", + encoding = "UTF-8", + jvm_opts = ["-client"], + source_version = "7", + target_version = "7", +) + filegroup( name = "srcs", srcs = [ -- cgit v1.2.3