diff options
-rwxr-xr-x | scripts/bootstrap/compile.sh | 1 | ||||
-rw-r--r-- | src/BUILD | 51 | ||||
-rw-r--r-- | src/create_embedded_tools.py | 176 | ||||
-rwxr-xr-x | src/create_embedded_tools.sh | 94 | ||||
-rw-r--r-- | src/embedded_tools.bzl | 50 |
5 files changed, 271 insertions, 101 deletions
diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh index 494c1a109e..4280e003e0 100755 --- a/scripts/bootstrap/compile.sh +++ b/scripts/bootstrap/compile.sh @@ -345,6 +345,7 @@ function run_bazel_jar() { --ignore_unsupported_sandboxing \ --startup_time=329 --extract_data_time=523 \ --rc_source=/dev/null --isatty=1 \ + --build_python_zip \ "${client_env[@]}" \ --client_cwd="$(get_cwd)" \ "${@}" @@ -1,5 +1,7 @@ # Packaging +load(":embedded_tools.bzl", "srcsfile") + md5_cmd = "set -e -o pipefail && cat $(SRCS) | sort | %s | awk '{ print $$1; }' > $@" # TODO(bazel-team): find a better way to handle dylib extensions. @@ -29,7 +31,7 @@ filegroup( "//src/main/tools:process-wrapper", "//src/main/tools:linux-sandbox", "//tools/osx:xcode-locator", - ] + embedded_tools, + ] + embedded_tools_target, outs = ["install_base_key" + suffix], cmd = select({ ":darwin": md5_cmd % "/sbin/md5", @@ -37,7 +39,7 @@ filegroup( ":freebsd": md5_cmd % "/sbin/md5", "//conditions:default": md5_cmd % "md5sum", }), -) for suffix, embedded_tools in { +) for suffix, embedded_tools_target in { "": [":embedded_tools"], "_with_jdk": [":embedded_tools_with_jdk"], "_notools": [], @@ -120,11 +122,17 @@ filegroup( ], ) -[genrule( - name = "embedded_tools" + suffix, +py_binary( + name = "create_embedded_tools", + srcs = ["create_embedded_tools.py"], +) + +# TODO(philwo): Clean this up, once a Bazel that can run py_binary and sh_binary +# in Skylark rules on Windows has been released. +[filegroup( + name = "embedded_tools" + suffix + "_srcs", srcs = [ "BUILD.tools", - ":create_embedded_tools.sh", "//tools:embedded_tools_srcs", "//third_party:gpl-srcs", "//third_party/java/j2objc:embedded_tools_srcs", @@ -209,13 +217,42 @@ filegroup( "@openjdk_linux//file", ], }) if (suffix == "_with_jdk") else []), - outs = ["embedded_tools" + suffix + ".zip"], - cmd = "$(location :create_embedded_tools.sh) $@ $(SRCS)", ) for suffix in [ "", "_with_jdk", ]] +[srcsfile( + name = "embedded_tools" + suffix + "_params", + srcs = [":embedded_tools" + suffix + "_srcs"], + out = "embedded_tools" + suffix + ".params", +) for suffix in [ + "", + "_with_jdk", +]] + +genrule( + name = "embedded_tools", + srcs = [ + ":embedded_tools_params", + ":embedded_tools_srcs", + ], + outs = ["embedded_tools.zip"], + cmd = "$(location :create_embedded_tools) \"$@\" $(location :embedded_tools_params)", + tools = [":create_embedded_tools"], +) + +genrule( + name = "embedded_tools_with_jdk", + srcs = [ + ":embedded_tools_with_jdk_params", + ":embedded_tools_with_jdk_srcs", + ], + outs = ["embedded_tools_with_jdk.zip"], + cmd = "$(location :create_embedded_tools) \"$@\" $(location :embedded_tools_with_jdk_params)", + tools = [":create_embedded_tools"], +) + [genrule( name = "package-zip" + suffix, srcs = ([":embedded_tools" + suffix + ".zip"] if embed else []) + [ diff --git a/src/create_embedded_tools.py b/src/create_embedded_tools.py new file mode 100644 index 0000000000..e9383140e1 --- /dev/null +++ b/src/create_embedded_tools.py @@ -0,0 +1,176 @@ +# pylint: disable=g-bad-file-header +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Creates the embedded_tools.zip that is part of the Bazel binary.""" + +import fnmatch +import os +import os.path +import re +import stat +import sys +import tarfile +import zipfile + +output_paths = [ + ('*tools/jdk/BUILD*', lambda x: 'tools/jdk/BUILD'), + ('*tools/platforms/platforms.BUILD', lambda x: 'platforms/BUILD'), + ('*tools/platforms/*', lambda x: 'platforms/' + os.path.basename(x)), + ('*JavaBuilder*_deploy.jar', lambda x: 'tools/jdk/' + os.path.basename(x)), + ('*JacocoCoverage*_deploy.jar', + lambda x: 'tools/jdk/JacocoCoverage_deploy.jar'), + ('*turbine_deploy.jar', lambda x: 'tools/jdk/turbine_deploy.jar'), + ('*javac-9-dev-r4023-2.jar', + lambda x: 'third_party/java/jdk/langtools/javac-9-dev-r4023-2.jar'), + ('*SingleJar_deploy.jar', + lambda x: 'tools/jdk/singlejar/SingleJar_deploy.jar'), + ('*GenClass_deploy.jar', lambda x: 'tools/jdk/GenClass_deploy.jar'), + ('*ExperimentalRunner_deploy.jar', + lambda x: 'tools/jdk/ExperimentalTestRunner_deploy.jar'), + ('*Runner_deploy.jar', lambda x: 'tools/jdk/TestRunner_deploy.jar'), + ('*singlejar', lambda x: 'tools/jdk/singlejar/singlejar'), + ('*launcher.exe', lambda x: 'tools/launcher/launcher.exe'), + ('*ijar.exe', lambda x: 'tools/jdk/ijar/ijar.exe'), + ('*ijar', lambda x: 'tools/jdk/ijar/ijar'), + ('*zipper.exe', lambda x: 'tools/zip/zipper/zipper.exe'), + ('*zipper', lambda x: 'tools/zip/zipper/zipper'), + ('*src/objc_tools/*', + lambda x: 'tools/objc/precomp_' + os.path.basename(x)), + ('*xcode*StdRedirect.dylib', lambda x: 'tools/objc/StdRedirect.dylib'), + ('*xcode*make_hashed_objlist.py', + lambda x: 'tools/objc/make_hashed_objlist.py'), + ('*xcode*realpath', lambda x: 'tools/objc/realpath'), + ('*xcode*xcode-locator', lambda x: 'tools/objc/xcode-locator'), + ('*src/tools/xcode/*.sh', lambda x: 'tools/objc/' + os.path.basename(x)), + ('*src/tools/xcode/*', + lambda x: 'tools/objc/' + os.path.basename(x) + '.sh'), + ('*external/openjdk_*/file/*.tar.gz', lambda x: 'jdk.tar.gz'), + ('*external/openjdk_*/file/*.zip', lambda x: 'jdk.zip'), + ('*', lambda x: re.sub(r'^.*bazel-out/[^/]*/bin/', '', x, count=1)), +] + + +def get_output_path(path): + for pattern, transformer in output_paths: + if fnmatch.fnmatch(path.replace('\\', '/'), pattern): + # BUILD.tools are stored as BUILD files. + return transformer(path).replace('/BUILD.tools', '/BUILD') + + +def is_mode_executable(mode): + return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) > 0 + + +def is_executable(path): + return is_mode_executable(os.stat(path)[stat.ST_MODE]) + + +def get_input_files(argsfile): + """Returns a sorted list of tuples (archive_file, input_file). + + This describes the files that should be put into the generated archive. + + Args: + argsfile: The file containing the list of input files. + """ + with open(argsfile, 'r') as f: + input_files = set(x.strip() for x in f.readlines()) + + result = {} + for input_file in input_files: + # If we have both a BUILD and a BUILD.tools file, take the latter only. + if (os.path.basename(input_file) == 'BUILD' and + input_file + '.tools' in input_files): + continue + + # This gives us the same behavior as the older bash version of this + # tool: If two input files map to the same output files, the one that + # comes last in the list of input files overrides all earlier ones. + result[get_output_path(input_file)] = input_file + + # By sorting the file list, the resulting ZIP file will not be reproducible + # and deterministic. + return sorted(result.items()) + + +def copy_jdk_into_archive(output_zip, archive_file, input_file): + # The JDK is special - it's extracted instead of copied. + if archive_file.endswith('.tar.gz'): + with tarfile.open(input_file, 'r', errorlevel=2) as jdk_tar: + while True: + jdk_tarinfo = jdk_tar.next() + if jdk_tarinfo is None: + break + # Rename the first folder to 'jdk', because Bazel looks for a + # bundled JDK in the embedded tools using that folder name. + filename = 'jdk/' + '/'.join(jdk_tarinfo.name.split('/')[1:]) + zipinfo = zipfile.ZipInfo(filename, (1980, 1, 1, 0, 0, 0)) + if jdk_tarinfo.isreg(): + if is_mode_executable(jdk_tarinfo.mode): + zipinfo.external_attr = 0o755 << 16 + else: + zipinfo.external_attr = 0o644 << 16 + zipinfo.compress_type = zipfile.ZIP_DEFLATED + output_zip.writestr(zipinfo, jdk_tar.extractfile(jdk_tarinfo).read()) + elif jdk_tarinfo.issym(): + # 0120000 originally comes from the definition of S_IFLNK and + # marks a symbolic link in the Zip file format. + zipinfo.external_attr = 0o120000 << 16 + output_zip.writestr(zipinfo, jdk_tarinfo.linkname) + else: + # Ignore directories, hard links, special files, ... + pass + elif archive_file.endswith('.zip'): + with zipfile.ZipFile(input_file, 'r') as jdk_zip: + for jdk_zipinfo in jdk_zip.infolist(): + # Rename the first folder to 'jdk', because Bazel looks for a + # bundled JDK in the embedded tools using that folder name. + filename = 'jdk/' + '/'.join(jdk_zipinfo.filename.split('/')[1:]) + zipinfo = zipfile.ZipInfo(filename, (1980, 1, 1, 0, 0, 0)) + if is_mode_executable(jdk_zipinfo.external_attr >> 16 & 0xFFFF): + zipinfo.external_attr = 0o755 << 16 + else: + zipinfo.external_attr = 0o644 << 16 + zipinfo.compress_type = jdk_zipinfo.compress_type + output_zip.writestr(zipinfo, jdk_zip.read(jdk_zipinfo)) + + +def main(): + output_zip = os.path.join(os.getcwd(), sys.argv[1]) + input_files = get_input_files(sys.argv[2]) + + # Copy all the input_files into output_zip. + with zipfile.ZipFile(output_zip, 'w', zipfile.ZIP_DEFLATED) as output_zip: + zipinfo = zipfile.ZipInfo('WORKSPACE', (1980, 1, 1, 0, 0, 0)) + zipinfo.external_attr = 0o644 << 16 + output_zip.writestr(zipinfo, 'workspace(name = "bazel_tools")\n') + + zipinfo = zipfile.ZipInfo('tools/defaults/BUILD', (1980, 1, 1, 0, 0, 0)) + zipinfo.external_attr = 0o644 << 16 + output_zip.writestr(zipinfo, '') + + for archive_file, input_file in input_files: + if os.path.basename(archive_file) in ('jdk.tar.gz', 'jdk.zip'): + copy_jdk_into_archive(output_zip, archive_file, input_file) + else: + zipinfo = zipfile.ZipInfo(archive_file, (1980, 1, 1, 0, 0, 0)) + zipinfo.external_attr = 0o755 << 16 if is_executable( + input_file) else 0o644 << 16 + zipinfo.compress_type = zipfile.ZIP_DEFLATED + with open(input_file, 'rb') as f: + output_zip.writestr(zipinfo, f.read()) + + +if __name__ == '__main__': + main() diff --git a/src/create_embedded_tools.sh b/src/create_embedded_tools.sh deleted file mode 100755 index b5aaa6bdba..0000000000 --- a/src/create_embedded_tools.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh - -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -# This script is used to create the directory tree embedded into the Bazel -# binary that is used as the default source for the @bazel_tools repository. -# It shuffles around files compiled in other rules, then zips them up. - -OUTPUT="${PWD}/$1" -shift - -TMP_DIR=${TMPDIR:-/tmp} -PACKAGE_DIR="$(mktemp -d ${TMP_DIR%%/}/bazel.XXXXXXXX)" -mkdir -p "${PACKAGE_DIR}" -trap "rm -fr \"${PACKAGE_DIR}\"" EXIT - -for i in $*; do - - case "$i" in - *tools/jdk/BUILD*) OUTPUT_PATH=tools/jdk/BUILD ;; - *tools/platforms/platforms.BUILD) OUTPUT_PATH=platforms/BUILD ;; - *tools/platforms/*) OUTPUT_PATH=platforms/${i##*/} ;; - *JavaBuilder*_deploy.jar) OUTPUT_PATH=tools/jdk/${i##*/} ;; - *JacocoCoverage*_deploy.jar) OUTPUT_PATH=tools/jdk/JacocoCoverage_deploy.jar ;; - *turbine_deploy.jar) OUTPUT_PATH=tools/jdk/turbine_deploy.jar ;; - *javac-9-dev-r4023-2.jar) OUTPUT_PATH=third_party/java/jdk/langtools/javac-9-dev-r4023-2.jar ;; - *javac7.jar) OUTPUT_PATH=third_party/java/jdk/langtools/javac7.jar ;; - *SingleJar_deploy.jar) OUTPUT_PATH=tools/jdk/singlejar/SingleJar_deploy.jar ;; - *GenClass_deploy.jar) OUTPUT_PATH=tools/jdk/GenClass_deploy.jar ;; - *ExperimentalRunner_deploy.jar) OUTPUT_PATH=tools/jdk/ExperimentalTestRunner_deploy.jar ;; - *Runner_deploy.jar) OUTPUT_PATH=tools/jdk/TestRunner_deploy.jar ;; - *singlejar) OUTPUT_PATH=tools/jdk/singlejar/singlejar ;; - *ijar.exe) OUTPUT_PATH=tools/jdk/ijar/ijar.exe ;; - *ijar) OUTPUT_PATH=tools/jdk/ijar/ijar ;; - *zipper.exe) OUTPUT_PATH=tools/zip/zipper/zipper.exe ;; - *zipper) OUTPUT_PATH=tools/zip/zipper/zipper ;; - *src/objc_tools/*) OUTPUT_PATH=tools/objc/precomp_${i##*/} ;; - *xcode*StdRedirect.dylib) OUTPUT_PATH=tools/objc/StdRedirect.dylib ;; - *xcode*make_hashed_objlist.py) OUTPUT_PATH=tools/objc/make_hashed_objlist.py ;; - *xcode*realpath) OUTPUT_PATH=tools/objc/realpath ;; - *xcode*xcode-locator) OUTPUT_PATH=tools/objc/xcode-locator ;; - *src/tools/xcode/*.sh) OUTPUT_PATH=tools/objc/${i##*/} ;; - *src/tools/xcode/*) OUTPUT_PATH=tools/objc/${i##*/}.sh ;; - *external/openjdk_*/file/*.tar.gz) OUTPUT_PATH=jdk.tar.gz ;; - *external/openjdk_*/file/*.zip) OUTPUT_PATH=jdk.zip ;; - *) OUTPUT_PATH=$(echo $i | sed 's_^.*bazel-out/[^/]*/bin/__') ;; - esac - - mkdir -p "${PACKAGE_DIR}/$(dirname "${OUTPUT_PATH}")" - cp "$i" "${PACKAGE_DIR}/${OUTPUT_PATH}" - chmod u+w "${PACKAGE_DIR}/${OUTPUT_PATH}" -done - -if [ -f ${PACKAGE_DIR}/jdk.tar.gz ]; then - tar xz -C ${PACKAGE_DIR} -f ${PACKAGE_DIR}/jdk.tar.gz - rm ${PACKAGE_DIR}/jdk.tar.gz - mv ${PACKAGE_DIR}/zulu* ${PACKAGE_DIR}/jdk -fi - -if [ -f ${PACKAGE_DIR}/jdk.zip ]; then - unzip -d ${PACKAGE_DIR} ${PACKAGE_DIR}/jdk.zip > /dev/null - rm ${PACKAGE_DIR}/jdk.zip - mv ${PACKAGE_DIR}/zulu* ${PACKAGE_DIR}/jdk -fi - -if [ ! -f ${PACKAGE_DIR}/third_party/java/jdk/langtools/javac-9-dev-r4023-2.jar ]; then - cp ${PACKAGE_DIR}/third_party/java/jdk/langtools/javac7.jar \ - ${PACKAGE_DIR}/third_party/java/jdk/langtools/javac-9-dev-r4023-2.jar -fi - -cat > "${PACKAGE_DIR}/WORKSPACE" <<EOF -workspace(name = "bazel_tools") -EOF -mkdir -p "${PACKAGE_DIR}/tools/defaults" -touch "${PACKAGE_DIR}/tools/defaults/BUILD" -for i in $(find "${PACKAGE_DIR}" -name BUILD.tools); do - mv "$i" "$(dirname "$i")/BUILD" -done -find "${PACKAGE_DIR}" -exec touch -t 198001010000.00 '{}' '+' -(cd "${PACKAGE_DIR}" && find . -type f | sort | zip -qDX@ "${OUTPUT}") diff --git a/src/embedded_tools.bzl b/src/embedded_tools.bzl new file mode 100644 index 0000000000..b36cc142cf --- /dev/null +++ b/src/embedded_tools.bzl @@ -0,0 +1,50 @@ +# pylint: disable=g-bad-file-header +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains Skylark rules used to build the embedded_tools.zip.""" + +def _embedded_tools(ctx): + # The list of arguments we pass to the script. + args_file = ctx.new_file(ctx.label.name + ".params") + ctx.file_action(output=args_file, content="\n".join([f.path for f in ctx.files.srcs])) + # Action to call the script. + ctx.action( + inputs=ctx.files.srcs, + outputs=[ctx.outputs.out], + arguments=[ctx.outputs.out.path, args_file.path], + progress_message="Creating embedded tools: %s" % ctx.outputs.out.short_path, + executable=ctx.executable.tool) + +embedded_tools = rule( + implementation=_embedded_tools, + attrs={ + "srcs": attr.label_list(allow_files=True), + "out": attr.output(mandatory=True), + "tool": attr.label(executable=True, cfg="host", allow_files=True, + default=Label("//src:create_embedded_tools_sh")) + } +) + +def _srcsfile(ctx): + ctx.file_action( + output=ctx.outputs.out, + content="\n".join([f.path for f in ctx.files.srcs])) + +srcsfile = rule( + implementation=_srcsfile, + attrs={ + "srcs": attr.label_list(allow_files=True), + "out": attr.output(mandatory=True), + } +) |