distfile: pack the archives needed later in the build

...and point --experimental_distdir there, so that offline builds
are again possible out of the distribution archive.

Related #5175.
Fixes #5202.
To be cherry-picked for #5056.

Change-Id: I634296e9d83e4e18ed966b42f35acc63061259d9
PiperOrigin-RevId: 197866998

5 files changed, 69 insertions, 2 deletions

diff --git a/BUILD b/BUILD
index 3687a2ed2e..2af8643ba4 100644
--- a/BUILD
+++ b/BUILD
@@ -40,7 +40,7 @@ filegroup(
 
 filegroup(
     name = "workspace-file",
-    srcs = [":WORKSPACE"],
+    srcs = [":WORKSPACE", ":distdir.bzl"],
     visibility = [
         "//src/test/shell/bazel:__subpackages__",
     ],
@@ -83,6 +83,7 @@ genrule(
         ":bazel-srcs",
         "//src:derived_java_srcs",
         "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:bootstrap_autocodec.tar",
+        "@additional_distfiles//:archives.tar",
     ],
     outs = ["bazel-distfile.zip"],
     cmd = "$(location :combine_distfiles) $@ $(SRCS)",
@@ -97,6 +98,7 @@ genrule(
         ":bazel-srcs",
         "//src:derived_java_srcs",
         "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:bootstrap_autocodec.tar",
+        "@additional_distfiles//:archives.tar",
     ],
     outs = ["bazel-distfile.tar"],
     cmd = "$(location :combine_distfiles_to_tar.sh) $@ $(SRCS)",
diff --git a/WORKSPACE b/WORKSPACE
index bcdce0caf7..a0669072cb 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -99,6 +99,21 @@ http_archive(
     strip_prefix = "desugar_jdk_libs-f5e6d80c6b4ec6b0a46603f72b015d45cf3c11cd",
 )
 
+load("//:distdir.bzl", "distdir_tar")
+distdir_tar(
+    name = "additional_distfiles",
+    dirname = "derived/distdir",
+    archives = ["f5e6d80c6b4ec6b0a46603f72b015d45cf3c11cd.zip"],
+    urls = {
+        "f5e6d80c6b4ec6b0a46603f72b015d45cf3c11cd.zip" :
+        ["https://github.com/google/desugar_jdk_libs/archive/f5e6d80c6b4ec6b0a46603f72b015d45cf3c11cd.zip"],
+    },
+    sha256 = {
+        "f5e6d80c6b4ec6b0a46603f72b015d45cf3c11cd.zip" :
+        "c80f3f3d442d8a6ca7adc83f90ecd638c3864087fdd6787ffac070b6f1cc8f9b",
+  },
+)
+
 # OpenJDK distributions used to create a version of Bazel bundled with the OpenJDK.
 http_file(
     name = "openjdk_linux",
diff --git a/distdir.bzl b/distdir.bzl
new file mode 100644
index 0000000000..f5a130de27
--- /dev/null
+++ b/distdir.bzl
@@ -0,0 +1,46 @@
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Defines a repository rule that generates an archive consisting of the specified files to fetch"""
+
+_BUILD="""
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
+pkg_tar(
+  name="archives",
+  srcs = {srcs},
+  package_dir = "{dirname}",
+  visibility = ["//visibility:public"],
+)
+
+"""
+
+def _distdir_tar_impl(ctx):
+  for name in ctx.attr.archives:
+      ctx.download(ctx.attr.urls[name], name, ctx.attr.sha256[name], False)
+  ctx.file("WORKSPACE", "")
+  ctx.file("BUILD",
+           _BUILD.format(srcs=ctx.attr.archives, dirname=ctx.attr.dirname))
+
+_distdir_tar_attrs = {
+    "archives" : attr.string_list(),
+    "sha256" : attr.string_dict(),
+    "urls" : attr.string_list_dict(),
+    "dirname" : attr.string(default="distdir"),
+}
+
+
+distdir_tar = repository_rule(
+    implementation = _distdir_tar_impl,
+    attrs = _distdir_tar_attrs,
+)
diff --git a/scripts/bootstrap/bootstrap.sh b/scripts/bootstrap/bootstrap.sh
index 61d4f71910..260e6ddc61 100755
--- a/scripts/bootstrap/bootstrap.sh
+++ b/scripts/bootstrap/bootstrap.sh
@@ -37,6 +37,7 @@ _BAZEL_ARGS="--java_toolchain=//src/java_tools/buildjar:bootstrap_toolchain \
       --nojava_header_compilation \
       --strategy=Javac=worker --worker_quit_after_build --ignore_unsupported_sandboxing \
       --compilation_mode=opt \
+      --experimental_distdir=derived/distdir \
       ${EXTRA_BAZEL_ARGS:-}"
 
 if [ -z "${BAZEL-}" ]; then
diff --git a/src/test/shell/testenv.sh b/src/test/shell/testenv.sh
index 44fc1a07cf..9de4521fea 100755
--- a/src/test/shell/testenv.sh
+++ b/src/test/shell/testenv.sh
@@ -80,6 +80,7 @@ BAZEL_RUNFILES="$TEST_SRCDIR/io_bazel"
 
 # WORKSPACE file
 workspace_file="${BAZEL_RUNFILES}/WORKSPACE"
+distdir_bzl_file="${BAZEL_RUNFILES}/distdir.bzl"
 
 # Bazel
 bazel_tree="$(rlocation io_bazel/src/test/shell/bazel/doc-srcs.zip)"
@@ -136,8 +137,10 @@ def list_source_repository(name):
   pass
 EOF
   touch src/test/shell/bazel/BUILD
-  rm -f WORKSPACE
+  rm -f WORKSPACE distdir.bzl
   ln -sf ${workspace_file} WORKSPACE
+  touch BUILD
+  ln -sf ${distdir_bzl_file} distdir.bzl
 }
 
 # This function copies the tools directory from Bazel.