Make cc_configure script more extensible

This commit adds a few extension points to cc_configure script with
the ultimate goal to allow downloading the compiler and other tools
used for C++ build before generating the crosstool.

Specifically, we make the following changes:
  - Expose the implementation of a cc_autoconf repository rule under a
    name of cc_autoconf_impl.
  - Extend cc_autoconf_impl to allow overriding paths to build
    tools (i.e. compilers, linkers, etc.)
  - Allow to put any extra artifacts into the generated crosstool
    repository. All files inside 'extra_tools' folder are added into
    compiler_files, all_files and linker_files properties of the
    generated crosstool, so that they are available for the crosstool.

With these extension one can do the following to download the
compilers used for the build and configure the crosstool:
  - Create a repository_rule for the toolchain with a custom
    implementation.
  - In the implementation function of the repository rule:
    + Download the compilers and put them into `extra_tools` folder.
    + Run cc_autoconf_impl with overriden_tools set to relative paths
      of the compiler and other build tools in the extra_tools folder.

Change-Id: I51af6b504578963b3e97bcdd1ccb6d0a5fed1c3e
PiperOrigin-RevId: 179675911

4 files changed, 33 insertions, 19 deletions

diff --git a/tools/cpp/BUILD.tpl b/tools/cpp/BUILD.tpl
index 8db745f06e..9f3fd764b5 100644
--- a/tools/cpp/BUILD.tpl
+++ b/tools/cpp/BUILD.tpl
@@ -32,6 +32,11 @@ filegroup(
     srcs = ["cc_wrapper.sh"],
 )
 
+filegroup(
+    name = "compiler_deps",
+    srcs = glob(["extra_tools/**"]) + ["%{cc_compiler_deps}"],
+)
+
 # This is the entry point for --crosstool_top.  Toolchains are found
 # by lopping off the name of --crosstool_top and searching for
 # the "${CPU}" entry in the toolchains attribute.
@@ -46,12 +51,12 @@ cc_toolchain_suite(
 
 cc_toolchain(
     name = "cc-compiler-%{name}",
-    all_files = "%{cc_compiler_deps}",
-    compiler_files = "%{cc_compiler_deps}",
+    all_files = ":compiler_deps",
+    compiler_files = ":compiler_deps",
     cpu = "%{name}",
     dwp_files = ":empty",
     dynamic_runtime_libs = [":empty"],
-    linker_files = "%{cc_compiler_deps}",
+    linker_files = ":compiler_deps",
     objcopy_files = ":empty",
     static_runtime_libs = [":empty"],
     strip_files = ":empty",
diff --git a/tools/cpp/cc_configure.bzl b/tools/cpp/cc_configure.bzl
index b14b00e82c..610f5a7d54 100644
--- a/tools/cpp/cc_configure.bzl
+++ b/tools/cpp/cc_configure.bzl
@@ -19,7 +19,7 @@ load("@bazel_tools//tools/cpp:osx_cc_configure.bzl", "configure_osx_toolchain")
 load("@bazel_tools//tools/cpp:unix_cc_configure.bzl", "configure_unix_toolchain")
 load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value")
 
-def _impl(repository_ctx):
+def cc_autoconf_impl(repository_ctx, overriden_tools = dict()):
   repository_ctx.symlink(
       Label("@bazel_tools//tools/cpp:dummy_toolchain.bzl"), "dummy_toolchain.bzl")
   env = repository_ctx.os.environ
@@ -32,16 +32,17 @@ def _impl(repository_ctx):
     repository_ctx.symlink(Label("@bazel_tools//tools/cpp:CROSSTOOL"), "CROSSTOOL")
     repository_ctx.symlink(Label("@bazel_tools//tools/cpp:BUILD.static"), "BUILD")
   elif cpu_value == "x64_windows":
+    # TODO(ibiryukov): overriden_tools are only supported in configure_unix_toolchain.
+    # We might want to add that to Windows too(at least for msys toolchain).
     configure_windows_toolchain(repository_ctx)
   elif (cpu_value == "darwin" and
       ("BAZEL_USE_CPP_ONLY_TOOLCHAIN" not in env or env["BAZEL_USE_CPP_ONLY_TOOLCHAIN"] != "1")):
-    configure_osx_toolchain(repository_ctx)
+    configure_osx_toolchain(repository_ctx, overriden_tools)
   else:
-    configure_unix_toolchain(repository_ctx, cpu_value)
-
+    configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools)
 
 cc_autoconf = repository_rule(
-    implementation=_impl,
+    implementation = cc_autoconf_impl,
     environ = [
         "ABI_LIBC_VERSION",
         "ABI_VERSION",
diff --git a/tools/cpp/osx_cc_configure.bzl b/tools/cpp/osx_cc_configure.bzl
index bac3c0dfc9..1c75c6e271 100644
--- a/tools/cpp/osx_cc_configure.bzl
+++ b/tools/cpp/osx_cc_configure.bzl
@@ -50,14 +50,14 @@ def _get_escaped_xcode_cxx_inc_directories(repository_ctx, cc, xcode_toolchains)
   return include_dirs
 
 
-def configure_osx_toolchain(repository_ctx):
+def configure_osx_toolchain(repository_ctx, overriden_tools):
   """Configure C++ toolchain on macOS."""
   xcode_toolchains = []
   (xcode_toolchains, xcodeloc_err) = run_xcode_locator(
       repository_ctx,
       Label("@bazel_tools//tools/osx:xcode_locator.m"))
   if xcode_toolchains:
-    cc = find_cc(repository_ctx)
+    cc = find_cc(repository_ctx, overriden_tools = {})
     tpl(repository_ctx, "osx_cc_wrapper.sh", {
         "%{cc}": escape_string(str(cc)),
         "%{env}": escape_string(get_env(repository_ctx))
@@ -95,4 +95,4 @@ def configure_osx_toolchain(repository_ctx):
         Label("@bazel_tools//tools/osx/crosstool:CROSSTOOL.tpl"),
         {"%{cxx_builtin_include_directory}": "\n".join(escaped_cxx_include_directories)})
   else:
-    configure_unix_toolchain(repository_ctx, cpu_value = "darwin")
+    configure_unix_toolchain(repository_ctx, cpu_value = "darwin", overriden_tools = overriden_tools)
diff --git a/tools/cpp/unix_cc_configure.bzl b/tools/cpp/unix_cc_configure.bzl
index e7c099eeed..1e5207145c 100644
--- a/tools/cpp/unix_cc_configure.bzl
+++ b/tools/cpp/unix_cc_configure.bzl
@@ -76,10 +76,15 @@ def _build_tool_path(d):
     lines.append("  tool_path {name: \"%s\" path: \"%s\" }" % (k, escape_string(d[k])))
   return "\n".join(lines)
 
+def _find_tool(repository_ctx, tool, overriden_tools):
+  """Find a tool for repository, taking overriden tools into account."""
+  if tool in overriden_tools:
+    return overriden_tools[tool]
+  return which(repository_ctx, tool, "/usr/bin/" + tool)
 
-def _get_tool_paths(repository_ctx, darwin, cc):
+def _get_tool_paths(repository_ctx, darwin, cc, overriden_tools):
   """Compute the path to the various tools. Doesn't %-escape the result!"""
-  return {k: which(repository_ctx, k, "/usr/bin/" + k)
+  return {k: _find_tool(repository_ctx, k, overriden_tools)
           for k in [
               "ld",
               "cpp",
@@ -351,9 +356,12 @@ def _coverage_feature(darwin):
     }
   """
 
-
-def find_cc(repository_ctx):
+def find_cc(repository_ctx, overriden_tools):
   """Find the C++ compiler. Doesn't %-escape the result."""
+
+  if "gcc" in overriden_tools:
+    return overriden_tools["gcc"]
+
   cc_name = "gcc"
   cc_environ = repository_ctx.os.environ.get("CC")
   cc_paren = ""
@@ -372,14 +380,14 @@ def find_cc(repository_ctx):
          + " environment variable") % cc_paren)
   return cc
 
-
-def configure_unix_toolchain(repository_ctx, cpu_value):
+def configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools):
   """Configure C++ toolchain on Unix platforms."""
   repository_ctx.file("tools/cpp/empty.cc", "int main() {}")
   darwin = cpu_value == "darwin"
-  cc = find_cc(repository_ctx)
+  cc = find_cc(repository_ctx, overriden_tools)
   tool_paths = _get_tool_paths(repository_ctx, darwin,
-                               "cc_wrapper.sh" if darwin else str(cc))
+                               "cc_wrapper.sh" if darwin else str(cc),
+                               overriden_tools)
   crosstool_content = _crosstool_content(repository_ctx, cc, cpu_value, darwin)
   opt_content = _opt_content(darwin)
   dbg_content = _dbg_content()