Improve scalac: use ijar and download scalac

--
Change-Id: I22fb6cc5a37f5c6c216dc3abbc915d1be3fd5e6b
Reviewed-on: https://bazel-review.googlesource.com/#/c/2410/
MOS_MIGRATED_REVID=109717610

5 files changed, 141 insertions, 43 deletions

diff --git a/WORKSPACE b/WORKSPACE
index 4bb08eda15..ebe5700f2f 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -43,3 +43,12 @@ bind(
     #   actual = "@androidndk//:files",
     actual = "//:dummy",
 )
+
+# only used for the scala rule
+new_http_archive(
+    name = "scala",
+    strip_prefix = "scala-2.11.7",
+    sha256 = "ffe4196f13ee98a66cf54baffb0940d29432b2bd820bd0781a8316eec22926d0",
+    url = "http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz",
+    build_file = "tools/build_defs/scala/scala.BUILD",
+)
diff --git a/tools/build_defs/scala/BUILD b/tools/build_defs/scala/BUILD
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools/build_defs/scala/BUILD
diff --git a/tools/build_defs/scala/README.md b/tools/build_defs/scala/README.md
index b72433939f..49b210bef7 100644
--- a/tools/build_defs/scala/README.md
+++ b/tools/build_defs/scala/README.md
@@ -3,7 +3,7 @@
 <div class="toc">
   <h2>Rules</h2>
   <ul>
-    <li><a href="#scala_library">scala_library</a></li>
+    <li><a href="#scala_library">scala_library/scala_macro_library</a></li>
     <li><a href="#scala_binary">scala_binary</a></li>
   </ul>
 </div>
@@ -11,23 +11,41 @@
 ## Overview
 
 This rule is used for building [Scala][scala] projects with Bazel. There are
-currently two rules, `scala_library` and `scala_binary`. More features will be
-added in the future, e.g. `scala_test`.
+currently three rules, `scala_library`, `scala_macro_library` and
+`scala_binary`. More features will be added in the future, e.g. `scala_test`.
+
+In order to use this build rule, you must add the following to your WORKSPACE
+file:
+```
+new_http_archive(
+    name = "scala",
+    strip_prefix = "scala-2.11.7",
+    sha256 = "ffe4196f13ee98a66cf54baffb0940d29432b2bd820bd0781a8316eec22926d0",
+    url = "http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz",
+    build_file = "tools/build_defs/scala/scala.BUILD",
+)
+```
 
 [scala]: http://www.scala-lang.org/
 
 <a name="scala_library"></a>
-## scala_library
+## scala_library / scala_macro_library
 
 ```python
 scala_library(name, srcs, deps, data, main_class, resources, scalacopts, jvm_flags)
+scala_macro_library(name, srcs, deps, data, main_class, resources, scalacopts, jvm_flags)
 ```
 
-`scala_library` generates a `.jar` file from `.scala` source files.
-In order to make a java rule use this jar file, use the `java_import` rule.
+`scala_library` generates a `.jar` file from `.scala` source files. This rule
+also creates an interface jar to avoid recompiling downstream targets unless
+then interface changes.
+
+`scala_macro_library` generates a `.jar` file from `.scala` source files when
+they contain macros. For macros, there are no interface jars because the macro
+code is executed at compile time. For best performance, you want very granular
+targets until such time as the zinc incremental compiler can be supported.
 
-The current implementation assumes that the files `/usr/bin/scalac` and
-`/usr/share/java/scala-library.jar` exist.
+In order to make a java rule use this jar file, use the `java_import` rule.
 
 <table class="table table-condensed table-bordered table-params">
   <colgroup>
@@ -124,7 +142,7 @@ The current implementation assumes that the files `/usr/bin/scalac` and
 scala_binary(name, srcs, deps, data, main_class, resources, scalacopts, jvm_flags)
 ```
 
-`scala_binary` generates a Scala executable. It may depend on `scala_library`
+`scala_binary` generates a Scala executable. It may depend on `scala_library`, `scala_macro_library`
 and `java_library` rules.
 
 A `scala_binary` requires a `main_class` attribute.
diff --git a/tools/build_defs/scala/scala.BUILD b/tools/build_defs/scala/scala.BUILD
new file mode 100644
index 0000000000..8d938609a7
--- /dev/null
+++ b/tools/build_defs/scala/scala.BUILD
@@ -0,0 +1,20 @@
+# scala.BUILD
+exports_files([
+  "bin/scala",
+  "bin/scalac",
+  "bin/scaladoc",
+  "lib/akka-actor_2.11-2.3.10.jar",
+  "lib/config-1.2.1.jar",
+  "lib/jline-2.12.1.jar",
+  "lib/scala-actors-2.11.0.jar",
+  "lib/scala-actors-migration_2.11-1.1.0.jar",
+  "lib/scala-compiler.jar",
+  "lib/scala-continuations-library_2.11-1.0.2.jar",
+  "lib/scala-continuations-plugin_2.11.7-1.0.2.jar",
+  "lib/scala-library.jar",
+  "lib/scala-parser-comscala-2.11.7/binators_2.11-1.0.4.jar",
+  "lib/scala-reflect.jar",
+  "lib/scala-swing_2.11-1.0.2.jar",
+  "lib/scala-xml_2.11-1.0.4.jar",
+  "lib/scalap-2.11.7.jar",
+])
diff --git a/tools/build_defs/scala/scala.bzl b/tools/build_defs/scala/scala.bzl
index f92bf55b63..758c441aef 100644
--- a/tools/build_defs/scala/scala.bzl
+++ b/tools/build_defs/scala/scala.bzl
@@ -17,10 +17,6 @@
 
 _scala_filetype = FileType([".scala"])
 
-# TODO(bazel-team): Add local_repository to properly declare the dependency.
-_scala_library_path = "/usr/share/java/scala-library.jar"
-_scalac_path = "/usr/bin/scalac"
-
 def _adjust_resources_path(path):
   dir_1, dir_2, rel_path = path.partition("resources")
   if rel_path:
@@ -30,40 +26,47 @@ def _adjust_resources_path(path):
     return dir_1 + dir_2, rel_path
   return "", path
 
-def _compile(ctx, jars):
+def _compile(ctx, jars, buildijar):
   res_cmd = ""
   for f in ctx.files.resources:
     c_dir, res_path = _adjust_resources_path(f.path)
     change_dir = "-C " + c_dir if c_dir else ""
     res_cmd = "\njar uf {out} " + change_dir + " " + res_path
+  ijar_cmd = ""
+  if buildijar:
+    ijar_cmd = "\n{ijar} {out} {ijar_out}".format(
+      ijar=ctx.file._ijar.path,
+      out=ctx.outputs.jar.path,
+      ijar_out=ctx.outputs.ijar.path)
   cmd = """
 set -e
 mkdir -p {out}_tmp
 {scalac} {scala_opts} {jvm_flags} -classpath "{jars}" $@ -d {out}_tmp
 # Make jar file deterministic by setting the timestamp of files
-touch -t 198001010000 $(find {out}_tmp)
+find {out}_tmp -exec touch -t 198001010000 {{}} \;
 touch -t 198001010000 {manifest}
 jar cmf {manifest} {out} -C {out}_tmp .
-""" + res_cmd
+""" + ijar_cmd + res_cmd
   cmd = cmd.format(
-      scalac=_scalac_path,
+      scalac=ctx.file._scalac.path,
       scala_opts=" ".join(ctx.attr.scalacopts),
       jvm_flags=" ".join(["-J" + flag for flag in ctx.attr.jvm_flags]),
       out=ctx.outputs.jar.path,
       manifest=ctx.outputs.manifest.path,
-      jars=":".join([j.path for j in jars]))
-
+      jars=":".join([j.path for j in jars]),)
+  outs = [ctx.outputs.jar]
+  if buildijar:
+    outs.extend([ctx.outputs.ijar])
   ctx.action(
       inputs=list(jars) + ctx.files.srcs + [ctx.outputs.manifest],
-      outputs=[ctx.outputs.jar],
+      outputs=outs,
       command=cmd,
       progress_message="scala %s" % ctx.label,
       arguments=[f.path for f in ctx.files.srcs])
 
-
 def _write_manifest(ctx):
-  cp = "/usr/share/java/scala-library.jar"
-  manifest = "Class-Path: %s\n" % cp
+  # TODO(bazel-team): I don't think this classpath is what you want
+  manifest = "Class-Path: %s\n" % ctx.file._scalalib.path
   if getattr(ctx.attr, "main_class", ""):
     manifest += "Main-Class: %s\n" % ctx.attr.main_class
 
@@ -85,42 +88,61 @@ java -cp {cp} {name} "$@"
       output=ctx.outputs.executable,
       content=content)
 
-
-def _collect_jars(ctx):
-  jars = set()
+def _collect_comp_run_jars(ctx):
+  compile_jars = set()
+  runtime_jars = set()
   for target in ctx.attr.deps:
-    if hasattr(target, "jar_files"):
-      jars += target.jar_files
-    elif hasattr(target, "java"):
-      jars += target.java.transitive_runtime_deps
-  return jars
-
+    if hasattr(target, "runtime_jar_files"):
+      runtime_jars += target.runtime_jar_files
+    if hasattr(target, "interface_jar_files"):
+      compile_jars += target.interface_jar_files
+    if hasattr(target, "java"):
+      runtime_jars += target.java.transitive_runtime_deps
+      #see JavaSkylarkApiProvider.java, this is just the compile-time deps
+      compile_jars += target.java.transitive_deps
+  return (compile_jars, runtime_jars)
 
 def _scala_library_impl(ctx):
-  jars = _collect_jars(ctx)
+  (cjars, rjars) = _collect_comp_run_jars(ctx)
   _write_manifest(ctx)
-  _compile(ctx, jars)
+  _compile(ctx, cjars, True)
 
-  jars += [ctx.outputs.jar]
+  cjars += [ctx.outputs.ijar]
+  rjars += [ctx.outputs.jar]
   runfiles = ctx.runfiles(
-      files = list(jars),
+      files = list(rjars),
       collect_data = True)
   return struct(
-      files=jars,
-      jar_files=jars,
+      runtime_jar_files=rjars,
+      interface_jar_files=cjars,
       runfiles=runfiles)
 
+def _scala_macro_library_impl(ctx):
+  (cjars, rjars) = _collect_comp_run_jars(ctx)
+  _write_manifest(ctx)
+  _compile(ctx, cjars, False)
+
+  rjars += [ctx.outputs.jar]
+  # macro code needs to be available at compiletime
+  cjars += [ctx.outputs.jar]
+  runfiles = ctx.runfiles(
+      files = list(rjars),
+      collect_data = True)
+  return struct(
+      runtime_jar_files=rjars,
+      interface_jar_files=cjars,
+      runfiles=runfiles)
 
 def _scala_binary_impl(ctx):
-  jars = _collect_jars(ctx)
+  (cjars, rjars) = _collect_comp_run_jars(ctx)
   _write_manifest(ctx)
-  _compile(ctx, jars)
+  _compile(ctx, cjars, False)
 
-  jars += [ctx.outputs.jar]
-  _write_launcher(ctx, jars)
+  rjars += [ctx.outputs.jar, ctx.file._scalalib]
+  _write_launcher(ctx, rjars)
 
   runfiles = ctx.runfiles(
-      files = list(jars) + [ctx.outputs.executable],
+      files = list(rjars) + [ctx.outputs.executable],
       collect_data = True)
   return struct(
       files=set([ctx.outputs.executable]),
@@ -139,6 +161,33 @@ scala_library = rule(
       "resources": attr.label_list(allow_files=True),
       "scalacopts": attr.string_list(),
       "jvm_flags": attr.string_list(),
+      "_ijar": attr.label(executable=True, default=Label("//tools/defaults:ijar"), single_file=True, allow_files=True),
+      "_scalac": attr.label(executable=True, default=Label("@scala//:bin/scalac"), single_file=True, allow_files=True),
+      "_scalalib": attr.label(default=Label("@scala//:lib/scala-library.jar"), single_file=True, allow_files=True),
+      },
+  outputs={
+      "jar": "%{name}_deploy.jar",
+      "ijar": "%{name}_ijar.jar",
+      "manifest": "%{name}_MANIFEST.MF",
+      },
+)
+
+scala_macro_library = rule(
+  implementation=_scala_macro_library_impl,
+  attrs={
+      "main_class": attr.string(),
+      "srcs": attr.label_list(
+          allow_files=_scala_filetype,
+          non_empty=True),
+      "deps": attr.label_list(),
+      "data": attr.label_list(allow_files=True, cfg=DATA_CFG),
+      "resources": attr.label_list(allow_files=True),
+      "scalacopts": attr.string_list(),
+      "jvm_flags": attr.string_list(),
+      "_ijar": attr.label(executable=True, default=Label("//tools/defaults:ijar"), single_file=True, allow_files=True),
+      "_scalac": attr.label(executable=True, default=Label("@scala//:bin/scalac"), single_file=True, allow_files=True),
+      "_scalalib": attr.label(default=Label("@scala//:lib/scala-library.jar"), single_file=True, allow_files=True),
+      "_scala-reflect": attr.label(default=Label("@scala//:lib/scala-reflect.jar"), single_file=True, allow_files=True),
       },
   outputs={
       "jar": "%{name}_deploy.jar",
@@ -158,6 +207,8 @@ scala_binary = rule(
       "resources": attr.label_list(allow_files=True),
       "scalacopts":attr.string_list(),
       "jvm_flags": attr.string_list(),
+      "_scalac": attr.label(executable=True, default=Label("@scala//:bin/scalac"), single_file=True, allow_files=True),
+      "_scalalib": attr.label(default=Label("@scala//:lib/scala-library.jar"), single_file=True, allow_files=True),
       },
   outputs={
       "jar": "%{name}_deploy.jar",