Android,Windows: support long paths in tooling

aar_resources_extractor now supports long paths
on Windows.

If the script needs to extract a file from the
AAR where the destination path is too long, the
script will:
1. create a temporary junction under a short path,
   pointing to the destination directory (which
   has a long path)
2. extract the file under the junction
3. delete the junction and the temp directory

See https://github.com/bazelbuild/bazel/issues/3659

Change-Id: Ie85665b360a6514afaac546aaec8869224fe9d06
PiperOrigin-RevId: 167545085

8 files changed, 269 insertions, 4 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java b/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java
index 61295fc8e6..e3d5a12733 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java
@@ -176,6 +176,7 @@ public class AarImport implements RuleConfiguredTargetFactory {
   private static Action[] createSingleFileExtractorActions(RuleContext ruleContext, Artifact aar,
       String filename, Artifact outputArtifact) {
     return new SpawnAction.Builder()
+        .useDefaultShellEnvironment()
         .setExecutable(ruleContext.getExecutablePrerequisite(AarImportBaseRule.ZIPPER, Mode.HOST))
         .setMnemonic("AarFileExtractor")
         .setProgressMessage("Extracting %s from %s", filename, aar.getFilename())
@@ -193,6 +194,7 @@ public class AarImport implements RuleConfiguredTargetFactory {
   private static Action[] createAarResourcesExtractorActions(
       RuleContext ruleContext, Artifact aar, Artifact outputTree) {
     return new SpawnAction.Builder()
+        .useDefaultShellEnvironment()
         .setExecutable(
             ruleContext.getExecutablePrerequisite(
                 AarImportBaseRule.AAR_RESOURCES_EXTRACTOR, Mode.HOST))
@@ -210,6 +212,7 @@ public class AarImport implements RuleConfiguredTargetFactory {
   private static Action[] createAarEmbeddedJarsExtractorActions(RuleContext ruleContext,
       Artifact aar, Artifact jarsTreeArtifact, Artifact singleJarParamFile) {
     return new SpawnAction.Builder()
+        .useDefaultShellEnvironment()
         .setExecutable(
             ruleContext.getExecutablePrerequisite(
                 AarImportBaseRule.AAR_EMBEDDED_JARS_EXTACTOR, Mode.HOST))
@@ -248,6 +251,7 @@ public class AarImport implements RuleConfiguredTargetFactory {
       Artifact outputZip) {
     SpawnAction.Builder actionBuilder =
         new SpawnAction.Builder()
+            .useDefaultShellEnvironment()
             .setExecutable(
                 ruleContext.getExecutablePrerequisite(
                     AarImportBaseRule.AAR_NATIVE_LIBS_ZIP_CREATOR, Mode.HOST))
@@ -276,7 +280,7 @@ public class AarImport implements RuleConfiguredTargetFactory {
 
   // Adds the appropriate SpawnAction options depending on if SingleJar is a jar or not.
   private static SpawnAction.Builder singleJarSpawnActionBuilder(RuleContext ruleContext) {
-    SpawnAction.Builder builder = new SpawnAction.Builder();
+    SpawnAction.Builder builder = new SpawnAction.Builder().useDefaultShellEnvironment();
     Artifact singleJar = JavaToolchainProvider.fromRuleContext(ruleContext).getSingleJar();
     if (singleJar.getFilename().endsWith(".jar")) {
       builder
diff --git a/src/test/py/bazel/BUILD b/src/test/py/bazel/BUILD
index 3be9bc2e0f..f93093b0a1 100644
--- a/src/test/py/bazel/BUILD
+++ b/src/test/py/bazel/BUILD
@@ -20,6 +20,7 @@ py_library(
     visibility = [
         "//src/test/py/bazel:__pkg__",
         "//third_party/def_parser:__pkg__",
+        "//tools/android:__pkg__",
     ],
 )
 
diff --git a/tools/android/BUILD b/tools/android/BUILD
index 75a9956e67..340847e92e 100644
--- a/tools/android/BUILD
+++ b/tools/android/BUILD
@@ -135,7 +135,10 @@ py_test(
 py_binary(
     name = "aar_resources_extractor",
     srcs = ["aar_resources_extractor.py"],
-    deps = ["//third_party/py/gflags"],
+    deps = [
+        ":junction_lib",
+        "//third_party/py/gflags",
+    ],
 )
 
 py_test(
@@ -155,6 +158,30 @@ py_test(
     deps = [":resource_extractor"],
 )
 
+py_library(
+    name = "junction_lib",
+    srcs = ["junction.py"],
+    visibility = ["//visibility:private"],
+)
+
+py_test(
+    name = "junction_test",
+    srcs = select({
+        "//src:windows": ["junction_test.py"],
+        "//src:windows_msvc": ["junction_test.py"],
+        "//conditions:default": ["dummy_test.py"],
+    }),
+    main = select({
+        "//src:windows": "junction_test.py",
+        "//src:windows_msvc": "junction_test.py",
+        "//conditions:default": "dummy_test.py",
+    }),
+    deps = [
+        ":junction_lib",
+        "//src/test/py/bazel:test_base",
+    ],
+)
+
 filegroup(
     name = "srcs",
     srcs = glob(["**"]) + ["//tools/android/emulator:srcs"],
diff --git a/tools/android/BUILD.tools b/tools/android/BUILD.tools
index fde4587543..7110448c63 100644
--- a/tools/android/BUILD.tools
+++ b/tools/android/BUILD.tools
@@ -189,7 +189,10 @@ py_binary(
 py_binary(
     name = "aar_resources_extractor",
     srcs = ["aar_resources_extractor.py"],
-    deps = ["//third_party/py/gflags"],
+    deps = [
+        ":junction_lib",
+        "//third_party/py/gflags",
+    ],
 )
 
 py_binary(
@@ -197,6 +200,12 @@ py_binary(
     srcs = ["resource_extractor.py"],
 )
 
+py_library(
+    name = "junction_lib",
+    srcs = ["junction.py"],
+    visibility = ["//visibility:private"],
+)
+
 alias(
     name = "android_runtest",
     actual = "fail.sh",
diff --git a/tools/android/aar_resources_extractor.py b/tools/android/aar_resources_extractor.py
index 0e259b8028..eb0d432027 100644
--- a/tools/android/aar_resources_extractor.py
+++ b/tools/android/aar_resources_extractor.py
@@ -26,6 +26,7 @@ import os
 import sys
 import zipfile
 
+from tools.android import junction
 from third_party.py import gflags
 
 FLAGS = gflags.FLAGS
@@ -37,10 +38,19 @@ gflags.MarkFlagAsRequired("output_res_dir")
 
 
 def ExtractResources(aar, output_res_dir):
+  """Extract resource from an `aar` file to the `output_res_dir` directory."""
   aar_contains_no_resources = True
+  output_res_dir_abs = os.path.normpath(
+      os.path.join(os.getcwd(), output_res_dir))
   for name in aar.namelist():
     if name.startswith("res/"):
-      aar.extract(name, output_res_dir)
+      fullpath = os.path.normpath(os.path.join(output_res_dir_abs, name))
+      if os.name == "nt" and len(fullpath) >= 260:  # MAX_PATH in <windows.h>
+        with junction.TempJunction(os.path.dirname(fullpath)) as juncpath:
+          shortpath = os.path.join(juncpath, os.path.basename(fullpath))
+          aar.extract(name, shortpath)
+      else:
+        aar.extract(name, output_res_dir)
       aar_contains_no_resources = False
   if aar_contains_no_resources:
     empty_xml_filename = output_res_dir + "/res/values/empty.xml"
diff --git a/tools/android/dummy_test.py b/tools/android/dummy_test.py
new file mode 100644
index 0000000000..9afac3f06f
--- /dev/null
+++ b/tools/android/dummy_test.py
@@ -0,0 +1,22 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Dummy empty test.
+
+You can use this as a dummy test for platform-specific tests on platforms that
+are not under test. For example, we use this dummy test on Linux in a
+Windows-specific py_test rule.
+"""
+
+if __name__ == "__main__":
+  pass
diff --git a/tools/android/junction.py b/tools/android/junction.py
new file mode 100644
index 0000000000..7dc58b8c17
--- /dev/null
+++ b/tools/android/junction.py
@@ -0,0 +1,115 @@
+# pylint: disable=g-direct-third-party-import
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A class that creates junctions in temp directories on Windows.
+
+Only use this class on Windows, do not use on other platforms. Other platforms
+support longer paths than Windows, and also support symlinks. Windows only
+supports junctions (directory symlinks).
+
+Junctions are useful if you need to shorten a long path. A long path is one that
+is at least MAX_PATH (260) letters long. This is a constant in Windows, see
+<windows.h> and API documentation for file-handling functions such as
+CreateFileA.
+"""
+
+import os
+import subprocess
+import tempfile
+
+
+class JunctionCreationError(Exception):
+  """Raised when TempJunction fails to create an NTFS junction."""
+
+  def __init__(self, path, target, stdout):
+    Exception.__init__(
+        self,
+        "Could not create junction \"%s\" -> \"%s\"\nError from mklink:\n%s" %
+        (path, target, stdout))
+
+
+class TempJunction(object):
+  r"""Junction in a temp directory.
+
+  This object creates a temp directory and a junction under it. The junction
+  points to a user-specified path.
+
+  Use this object if you want to write files under long paths (absolute path at
+  least MAX_PATH (260) chars long). Pass the directory you want to "shorten" as
+  the initializer's argument. This object will create a junction under a shorter
+  path, that points to the long directory. The combined path of the junction and
+  files under it are more likely to be short than the original paths were.
+
+  Usage example:
+    with TempJunction("C:/some/long/path") as junc:
+      # `junc` created a temp directory and a junction in it that points to
+      # \\?\C:\some\long\path, and is itself shorter than that
+      shortpath = os.path.join(junc, "file.txt")
+      with open(shortpath, "w") as f:
+        ...do something with f...
+    # `junc` deleted the junction and its parent temp directory upon leaving
+    # the `with` statement's body
+    ...do something else...
+  """
+
+  def __init__(self, junction_target, testonly_mkdtemp=None):
+    """Initialize this object.
+
+    Args:
+      junction_target: string; an absolute Windows path; the __enter__ method
+        creates a junction that points to this path
+      testonly_mkdtemp: function(); for testing only; a custom function that
+        returns a temp directory path, you can use it to mock out
+        tempfile.mkdtemp
+    """
+    self._target = os.path.normpath(junction_target)
+    self._junction = None
+    self._mkdtemp = testonly_mkdtemp or tempfile.mkdtemp
+
+  def __enter__(self):
+    """Creates a temp directory and a junction in it, pointing to self._target.
+
+    This method is automatically called upon entering a `with` statement's body.
+
+    Returns:
+      The full path to the junction.
+    Raises:
+      JunctionCreationError: if `mklink` fails to create a junction
+    """
+    result = os.path.normpath(os.path.join(self._mkdtemp(), "j"))
+    proc = subprocess.Popen(
+        "cmd.exe /C mklink /J \"%s\" \"\\\\?\\%s\"" %
+        (result, os.path.normpath(self._target)),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+    exitcode = proc.wait()
+    if exitcode != 0:
+      stdout = proc.communicate()[0]
+      raise JunctionCreationError(result, self._target, stdout)
+    self._junction = result
+    return result
+
+  def __exit__(self, unused_type, unused_value, unused_traceback):
+    """Deletes the junction and its parent directory.
+
+    This method is automatically called upon leaving a `with` statement's body.
+
+    Args:
+      unused_type: unused
+      unused_value: unused
+      unused_traceback: unused
+    """
+    if self._junction:
+      os.rmdir(self._junction)
+      os.rmdir(os.path.dirname(self._junction))
diff --git a/tools/android/junction_test.py b/tools/android/junction_test.py
new file mode 100644
index 0000000000..0ee1b08037
--- /dev/null
+++ b/tools/android/junction_test.py
@@ -0,0 +1,77 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for TempJunction."""
+
+import os
+import unittest
+
+from src.test.py.bazel import test_base
+from tools.android import junction
+
+
+class JunctionTest(test_base.TestBase):
+  """Unit tests for junction.py."""
+
+  def testCreateJunction(self):
+
+    def tempdir():
+      return self.ScratchDir("junc temp")
+
+    target = self.ScratchDir("junc target")
+    # Make the `target` path a non-normalized Windows path with a space in it.
+    # TempJunction should still work.
+    target = os.path.dirname(target) + "/junc target"
+    juncpath = None
+    with junction.TempJunction(target, testonly_mkdtemp=tempdir) as j:
+      juncpath = j
+      # Ensure that `j` created the junction.
+      self.assertTrue(os.path.exists(target))
+      self.assertTrue(os.path.exists(juncpath))
+      # Create a file under the junction.
+      filepath = os.path.join(juncpath, "file.txt")
+      with open(filepath, "w") as f:
+        f.write("hello")
+      # Ensure we can reach the file via the junction and the target directory.
+      self.assertTrue(os.path.exists(os.path.join(target, "file.txt")))
+      self.assertTrue(os.path.exists(os.path.join(juncpath, "file.txt")))
+    # Ensure that after the `with` block the junction and temp directories no
+    # longer exist, but we can still reach the file via the target directory.
+    self.assertTrue(os.path.exists(os.path.join(target, "file.txt")))
+    self.assertFalse(os.path.exists(os.path.join(juncpath, "file.txt")))
+    self.assertFalse(os.path.exists(juncpath))
+    self.assertFalse(os.path.exists(os.path.dirname(juncpath)))
+
+  def testCannotCreateJunction(self):
+
+    def tempdir():
+      return self.ScratchDir("junc temp")
+
+    target = self.ScratchDir("junc target")
+    # Make the `target` path a non-normalized Windows path with a space in it.
+    # TempJunction should still work.
+    target = os.path.dirname(target) + "/junc target"
+    with junction.TempJunction(target, testonly_mkdtemp=tempdir) as j:
+      self.assertTrue(os.path.exists(j))
+      try:
+        # Ensure that TempJunction raises a JunctionCreationError if it cannot
+        # create a junction. In this case the junction already exists in that
+        # directory.
+        with junction.TempJunction(target, testonly_mkdtemp=tempdir) as _:
+          self.fail("Expected exception")
+      except junction.JunctionCreationError:
+        pass  # expected
+
+
+if __name__ == "__main__":
+  unittest.main()