From c77f891e7c214a39f9f196cd0a308f9ea5819ad9 Mon Sep 17 00:00:00 2001 From: Laszlo Csomor Date: Tue, 5 Sep 2017 09:35:39 +0200 Subject: Android,Windows: support long paths in tooling aar_resources_extractor now supports long paths on Windows. If the script needs to extract a file from the AAR where the destination path is too long, the script will: 1. create a temporary junction under a short path, pointing to the destination directory (which has a long path) 2. extract the file under the junction 3. delete the junction and the temp directory See https://github.com/bazelbuild/bazel/issues/3659 Change-Id: Ie85665b360a6514afaac546aaec8869224fe9d06 PiperOrigin-RevId: 167545085 --- .../build/lib/rules/android/AarImport.java | 6 +- src/test/py/bazel/BUILD | 1 + tools/android/BUILD | 29 +++++- tools/android/BUILD.tools | 11 +- tools/android/aar_resources_extractor.py | 12 ++- tools/android/dummy_test.py | 22 ++++ tools/android/junction.py | 115 +++++++++++++++++++++ tools/android/junction_test.py | 77 ++++++++++++++ 8 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 tools/android/dummy_test.py create mode 100644 tools/android/junction.py create mode 100644 tools/android/junction_test.py diff --git a/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java b/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java index 61295fc8e6..e3d5a12733 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java +++ b/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java @@ -176,6 +176,7 @@ public class AarImport implements RuleConfiguredTargetFactory { private static Action[] createSingleFileExtractorActions(RuleContext ruleContext, Artifact aar, String filename, Artifact outputArtifact) { return new SpawnAction.Builder() + .useDefaultShellEnvironment() .setExecutable(ruleContext.getExecutablePrerequisite(AarImportBaseRule.ZIPPER, Mode.HOST)) .setMnemonic("AarFileExtractor") .setProgressMessage("Extracting %s from %s", filename, aar.getFilename()) @@ -193,6 +194,7 @@ public class AarImport implements RuleConfiguredTargetFactory { private static Action[] createAarResourcesExtractorActions( RuleContext ruleContext, Artifact aar, Artifact outputTree) { return new SpawnAction.Builder() + .useDefaultShellEnvironment() .setExecutable( ruleContext.getExecutablePrerequisite( AarImportBaseRule.AAR_RESOURCES_EXTRACTOR, Mode.HOST)) @@ -210,6 +212,7 @@ public class AarImport implements RuleConfiguredTargetFactory { private static Action[] createAarEmbeddedJarsExtractorActions(RuleContext ruleContext, Artifact aar, Artifact jarsTreeArtifact, Artifact singleJarParamFile) { return new SpawnAction.Builder() + .useDefaultShellEnvironment() .setExecutable( ruleContext.getExecutablePrerequisite( AarImportBaseRule.AAR_EMBEDDED_JARS_EXTACTOR, Mode.HOST)) @@ -248,6 +251,7 @@ public class AarImport implements RuleConfiguredTargetFactory { Artifact outputZip) { SpawnAction.Builder actionBuilder = new SpawnAction.Builder() + .useDefaultShellEnvironment() .setExecutable( ruleContext.getExecutablePrerequisite( AarImportBaseRule.AAR_NATIVE_LIBS_ZIP_CREATOR, Mode.HOST)) @@ -276,7 +280,7 @@ public class AarImport implements RuleConfiguredTargetFactory { // Adds the appropriate SpawnAction options depending on if SingleJar is a jar or not. private static SpawnAction.Builder singleJarSpawnActionBuilder(RuleContext ruleContext) { - SpawnAction.Builder builder = new SpawnAction.Builder(); + SpawnAction.Builder builder = new SpawnAction.Builder().useDefaultShellEnvironment(); Artifact singleJar = JavaToolchainProvider.fromRuleContext(ruleContext).getSingleJar(); if (singleJar.getFilename().endsWith(".jar")) { builder diff --git a/src/test/py/bazel/BUILD b/src/test/py/bazel/BUILD index 3be9bc2e0f..f93093b0a1 100644 --- a/src/test/py/bazel/BUILD +++ b/src/test/py/bazel/BUILD @@ -20,6 +20,7 @@ py_library( visibility = [ "//src/test/py/bazel:__pkg__", "//third_party/def_parser:__pkg__", + "//tools/android:__pkg__", ], ) diff --git a/tools/android/BUILD b/tools/android/BUILD index 75a9956e67..340847e92e 100644 --- a/tools/android/BUILD +++ b/tools/android/BUILD @@ -135,7 +135,10 @@ py_test( py_binary( name = "aar_resources_extractor", srcs = ["aar_resources_extractor.py"], - deps = ["//third_party/py/gflags"], + deps = [ + ":junction_lib", + "//third_party/py/gflags", + ], ) py_test( @@ -155,6 +158,30 @@ py_test( deps = [":resource_extractor"], ) +py_library( + name = "junction_lib", + srcs = ["junction.py"], + visibility = ["//visibility:private"], +) + +py_test( + name = "junction_test", + srcs = select({ + "//src:windows": ["junction_test.py"], + "//src:windows_msvc": ["junction_test.py"], + "//conditions:default": ["dummy_test.py"], + }), + main = select({ + "//src:windows": "junction_test.py", + "//src:windows_msvc": "junction_test.py", + "//conditions:default": "dummy_test.py", + }), + deps = [ + ":junction_lib", + "//src/test/py/bazel:test_base", + ], +) + filegroup( name = "srcs", srcs = glob(["**"]) + ["//tools/android/emulator:srcs"], diff --git a/tools/android/BUILD.tools b/tools/android/BUILD.tools index fde4587543..7110448c63 100644 --- a/tools/android/BUILD.tools +++ b/tools/android/BUILD.tools @@ -189,7 +189,10 @@ py_binary( py_binary( name = "aar_resources_extractor", srcs = ["aar_resources_extractor.py"], - deps = ["//third_party/py/gflags"], + deps = [ + ":junction_lib", + "//third_party/py/gflags", + ], ) py_binary( @@ -197,6 +200,12 @@ py_binary( srcs = ["resource_extractor.py"], ) +py_library( + name = "junction_lib", + srcs = ["junction.py"], + visibility = ["//visibility:private"], +) + alias( name = "android_runtest", actual = "fail.sh", diff --git a/tools/android/aar_resources_extractor.py b/tools/android/aar_resources_extractor.py index 0e259b8028..eb0d432027 100644 --- a/tools/android/aar_resources_extractor.py +++ b/tools/android/aar_resources_extractor.py @@ -26,6 +26,7 @@ import os import sys import zipfile +from tools.android import junction from third_party.py import gflags FLAGS = gflags.FLAGS @@ -37,10 +38,19 @@ gflags.MarkFlagAsRequired("output_res_dir") def ExtractResources(aar, output_res_dir): + """Extract resource from an `aar` file to the `output_res_dir` directory.""" aar_contains_no_resources = True + output_res_dir_abs = os.path.normpath( + os.path.join(os.getcwd(), output_res_dir)) for name in aar.namelist(): if name.startswith("res/"): - aar.extract(name, output_res_dir) + fullpath = os.path.normpath(os.path.join(output_res_dir_abs, name)) + if os.name == "nt" and len(fullpath) >= 260: # MAX_PATH in + with junction.TempJunction(os.path.dirname(fullpath)) as juncpath: + shortpath = os.path.join(juncpath, os.path.basename(fullpath)) + aar.extract(name, shortpath) + else: + aar.extract(name, output_res_dir) aar_contains_no_resources = False if aar_contains_no_resources: empty_xml_filename = output_res_dir + "/res/values/empty.xml" diff --git a/tools/android/dummy_test.py b/tools/android/dummy_test.py new file mode 100644 index 0000000000..9afac3f06f --- /dev/null +++ b/tools/android/dummy_test.py @@ -0,0 +1,22 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Dummy empty test. + +You can use this as a dummy test for platform-specific tests on platforms that +are not under test. For example, we use this dummy test on Linux in a +Windows-specific py_test rule. +""" + +if __name__ == "__main__": + pass diff --git a/tools/android/junction.py b/tools/android/junction.py new file mode 100644 index 0000000000..7dc58b8c17 --- /dev/null +++ b/tools/android/junction.py @@ -0,0 +1,115 @@ +# pylint: disable=g-direct-third-party-import +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A class that creates junctions in temp directories on Windows. + +Only use this class on Windows, do not use on other platforms. Other platforms +support longer paths than Windows, and also support symlinks. Windows only +supports junctions (directory symlinks). + +Junctions are useful if you need to shorten a long path. A long path is one that +is at least MAX_PATH (260) letters long. This is a constant in Windows, see + and API documentation for file-handling functions such as +CreateFileA. +""" + +import os +import subprocess +import tempfile + + +class JunctionCreationError(Exception): + """Raised when TempJunction fails to create an NTFS junction.""" + + def __init__(self, path, target, stdout): + Exception.__init__( + self, + "Could not create junction \"%s\" -> \"%s\"\nError from mklink:\n%s" % + (path, target, stdout)) + + +class TempJunction(object): + r"""Junction in a temp directory. + + This object creates a temp directory and a junction under it. The junction + points to a user-specified path. + + Use this object if you want to write files under long paths (absolute path at + least MAX_PATH (260) chars long). Pass the directory you want to "shorten" as + the initializer's argument. This object will create a junction under a shorter + path, that points to the long directory. The combined path of the junction and + files under it are more likely to be short than the original paths were. + + Usage example: + with TempJunction("C:/some/long/path") as junc: + # `junc` created a temp directory and a junction in it that points to + # \\?\C:\some\long\path, and is itself shorter than that + shortpath = os.path.join(junc, "file.txt") + with open(shortpath, "w") as f: + ...do something with f... + # `junc` deleted the junction and its parent temp directory upon leaving + # the `with` statement's body + ...do something else... + """ + + def __init__(self, junction_target, testonly_mkdtemp=None): + """Initialize this object. + + Args: + junction_target: string; an absolute Windows path; the __enter__ method + creates a junction that points to this path + testonly_mkdtemp: function(); for testing only; a custom function that + returns a temp directory path, you can use it to mock out + tempfile.mkdtemp + """ + self._target = os.path.normpath(junction_target) + self._junction = None + self._mkdtemp = testonly_mkdtemp or tempfile.mkdtemp + + def __enter__(self): + """Creates a temp directory and a junction in it, pointing to self._target. + + This method is automatically called upon entering a `with` statement's body. + + Returns: + The full path to the junction. + Raises: + JunctionCreationError: if `mklink` fails to create a junction + """ + result = os.path.normpath(os.path.join(self._mkdtemp(), "j")) + proc = subprocess.Popen( + "cmd.exe /C mklink /J \"%s\" \"\\\\?\\%s\"" % + (result, os.path.normpath(self._target)), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + exitcode = proc.wait() + if exitcode != 0: + stdout = proc.communicate()[0] + raise JunctionCreationError(result, self._target, stdout) + self._junction = result + return result + + def __exit__(self, unused_type, unused_value, unused_traceback): + """Deletes the junction and its parent directory. + + This method is automatically called upon leaving a `with` statement's body. + + Args: + unused_type: unused + unused_value: unused + unused_traceback: unused + """ + if self._junction: + os.rmdir(self._junction) + os.rmdir(os.path.dirname(self._junction)) diff --git a/tools/android/junction_test.py b/tools/android/junction_test.py new file mode 100644 index 0000000000..0ee1b08037 --- /dev/null +++ b/tools/android/junction_test.py @@ -0,0 +1,77 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for TempJunction.""" + +import os +import unittest + +from src.test.py.bazel import test_base +from tools.android import junction + + +class JunctionTest(test_base.TestBase): + """Unit tests for junction.py.""" + + def testCreateJunction(self): + + def tempdir(): + return self.ScratchDir("junc temp") + + target = self.ScratchDir("junc target") + # Make the `target` path a non-normalized Windows path with a space in it. + # TempJunction should still work. + target = os.path.dirname(target) + "/junc target" + juncpath = None + with junction.TempJunction(target, testonly_mkdtemp=tempdir) as j: + juncpath = j + # Ensure that `j` created the junction. + self.assertTrue(os.path.exists(target)) + self.assertTrue(os.path.exists(juncpath)) + # Create a file under the junction. + filepath = os.path.join(juncpath, "file.txt") + with open(filepath, "w") as f: + f.write("hello") + # Ensure we can reach the file via the junction and the target directory. + self.assertTrue(os.path.exists(os.path.join(target, "file.txt"))) + self.assertTrue(os.path.exists(os.path.join(juncpath, "file.txt"))) + # Ensure that after the `with` block the junction and temp directories no + # longer exist, but we can still reach the file via the target directory. + self.assertTrue(os.path.exists(os.path.join(target, "file.txt"))) + self.assertFalse(os.path.exists(os.path.join(juncpath, "file.txt"))) + self.assertFalse(os.path.exists(juncpath)) + self.assertFalse(os.path.exists(os.path.dirname(juncpath))) + + def testCannotCreateJunction(self): + + def tempdir(): + return self.ScratchDir("junc temp") + + target = self.ScratchDir("junc target") + # Make the `target` path a non-normalized Windows path with a space in it. + # TempJunction should still work. + target = os.path.dirname(target) + "/junc target" + with junction.TempJunction(target, testonly_mkdtemp=tempdir) as j: + self.assertTrue(os.path.exists(j)) + try: + # Ensure that TempJunction raises a JunctionCreationError if it cannot + # create a junction. In this case the junction already exists in that + # directory. + with junction.TempJunction(target, testonly_mkdtemp=tempdir) as _: + self.fail("Expected exception") + except junction.JunctionCreationError: + pass # expected + + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3