python,runfiles: move to different package

Move the Python runfiles library from
`@bazel_tools//tools/runfiles:py-runfiles` to
`@bazel_tools//tools/python/runfiles:runfiles`

Also rename the testdata runfiles.py to foo.py.
This file was not a mock runfiles library, just a
client file using the runfiles library that was
also called runfiles.py

Fixes https://github.com/bazelbuild/bazel/issues/4878

Change-Id: I874b230c93679d4454ac91e816932c8272ecc5c7

Closes #4981.

Change-Id: I908e0ab7ec61225e82f70793b1a05432e7f0b07e
PiperOrigin-RevId: 192256481

5 files changed, 462 insertions, 1 deletions

diff --git a/tools/python/BUILD b/tools/python/BUILD
index 6933f9d419..ab09128e0c 100644
--- a/tools/python/BUILD
+++ b/tools/python/BUILD
@@ -6,10 +6,28 @@ sh_binary(
 )
 
 filegroup(
-    name = "srcs",
+    name = "srcs_and_embedded_tools",
     srcs = [
         # Tools are build from the workspace for tests.
         "2to3.sh",
         "BUILD",
     ],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "srcs",
+    srcs = [
+        ":srcs_and_embedded_tools",
+        "//tools/python/runfiles:srcs",
+    ],
+)
+
+filegroup(
+    name = "embedded_tools",
+    srcs = [
+        ":srcs_and_embedded_tools",
+        "//tools/python/runfiles:embedded_tools",
+    ],
+    visibility = ["//tools:__pkg__"],
 )
diff --git a/tools/python/runfiles/BUILD b/tools/python/runfiles/BUILD
new file mode 100644
index 0000000000..79273e85ce
--- /dev/null
+++ b/tools/python/runfiles/BUILD
@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:private"])
+
+filegroup(
+    name = "srcs",
+    srcs = glob(
+        ["**"],
+        exclude = [
+            ".*",
+            "*~",
+        ],
+    ),
+    visibility = ["//tools/python:__pkg__"],
+)
+
+filegroup(
+    name = "embedded_tools",
+    srcs = [
+        "BUILD.tools",
+        "runfiles.py",
+    ],
+    visibility = ["//tools/python:__pkg__"],
+)
+
+py_library(
+    name = "runfiles",
+    testonly = 1,
+    srcs = ["runfiles.py"],
+)
+
+py_test(
+    name = "runfiles_test",
+    srcs = ["runfiles_test.py"],
+    visibility = ["//visibility:public"],
+    deps = [":runfiles"],
+)
diff --git a/tools/python/runfiles/BUILD.tools b/tools/python/runfiles/BUILD.tools
new file mode 100644
index 0000000000..4080d68774
--- /dev/null
+++ b/tools/python/runfiles/BUILD.tools
@@ -0,0 +1,5 @@
+py_library(
+    name = "runfiles",
+    srcs = ["runfiles.py"],
+    visibility = ["//visibility:public"],
+)
diff --git a/tools/python/runfiles/runfiles.py b/tools/python/runfiles/runfiles.py
new file mode 100644
index 0000000000..17e4121a0f
--- /dev/null
+++ b/tools/python/runfiles/runfiles.py
@@ -0,0 +1,221 @@
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Runfiles lookup library for Bazel-built Python binaries and tests.
+
+Usage:
+
+from bazel_tools.tools.python.runfiles import runfiles
+
+r = runfiles.Create()
+with open(r.Rlocation("io_bazel/foo/bar.txt"), "r") as f:
+  contents = f.readlines()
+
+The code above creates a manifest- or directory-based implementations based on
+the environment variables in os.environ. See `Create()` for more info.
+
+If you want to explicitly create a manifest- or directory-based
+implementations, you can do so as follows:
+
+  r1 = runfiles.CreateManifestBased("path/to/foo.runfiles_manifest")
+
+  r2 = runfiles.CreateDirectoryBased("path/to/foo.runfiles/")
+
+If you want to start subprocesses that also need runfiles, you need to set the
+right environment variables for them:
+
+  import subprocess
+  from bazel_tools.tools.python.runfiles import runfiles
+
+  r = runfiles.Create()
+  env = {}
+  ...
+  env.update(r.EnvVars())
+  p = subprocess.Popen([r.Rlocation("path/to/binary")], env, ...)
+"""
+
+import os
+import posixpath
+
+
+def CreateManifestBased(manifest_path):
+  return _Runfiles(_ManifestBased(manifest_path))
+
+
+def CreateDirectoryBased(runfiles_dir_path):
+  return _Runfiles(_DirectoryBased(runfiles_dir_path))
+
+
+def Create(env=None):
+  """Returns a new `Runfiles` instance.
+
+  The returned object is either:
+  - manifest-based, meaning it looks up runfile paths from a manifest file, or
+  - directory-based, meaning it looks up runfile paths under a given directory
+    path
+
+  If `env` contains "RUNFILES_MANIFEST_FILE" with non-empty value, this method
+  returns a manifest-based implementation. The object eagerly reads and caches
+  the whole manifest file upon instantiation; this may be relevant for
+  performance consideration.
+
+  Otherwise, if `env` contains "RUNFILES_DIR" with non-empty value (checked in
+  this priority order), this method returns a directory-based implementation.
+
+  If neither cases apply, this method returns null.
+
+  Args:
+    env: {string: string}; optional; the map of environment variables. If None,
+        this function uses the environment variable map of this process.
+  Raises:
+    IOError: if some IO error occurs.
+  """
+  env_map = os.environ if env is None else env
+  manifest = env_map.get("RUNFILES_MANIFEST_FILE")
+  if manifest:
+    return CreateManifestBased(manifest)
+
+  directory = env_map.get("RUNFILES_DIR")
+  if directory:
+    return CreateDirectoryBased(directory)
+
+  return None
+
+
+class _Runfiles(object):
+  """Returns the runtime location of runfiles.
+
+  Runfiles are data-dependencies of Bazel-built binaries and tests.
+  """
+
+  def __init__(self, strategy):
+    self._strategy = strategy
+
+  def Rlocation(self, path):
+    """Returns the runtime path of a runfile.
+
+    Runfiles are data-dependencies of Bazel-built binaries and tests.
+
+    The returned path may not be valid. The caller should check the path's
+    validity and that the path exists.
+
+    The function may return None. In that case the caller can be sure that the
+    rule does not know about this data-dependency.
+
+    Args:
+      path: string; runfiles-root-relative path of the runfile
+    Returns:
+      the path to the runfile, which the caller should check for existence, or
+      None if the method doesn't know about this runfile
+    Raises:
+      TypeError: if `path` is not a string
+      ValueError: if `path` is None or empty, or it's absolute or contains
+        uplevel references
+    """
+    if not path:
+      raise ValueError()
+    if not isinstance(path, str):
+      raise TypeError()
+    if ".." in path:
+      raise ValueError("path contains uplevel references: \"%s\"" % path)
+    if path[0] == "\\":
+      raise ValueError("path is absolute without a drive letter: \"%s\"" % path)
+    if os.path.isabs(path):
+      return path
+    return self._strategy.RlocationChecked(path)
+
+  def EnvVars(self):
+    """Returns environment variables for subprocesses.
+
+    The caller should set the returned key-value pairs in the environment of
+    subprocesses in case those subprocesses are also Bazel-built binaries that
+    need to use runfiles.
+
+    Returns:
+      {string: string}; a dict; keys are environment variable names, values are
+      the values for these environment variables
+    """
+    return self._strategy.EnvVars()
+
+
+class _ManifestBased(object):
+  """`Runfiles` strategy that parses a runfiles-manifest to look up runfiles."""
+
+  def __init__(self, path):
+    if not path:
+      raise ValueError()
+    if not isinstance(path, str):
+      raise TypeError()
+    self._path = path
+    self._runfiles = _ManifestBased._LoadRunfiles(path)
+
+  def RlocationChecked(self, path):
+    return self._runfiles.get(path)
+
+  @staticmethod
+  def _LoadRunfiles(path):
+    """Loads the runfiles manifest."""
+    result = {}
+    with open(path, "r") as f:
+      for line in f:
+        line = line.strip()
+        if line:
+          tokens = line.split(" ", 1)
+          if len(tokens) == 1:
+            result[line] = line
+          else:
+            result[tokens[0]] = tokens[1]
+    return result
+
+  def _GetRunfilesDir(self):
+    if self._path.endswith("/MANIFEST") or self._path.endswith("\\MANIFEST"):
+      return self._path[:-len("/MANIFEST")]
+    elif self._path.endswith(".runfiles_manifest"):
+      return self._path[:-len("_manifest")]
+    else:
+      return ""
+
+  def EnvVars(self):
+    directory = self._GetRunfilesDir()
+    return {
+        "RUNFILES_MANIFEST_FILE": self._path,
+        "RUNFILES_DIR": directory,
+        # TODO(laszlocsomor): remove JAVA_RUNFILES once the Java launcher can
+        # pick up RUNFILES_DIR.
+        "JAVA_RUNFILES": directory,
+    }
+
+
+class _DirectoryBased(object):
+  """`Runfiles` strategy that appends runfiles paths to the runfiles root."""
+
+  def __init__(self, path):
+    if not path:
+      raise ValueError()
+    if not isinstance(path, str):
+      raise TypeError()
+    self._runfiles_root = path
+
+  def RlocationChecked(self, path):
+    # Use posixpath instead of os.path, because Bazel only creates a runfiles
+    # tree on Unix platforms, so `Create()` will only create a directory-based
+    # runfiles strategy on those platforms.
+    return posixpath.join(self._runfiles_root, path)
+
+  def EnvVars(self):
+    return {
+        "RUNFILES_DIR": self._runfiles_root,
+        # TODO(laszlocsomor): remove JAVA_RUNFILES once the Java launcher can
+        # pick up RUNFILES_DIR.
+        "JAVA_RUNFILES": self._runfiles_root,
+    }
diff --git a/tools/python/runfiles/runfiles_test.py b/tools/python/runfiles/runfiles_test.py
new file mode 100644
index 0000000000..bbe1c84be4
--- /dev/null
+++ b/tools/python/runfiles/runfiles_test.py
@@ -0,0 +1,182 @@
+# pylint: disable=g-bad-file-header
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import unittest
+
+from tools.python.runfiles import runfiles
+
+
+class RunfilesTest(unittest.TestCase):
+  # """Unit tests for `runfiles.Runfiles`."""
+
+  def testRlocationArgumentValidation(self):
+    r = runfiles.Create({"RUNFILES_DIR": "whatever"})
+    self.assertRaises(ValueError, lambda: r.Rlocation(None))
+    self.assertRaises(ValueError, lambda: r.Rlocation(""))
+    self.assertRaises(TypeError, lambda: r.Rlocation(1))
+    self.assertRaisesRegexp(ValueError, "contains uplevel",
+                            lambda: r.Rlocation("foo/.."))
+    self.assertRaisesRegexp(ValueError, "is absolute without a drive letter",
+                            lambda: r.Rlocation("\\foo"))
+
+  def testCreatesManifestBasedRunfiles(self):
+    with _MockFile(contents=["a/b c/d"]) as mf:
+      r = runfiles.Create({
+          "RUNFILES_MANIFEST_FILE": mf.Path(),
+          "RUNFILES_DIR": "ignored when RUNFILES_MANIFEST_FILE has a value",
+          "TEST_SRCDIR": "always ignored",
+      })
+      self.assertEqual(r.Rlocation("a/b"), "c/d")
+      self.assertIsNone(r.Rlocation("foo"))
+
+  def testManifestBasedRunfilesEnvVars(self):
+    with _MockFile(name="MANIFEST") as mf:
+      r = runfiles.Create({
+          "RUNFILES_MANIFEST_FILE": mf.Path(),
+          "TEST_SRCDIR": "always ignored",
+      })
+      self.assertDictEqual(
+          r.EnvVars(), {
+              "RUNFILES_MANIFEST_FILE": mf.Path(),
+              "RUNFILES_DIR": mf.Path()[:-len("/MANIFEST")],
+              "JAVA_RUNFILES": mf.Path()[:-len("/MANIFEST")],
+          })
+
+    with _MockFile(name="foo.runfiles_manifest") as mf:
+      r = runfiles.Create({
+          "RUNFILES_MANIFEST_FILE": mf.Path(),
+          "TEST_SRCDIR": "always ignored",
+      })
+      self.assertDictEqual(
+          r.EnvVars(), {
+              "RUNFILES_MANIFEST_FILE":
+                  mf.Path(),
+              "RUNFILES_DIR": (
+                  mf.Path()[:-len("foo.runfiles_manifest")] + "foo.runfiles"),
+              "JAVA_RUNFILES": (
+                  mf.Path()[:-len("foo.runfiles_manifest")] + "foo.runfiles"),
+          })
+
+    with _MockFile(name="x_manifest") as mf:
+      r = runfiles.Create({
+          "RUNFILES_MANIFEST_FILE": mf.Path(),
+          "TEST_SRCDIR": "always ignored",
+      })
+      self.assertDictEqual(
+          r.EnvVars(), {
+              "RUNFILES_MANIFEST_FILE": mf.Path(),
+              "RUNFILES_DIR": "",
+              "JAVA_RUNFILES": "",
+          })
+
+  def testCreatesDirectoryBasedRunfiles(self):
+    r = runfiles.Create({
+        "RUNFILES_DIR": "runfiles/dir",
+        "TEST_SRCDIR": "always ignored",
+    })
+    self.assertEqual(r.Rlocation("a/b"), "runfiles/dir/a/b")
+    self.assertEqual(r.Rlocation("foo"), "runfiles/dir/foo")
+
+  def testDirectoryBasedRunfilesEnvVars(self):
+    r = runfiles.Create({
+        "RUNFILES_DIR": "runfiles/dir",
+        "TEST_SRCDIR": "always ignored",
+    })
+    self.assertDictEqual(r.EnvVars(), {
+        "RUNFILES_DIR": "runfiles/dir",
+        "JAVA_RUNFILES": "runfiles/dir",
+    })
+
+  def testFailsToCreateManifestBasedBecauseManifestDoesNotExist(self):
+
+    def _Run():
+      runfiles.Create({"RUNFILES_MANIFEST_FILE": "non-existing path"})
+
+    self.assertRaisesRegexp(IOError, "non-existing path", _Run)
+
+  def testFailsToCreateAnyRunfilesBecauseEnvvarsAreNotDefined(self):
+    with _MockFile(contents=["a b"]) as mf:
+      runfiles.Create({
+          "RUNFILES_MANIFEST_FILE": mf.Path(),
+          "RUNFILES_DIR": "whatever",
+          "TEST_SRCDIR": "always ignored",
+      })
+    runfiles.Create({
+        "RUNFILES_DIR": "whatever",
+        "TEST_SRCDIR": "always ignored",
+    })
+    self.assertIsNone(runfiles.Create({"TEST_SRCDIR": "always ignored"}))
+    self.assertIsNone(runfiles.Create({"FOO": "bar"}))
+
+  def testManifestBasedRlocation(self):
+    with _MockFile(contents=[
+        "Foo/runfile1", "Foo/runfile2 C:/Actual Path\\runfile2",
+        "Foo/Bar/runfile3 D:\\the path\\run file 3.txt"
+    ]) as mf:
+      r = runfiles.CreateManifestBased(mf.Path())
+      self.assertEqual(r.Rlocation("Foo/runfile1"), "Foo/runfile1")
+      self.assertEqual(r.Rlocation("Foo/runfile2"), "C:/Actual Path\\runfile2")
+      self.assertEqual(
+          r.Rlocation("Foo/Bar/runfile3"), "D:\\the path\\run file 3.txt")
+      self.assertIsNone(r.Rlocation("unknown"))
+      if RunfilesTest.IsWindows():
+        self.assertEqual(r.Rlocation("c:/foo"), "c:/foo")
+        self.assertEqual(r.Rlocation("c:\\foo"), "c:\\foo")
+      else:
+        self.assertEqual(r.Rlocation("/foo"), "/foo")
+
+  def testDirectoryBasedRlocation(self):
+    # The _DirectoryBased strategy simply joins the runfiles directory and the
+    # runfile's path on a "/". This strategy does not perform any normalization,
+    # nor does it check that the path exists.
+    r = runfiles.CreateDirectoryBased("foo/bar baz//qux/")
+    self.assertEqual(r.Rlocation("arg"), "foo/bar baz//qux/arg")
+    if RunfilesTest.IsWindows():
+      self.assertEqual(r.Rlocation("c:/foo"), "c:/foo")
+      self.assertEqual(r.Rlocation("c:\\foo"), "c:\\foo")
+    else:
+      self.assertEqual(r.Rlocation("/foo"), "/foo")
+
+  @staticmethod
+  def IsWindows():
+    return os.name == "nt"
+
+
+class _MockFile(object):
+
+  def __init__(self, name=None, contents=None):
+    self._contents = contents or []
+    self._name = name or "x"
+    self._path = None
+
+  def __enter__(self):
+    tmpdir = os.environ.get("TEST_TMPDIR")
+    self._path = os.path.join(tempfile.mkdtemp(dir=tmpdir), self._name)
+    with open(self._path, "wt") as f:
+      f.writelines(l + "\n" for l in self._contents)
+    return self
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    os.remove(self._path)
+    os.rmdir(os.path.dirname(self._path))
+
+  def Path(self):
+    return self._path
+
+
+if __name__ == "__main__":
+  unittest.main()