Remove hack for building Python support with Bazel.

This change makes use of new imports attribute for Bazel's Python rules, which
enable adding directories to the PYTHONPATH. This allows us to remove
the hack for building protobuf's Python support with Bazel and now
allows projects to include protobuf using a Bazel external repository
rather than requiring it to be imported directly into the source tree as
//google/protobuf.

This change also updates the protobuf BUILD file to use a named
repository, @python//, for including Python headers rather than
//util/python. This allows projects to specify their own package for
Python headers when including protobuf with an external repository.

Fixes #1230

4 files changed, 51 insertions, 96 deletions

diff --git a/BUILD b/BUILD
index c4de1c4c..9cbddd5b 100644
--- a/BUILD
+++ b/BUILD
@@ -22,7 +22,6 @@ load(
     "protobuf",
     "cc_proto_library",
     "py_proto_library",
-    "internal_copied_filegroup",
     "internal_protobuf_py_tests",
 )
 
@@ -484,25 +483,7 @@ java_library(
 # Python support
 ################################################################################
 
-# Hack:
-# protoc generated files contain imports like:
-#   "from google.protobuf.xxx import yyy"
-# However, the sources files of the python runtime are not directly under
-# "google/protobuf" (they are under python/google/protobuf).  We workaround
-# this by copying runtime source files into the desired location to workaround
-# the import issue. Ideally py_library should support something similiar to the
-# "include" attribute in cc_library to inject the PYTHON_PATH for all libraries
-# that depend on the target.
-#
-# If you use python protobuf as a third_party library in your bazel managed
-# project:
-# 1) Please import the whole package to //google/protobuf in your
-# project. Otherwise, bazel disallows generated files out of the current
-# package, thus we won't be able to copy protobuf runtime files into
-# //google/protobuf/.
-# 2) The runtime also requires "six" for Python2/3 compatibility, please see the
-# WORKSPACE file and bind "six" to your workspace as well.
-internal_copied_filegroup(
+py_library(
     name = "python_srcs",
     srcs = glob(
         [
@@ -514,7 +495,7 @@ internal_copied_filegroup(
             "python/google/protobuf/internal/test_util.py",
         ],
     ),
-    include = "python",
+    imports = ["python"],
 )
 
 cc_binary(
@@ -527,7 +508,7 @@ cc_binary(
     linkstatic = 1,
     deps = select({
         "//conditions:default": [],
-        ":use_fast_cpp_protos": ["//util/python:python_headers"],
+        ":use_fast_cpp_protos": ["//external:python_headers"],
     }),
 )
 
@@ -553,7 +534,7 @@ cc_binary(
         ":protobuf",
     ] + select({
         "//conditions:default": [],
-        ":use_fast_cpp_protos": ["//util/python:python_headers"],
+        ":use_fast_cpp_protos": ["//external:python_headers"],
     }),
 )
 
@@ -584,23 +565,14 @@ py_proto_library(
     }),
     default_runtime = "",
     protoc = ":protoc",
-    py_extra_srcs = [":python_srcs"],
-    py_libs = ["//external:six"],
+    py_libs = [
+        ":python_srcs",
+        "//external:six"
+    ],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
 )
 
-internal_copied_filegroup(
-    name = "python_test_srcs",
-    srcs = glob(
-        [
-            "python/google/protobuf/internal/*_test.py",
-            "python/google/protobuf/internal/test_util.py",
-        ],
-    ),
-    include = "python",
-)
-
 py_proto_library(
     name = "python_common_test_protos",
     srcs = LITE_TEST_PROTOS + TEST_PROTOS,
@@ -624,7 +596,13 @@ py_proto_library(
 
 py_library(
     name = "python_tests",
-    srcs = [":python_test_srcs"],
+    srcs = glob(
+        [
+            "python/google/protobuf/internal/*_test.py",
+            "python/google/protobuf/internal/test_util.py",
+        ],
+    ),
+    imports = ["python"],
     srcs_version = "PY2AND3",
     deps = [
         ":protobuf_python",
diff --git a/WORKSPACE b/WORKSPACE
index c3f18cc7..1e8e0a7f 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,28 +1,33 @@
 new_http_archive(
-  name = "gmock_archive",
-  url = "https://googlemock.googlecode.com/files/gmock-1.7.0.zip",
-  sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b",
-  build_file = "gmock.BUILD",
+    name = "gmock_archive",
+    url = "https://googlemock.googlecode.com/files/gmock-1.7.0.zip",
+    sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b",
+    build_file = "gmock.BUILD",
 )
 
 new_http_archive(
-  name = "six_archive",
-  url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55",
-  sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
-  build_file = "six.BUILD",
+    name = "six_archive",
+    url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55",
+    sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
+    build_file = "six.BUILD",
 )
 
 bind(
-  name = "gtest",
-  actual = "@gmock_archive//:gtest",
+    name = "python_headers",
+    actual = "//util/python:python_headers",
 )
 
 bind(
-  name = "gtest_main",
-  actual = "@gmock_archive//:gtest_main",
+    name = "gtest",
+    actual = "@gmock_archive//:gtest",
 )
 
 bind(
-  name = "six",
-  actual = "@six_archive//:six",
+    name = "gtest_main",
+    actual = "@gmock_archive//:gtest_main",
+)
+
+bind(
+    name = "six",
+    actual = "@six_archive//:six",
 )
diff --git a/protobuf.bzl b/protobuf.bzl
index e5af3339..71eaba22 100644
--- a/protobuf.bzl
+++ b/protobuf.bzl
@@ -117,10 +117,10 @@ def cc_proto_library(
         deps=[],
         cc_libs=[],
         include=None,
-        protoc="//google/protobuf:protoc",
+        protoc="//:protoc",
         internal_bootstrap_hack=False,
         use_grpc_plugin=False,
-        default_runtime="//google/protobuf:protobuf",
+        default_runtime="//:protobuf",
         **kargs):
   """Bazel rule to create a C++ protobuf library from proto source files
 
@@ -199,35 +199,6 @@ def cc_proto_library(
       includes=includes,
       **kargs)
 
-def internal_copied_filegroup(
-        name,
-        srcs,
-        include,
-        **kargs):
-  """Bazel rule to fix sources file to workaround with python path issues.
-
-  Args:
-    name: the name of the internal_copied_filegroup rule, which will be the
-        name of the generated filegroup.
-    srcs: the source files to be copied.
-    include: the expected import root of the source.
-    **kargs: extra arguments that will be passed into the filegroup.
-  """
-  outs = [_RelativeOutputPath(s, include) for s in srcs]
-
-  native.genrule(
-      name=name+"_genrule",
-      srcs=srcs,
-      outs=outs,
-      cmd=" && ".join(["cp $(location %s) $(location %s)" %
-                       (s, _RelativeOutputPath(s, include))
-                       for s in srcs]))
-
-  native.filegroup(
-      name=name,
-      srcs=outs,
-      **kargs)
-
 def py_proto_library(
         name,
         srcs=[],
@@ -235,8 +206,8 @@ def py_proto_library(
         py_libs=[],
         py_extra_srcs=[],
         include=None,
-        default_runtime="//google/protobuf:protobuf_python",
-        protoc="//google/protobuf:protoc",
+        default_runtime="//:protobuf_python",
+        protoc="//:protoc",
         **kargs):
   """Bazel rule to create a Python protobuf library from proto source files
 
@@ -276,15 +247,6 @@ def py_proto_library(
       visibility=["//visibility:public"],
   )
 
-  if include != None:
-    # Copy the output files to the desired location to make the import work.
-    internal_copied_filegroup_name=name + "_internal_copied_filegroup"
-    internal_copied_filegroup(
-        name=internal_copied_filegroup_name,
-        srcs=outs,
-        include=include)
-    outs=[internal_copied_filegroup_name]
-
   if default_runtime and not default_runtime in py_libs + deps:
     py_libs += [default_runtime]
 
@@ -292,6 +254,7 @@ def py_proto_library(
       name=name,
       srcs=outs+py_extra_srcs,
       deps=py_libs+deps,
+      imports=includes,
       **kargs)
 
 def internal_protobuf_py_tests(
@@ -308,8 +271,7 @@ def internal_protobuf_py_tests(
 
   """
   for m in modules:
-    s = _RelativeOutputPath(
-        "python/google/protobuf/internal/%s.py" % m, "python")
+    s = "python/google/protobuf/internal/%s.py" % m
     native.py_test(
         name="py_%s" % m,
         srcs=[s],
diff --git a/util/python/BUILD b/util/python/BUILD
index 358c381c..3ac03856 100644
--- a/util/python/BUILD
+++ b/util/python/BUILD
@@ -1,7 +1,17 @@
 # This is a placeholder for python headers. Projects needing to use
 # fast cpp protos in protobuf's python interface should build with
 # --define=use_fast_cpp_protos=true, and in addition, provide
-# //util/python:python_headers dependency that in turn provides Python.h.
+# //external:python_headers dependency that in turn provides Python.h.
+#
+# Projects that include protobuf using a Bazel external repository will need to
+# add a workspace rule to their WORKSPACE files to add an external workspace
+# that includes the Python headers. For example, the protobuf WORKSPACE file
+# includes the following local_repository rule that points to this directory:
+#
+# new_local_repository(
+#   name = "python",
+#   path = __workspace_dir__ + "/util/python",
+# )
 cc_library(
     name = "python_headers",
     visibility = ["//visibility:public"],