Merge remote-tracking branch 'upstream/master' into upmerge

10 files changed, 197 insertions, 27 deletions

diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst
index d491311886..3dfae50b4b 100644
--- a/src/python/grpcio/README.rst
+++ b/src/python/grpcio/README.rst
@@ -42,3 +42,17 @@ package named :code:`python-dev`).
 
 Note that :code:`$REPO_ROOT` can be assigned to whatever directory name floats
 your fancy.
+
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+Help, I ...
+
+* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install
+  grpc!**
+
+  This is likely because :code:`pip` doesn't own the offending dependency,
+  which in turn is likely because your operating system's package manager owns
+  it. You'll need to force the installation of the dependency:
+
+  :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY`
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi
index 757f1245e8..305475c006 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,9 @@
 cdef class CompletionQueue:
 
   cdef grpc_completion_queue *c_completion_queue
-  cdef object poll_condition
-  cdef bint is_polling
+  cdef object pluck_condition
+  cdef int num_plucking
+  cdef int num_polling
   cdef bint is_shutting_down
   cdef bint is_shutdown
 
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
index b299dfee22..c139147114 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
@@ -39,8 +39,9 @@ cdef class CompletionQueue:
     self.c_completion_queue = grpc_completion_queue_create(NULL)
     self.is_shutting_down = False
     self.is_shutdown = False
-    self.poll_condition = threading.Condition()
-    self.is_polling = False
+    self.pluck_condition = threading.Condition()
+    self.num_plucking = 0
+    self.num_polling = 0
 
   cdef _interpret_event(self, grpc_event event):
     cdef OperationTag tag = None
@@ -87,19 +88,15 @@ cdef class CompletionQueue:
       c_deadline = deadline.c_time
     cdef grpc_event event
 
-    # Poll within a critical section
-    # TODO(atash) consider making queue polling contention a hard error to
-    # enable easier bug discovery
-    with self.poll_condition:
-      while self.is_polling:
-        self.poll_condition.wait(float(deadline) - time.time())
-      self.is_polling = True
+    # Poll within a critical section to detect contention
+    with self.pluck_condition:
+      assert self.num_plucking == 0, 'cannot simultaneously pluck and poll'
+      self.num_polling += 1
     with nogil:
       event = grpc_completion_queue_next(
           self.c_completion_queue, c_deadline, NULL)
-    with self.poll_condition:
-      self.is_polling = False
-      self.poll_condition.notify()
+    with self.pluck_condition:
+      self.num_polling -= 1
     return self._interpret_event(event)
 
   def pluck(self, OperationTag tag, Timespec deadline=None):
@@ -111,19 +108,18 @@ cdef class CompletionQueue:
       c_deadline = deadline.c_time
     cdef grpc_event event
 
-    # Poll within a critical section
-    # TODO(atash) consider making queue polling contention a hard error to
-    # enable easier bug discovery
-    with self.poll_condition:
-      while self.is_polling:
-        self.poll_condition.wait(float(deadline) - time.time())
-      self.is_polling = True
+    # Pluck within a critical section to detect contention
+    with self.pluck_condition:
+      assert self.num_polling == 0, 'cannot simultaneously pluck and poll'
+      assert self.num_plucking < GRPC_MAX_COMPLETION_QUEUE_PLUCKERS, (
+          'cannot pluck more than {} times simultaneously'.format(
+              GRPC_MAX_COMPLETION_QUEUE_PLUCKERS))
+      self.num_plucking += 1
     with nogil:
       event = grpc_completion_queue_pluck(
           self.c_completion_queue, <cpython.PyObject *>tag, c_deadline, NULL)
-    with self.poll_condition:
-      self.is_polling = False
-      self.poll_condition.notify()
+    with self.pluck_condition:
+      self.num_plucking -= 1
     return self._interpret_event(event)
 
   def shutdown(self):
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
index 800d0ea2f6..dbf0045710 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
@@ -138,6 +138,8 @@ cdef extern from "grpc/_cython/loader.h":
   const int GRPC_WRITE_NO_COMPRESS
   const int GRPC_WRITE_USED_MASK
 
+  const int GRPC_MAX_COMPLETION_QUEUE_PLUCKERS
+
   ctypedef struct grpc_completion_queue:
     # We don't care about the internals (and in fact don't know them)
     pass
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c
index 4b1860ce8c..8bd6ae6372 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.c
+++ b/src/python/grpcio/grpc/_cython/imports.generated.c
@@ -220,6 +220,7 @@ gpr_event_get_type gpr_event_get_import;
 gpr_event_wait_type gpr_event_wait_import;
 gpr_ref_init_type gpr_ref_init_import;
 gpr_ref_type gpr_ref_import;
+gpr_ref_non_zero_type gpr_ref_non_zero_import;
 gpr_refn_type gpr_refn_import;
 gpr_unref_type gpr_unref_import;
 gpr_stats_init_type gpr_stats_init_import;
@@ -485,6 +486,7 @@ void pygrpc_load_imports(HMODULE library) {
   gpr_event_wait_import = (gpr_event_wait_type) GetProcAddress(library, "gpr_event_wait");
   gpr_ref_init_import = (gpr_ref_init_type) GetProcAddress(library, "gpr_ref_init");
   gpr_ref_import = (gpr_ref_type) GetProcAddress(library, "gpr_ref");
+  gpr_ref_non_zero_import = (gpr_ref_non_zero_type) GetProcAddress(library, "gpr_ref_non_zero");
   gpr_refn_import = (gpr_refn_type) GetProcAddress(library, "gpr_refn");
   gpr_unref_import = (gpr_unref_type) GetProcAddress(library, "gpr_unref");
   gpr_stats_init_import = (gpr_stats_init_type) GetProcAddress(library, "gpr_stats_init");
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h
index a395dce7d6..b70dcccd17 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.h
+++ b/src/python/grpcio/grpc/_cython/imports.generated.h
@@ -610,6 +610,9 @@ extern gpr_ref_init_type gpr_ref_init_import;
 typedef void(*gpr_ref_type)(gpr_refcount *r);
 extern gpr_ref_type gpr_ref_import;
 #define gpr_ref gpr_ref_import
+typedef void(*gpr_ref_non_zero_type)(gpr_refcount *r);
+extern gpr_ref_non_zero_type gpr_ref_non_zero_import;
+#define gpr_ref_non_zero gpr_ref_non_zero_import
 typedef void(*gpr_refn_type)(gpr_refcount *r, int n);
 extern gpr_refn_type gpr_refn_import;
 #define gpr_refn gpr_refn_import
diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py
index 4f1ddb57fc..38a5432e79 100644
--- a/src/python/grpcio/tests/_runner.py
+++ b/src/python/grpcio/tests/_runner.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -143,10 +143,17 @@ class Runner(object):
 
   def run(self, suite):
     """See setuptools' test_runner setup argument for information."""
+    # only run test cases with id starting with given prefix
+    testcase_filter = os.getenv('GPRC_PYTHON_TESTRUNNER_FILTER')
+    filtered_cases = []
+    for case in _loader.iterate_suite_cases(suite):
+      if not testcase_filter or case.id().startswith(testcase_filter):
+        filtered_cases.append(case)
+
     # Ensure that every test case has no collision with any other test case in
     # the augmented results.
     augmented_cases = [AugmentedCase(case, uuid.uuid4())
-                       for case in _loader.iterate_suite_cases(suite)]
+                       for case in filtered_cases]
     case_id_by_case = dict((augmented_case.case, augmented_case.id)
                            for augmented_case in augmented_cases)
     result_out = StringIO.StringIO()
diff --git a/src/python/grpcio/tests/tests.json b/src/python/grpcio/tests/tests.json
new file mode 100644
index 0000000000..388d040d5c
--- /dev/null
+++ b/src/python/grpcio/tests/tests.json
@@ -0,0 +1,62 @@
+[
+  "_base_interface_test.AsyncEasyTest", 
+  "_base_interface_test.AsyncPeasyTest", 
+  "_base_interface_test.SyncEasyTest", 
+  "_base_interface_test.SyncPeasyTest", 
+  "_beta_features_test.BetaFeaturesTest", 
+  "_beta_features_test.ContextManagementAndLifecycleTest", 
+  "_channel_test.ChannelTest", 
+  "_connectivity_channel_test.ChannelConnectivityTest", 
+  "_core_over_links_base_interface_test.AsyncEasyTest", 
+  "_core_over_links_base_interface_test.AsyncPeasyTest", 
+  "_core_over_links_base_interface_test.SyncEasyTest", 
+  "_core_over_links_base_interface_test.SyncPeasyTest", 
+  "_crust_over_core_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", 
+  "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", 
+  "_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", 
+  "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", 
+  "_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", 
+  "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", 
+  "_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", 
+  "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", 
+  "_implementations_test.ChannelCredentialsTest", 
+  "_insecure_interop_test.InsecureInteropTest", 
+  "_intermediary_low_test.CancellationTest", 
+  "_intermediary_low_test.EchoTest", 
+  "_intermediary_low_test.ExpirationTest", 
+  "_intermediary_low_test.LonelyClientTest", 
+  "_later_test.LaterTest", 
+  "_logging_pool_test.LoggingPoolTest", 
+  "_lonely_invocation_link_test.LonelyInvocationLinkTest", 
+  "_low_test.HangingServerShutdown", 
+  "_low_test.InsecureServerInsecureClient", 
+  "_not_found_test.NotFoundTest", 
+  "_sanity_test.Sanity", 
+  "_secure_interop_test.SecureInteropTest", 
+  "_transmission_test.RoundTripTest", 
+  "_transmission_test.TransmissionTest", 
+  "_utilities_test.ChannelConnectivityTest", 
+  "beta_python_plugin_test.PythonPluginTest", 
+  "cygrpc_test.InsecureServerInsecureClient", 
+  "cygrpc_test.SecureServerSecureClient", 
+  "cygrpc_test.TypeSmokeTest"
+]
\ No newline at end of file
diff --git a/src/python/grpcio/tests/unit/_sanity/__init__.py b/src/python/grpcio/tests/unit/_sanity/__init__.py
new file mode 100644
index 0000000000..2f88fa0412
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_sanity/__init__.py
@@ -0,0 +1,30 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
diff --git a/src/python/grpcio/tests/unit/_sanity/_sanity_test.py b/src/python/grpcio/tests/unit/_sanity/_sanity_test.py
new file mode 100644
index 0000000000..0a5a715c0e
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_sanity/_sanity_test.py
@@ -0,0 +1,53 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import json
+import unittest
+
+import tests
+
+
+class Sanity(unittest.TestCase):
+
+  def testTestsJsonUpToDate(self):
+    """Autodiscovers all test suites and checks that tests.json is up to date"""
+    loader = tests.Loader()
+    loader.loadTestsFromNames(['tests'])
+    test_suite_names = [
+        test_case_class.id().rsplit('.', 1)[0]
+        for test_case_class in tests._loader.iterate_suite_cases(loader.suite)]
+    test_suite_names = sorted(set(test_suite_names))
+
+    with open('src/python/grpcio/tests/tests.json') as tests_json_file:
+      tests_json = json.load(tests_json_file)
+    self.assertListEqual(test_suite_names, tests_json)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)