Merge github.com:grpc/grpc into latent-see

12 files changed, 101 insertions, 61 deletions

diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index e742ec2571..d3be97ecbc 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -857,6 +857,7 @@ src/core/statistics/census_rpc_stats.h \
 src/core/surface/api_trace.h \
 src/core/surface/byte_buffer_queue.h \
 src/core/surface/call.h \
+src/core/surface/call_test_only.h \
 src/core/surface/channel.h \
 src/core/surface/completion_queue.h \
 src/core/surface/event_string.h \
diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh
index 8b7809f2e2..6e3166ce57 100755
--- a/tools/jenkins/build_docker_and_run_tests.sh
+++ b/tools/jenkins/build_docker_and_run_tests.sh
@@ -53,8 +53,8 @@ DOCKER_IMAGE_NAME=grpc_jenkins_slave${docker_suffix}_`sha1sum tools/jenkins/grpc
 # Make sure docker image has been built. Should be instantaneous if so.
 docker build -t $DOCKER_IMAGE_NAME tools/jenkins/grpc_jenkins_slave$docker_suffix
 
-# Make sure the CID file is gone.
-rm -f docker.cid
+# Choose random name for docker container
+CONTAINER_NAME="run_tests_$(uuidgen)"
 
 # Run tests inside docker
 docker run \
@@ -70,23 +70,21 @@ docker run \
   -v /var/run/docker.sock:/var/run/docker.sock \
   -v $(which docker):/bin/docker \
   -w /var/local/git/grpc \
-  --cidfile=docker.cid \
+  --name=$CONTAINER_NAME \
   $DOCKER_IMAGE_NAME \
   bash -l /var/local/jenkins/grpc/tools/jenkins/docker_run_tests.sh || DOCKER_FAILED="true"
 
-DOCKER_CID=`cat docker.cid`
-
 if [ "$XML_REPORT" != "" ]
 then
-  docker cp "$DOCKER_CID:/var/local/git/grpc/$XML_REPORT" $git_root
+  docker cp "$CONTAINER_NAME:/var/local/git/grpc/$XML_REPORT" $git_root
 fi
 
-docker cp "$DOCKER_CID:/var/local/git/grpc/reports.zip" $git_root || true
+docker cp "$CONTAINER_NAME:/var/local/git/grpc/reports.zip" $git_root || true
 unzip $git_root/reports.zip -d $git_root || true
 rm -f reports.zip
 
 # remove the container, possibly killing it first
-docker rm -f $DOCKER_CID || true
+docker rm -f $CONTAINER_NAME || true
 
 if [ "$DOCKER_FAILED" != "" ] && [ "$XML_REPORT" == "" ]
 then
diff --git a/tools/jenkins/build_interop_image.sh b/tools/jenkins/build_interop_image.sh
index 3664eed84d..166efbd9e2 100755
--- a/tools/jenkins/build_interop_image.sh
+++ b/tools/jenkins/build_interop_image.sh
@@ -77,7 +77,7 @@ docker build -t $BASE_IMAGE --force-rm=true tools/jenkins/$BASE_NAME || exit $?
 # Create a local branch so the child Docker script won't complain
 git branch -f jenkins-docker
 
-CIDFILE=`mktemp -u --suffix=.cid`
+CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)"
 
 # Prepare image for interop tests, commit it on success.
 (docker run \
@@ -85,17 +85,14 @@ CIDFILE=`mktemp -u --suffix=.cid`
   -i $TTY_FLAG \
   $MOUNT_ARGS \
   -v /tmp/ccache:/tmp/ccache \
-  --cidfile=$CIDFILE \
+  --name=$CONTAINER_NAME \
   $BASE_IMAGE \
   bash -l /var/local/jenkins/grpc/tools/jenkins/$BASE_NAME/build_interop.sh \
-  && docker commit `cat $CIDFILE` $INTEROP_IMAGE \
+  && docker commit $CONTAINER_NAME $INTEROP_IMAGE \
   && echo "Successfully built image $INTEROP_IMAGE")
 EXITCODE=$?
 
 # remove intermediate container, possibly killing it first
-docker rm -f `cat $CIDFILE`
-
-# remove the cidfile
-rm -rf `cat $CIDFILE`
+docker rm -f $CONTAINER_NAME
 
 exit $EXITCODE
diff --git a/tools/jenkins/grpc_interop_node/Dockerfile b/tools/jenkins/grpc_interop_node/Dockerfile
index 587227b942..db5aff844d 100644
--- a/tools/jenkins/grpc_interop_node/Dockerfile
+++ b/tools/jenkins/grpc_interop_node/Dockerfile
@@ -48,6 +48,7 @@ RUN apt-get update && apt-get install -y \
   libc6-dbg \
   libc6-dev \
   libgtest-dev \
+  libssl-dev \
   libtool \
   make \
   strace \
diff --git a/tools/jenkins/grpc_interop_node/build_interop.sh b/tools/jenkins/grpc_interop_node/build_interop.sh
index 84e25e3308..3b69715c9a 100755
--- a/tools/jenkins/grpc_interop_node/build_interop.sh
+++ b/tools/jenkins/grpc_interop_node/build_interop.sh
@@ -45,5 +45,4 @@ make install-certs
 
 # build Node interop client & server
 npm install -g node-gyp
-make install_c -C /var/local/git/grpc
-(cd src/node && npm install && node-gyp rebuild)
+(npm install && node-gyp rebuild)
diff --git a/tools/jenkins/grpc_jenkins_slave/Dockerfile b/tools/jenkins/grpc_jenkins_slave/Dockerfile
index 5f2b425c8c..129a8db24f 100644
--- a/tools/jenkins/grpc_jenkins_slave/Dockerfile
+++ b/tools/jenkins/grpc_jenkins_slave/Dockerfile
@@ -157,6 +157,12 @@ RUN apt-get update && apt-get install -y \
 
 RUN apt-get install -y libzookeeper-mt-dev
 
+##################
+# Docker "inception".
+# Note this is quite the ugly hack.
+# This makes sure that the docker binary we inject has its dependencies.
+RUN curl https://get.docker.com/ | sh
+RUN apt-get remove --purge -y docker-engine
 
 RUN mkdir /var/local/jenkins
 
diff --git a/tools/run_tests/dockerjob.py b/tools/run_tests/dockerjob.py
index 11686d46b0..1d67fe3033 100755
--- a/tools/run_tests/dockerjob.py
+++ b/tools/run_tests/dockerjob.py
@@ -38,24 +38,32 @@ import subprocess
 
 _DEVNULL = open(os.devnull, 'w')
 
-def wait_for_file(filepath, timeout_seconds=15):
-  """Wait until given file exists and returns its content."""
-  started = time.time()
-  while time.time() - started < timeout_seconds:
-    if os.path.isfile(filepath):
-      with open(filepath, 'r') as f:
-        content = f.read()
-        # make sure we don't return empty content
-        if content:
-          return content
-    time.sleep(1)
-  raise Exception('Failed to read file %s.' % filepath)
+
+def random_name(base_name):
+  """Randomizes given base name."""
+  return '%s_%s' % (base_name, uuid.uuid4())
+
+
+def docker_kill(cid):
+  """Kills a docker container. Returns True if successful."""
+  return subprocess.call(['docker','kill', str(cid)],
+                         stdout=_DEVNULL,
+                         stderr=subprocess.STDOUT) == 0
 
 
-def docker_mapped_port(cid, port):
+def docker_mapped_port(cid, port, timeout_seconds=15):
   """Get port mapped to internal given internal port for given container."""
-  output = subprocess.check_output('docker port %s %s' % (cid, port), shell=True)
-  return int(output.split(':', 2)[1])
+  started = time.time()
+  while time.time() - started < timeout_seconds:
+    try:
+      output = subprocess.check_output('docker port %s %s' % (cid, port),
+                                       stderr=_DEVNULL,
+                                       shell=True)
+      return int(output.split(':', 2)[1])
+    except subprocess.CalledProcessError as e:
+      pass
+  raise Exception('Failed to get exposed port %s for container %s.' %
+                  (port, cid))
 
 
 def finish_jobs(jobs):
@@ -71,7 +79,7 @@ def image_exists(image):
   """Returns True if given docker image exists."""
   return subprocess.call(['docker','inspect', image],
                          stdout=_DEVNULL,
-                         stderr=_DEVNULL) == 0
+                         stderr=subprocess.STDOUT) == 0
 
 
 def remove_image(image, skip_nonexistent=False, max_retries=10):
@@ -79,7 +87,9 @@ def remove_image(image, skip_nonexistent=False, max_retries=10):
   if skip_nonexistent and not image_exists(image):
     return True
   for attempt in range(0, max_retries):
-    if subprocess.call(['docker','rmi', '-f', image]) == 0:
+    if subprocess.call(['docker','rmi', '-f', image],
+                       stdout=_DEVNULL,
+                       stderr=subprocess.STDOUT) == 0:
       return True
     time.sleep(2)
   print 'Failed to remove docker image %s' % image
@@ -92,23 +102,16 @@ class DockerJob:
   def __init__(self, spec):
     self._spec = spec
     self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={}, xml_report=None)
-    self._cidfile = spec.cidfile
-    self._cid = None
-
-  def cid(self):
-    """Gets cid of this container"""
-    if not self._cid:
-      self._cid = wait_for_file(self._cidfile)
-    return self._cid
+    self._container_name = spec.container_name
 
   def mapped_port(self, port):
-    return docker_mapped_port(self.cid(), port)
+    return docker_mapped_port(self._container_name, port)
 
   def kill(self, suppress_failure=False):
     """Sends kill signal to the container."""
     if suppress_failure:
       self._job.suppress_failure_message()
-    return subprocess.call(['docker','kill', self.cid()]) == 0
+    return docker_kill(self._container_name)
 
   def is_running(self):
     """Polls a job and returns True if given job is still running."""
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index 87be703b4c..17a63c02e8 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -135,13 +135,14 @@ class JobSpec(object):
 
   def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None,
                cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0,
-               timeout_retries=0):
+               timeout_retries=0, kill_handler=None):
     """
     Arguments:
       cmdline: a list of arguments to pass as the command line
       environ: a dictionary of environment variables to set in the child process
       hash_targets: which files to include in the hash representing the jobs version
                     (or empty, indicating the job should not be hashed)
+      kill_handler: a handler that will be called whenever job.kill() is invoked
     """
     if environ is None:
       environ = {}
@@ -156,6 +157,7 @@ class JobSpec(object):
     self.timeout_seconds = timeout_seconds
     self.flake_retries = flake_retries
     self.timeout_retries = timeout_retries
+    self.kill_handler = kill_handler
 
   def identity(self):
     return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets)
@@ -254,6 +256,8 @@ class Job(object):
   def kill(self):
     if self._state == _RUNNING:
       self._state = _KILLED
+      if self._spec.kill_handler:
+        self._spec.kill_handler(self)
       self._process.terminate()
 
   def suppress_failure_message(self):
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index f328f4642e..6daa967bba 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -71,6 +71,7 @@ class CXXLanguage:
     self.client_cmdline_base = ['bins/opt/interop_client']
     self.client_cwd = None
     self.server_cwd = None
+    self.safename = 'cxx'
 
   def cloud_to_prod_args(self):
     return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS +
@@ -96,6 +97,7 @@ class CSharpLanguage:
     self.client_cmdline_base = ['mono', 'Grpc.IntegrationTesting.Client.exe']
     self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug'
     self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug'
+    self.safename = str(self)
 
   def cloud_to_prod_args(self):
     return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS +
@@ -121,6 +123,7 @@ class JavaLanguage:
     self.client_cmdline_base = ['./run-test-client.sh']
     self.client_cwd = '../grpc-java'
     self.server_cwd = '../grpc-java'
+    self.safename = str(self)
 
   def cloud_to_prod_args(self):
     return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS +
@@ -147,6 +150,7 @@ class GoLanguage:
     # TODO: this relies on running inside docker
     self.client_cwd = '/go/src/google.golang.org/grpc/interop/client'
     self.server_cwd = '/go/src/google.golang.org/grpc/interop/server'
+    self.safename = str(self)
 
   def cloud_to_prod_args(self):
     return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS +
@@ -172,6 +176,7 @@ class NodeLanguage:
     self.client_cmdline_base = ['node', 'src/node/interop/interop_client.js']
     self.client_cwd = None
     self.server_cwd = None
+    self.safename = str(self)
 
   def cloud_to_prod_args(self):
     return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS +
@@ -196,6 +201,7 @@ class PHPLanguage:
   def __init__(self):
     self.client_cmdline_base = ['src/php/bin/interop_client.sh']
     self.client_cwd = None
+    self.safename = str(self)
 
   def cloud_to_prod_args(self):
     return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS +
@@ -218,6 +224,7 @@ class RubyLanguage:
     self.client_cmdline_base = ['ruby', 'src/ruby/bin/interop/interop_client.rb']
     self.client_cwd = None
     self.server_cwd = None
+    self.safename = str(self)
 
   def cloud_to_prod_args(self):
     return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS +
@@ -251,11 +258,9 @@ _LANGUAGES = {
 # languages supported as cloud_to_cloud servers
 _SERVERS = ['c++', 'node', 'csharp', 'java', 'go', 'ruby']
 
-# TODO(jtattermusch): add empty_stream once PHP starts supporting it.
 # TODO(jtattermusch): add timeout_on_sleeping_server once java starts supporting it.
-# TODO(jtattermusch): add support for auth tests.
 _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong',
-               'client_streaming', 'server_streaming',
+               'empty_stream', 'client_streaming', 'server_streaming',
                'cancel_after_begin', 'cancel_after_first_response']
 
 _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds',
@@ -321,17 +326,29 @@ def add_auth_options(language, test_case, cmdline, env):
   return (cmdline, env)
 
 
+def _job_kill_handler(job):
+  if job._spec.container_name:
+    dockerjob.docker_kill(job._spec.container_name)
+
+
 def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
   """Creates jobspec for cloud-to-prod interop test"""
   cmdline = language.cloud_to_prod_args() + ['--test_case=%s' % test_case]
   cwd = language.client_cwd
   environ = language.cloud_to_prod_env()
+  container_name = None
   if auth:
     cmdline, environ = add_auth_options(language, test_case, cmdline, environ)
   cmdline = bash_login_cmdline(cmdline)
 
   if docker_image:
-    cmdline = docker_run_cmdline(cmdline, image=docker_image, cwd=cwd, environ=environ)
+    container_name = dockerjob.random_name('interop_client_%s' % language.safename)
+    cmdline = docker_run_cmdline(cmdline,
+                                 image=docker_image,
+                                 cwd=cwd,
+                                 environ=environ,
+                                 docker_args=['--net=host',
+                                              '--name', container_name])
     cwd = None
     environ = None
 
@@ -343,7 +360,9 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
           shortname="%s:%s:%s" % (suite_name, language, test_case),
           timeout_seconds=2*60,
           flake_retries=5 if args.allow_flakes else 0,
-          timeout_retries=2 if args.allow_flakes else 0)
+          timeout_retries=2 if args.allow_flakes else 0,
+          kill_handler=_job_kill_handler)
+  test_job.container_name = container_name
   return test_job
 
 
@@ -356,11 +375,14 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
                                 '--server_port=%s' % server_port ])
   cwd = language.client_cwd
   if docker_image:
+    container_name = dockerjob.random_name('interop_client_%s' % language.safename)
     cmdline = docker_run_cmdline(cmdline,
                                  image=docker_image,
                                  cwd=cwd,
-                                 docker_args=['--net=host'])
+                                 docker_args=['--net=host',
+                                              '--name', container_name])
     cwd = None
+
   test_job = jobset.JobSpec(
           cmdline=cmdline,
           cwd=cwd,
@@ -368,34 +390,36 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
                                                  test_case),
           timeout_seconds=2*60,
           flake_retries=5 if args.allow_flakes else 0,
-          timeout_retries=2 if args.allow_flakes else 0)
+          timeout_retries=2 if args.allow_flakes else 0,
+          kill_handler=_job_kill_handler)
+  test_job.container_name = container_name
   return test_job
 
 
 def server_jobspec(language, docker_image):
   """Create jobspec for running a server"""
-  cidfile = tempfile.mktemp()
+  container_name = dockerjob.random_name('interop_server_%s' % language.safename)
   cmdline = bash_login_cmdline(language.server_args() +
                                ['--port=%s' % _DEFAULT_SERVER_PORT])
   docker_cmdline = docker_run_cmdline(cmdline,
                                       image=docker_image,
                                       cwd=language.server_cwd,
                                       docker_args=['-p', str(_DEFAULT_SERVER_PORT),
-                                                   '--cidfile', cidfile])
+                                                   '--name', container_name])
   server_job = jobset.JobSpec(
           cmdline=docker_cmdline,
-          shortname="interop_server:%s" % language,
+          shortname="interop_server_%s" % language,
           timeout_seconds=30*60)
-  server_job.cidfile = cidfile
+  server_job.container_name = container_name
   return server_job
 
 
 def build_interop_image_jobspec(language, tag=None):
   """Creates jobspec for building interop docker image for a language"""
-  safelang = str(language).replace("+", "x")
   if not tag:
-    tag = 'grpc_interop_%s:%s' % (safelang, uuid.uuid4())
-  env = {'INTEROP_IMAGE': tag, 'BASE_NAME': 'grpc_interop_%s' % safelang}
+    tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4())
+  env = {'INTEROP_IMAGE': tag,
+         'BASE_NAME': 'grpc_interop_%s' % language.safename}
   if not args.travis:
     env['TTY_FLAG'] = '-t'
   build_job = jobset.JobSpec(
diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/run_node.sh
index 780969089d..0a11e87c37 100755
--- a/tools/run_tests/run_node.sh
+++ b/tools/run_tests/run_node.sh
@@ -46,6 +46,8 @@ then
   lcov --base-directory . --directory . -c -o coverage.info
   genhtml -o ../reports/node_ext_coverage --num-spaces 2 \
     -t 'Node gRPC test coverage' coverage.info
+  echo '<html><head><meta http-equiv="refresh" content="0;URL=lcov-report/index.html"></head></html>' > \
+    ../reports/node_coverage/index.html
 else
   ./node_modules/mocha/bin/mocha --timeout 8000 src/node/test
 fi
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index a3bed4ef87..ce1e0887f2 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -101,7 +101,8 @@ class SimpleConfig(object):
                           timeout_seconds=self.timeout_seconds,
                           hash_targets=hash_targets
                               if self.allow_hashing else None,
-                          flake_retries=5 if args.allow_flakes else 0)
+                          flake_retries=5 if args.allow_flakes else 0,
+                          timeout_retries=3 if args.allow_flakes else 0)
 
 
 # ValgrindConfig: compile with some CONFIG=config, but use valgrind to run
@@ -121,7 +122,7 @@ class ValgrindConfig(object):
                           shortname='valgrind %s' % cmdline[0],
                           hash_targets=None,
                           flake_retries=5 if args.allow_flakes else 0,
-                          timeout_retries=2 if args.allow_flakes else 0)
+                          timeout_retries=3 if args.allow_flakes else 0)
 
 
 def get_c_tests(travis, test_lang) :
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index c16477ccf8..32c490376d 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -12369,6 +12369,7 @@
       "src/core/surface/api_trace.h", 
       "src/core/surface/byte_buffer_queue.h", 
       "src/core/surface/call.h", 
+      "src/core/surface/call_test_only.h", 
       "src/core/surface/channel.h", 
       "src/core/surface/completion_queue.h", 
       "src/core/surface/event_string.h", 
@@ -12608,6 +12609,7 @@
       "src/core/surface/call.h", 
       "src/core/surface/call_details.c", 
       "src/core/surface/call_log_batch.c", 
+      "src/core/surface/call_test_only.h", 
       "src/core/surface/channel.c", 
       "src/core/surface/channel.h", 
       "src/core/surface/channel_connectivity.c", 
@@ -12861,6 +12863,7 @@
       "src/core/surface/api_trace.h", 
       "src/core/surface/byte_buffer_queue.h", 
       "src/core/surface/call.h", 
+      "src/core/surface/call_test_only.h", 
       "src/core/surface/channel.h", 
       "src/core/surface/completion_queue.h", 
       "src/core/surface/event_string.h", 
@@ -13070,6 +13073,7 @@
       "src/core/surface/call.h", 
       "src/core/surface/call_details.c", 
       "src/core/surface/call_log_batch.c", 
+      "src/core/surface/call_test_only.h", 
       "src/core/surface/channel.c", 
       "src/core/surface/channel.h", 
       "src/core/surface/channel_connectivity.c",