Merge github.com:grpc/grpc into mergabit

9 files changed, 103 insertions, 22 deletions

diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh
new file mode 100755
index 0000000000..55f4c52ec2
--- /dev/null
+++ b/tools/distrib/clang_format_code.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -ex
+
+# change to root directory
+cd $(dirname $0)/../..
+
+# build clang-format docker image
+docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format
+
+# run clang-format against the checked out codebase
+docker run -e TEST=$TEST --rm=true -v `pwd`:/local-code -t grpc_clang_format /clang_format_all_the_things.sh
+
diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile
new file mode 100644
index 0000000000..a0fff2f2b5
--- /dev/null
+++ b/tools/dockerfile/grpc_clang_format/Dockerfile
@@ -0,0 +1,6 @@
+FROM ubuntu:vivid
+RUN apt-get update
+RUN apt-get -y install clang-format-3.6
+ADD clang_format_all_the_things.sh /
+CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"]
+
diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
new file mode 100755
index 0000000000..5aa265800d
--- /dev/null
+++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# directories to run against
+DIRS="src/core src/cpp test/core test/cpp include"
+
+# file matching patterns to check
+GLOB="*.h *.cpp"
+
+# clang format command
+CLANG_FORMAT=clang-format-3.6
+
+files=
+for dir in $DIRS
+do
+  for glob in $GLOB
+  do
+    files="$files `find /local-code/$dir -name $glob`"
+  done
+done
+
+if [ "x$TEST" = "x" ]
+then
+  echo $files | xargs $CLANG_FORMAT -i
+else
+  for file in $files
+  do
+    $CLANG_FORMAT $file | diff $file -
+  done
+fi
+
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 75e6f5119f..a7a3da4d20 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1067,6 +1067,7 @@ include/grpc/support/tls_gcc.h \
 include/grpc/support/tls_msvc.h \
 include/grpc/support/tls_pthread.h \
 include/grpc/support/useful.h \
+src/core/support/block_annotate.h \
 src/core/support/env.h \
 src/core/support/file.h \
 src/core/support/murmur_hash.h \
diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh
index fa6bd44e18..9f97f39e75 100755
--- a/tools/jenkins/build_docker_and_run_tests.sh
+++ b/tools/jenkins/build_docker_and_run_tests.sh
@@ -60,6 +60,8 @@ docker run \
   -i $TTY_FLAG \
   -v "$git_root:/var/local/jenkins/grpc" \
   -v /tmp/ccache:/tmp/ccache \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v $(which docker):/bin/docker \
   -w /var/local/git/grpc \
   --cidfile=docker.cid \
   $DOCKER_IMAGE_NAME \
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index c0b9c02b3e..fb3632e4ad 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -72,6 +72,7 @@ _COLORS = {
     'yellow': [ 33, 0 ],
     'lightgray': [ 37, 0],
     'gray': [ 30, 1 ],
+    'purple': [ 35, 0 ],
     }
 
 
@@ -81,6 +82,7 @@ _CLEAR_LINE = '\x1b[2K'
 
 _TAG_COLOR = {
     'FAILED': 'red',
+    'FLAKE': 'purple',
     'WARNING': 'yellow',
     'TIMEOUT': 'red',
     'PASSED': 'green',
@@ -131,7 +133,7 @@ class JobSpec(object):
   """Specifies what to run for a job."""
 
   def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None,
-               cwd=None, shell=False, timeout_seconds=5*60):
+               cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0):
     """
     Arguments:
       cmdline: a list of arguments to pass as the command line
@@ -150,6 +152,7 @@ class JobSpec(object):
     self.cwd = cwd
     self.shell = shell
     self.timeout_seconds = timeout_seconds
+    self.flake_retries = flake_retries
 
   def identity(self):
     return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets)
@@ -167,25 +170,28 @@ class Job(object):
   def __init__(self, spec, bin_hash, newline_on_success, travis, add_env, xml_report):
     self._spec = spec
     self._bin_hash = bin_hash
+    self._newline_on_success = newline_on_success
+    self._travis = travis
+    self._add_env = add_env.copy()
+    self._xml_test = ET.SubElement(xml_report, 'testcase',
+                                   name=self._spec.shortname) if xml_report is not None else None
+    self._retries = 0
+    message('START', spec.shortname, do_newline=self._travis)
+    self.start()
+
+  def start(self):
     self._tempfile = tempfile.TemporaryFile()
-    env = os.environ.copy()
-    for k, v in spec.environ.iteritems():
-      env[k] = v
-    for k, v in add_env.iteritems():
-      env[k] = v
+    env = dict(os.environ)
+    env.update(self._spec.environ)
+    env.update(self._add_env)
     self._start = time.time()
-    message('START', spec.shortname, do_newline=travis)
-    self._process = subprocess.Popen(args=spec.cmdline,
+    self._process = subprocess.Popen(args=self._spec.cmdline,
                                      stderr=subprocess.STDOUT,
                                      stdout=self._tempfile,
-                                     cwd=spec.cwd,
-                                     shell=spec.shell,
+                                     cwd=self._spec.cwd,
+                                     shell=self._spec.shell,
                                      env=env)
     self._state = _RUNNING
-    self._newline_on_success = newline_on_success
-    self._travis = travis
-    self._xml_test = ET.SubElement(xml_report, 'testcase',
-                                   name=self._spec.shortname) if xml_report is not None else None
 
   def state(self, update_cache):
     """Poll current state of the job. Prints messages at completion."""
@@ -202,15 +208,22 @@ class Job(object):
         self._xml_test.set('time', str(elapsed))
         ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
       if self._process.returncode != 0:
-        self._state = _FAILURE
-        message('FAILED', '%s [ret=%d, pid=%d]' % (
+        if self._retries < self._spec.flake_retries:
+          message('FLAKE', '%s [ret=%d, pid=%d]' % (
             self._spec.shortname, self._process.returncode, self._process.pid),
             stdout, do_newline=True)
-        if self._xml_test is not None:
-          ET.SubElement(self._xml_test, 'failure', message='Failure').text
+          self._retries += 1
+          self.start()
+        else:
+          self._state = _FAILURE
+          message('FAILED', '%s [ret=%d, pid=%d]' % (
+              self._spec.shortname, self._process.returncode, self._process.pid),
+              stdout, do_newline=True)
+          if self._xml_test is not None:
+            ET.SubElement(self._xml_test, 'failure', message='Failure').text
       else:
         self._state = _SUCCESS
-        message('PASSED', '%s [time=%.1fsec]' % (self._spec.shortname, elapsed),
+        message('PASSED', '%s [time=%.1fsec; retries=%d]' % (self._spec.shortname, elapsed, self._retries),
                 do_newline=self._newline_on_success or self._travis)
         if self._bin_hash:
           update_cache.finished(self._spec.identity(), self._bin_hash)
diff --git a/tools/run_tests/run_sanity.sh b/tools/run_tests/run_sanity.sh
index 4b367dcbc7..1e8fe5c1a7 100755
--- a/tools/run_tests/run_sanity.sh
+++ b/tools/run_tests/run_sanity.sh
@@ -58,3 +58,5 @@ if [ -f cache.mk ] ; then
 fi
 
 ./tools/buildgen/generate_projects.sh
+./tools/distrib/clang_format_code.sh
+
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index d271137ca1..cbbb58d7e7 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -40,6 +40,7 @@ import os
 import platform
 import random
 import re
+import socket
 import subprocess
 import sys
 import time
@@ -99,7 +100,8 @@ class SimpleConfig(object):
                           environ=actual_environ,
                           timeout_seconds=self.timeout_seconds,
                           hash_targets=hash_targets
-                              if self.allow_hashing else None)
+                              if self.allow_hashing else None,
+                          flake_retries=5 if args.allow_flakes else 0)
 
 
 # ValgrindConfig: compile with some CONFIG=config, but use valgrind to run
@@ -117,7 +119,8 @@ class ValgrindConfig(object):
     return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool] +
                           self.args + cmdline,
                           shortname='valgrind %s' % cmdline[0],
-                          hash_targets=None)
+                          hash_targets=None,
+                          flake_retries=5 if args.allow_flakes else 0)
 
 
 def get_c_tests(travis, test_lang) :
@@ -555,6 +558,11 @@ argp.add_argument('--use_docker',
                   help="Run all the tests under docker. That provides " +
                   "additional isolation and prevents the need to installs " +
                   "language specific prerequisites. Only available on Linux.")
+argp.add_argument('--allow_flakes',
+                  default=False,
+                  action='store_const',
+                  const=True,
+                  help="Allow flaky tests to show as passing (re-runs failed tests up to five times)")
 argp.add_argument('-a', '--antagonists', default=0, type=int)
 argp.add_argument('-x', '--xml_report', default=None, type=str,
         help='Generates a JUnit-compatible XML report')
@@ -617,7 +625,7 @@ if platform.system() == 'Windows':
     # better do parallel compilation
     extra_args.extend(["/m"])
     # disable PDB generation: it's broken, and we don't need it during CI
-    extra_args.extend(["/p:GenerateDebugInformation=false", "/p:DebugInformationFormat=None"])
+    extra_args.extend(["/p:Jenkins=true"])
     return [
       jobset.JobSpec(['vsprojects\\build.bat',
                       'vsprojects\\%s.sln' % target,
@@ -736,6 +744,10 @@ def _start_port_server(port_server_port):
         urllib2.urlopen('http://localhost:%d/get' % port_server_port,
                         timeout=1).read()
         break
+      except socket.timeout:
+        print "waiting for port_server"
+        time.sleep(0.5)
+        waits += 1
       except urllib2.URLError:
         print "waiting for port_server"
         time.sleep(0.5)
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index 071d7943e4..1e5dd11fcd 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -12161,6 +12161,7 @@
       "include/grpc/support/tls_msvc.h", 
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
+      "src/core/support/block_annotate.h", 
       "src/core/support/env.h", 
       "src/core/support/file.h", 
       "src/core/support/murmur_hash.h", 
@@ -12201,6 +12202,7 @@
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
       "src/core/support/alloc.c", 
+      "src/core/support/block_annotate.h", 
       "src/core/support/cmdline.c", 
       "src/core/support/cpu_iphone.c", 
       "src/core/support/cpu_linux.c",