Merge github.com:grpc/grpc into flaky-timeouts

11 files changed, 78 insertions, 12 deletions

diff --git a/tools/buildgen/build-cleaner.py b/tools/buildgen/build-cleaner.py
index 6c9c6b3749..8288a8998d 100755
--- a/tools/buildgen/build-cleaner.py
+++ b/tools/buildgen/build-cleaner.py
@@ -88,7 +88,7 @@ for filename in sys.argv[1:]:
     if grp not in js: continue
     js[grp] = sorted([clean_elem(x) for x in js[grp]],
                      key=lambda x: (x.get('language', '_'), x['name']))
-  output = yaml.dump(js, indent=2, width=80)
+  output = yaml.dump(js, indent=2, width=80, default_flow_style=False)
   # massage out trailing whitespace
   lines = []
   for line in output.splitlines():
diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh
new file mode 100755
index 0000000000..55f4c52ec2
--- /dev/null
+++ b/tools/distrib/clang_format_code.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -ex
+
+# change to root directory
+cd $(dirname $0)/../..
+
+# build clang-format docker image
+docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format
+
+# run clang-format against the checked out codebase
+docker run -e TEST=$TEST --rm=true -v `pwd`:/local-code -t grpc_clang_format /clang_format_all_the_things.sh
+
diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile
new file mode 100644
index 0000000000..a0fff2f2b5
--- /dev/null
+++ b/tools/dockerfile/grpc_clang_format/Dockerfile
@@ -0,0 +1,6 @@
+FROM ubuntu:vivid
+RUN apt-get update
+RUN apt-get -y install clang-format-3.6
+ADD clang_format_all_the_things.sh /
+CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"]
+
diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
new file mode 100755
index 0000000000..5aa265800d
--- /dev/null
+++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# directories to run against
+DIRS="src/core src/cpp test/core test/cpp include"
+
+# file matching patterns to check
+GLOB="*.h *.cpp"
+
+# clang format command
+CLANG_FORMAT=clang-format-3.6
+
+files=
+for dir in $DIRS
+do
+  for glob in $GLOB
+  do
+    files="$files `find /local-code/$dir -name $glob`"
+  done
+done
+
+if [ "x$TEST" = "x" ]
+then
+  echo $files | xargs $CLANG_FORMAT -i
+else
+  for file in $files
+  do
+    $CLANG_FORMAT $file | diff $file -
+  done
+fi
+
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 75e6f5119f..a7a3da4d20 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1067,6 +1067,7 @@ include/grpc/support/tls_gcc.h \
 include/grpc/support/tls_msvc.h \
 include/grpc/support/tls_pthread.h \
 include/grpc/support/useful.h \
+src/core/support/block_annotate.h \
 src/core/support/env.h \
 src/core/support/file.h \
 src/core/support/murmur_hash.h \
diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh
index fa6bd44e18..9f97f39e75 100755
--- a/tools/jenkins/build_docker_and_run_tests.sh
+++ b/tools/jenkins/build_docker_and_run_tests.sh
@@ -60,6 +60,8 @@ docker run \
   -i $TTY_FLAG \
   -v "$git_root:/var/local/jenkins/grpc" \
   -v /tmp/ccache:/tmp/ccache \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v $(which docker):/bin/docker \
   -w /var/local/git/grpc \
   --cidfile=docker.cid \
   $DOCKER_IMAGE_NAME \
diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh
index a55f1de996..c4a01a7d66 100755
--- a/tools/jenkins/run_jenkins.sh
+++ b/tools/jenkins/run_jenkins.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -31,8 +31,6 @@
 # This script is invoked by Jenkins and triggers a test run based on
 # env variable settings.
 #
-# Bootstrap into bash
-[ -z $1 ] && exec bash $0 bootstrapped
 # Setting up rvm environment BEFORE we set -ex.
 [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
 # To prevent cygwin bash complaining about empty lines ending with \r
@@ -56,11 +54,11 @@ if [ "$platform" == "linux" ]
 then
   echo "building $language on Linux"
 
-  ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml || true
+  ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml $@ || true
 
 elif [ "$platform" == "interop" ]
 then
-  python tools/run_tests/run_interops.py --language=$language
+  python tools/run_tests/run_interops.py --language=$language $@
 elif [ "$platform" == "windows" ]
 then
   echo "building $language on Windows"
@@ -72,19 +70,19 @@ then
   /cygdrive/c/nuget/nuget.exe restore vsprojects/grpc.sln
   /cygdrive/c/nuget/nuget.exe restore src/csharp/Grpc.sln
 
-  python tools/run_tests/run_tests.py -t -l $language -x report.xml || true
+  python tools/run_tests/run_tests.py -t -l $language -x report.xml $@ || true
 
 elif [ "$platform" == "macos" ]
 then
   echo "building $language on MacOS"
 
-  ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml || true
+  ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || true
 
 elif [ "$platform" == "freebsd" ]
 then
   echo "building $language on FreeBSD"
 
-  MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml || true
+  MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || true
 else
   echo "Unknown platform $platform"
   exit 1
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index f6fdc80595..068deb0b17 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -134,7 +134,7 @@ class JobSpec(object):
   """Specifies what to run for a job."""
 
   def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None,
-               cwd=None, shell=False, timeout_seconds=5*60, flake_retries=5):
+               cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0):
     """
     Arguments:
       cmdline: a list of arguments to pass as the command line
diff --git a/tools/run_tests/run_sanity.sh b/tools/run_tests/run_sanity.sh
index 4b367dcbc7..1e8fe5c1a7 100755
--- a/tools/run_tests/run_sanity.sh
+++ b/tools/run_tests/run_sanity.sh
@@ -58,3 +58,5 @@ if [ -f cache.mk ] ; then
 fi
 
 ./tools/buildgen/generate_projects.sh
+./tools/distrib/clang_format_code.sh
+
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index ac5eef663e..cbbb58d7e7 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -40,6 +40,7 @@ import os
 import platform
 import random
 import re
+import socket
 import subprocess
 import sys
 import time
@@ -99,7 +100,8 @@ class SimpleConfig(object):
                           environ=actual_environ,
                           timeout_seconds=self.timeout_seconds,
                           hash_targets=hash_targets
-                              if self.allow_hashing else None)
+                              if self.allow_hashing else None,
+                          flake_retries=5 if args.allow_flakes else 0)
 
 
 # ValgrindConfig: compile with some CONFIG=config, but use valgrind to run
@@ -117,7 +119,8 @@ class ValgrindConfig(object):
     return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool] +
                           self.args + cmdline,
                           shortname='valgrind %s' % cmdline[0],
-                          hash_targets=None)
+                          hash_targets=None,
+                          flake_retries=5 if args.allow_flakes else 0)
 
 
 def get_c_tests(travis, test_lang) :
@@ -555,6 +558,11 @@ argp.add_argument('--use_docker',
                   help="Run all the tests under docker. That provides " +
                   "additional isolation and prevents the need to installs " +
                   "language specific prerequisites. Only available on Linux.")
+argp.add_argument('--allow_flakes',
+                  default=False,
+                  action='store_const',
+                  const=True,
+                  help="Allow flaky tests to show as passing (re-runs failed tests up to five times)")
 argp.add_argument('-a', '--antagonists', default=0, type=int)
 argp.add_argument('-x', '--xml_report', default=None, type=str,
         help='Generates a JUnit-compatible XML report')
@@ -736,6 +744,10 @@ def _start_port_server(port_server_port):
         urllib2.urlopen('http://localhost:%d/get' % port_server_port,
                         timeout=1).read()
         break
+      except socket.timeout:
+        print "waiting for port_server"
+        time.sleep(0.5)
+        waits += 1
       except urllib2.URLError:
         print "waiting for port_server"
         time.sleep(0.5)
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index 071d7943e4..1e5dd11fcd 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -12161,6 +12161,7 @@
       "include/grpc/support/tls_msvc.h", 
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
+      "src/core/support/block_annotate.h", 
       "src/core/support/env.h", 
       "src/core/support/file.h", 
       "src/core/support/murmur_hash.h", 
@@ -12201,6 +12202,7 @@
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
       "src/core/support/alloc.c", 
+      "src/core/support/block_annotate.h", 
       "src/core/support/cmdline.c", 
       "src/core/support/cpu_iphone.c", 
       "src/core/support/cpu_linux.c",