Merge branch 'master' into make_clang_great_again_v2

6 files changed, 244 insertions, 5 deletions

diff --git a/tools/distrib/check_include_guards.py b/tools/distrib/check_include_guards.py
new file mode 100755
index 0000000000..977f40e0b3
--- /dev/null
+++ b/tools/distrib/check_include_guards.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python2.7
+
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import os
+import re
+import sys
+import subprocess
+
+
+def build_valid_guard(fpath):
+  prefix = 'GRPC_' if not fpath.startswith('include/') else ''
+  return prefix + '_'.join(fpath.replace('++', 'XX').replace('.', '_').upper().split('/')[1:])
+
+
+def load(fpath):
+  with open(fpath, 'r') as f:
+    return f.read()
+
+
+def save(fpath, contents):
+  with open(fpath, 'w') as f:
+    f.write(contents)
+
+
+class GuardValidator(object):
+  def __init__(self):
+    self.ifndef_re = re.compile(r'#ifndef ([A-Z][A-Z_1-9]*)')
+    self.define_re = re.compile(r'#define ([A-Z][A-Z_1-9]*)')
+    self.endif_c_re = re.compile(r'#endif /\* ([A-Z][A-Z_1-9]*) \*/')
+    self.endif_cpp_re = re.compile(r'#endif  // ([A-Z][A-Z_1-9]*)')
+    self.failed = False
+
+  def fail(self, fpath, regexp, fcontents, match_txt, correct, fix):
+    cpp_header = 'grpc++' in fpath
+    self.failed = True
+    invalid_guards_msg_template = (
+        '{0}: Missing preprocessor guards (RE {1}). '
+        'Please wrap your code around the following guards:\n'
+        '#ifndef {2}\n'
+        '#define {2}\n'
+        '...\n'
+        '... epic code ...\n'
+        '...\n') + ('#endif  // {2}' if cpp_header else '#endif /* {2} */')
+    if not match_txt:
+      print invalid_guards_msg_template.format(fpath, regexp.pattern,
+                                               build_valid_guard(fpath))
+      return fcontents
+
+    print ('{}: Wrong preprocessor guards (RE {}):'
+           '\n\tFound {}, expected {}').format(
+        fpath, regexp.pattern, match_txt, correct)
+    if fix:
+      print 'Fixing {}...\n'.format(fpath)
+      fixed_fcontents = re.sub(match_txt, correct, fcontents)
+      if fixed_fcontents:
+        self.failed = False
+      return fixed_fcontents
+    else:
+      print
+    return fcontents
+
+  def check(self, fpath, fix):
+    cpp_header = 'grpc++' in fpath
+    valid_guard = build_valid_guard(fpath)
+
+    fcontents = load(fpath)
+
+    match = self.ifndef_re.search(fcontents)
+    if match.lastindex is None:
+      # No ifndef. Request manual addition with hints
+      self.fail(fpath, match.re, match.string, '', '', False)
+      return False  # failed
+
+    # Does the guard end with a '_H'?
+    running_guard = match.group(1)
+    if not running_guard.endswith('_H'):
+      fcontents = self.fail(fpath, match.re, match.string, match.group(1),
+                            valid_guard, fix)
+      if fix: save(fpath, fcontents)
+
+    # Is it the expected one based on the file path?
+    if running_guard != valid_guard:
+      fcontents = self.fail(fpath, match.re, match.string, match.group(1),
+                            valid_guard, fix)
+      if fix: save(fpath, fcontents)
+
+    # Is there a #define? Is it the same as the #ifndef one?
+    match = self.define_re.search(fcontents)
+    if match.lastindex is None:
+      # No define. Request manual addition with hints
+      self.fail(fpath, match.re, match.string, '', '', False)
+      return False  # failed
+
+    # Is the #define guard the same as the #ifndef guard?
+    if match.group(1) != running_guard:
+      fcontents = self.fail(fpath, match.re, match.string, match.group(1),
+                            valid_guard, fix)
+      if fix: save(fpath, fcontents)
+
+    # Is there a properly commented #endif?
+    endif_re = self.endif_cpp_re if cpp_header else self.endif_c_re
+    flines = fcontents.rstrip().splitlines()
+    match = endif_re.search(flines[-1])
+    if not match:
+      # No endif. Check if we have the last line as just '#endif' and if so
+      # replace it with a properly commented one.
+      if flines[-1] == '#endif':
+        flines[-1] = ('#endif' +
+                      ('  // {}\n'.format(valid_guard) if cpp_header
+                       else ' /* {} */\n'.format(valid_guard)))
+        if fix:
+            fcontents = '\n'.join(flines)
+            save(fpath, fcontents)
+      else:
+        # something else is wrong, bail out
+        self.fail(fpath, endif_re, flines[-1], '', '', False)
+    elif match.group(1) != running_guard:
+      # Is the #endif guard the same as the #ifndef and #define guards?
+      fcontents = self.fail(fpath, endif_re, fcontents, match.group(1),
+                            valid_guard, fix)
+      if fix: save(fpath, fcontents)
+
+    return not self.failed  # Did the check succeed? (ie, not failed)
+
+# find our home
+ROOT = os.path.abspath(
+    os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+os.chdir(ROOT)
+
+# parse command line
+argp = argparse.ArgumentParser(description='include guard checker')
+argp.add_argument('-f', '--fix',
+                  default=False,
+                  action='store_true');
+argp.add_argument('--precommit',
+                  default=False,
+                  action='store_true')
+args = argp.parse_args()
+
+KNOWN_BAD = set([
+    'src/core/proto/grpc/lb/v0/load_balancer.pb.h',
+])
+
+
+grep_filter = r"grep -E '^(include|src/core)/.*\.h$'"
+if args.precommit:
+  git_command = 'git diff --name-only HEAD'
+else:
+  git_command = 'git ls-tree -r --name-only -r HEAD'
+
+FILE_LIST_COMMAND = ' | '.join((git_command, grep_filter))
+
+# scan files
+ok = True
+filename_list = []
+try:
+  filename_list = subprocess.check_output(FILE_LIST_COMMAND,
+                                          shell=True).splitlines()
+except subprocess.CalledProcessError:
+  sys.exit(0)
+
+validator = GuardValidator()
+
+for filename in filename_list:
+  if filename in KNOWN_BAD: continue
+  ok = validator.check(filename, args.fix)
+
+sys.exit(0 if ok else 1)
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 71b7af2a22..a06d4ecb42 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1112,6 +1112,7 @@ include/grpc/impl/codegen/sync_posix.h \
 include/grpc/impl/codegen/sync_win32.h \
 include/grpc/impl/codegen/time.h \
 src/core/profiling/timers.h \
+src/core/support/backoff.h \
 src/core/support/block_annotate.h \
 src/core/support/env.h \
 src/core/support/load_file.h \
@@ -1126,6 +1127,7 @@ src/core/profiling/basic_timers.c \
 src/core/profiling/stap_timers.c \
 src/core/support/alloc.c \
 src/core/support/avl.c \
+src/core/support/backoff.c \
 src/core/support/cmdline.c \
 src/core/support/cpu_iphone.c \
 src/core/support/cpu_linux.c \
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index f81909ab88..dc11c0bd51 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -80,7 +80,7 @@ class Config(object):
     self.timeout_multiplier = timeout_multiplier
 
   def job_spec(self, cmdline, hash_targets, timeout_seconds=5*60,
-               shortname=None, environ={}, cpu_cost=1.0):
+               shortname=None, environ={}, cpu_cost=1.0, flaky=False):
     """Construct a jobset.JobSpec for a test under this config
 
        Args:
@@ -102,7 +102,7 @@ class Config(object):
                           timeout_seconds=(self.timeout_multiplier * timeout_seconds if timeout_seconds else None),
                           hash_targets=hash_targets
                               if self.allow_hashing else None,
-                          flake_retries=5 if args.allow_flakes else 0,
+                          flake_retries=5 if flaky or args.allow_flakes else 0,
                           timeout_retries=3 if args.allow_flakes else 0)
 
 
@@ -189,6 +189,7 @@ class CLanguage(object):
           out.append(self.config.job_spec(cmdline, [binary],
                                           shortname=' '.join(cmdline),
                                           cpu_cost=target['cpu_cost'],
+                                          flaky=target.get('flaky', False),
                                           environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
                                                    _ROOT + '/src/core/tsi/test_creds/ca.pem'}))
       elif self.args.regex == '.*' or self.platform == 'windows':
@@ -882,7 +883,7 @@ if args.use_docker:
       sys.exit(1)
   else:
     dockerfile_dir = next(iter(dockerfile_dirs))
-    
+
   child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ]
   run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:])
 
diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml
index 3840f4d8df..f155c8da45 100644
--- a/tools/run_tests/sanity/sanity_tests.yaml
+++ b/tools/run_tests/sanity/sanity_tests.yaml
@@ -8,3 +8,4 @@
 - script: tools/distrib/clang_format_code.sh
 - script: tools/distrib/check_trailing_newlines.sh
 - script: tools/distrib/check_nanopb_output.sh
+- script: tools/distrib/check_include_guards.py
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index ae227005ad..0f54d5625b 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -411,6 +411,20 @@
     ], 
     "headers": [], 
     "language": "c", 
+    "name": "gpr_backoff_test", 
+    "src": [
+      "test/core/support/backoff_test.c"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util"
+    ], 
+    "headers": [], 
+    "language": "c", 
     "name": "gpr_cmdline_test", 
     "src": [
       "test/core/support/cmdline_test.c"
@@ -3746,6 +3760,7 @@
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
       "src/core/profiling/timers.h", 
+      "src/core/support/backoff.h", 
       "src/core/support/block_annotate.h", 
       "src/core/support/env.h", 
       "src/core/support/load_file.h", 
@@ -3807,6 +3822,8 @@
       "src/core/profiling/timers.h", 
       "src/core/support/alloc.c", 
       "src/core/support/avl.c", 
+      "src/core/support/backoff.c", 
+      "src/core/support/backoff.h", 
       "src/core/support/block_annotate.h", 
       "src/core/support/cmdline.c", 
       "src/core/support/cpu_iphone.c", 
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index 2be7d8a48a..de6496028a 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -466,6 +466,27 @@
     "flaky": false, 
     "gtest": false, 
     "language": "c", 
+    "name": "gpr_backoff_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ]
+  }, 
+  {
+    "args": [], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "gtest": false, 
+    "language": "c", 
     "name": "gpr_cmdline_test", 
     "platforms": [
       "linux", 
@@ -1272,7 +1293,7 @@
     ], 
     "cpu_cost": 0.1, 
     "exclude_configs": [], 
-    "flaky": false, 
+    "flaky": true, 
     "gtest": false, 
     "language": "c", 
     "name": "lb_policies_test", 
@@ -2295,7 +2316,7 @@
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 10, 
+    "cpu_cost": 0.5, 
     "exclude_configs": [], 
     "flaky": false, 
     "gtest": false,