Merge branch 'master' into calloptions-fluent

11 files changed, 277 insertions, 48 deletions

diff --git a/tools/buildgen/build-cleaner.py b/tools/buildgen/build-cleaner.py
index 37fedec6ad..f09a01fc57 100755
--- a/tools/buildgen/build-cleaner.py
+++ b/tools/buildgen/build-cleaner.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python2.7
-# Copyright 2015-2016, Google Inc.
+# Copyright 2015, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -38,9 +38,9 @@ import yaml
 TEST = (os.environ.get('TEST', 'false') == 'true')
 
 _TOP_LEVEL_KEYS = ['settings', 'proto_deps', 'filegroups', 'libs', 'targets', 'vspackages']
-_VERSION_KEYS = ['major', 'minor', 'micro', 'build']
 _ELEM_KEYS = [
     'name',
+    'gtest',
     'cpu_cost',
     'flaky',
     'build',
@@ -83,8 +83,6 @@ for filename in sys.argv[1:]:
   with open(filename) as f:
     js = yaml.load(f)
   js = rebuild_as_ordered_dict(js, _TOP_LEVEL_KEYS)
-  js['settings']['version'] = rebuild_as_ordered_dict(
-      js['settings']['version'], _VERSION_KEYS)
   for grp in ['filegroups', 'libs', 'targets']:
     if grp not in js: continue
     js[grp] = sorted([clean_elem(x) for x in js[grp]],
@@ -101,4 +99,3 @@ for filename in sys.argv[1:]:
   else:
     with open(filename, 'w') as f:
       f.write(output)
-
diff --git a/tools/buildgen/bunch.py b/tools/buildgen/bunch.py
index 3f5af53778..9d9dafaad0 100755
--- a/tools/buildgen/bunch.py
+++ b/tools/buildgen/bunch.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016, Google Inc.
+# Copyright 2015, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh
index 4e7ba9ebb9..c631713049 100644
--- a/tools/buildgen/generate_build_additions.sh
+++ b/tools/buildgen/generate_build_additions.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright 2015-2016, Google Inc.
+# Copyright 2015, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py
index 965dd292af..5f3af7738b 100755
--- a/tools/buildgen/generate_projects.py
+++ b/tools/buildgen/generate_projects.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python2.7
 
-# Copyright 2015-2016, Google Inc.
+# Copyright 2015, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -45,12 +45,12 @@ import jobset
 os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..'))
 
 argp = argparse.ArgumentParser()
-argp.add_argument('json', nargs='+')
+argp.add_argument('build_files', nargs='+', default=[])
 argp.add_argument('--templates', nargs='+', default=[])
 argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int)
 args = argp.parse_args()
 
-json = args.json
+json = args.build_files
 
 test = {} if 'TEST' in os.environ else None
 
@@ -62,21 +62,31 @@ if not templates:
     for f in files:
       templates.append(os.path.join(root, f))
 
+pre_jobs = []
+base_cmd = ['python2.7', 'tools/buildgen/mako_renderer.py']
+cmd = base_cmd[:]
+for plugin in plugins:
+  cmd.append('-p')
+  cmd.append(plugin)
+for js in json:
+  cmd.append('-d')
+  cmd.append(js)
+cmd.append('-w')
+preprocessed_build = '.preprocessed_build'
+cmd.append(preprocessed_build)
+pre_jobs.append(jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None))
+
 jobs = []
-for template in templates:
+for template in reversed(sorted(templates)):
   root, f = os.path.split(template)
   if os.path.splitext(f)[1] == '.template':
     out_dir = '.' + root[len('templates'):]
     out = out_dir + '/' + os.path.splitext(f)[0]
     if not os.path.exists(out_dir):
       os.makedirs(out_dir)
-    cmd = ['python2.7', 'tools/buildgen/mako_renderer.py']
-    for plugin in plugins:
-      cmd.append('-p')
-      cmd.append(plugin)
-    for js in json:
-      cmd.append('-d')
-      cmd.append(js)
+    cmd = base_cmd[:]
+    cmd.append('-P')
+    cmd.append(preprocessed_build)
     cmd.append('-o')
     if test is None:
       cmd.append(out)
@@ -88,6 +98,7 @@ for template in templates:
     cmd.append(root + '/' + f)
     jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None))
 
+jobset.run(pre_jobs, maxjobs=args.jobs)
 jobset.run(jobs, maxjobs=args.jobs)
 
 if test is not None:
diff --git a/tools/buildgen/mako_renderer.py b/tools/buildgen/mako_renderer.py
index f1b28d352e..f629e68eb9 100755
--- a/tools/buildgen/mako_renderer.py
+++ b/tools/buildgen/mako_renderer.py
@@ -38,6 +38,7 @@ Just a wrapper around the mako rendering library.
 import getopt
 import imp
 import os
+import cPickle as pickle
 import shutil
 import sys
 
@@ -66,21 +67,23 @@ def out(msg):
 
 
 def showhelp():
-  out('mako-renderer.py [-o out] [-m cache] [-d dict] [-d dict...] template')
+  out('mako-renderer.py [-o out] [-m cache] [-P preprocessed_input] [-d dict] [-d dict...]'
+      ' [-t template] [-w preprocessed_output]')
 
 
 def main(argv):
   got_input = False
   module_directory = None
+  preprocessed_output = None
   dictionary = {}
   json_dict = {}
   got_output = False
-  output_file = sys.stdout
   plugins = []
   output_name = None
+  got_preprocessed_input = False
 
   try:
-    opts, args = getopt.getopt(argv, 'hm:d:o:p:')
+    opts, args = getopt.getopt(argv, 'hm:d:o:p:t:P:w:')
   except getopt.GetoptError:
     out('Unknown option')
     showhelp()
@@ -104,18 +107,31 @@ def main(argv):
         showhelp()
         sys.exit(4)
       module_directory = arg
+    elif opt == '-P':
+      assert not got_preprocessed_input
+      assert json_dict == {}
+      sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'plugins')))
+      with open(arg, 'r') as dict_file:
+        dictionary = pickle.load(dict_file)
+      got_preprocessed_input = True
     elif opt == '-d':
-      dict_file = open(arg, 'r')
-      bunch.merge_json(json_dict, yaml.load(dict_file.read()))
-      dict_file.close()
+      assert not got_preprocessed_input
+      with open(arg, 'r') as dict_file:
+        bunch.merge_json(json_dict, yaml.load(dict_file.read()))
     elif opt == '-p':
       plugins.append(import_plugin(arg))
+    elif opt == '-w':
+      preprocessed_output = arg
 
-  for plugin in plugins:
-    plugin.mako_plugin(json_dict)
+  if not got_preprocessed_input:
+    for plugin in plugins:
+      plugin.mako_plugin(json_dict)
+    for k, v in json_dict.items():
+      dictionary[k] = bunch.to_bunch(v)
 
-  for k, v in json_dict.items():
-    dictionary[k] = bunch.to_bunch(v)
+  if preprocessed_output:
+    with open(preprocessed_output, 'w') as dict_file:
+      pickle.dump(dictionary, dict_file)
 
   cleared_dir = False
   for arg in args:
@@ -168,11 +184,9 @@ def main(argv):
           with open(item_output_name, 'w') as output_file:
             template.render_context(Context(output_file, **args))
 
-  if not got_input:
+  if not got_input and not preprocessed_output:
     out('Got nothing to do')
     showhelp()
 
-  output_file.close()
-
 if __name__ == '__main__':
   main(sys.argv[1:])
diff --git a/tools/buildgen/plugins/expand_bin_attrs.py b/tools/buildgen/plugins/expand_bin_attrs.py
index 735c60ea99..dc72bf3b9d 100755
--- a/tools/buildgen/plugins/expand_bin_attrs.py
+++ b/tools/buildgen/plugins/expand_bin_attrs.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016, Google Inc.
+# Copyright 2015, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -52,6 +52,7 @@ def mako_plugin(dictionary):
     tgt['ci_platforms'] = sorted(tgt.get('ci_platforms', tgt['platforms']))
     tgt['boringssl'] = tgt.get('boringssl', False)
     tgt['zlib'] = tgt.get('zlib', False)
+    tgt['gtest'] = tgt.get('gtest', False)
 
   libs = dictionary.get('libs')
   for lib in libs:
diff --git a/tools/buildgen/plugins/expand_filegroups.py b/tools/buildgen/plugins/expand_filegroups.py
index 156bdc4417..14cb616027 100755
--- a/tools/buildgen/plugins/expand_filegroups.py
+++ b/tools/buildgen/plugins/expand_filegroups.py
@@ -42,6 +42,9 @@ def excluded(filename, exclude_res):
   return False
 
 
+FILEGROUP_LISTS = ['src', 'headers', 'public_headers']
+
+
 def mako_plugin(dictionary):
   """The exported plugin code for expand_filegroups.
 
@@ -54,21 +57,60 @@ def mako_plugin(dictionary):
   filegroups_list = dictionary.get('filegroups')
   filegroups = {}
 
-  for fg in filegroups_list:
-    filegroups[fg['name']] = fg
+  todo = filegroups_list[:]
+  skips = 0
+
+  while todo:
+    assert skips != len(todo), "infinite loop in filegroup uses clauses"
+    # take the first element of the todo list
+    cur = todo[0]
+    todo = todo[1:]
+    # check all uses filegroups are present (if no, skip and come back later)
+    skip = False
+    for uses in cur.get('uses', []):
+      if uses not in filegroups:
+        skip = True
+    if skip:
+      skips += 1
+      todo.append(cur)
+    else:
+      skips = 0
+      assert 'plugins' not in cur
+      plugins = []
+      for uses in cur.get('uses', []):
+        for plugin in filegroups[uses]['plugins']:
+          if plugin not in plugins:
+            plugins.append(plugin)
+        for lst in FILEGROUP_LISTS:
+          vals = cur.get(lst, [])
+          vals.extend(filegroups[uses].get(lst, []))
+          cur[lst] = vals
+      cur_plugin_name = cur.get('plugin')
+      if cur_plugin_name:
+        plugins.append(cur_plugin_name)
+      cur['plugins'] = plugins
+      filegroups[cur['name']] = cur
+
+  # the above expansion can introduce duplicate filenames: contract them here
+  for fg in filegroups.itervalues():
+    for lst in FILEGROUP_LISTS:
+      fg[lst] = sorted(list(set(fg.get(lst, []))))
 
   for lib in libs:
+    assert 'plugins' not in lib
+    plugins = []
     for fg_name in lib.get('filegroups', []):
       fg = filegroups[fg_name]
-
-      src = lib.get('src', [])
-      src.extend(fg.get('src', []))
-      lib['src'] = src
-
-      headers = lib.get('headers', [])
-      headers.extend(fg.get('headers', []))
-      lib['headers'] = headers
-
-      public_headers = lib.get('public_headers', [])
-      public_headers.extend(fg.get('public_headers', []))
-      lib['public_headers'] = public_headers
+      for plugin in fg['plugins']:
+        if plugin not in plugins:
+          plugins.append(plugin)
+      for lst in FILEGROUP_LISTS:
+        vals = lib.get(lst, [])
+        vals.extend(fg.get(lst, []))
+        lib[lst] = vals
+      lib['plugins'] = plugins
+    if lib.get('generate_plugin_registry', False):
+      lib['src'].append('src/core/plugin_registry/%s_plugin_registry.c' %
+                        lib['name'])
+    for lst in FILEGROUP_LISTS:
+      lib[lst] = sorted(list(set(lib.get(lst, []))))
diff --git a/tools/buildgen/plugins/expand_version.py b/tools/buildgen/plugins/expand_version.py
new file mode 100755
index 0000000000..dd77f7af12
--- /dev/null
+++ b/tools/buildgen/plugins/expand_version.py
@@ -0,0 +1,107 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Buildgen .proto files list plugin.
+
+This parses the list of targets from the yaml build file, and creates
+a list called "protos" that contains all of the proto file names.
+
+"""
+
+
+import re
+
+LANGUAGES = [
+  'core', 
+  'cpp', 
+  'csharp', 
+  'node', 
+  'objc', 
+  'php',
+  'python', 
+  'ruby', 
+  ]
+
+class Version:
+
+  def __init__(self, s):
+    self.tag = None
+    if '-' in s:
+      s, self.tag = s.split('-')
+    self.major, self.minor, self.patch = [int(x) for x in s.split('.')]
+
+  def __str__(self):
+    """Version string in a somewhat idiomatic style for most languages"""
+    s = '%d.%d.%d' % (self.major, self.minor, self.patch)
+    if self.tag:
+      s += '-%s' % self.tag
+    return s
+
+  def pep440(self):
+    """Version string in Python PEP440 style"""
+    s = '%d.%d.%d' % (self.major, self.minor, self.patch)
+    if self.tag:
+      # we need to translate from grpc version tags to pep440 version
+      # tags; this code is likely to be a little ad-hoc
+      if self.tag == 'dev':
+        s += '.dev0'
+      elif len(self.tag) >= 3 and self.tag[0:3] == 'pre':
+        s += 'rc%d' % int(self.tag[3:])
+      else:
+        raise Exception('Don\'t know how to translate version tag "%s" to pep440' % self.tag)
+    return s
+
+  def ruby(self):
+    """Version string in Ruby style"""
+    if self.tag:
+      return '%d.%d.%d.%s' % (self.major, self.minor, self.patch, self.tag)
+    else:
+      return '%d.%d.%d' % (self.major, self.minor, self.patch)
+
+  def php(self):
+    """Version string in PHP style"""
+    """PECL does not allow tag in version string"""
+    return '%d.%d.%d' % (self.major, self.minor, self.patch)
+
+def mako_plugin(dictionary):
+  """Expand version numbers:
+     - for each language, ensure there's a language_version tag in
+       settings (defaulting to the master version tag)
+     - expand version strings to major, minor, patch, and tag
+  """
+
+  settings = dictionary['settings']
+  master_version = Version(settings['version'])
+  settings['version'] = master_version
+  for language in LANGUAGES:
+    version_tag = '%s_version' % language
+    if version_tag in settings:
+      settings[version_tag] = Version(settings[version_tag])
+    else:
+      settings[version_tag] = master_version
diff --git a/tools/buildgen/plugins/list_api.py b/tools/buildgen/plugins/list_api.py
index 3396dcd39b..ff937a0ab8 100755
--- a/tools/buildgen/plugins/list_api.py
+++ b/tools/buildgen/plugins/list_api.py
@@ -37,7 +37,7 @@ import sys
 import yaml
 
 
-_RE_API = r'(?:GPR_API|GRPC_API|CENSUS_API)([^;]*);'
+_RE_API = r'(?:GPRAPI|GRPCAPI|CENSUSAPI)([^;]*);'
 
 
 def list_c_apis(filenames):
diff --git a/tools/buildgen/plugins/make_fuzzer_tests.py b/tools/buildgen/plugins/make_fuzzer_tests.py
new file mode 100644
index 0000000000..806489bcd2
--- /dev/null
+++ b/tools/buildgen/plugins/make_fuzzer_tests.py
@@ -0,0 +1,57 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Create tests for each fuzzer"""
+
+import copy
+import glob
+
+def mako_plugin(dictionary):
+  targets = dictionary['targets']
+  tests = dictionary['tests']
+  for tgt in targets:
+    if tgt['build'] == 'fuzzer':
+      new_target = copy.deepcopy(tgt)
+      new_target['build'] = 'test'
+      new_target['name'] += '_one_entry'
+      new_target['run'] = False
+      new_target['deps'].insert(0, 'one_input_fuzzer')
+      targets.append(new_target)
+      for corpus in new_target['corpus_dirs']:
+        for fn in sorted(glob.glob('%s/*' % corpus)):
+          tests.append({
+              'name': new_target['name'],
+              'args': [fn],
+              'exclude_configs': [],
+              'platforms': ['linux', 'mac', 'windows', 'posix'],
+              'ci_platforms': ['linux', 'mac', 'windows', 'posix'],
+              'flaky': False,
+              'language': 'c',
+              'cpu_cost': 0.1,
+          })
diff --git a/tools/buildgen/plugins/transitive_dependencies.py b/tools/buildgen/plugins/transitive_dependencies.py
index 01e7f61ea9..176c8fa896 100644
--- a/tools/buildgen/plugins/transitive_dependencies.py
+++ b/tools/buildgen/plugins/transitive_dependencies.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016, Google Inc.
+# Copyright 2015, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without