diff options
Diffstat (limited to 'tools/buildgen')
| -rwxr-xr-x | tools/buildgen/build-cleaner.py | 2 | ||||
| -rwxr-xr-x | tools/buildgen/bunch.py | 2 | ||||
| -rw-r--r-- | tools/buildgen/generate_build_additions.sh | 6 | ||||
| -rwxr-xr-x | tools/buildgen/generate_projects.py | 6 | ||||
| -rwxr-xr-x | tools/buildgen/mako_renderer.py | 14 | ||||
| -rwxr-xr-x | tools/buildgen/plugins/expand_bin_attrs.py | 2 | ||||
| -rwxr-xr-x | tools/buildgen/plugins/expand_filegroups.py | 118 | ||||
| -rwxr-xr-x | tools/buildgen/plugins/expand_version.py | 5 | ||||
| -rw-r--r-- | tools/buildgen/plugins/make_fuzzer_tests.py | 58 | ||||
| -rw-r--r-- | tools/buildgen/plugins/transitive_dependencies.py | 2 |
10 files changed, 192 insertions, 23 deletions
diff --git a/tools/buildgen/build-cleaner.py b/tools/buildgen/build-cleaner.py index 12054da238..f09a01fc57 100755 --- a/tools/buildgen/build-cleaner.py +++ b/tools/buildgen/build-cleaner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015-2016, Google Inc. +# Copyright 2015, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/buildgen/bunch.py b/tools/buildgen/bunch.py index 3f5af53778..9d9dafaad0 100755 --- a/tools/buildgen/bunch.py +++ b/tools/buildgen/bunch.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016, Google Inc. +# Copyright 2015, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index 4e7ba9ebb9..9a1a7a7249 100644 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright 2015-2016, Google Inc. +# Copyright 2015, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -34,7 +34,8 @@ gen_build_yaml_dirs=" \ src/zlib \ test/core/bad_client \ test/core/bad_ssl \ - test/core/end2end" + test/core/end2end \ + test/cpp/qps" gen_build_files="" for gen_build_yaml in $gen_build_yaml_dirs do @@ -42,4 +43,3 @@ do $gen_build_yaml/gen_build_yaml.py > $output_file gen_build_files="$gen_build_files $output_file" done - diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 0602d93e56..5e78ad52d6 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -1,6 +1,6 @@ #!/usr/bin/env python2.7 -# Copyright 2015-2016, Google Inc. +# Copyright 2015, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -47,6 +47,7 @@ os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) argp = argparse.ArgumentParser() argp.add_argument('build_files', nargs='+', default=[]) argp.add_argument('--templates', nargs='+', default=[]) +argp.add_argument('--output_merged', default=None, type=str) argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) args = argp.parse_args() @@ -74,6 +75,9 @@ for js in json: cmd.append('-w') preprocessed_build = '.preprocessed_build' cmd.append(preprocessed_build) +if args.output_merged is not None: + cmd.append('-M') + cmd.append(args.output_merged) pre_jobs.append(jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None)) jobs = [] diff --git a/tools/buildgen/mako_renderer.py b/tools/buildgen/mako_renderer.py index 5f23f123c2..866e6fdb06 100755 --- a/tools/buildgen/mako_renderer.py +++ b/tools/buildgen/mako_renderer.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015-2016, Google Inc. +# Copyright 2015, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -81,9 +81,10 @@ def main(argv): plugins = [] output_name = None got_preprocessed_input = False + output_merged = None try: - opts, args = getopt.getopt(argv, 'hm:d:o:p:t:P:w:') + opts, args = getopt.getopt(argv, 'hM:m:d:o:p:t:P:w:') except getopt.GetoptError: out('Unknown option') showhelp() @@ -107,6 +108,12 @@ def main(argv): showhelp() sys.exit(4) module_directory = arg + elif opt == '-M': + if output_merged is not None: + out('Got more than one output merged path') + showhelp() + sys.exit(5) + output_merged = arg elif opt == '-P': assert not got_preprocessed_input assert json_dict == {} @@ -126,6 +133,9 @@ def main(argv): if not got_preprocessed_input: for plugin in plugins: plugin.mako_plugin(json_dict) + if output_merged: + with open(output_merged, 'w') as yaml_file: + yaml_file.write(yaml.dump(json_dict)) for k, v in json_dict.items(): dictionary[k] = bunch.to_bunch(v) diff --git a/tools/buildgen/plugins/expand_bin_attrs.py b/tools/buildgen/plugins/expand_bin_attrs.py index c30df2ad89..dc72bf3b9d 100755 --- a/tools/buildgen/plugins/expand_bin_attrs.py +++ b/tools/buildgen/plugins/expand_bin_attrs.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016, Google Inc. +# Copyright 2015, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/buildgen/plugins/expand_filegroups.py b/tools/buildgen/plugins/expand_filegroups.py index 156bdc4417..477e69c869 100755 --- a/tools/buildgen/plugins/expand_filegroups.py +++ b/tools/buildgen/plugins/expand_filegroups.py @@ -42,6 +42,24 @@ def excluded(filename, exclude_res): return False +def uniquify(lst): + out = [] + for el in lst: + if el not in out: + out.append(el) + return out + + +FILEGROUP_LISTS = ['src', 'headers', 'public_headers', 'deps'] + + +FILEGROUP_DEFAULTS = { + 'language': 'c', + 'boringssl': False, + 'zlib': False, +} + + def mako_plugin(dictionary): """The exported plugin code for expand_filegroups. @@ -51,24 +69,98 @@ def mako_plugin(dictionary): """ libs = dictionary.get('libs') + targets = dictionary.get('targets') filegroups_list = dictionary.get('filegroups') filegroups = {} for fg in filegroups_list: - filegroups[fg['name']] = fg + for lst in FILEGROUP_LISTS: + fg[lst] = fg.get(lst, []) + fg['own_%s' % lst] = list(fg[lst]) + for attr, val in FILEGROUP_DEFAULTS.iteritems(): + if attr not in fg: + fg[attr] = val - for lib in libs: - for fg_name in lib.get('filegroups', []): - fg = filegroups[fg_name] + todo = list(filegroups_list) + skips = 0 + + while todo: + assert skips != len(todo), "infinite loop in filegroup uses clauses" + # take the first element of the todo list + cur = todo[0] + todo = todo[1:] + # check all uses filegroups are present (if no, skip and come back later) + skip = False + for uses in cur.get('uses', []): + if uses not in filegroups: + skip = True + if skip: + skips += 1 + todo.append(cur) + else: + skips = 0 + assert 'plugins' not in cur + plugins = [] + for uses in cur.get('uses', []): + for plugin in filegroups[uses]['plugins']: + if plugin not in plugins: + plugins.append(plugin) + for lst in FILEGROUP_LISTS: + vals = cur.get(lst, []) + vals.extend(filegroups[uses].get(lst, [])) + cur[lst] = vals + cur_plugin_name = cur.get('plugin') + if cur_plugin_name: + plugins.append(cur_plugin_name) + cur['plugins'] = plugins + filegroups[cur['name']] = cur - src = lib.get('src', []) - src.extend(fg.get('src', [])) - lib['src'] = src + # build reverse dependency map + things = {} + for thing in dictionary['libs'] + dictionary['targets'] + dictionary['filegroups']: + things[thing['name']] = thing + thing['used_by'] = [] + thing_deps = lambda t: t.get('uses', []) + t.get('filegroups', []) + t.get('deps', []) + for thing in things.itervalues(): + done = set() + todo = thing_deps(thing) + while todo: + cur = todo[0] + todo = todo[1:] + if cur in done: continue + things[cur]['used_by'].append(thing['name']) + todo.extend(thing_deps(things[cur])) + done.add(cur) - headers = lib.get('headers', []) - headers.extend(fg.get('headers', [])) - lib['headers'] = headers + # the above expansion can introduce duplicate filenames: contract them here + for fg in filegroups.itervalues(): + for lst in FILEGROUP_LISTS: + fg[lst] = uniquify(fg.get(lst, [])) - public_headers = lib.get('public_headers', []) - public_headers.extend(fg.get('public_headers', [])) - lib['public_headers'] = public_headers + for tgt in dictionary['targets']: + for lst in FILEGROUP_LISTS: + tgt[lst] = tgt.get(lst, []) + tgt['own_%s' % lst] = list(tgt[lst]) + + for lib in libs + targets: + assert 'plugins' not in lib + plugins = [] + for lst in FILEGROUP_LISTS: + vals = lib.get(lst, []) + lib[lst] = list(vals) + lib['own_%s' % lst] = list(vals) + for fg_name in lib.get('filegroups', []): + fg = filegroups[fg_name] + for plugin in fg['plugins']: + if plugin not in plugins: + plugins.append(plugin) + for lst in FILEGROUP_LISTS: + vals = lib.get(lst, []) + vals.extend(fg.get(lst, [])) + lib[lst] = vals + lib['plugins'] = plugins + if lib.get('generate_plugin_registry', False): + lib['src'].append('src/core/plugin_registry/%s_plugin_registry.c' % + lib['name']) + for lst in FILEGROUP_LISTS: + lib[lst] = uniquify(lib.get(lst, [])) diff --git a/tools/buildgen/plugins/expand_version.py b/tools/buildgen/plugins/expand_version.py index b55e1b15ff..dd77f7af12 100755 --- a/tools/buildgen/plugins/expand_version.py +++ b/tools/buildgen/plugins/expand_version.py @@ -84,6 +84,11 @@ class Version: else: return '%d.%d.%d' % (self.major, self.minor, self.patch) + def php(self): + """Version string in PHP style""" + """PECL does not allow tag in version string""" + return '%d.%d.%d' % (self.major, self.minor, self.patch) + def mako_plugin(dictionary): """Expand version numbers: - for each language, ensure there's a language_version tag in diff --git a/tools/buildgen/plugins/make_fuzzer_tests.py b/tools/buildgen/plugins/make_fuzzer_tests.py new file mode 100644 index 0000000000..9d0006973a --- /dev/null +++ b/tools/buildgen/plugins/make_fuzzer_tests.py @@ -0,0 +1,58 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Create tests for each fuzzer""" + +import copy +import glob + +def mako_plugin(dictionary): + targets = dictionary['targets'] + tests = dictionary['tests'] + for tgt in targets: + if tgt['build'] == 'fuzzer': + new_target = copy.deepcopy(tgt) + new_target['build'] = 'test' + new_target['name'] += '_one_entry' + new_target['run'] = False + new_target['src'].append('test/core/util/one_corpus_entry_fuzzer.c') + new_target['own_src'].append('test/core/util/one_corpus_entry_fuzzer.c') + targets.append(new_target) + for corpus in new_target['corpus_dirs']: + for fn in sorted(glob.glob('%s/*' % corpus)): + tests.append({ + 'name': new_target['name'], + 'args': [fn], + 'exclude_configs': [], + 'platforms': ['linux'], + 'ci_platforms': ['linux'], + 'flaky': False, + 'language': 'c', + 'cpu_cost': 0.1, + }) diff --git a/tools/buildgen/plugins/transitive_dependencies.py b/tools/buildgen/plugins/transitive_dependencies.py index 01e7f61ea9..176c8fa896 100644 --- a/tools/buildgen/plugins/transitive_dependencies.py +++ b/tools/buildgen/plugins/transitive_dependencies.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016, Google Inc. +# Copyright 2015, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without |