diff options
author | 2016-03-21 07:08:50 -0700 | |
---|---|---|
committer | 2016-03-21 07:08:50 -0700 | |
commit | 3448887c901aab0db2b44b73f7840ed5b30b3c7d (patch) | |
tree | 43cae1a2fbfa7513e15f0b6896802e8735ea0769 /tools | |
parent | 6c59e1682606b47651b21977481e762a3b88f028 (diff) | |
parent | ddd4d7f10f58bded28b47f09938780633c71fcaa (diff) |
Merge github.com:grpc/grpc into filter-selection
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/distrib/check_include_guards.py | 197 | ||||
-rwxr-xr-x | tools/distrib/check_nanopb_output.sh | 2 | ||||
-rwxr-xr-x | tools/distrib/python_wrapper.sh | 47 | ||||
-rw-r--r-- | tools/dockerfile/test/cxx_wheezy_x64/Dockerfile (renamed from tools/dockerfile/test/cxx_squeeze_x64/Dockerfile) | 21 | ||||
-rwxr-xr-x | tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh (renamed from tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh) | 0 | ||||
-rw-r--r-- | tools/dockerfile/test/sanity/Dockerfile | 8 | ||||
-rwxr-xr-x | tools/openssl/use_openssl.sh | 4 | ||||
-rwxr-xr-x | tools/run_tests/run_tests.py | 12 | ||||
-rw-r--r-- | tools/run_tests/sanity/sanity_tests.yaml | 1 | ||||
-rw-r--r-- | tools/run_tests/sources_and_headers.json | 6 | ||||
-rw-r--r-- | tools/run_tests/tests.json | 10 |
11 files changed, 289 insertions, 19 deletions
diff --git a/tools/distrib/check_include_guards.py b/tools/distrib/check_include_guards.py new file mode 100755 index 0000000000..977f40e0b3 --- /dev/null +++ b/tools/distrib/check_include_guards.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python2.7 + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import os +import re +import sys +import subprocess + + +def build_valid_guard(fpath): + prefix = 'GRPC_' if not fpath.startswith('include/') else '' + return prefix + '_'.join(fpath.replace('++', 'XX').replace('.', '_').upper().split('/')[1:]) + + +def load(fpath): + with open(fpath, 'r') as f: + return f.read() + + +def save(fpath, contents): + with open(fpath, 'w') as f: + f.write(contents) + + +class GuardValidator(object): + def __init__(self): + self.ifndef_re = re.compile(r'#ifndef ([A-Z][A-Z_1-9]*)') + self.define_re = re.compile(r'#define ([A-Z][A-Z_1-9]*)') + self.endif_c_re = re.compile(r'#endif /\* ([A-Z][A-Z_1-9]*) \*/') + self.endif_cpp_re = re.compile(r'#endif // ([A-Z][A-Z_1-9]*)') + self.failed = False + + def fail(self, fpath, regexp, fcontents, match_txt, correct, fix): + cpp_header = 'grpc++' in fpath + self.failed = True + invalid_guards_msg_template = ( + '{0}: Missing preprocessor guards (RE {1}). ' + 'Please wrap your code around the following guards:\n' + '#ifndef {2}\n' + '#define {2}\n' + '...\n' + '... epic code ...\n' + '...\n') + ('#endif // {2}' if cpp_header else '#endif /* {2} */') + if not match_txt: + print invalid_guards_msg_template.format(fpath, regexp.pattern, + build_valid_guard(fpath)) + return fcontents + + print ('{}: Wrong preprocessor guards (RE {}):' + '\n\tFound {}, expected {}').format( + fpath, regexp.pattern, match_txt, correct) + if fix: + print 'Fixing {}...\n'.format(fpath) + fixed_fcontents = re.sub(match_txt, correct, fcontents) + if fixed_fcontents: + self.failed = False + return fixed_fcontents + else: + print + return fcontents + + def check(self, fpath, fix): + cpp_header = 'grpc++' in fpath + valid_guard = build_valid_guard(fpath) + + fcontents = load(fpath) + + match = self.ifndef_re.search(fcontents) + if match.lastindex is None: + # No ifndef. Request manual addition with hints + self.fail(fpath, match.re, match.string, '', '', False) + return False # failed + + # Does the guard end with a '_H'? + running_guard = match.group(1) + if not running_guard.endswith('_H'): + fcontents = self.fail(fpath, match.re, match.string, match.group(1), + valid_guard, fix) + if fix: save(fpath, fcontents) + + # Is it the expected one based on the file path? + if running_guard != valid_guard: + fcontents = self.fail(fpath, match.re, match.string, match.group(1), + valid_guard, fix) + if fix: save(fpath, fcontents) + + # Is there a #define? Is it the same as the #ifndef one? + match = self.define_re.search(fcontents) + if match.lastindex is None: + # No define. Request manual addition with hints + self.fail(fpath, match.re, match.string, '', '', False) + return False # failed + + # Is the #define guard the same as the #ifndef guard? + if match.group(1) != running_guard: + fcontents = self.fail(fpath, match.re, match.string, match.group(1), + valid_guard, fix) + if fix: save(fpath, fcontents) + + # Is there a properly commented #endif? + endif_re = self.endif_cpp_re if cpp_header else self.endif_c_re + flines = fcontents.rstrip().splitlines() + match = endif_re.search(flines[-1]) + if not match: + # No endif. Check if we have the last line as just '#endif' and if so + # replace it with a properly commented one. + if flines[-1] == '#endif': + flines[-1] = ('#endif' + + (' // {}\n'.format(valid_guard) if cpp_header + else ' /* {} */\n'.format(valid_guard))) + if fix: + fcontents = '\n'.join(flines) + save(fpath, fcontents) + else: + # something else is wrong, bail out + self.fail(fpath, endif_re, flines[-1], '', '', False) + elif match.group(1) != running_guard: + # Is the #endif guard the same as the #ifndef and #define guards? + fcontents = self.fail(fpath, endif_re, fcontents, match.group(1), + valid_guard, fix) + if fix: save(fpath, fcontents) + + return not self.failed # Did the check succeed? (ie, not failed) + +# find our home +ROOT = os.path.abspath( + os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(ROOT) + +# parse command line +argp = argparse.ArgumentParser(description='include guard checker') +argp.add_argument('-f', '--fix', + default=False, + action='store_true'); +argp.add_argument('--precommit', + default=False, + action='store_true') +args = argp.parse_args() + +KNOWN_BAD = set([ + 'src/core/proto/grpc/lb/v0/load_balancer.pb.h', +]) + + +grep_filter = r"grep -E '^(include|src/core)/.*\.h$'" +if args.precommit: + git_command = 'git diff --name-only HEAD' +else: + git_command = 'git ls-tree -r --name-only -r HEAD' + +FILE_LIST_COMMAND = ' | '.join((git_command, grep_filter)) + +# scan files +ok = True +filename_list = [] +try: + filename_list = subprocess.check_output(FILE_LIST_COMMAND, + shell=True).splitlines() +except subprocess.CalledProcessError: + sys.exit(0) + +validator = GuardValidator() + +for filename in filename_list: + if filename in KNOWN_BAD: continue + ok = validator.check(filename, args.fix) + +sys.exit(0 if ok else 1) diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 5f49ebb93e..51c4d75041 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -30,8 +30,6 @@ set -ex -apt-get install -y autoconf automake libtool curl python-virtualenv - readonly NANOPB_TMP_OUTPUT="$(mktemp -d)" # install protoc version 3 diff --git a/tools/distrib/python_wrapper.sh b/tools/distrib/python_wrapper.sh new file mode 100755 index 0000000000..347a715c85 --- /dev/null +++ b/tools/distrib/python_wrapper.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +for p in python2.7 python2.6 python2 python not_found ; do + + python=`which $p || echo not_found` + + if [ -x "$python" ] ; then + break + fi + +done + +if [ -x "$python" ] ; then + exec $python $@ +else + echo "No acceptable version of python found on the system" + exit 1 +fi diff --git a/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile index b7f95aaa8d..6f330f9166 100644 --- a/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM debian:squeeze +FROM debian:wheezy # Install Git and basic packages. RUN apt-get update && apt-get install -y \ @@ -40,6 +40,7 @@ RUN apt-get update && apt-get install -y \ gcc \ gcc-multilib \ git \ + golang \ gyp \ lcov \ libc6 \ @@ -62,16 +63,18 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean -# libgflags-dev is not available on squeezy -RUN apt-get update && apt-get -y install libgtest-dev libc++-dev clang && apt-get clean -RUN apt-get update && apt-get -y install python-pip && apt-get clean -RUN pip install argparse +RUN apt-get update && apt-get install -y \ + gcc-4.4 \ + gcc-4.4-multilib -RUN wget http://openssl.org/source/openssl-1.0.2f.tar.gz +RUN wget https://openssl.org/source/old/1.0.2/openssl-1.0.2f.tar.gz -ENV POST_GIT_STEP tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh +ENV POST_GIT_STEP tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc @@ -81,6 +84,10 @@ RUN ln -s /usr/bin/ccache /usr/local/bin/c++ RUN ln -s /usr/bin/ccache /usr/local/bin/clang RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ +#====================== +# Zookeeper dependencies +# TODO(jtattermusch): is zookeeper still needed? +RUN apt-get install -y libzookeeper-mt-dev RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh b/tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh index dfde93b1bd..dfde93b1bd 100755 --- a/tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh +++ b/tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 1935f67560..4a69cd8c00 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -65,7 +65,13 @@ RUN apt-get update && apt-get install -y time && apt-get clean #======================== # Sanity test dependencies -RUN apt-get update && apt-get install -y python-pip +RUN apt-get update && apt-get install -y \ + python-pip \ + autoconf \ + automake \ + libtool \ + curl \ + python-virtualenv RUN pip install simplejson mako #=================== diff --git a/tools/openssl/use_openssl.sh b/tools/openssl/use_openssl.sh index 3098217ec1..9318b34257 100755 --- a/tools/openssl/use_openssl.sh +++ b/tools/openssl/use_openssl.sh @@ -38,8 +38,8 @@ CC=${CC:-cc} # allow openssl to be pre-downloaded if [ ! -e third_party/openssl-1.0.2f.tar.gz ] then - echo "Downloading http://openssl.org/source/openssl-1.0.2f.tar.gz to third_party/openssl-1.0.2f.tar.gz" - wget http://openssl.org/source/openssl-1.0.2f.tar.gz -O third_party/openssl-1.0.2f.tar.gz + echo "Downloading https://openssl.org/source/old/1.0.2/openssl-1.0.2f.tar.gz to third_party/openssl-1.0.2f.tar.gz" + wget https://openssl.org/source/old/1.0.2/openssl-1.0.2f.tar.gz -O third_party/openssl-1.0.2f.tar.gz fi # clean openssl directory diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 99bf5df1f9..dc11c0bd51 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -80,7 +80,7 @@ class Config(object): self.timeout_multiplier = timeout_multiplier def job_spec(self, cmdline, hash_targets, timeout_seconds=5*60, - shortname=None, environ={}, cpu_cost=1.0): + shortname=None, environ={}, cpu_cost=1.0, flaky=False): """Construct a jobset.JobSpec for a test under this config Args: @@ -102,7 +102,7 @@ class Config(object): timeout_seconds=(self.timeout_multiplier * timeout_seconds if timeout_seconds else None), hash_targets=hash_targets if self.allow_hashing else None, - flake_retries=5 if args.allow_flakes else 0, + flake_retries=5 if flaky or args.allow_flakes else 0, timeout_retries=3 if args.allow_flakes else 0) @@ -189,6 +189,7 @@ class CLanguage(object): out.append(self.config.job_spec(cmdline, [binary], shortname=' '.join(cmdline), cpu_cost=target['cpu_cost'], + flaky=target.get('flaky', False), environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': _ROOT + '/src/core/tsi/test_creds/ca.pem'})) elif self.args.regex == '.*' or self.platform == 'windows': @@ -231,6 +232,9 @@ class CLanguage(object): def _clang_make_options(self): return ['CC=clang', 'CXX=clang++', 'LD=clang', 'LDXX=clang++'] + def _gcc44_make_options(self): + return ['CC=gcc-4.4', 'CXX=g++-4.4', 'LD=gcc-4.4', 'LDXX=g++-4.4'] + def _compiler_options(self, use_docker, compiler): """Returns docker distro and make options to use for given compiler.""" if _is_use_docker_child(): @@ -241,7 +245,7 @@ class CLanguage(object): if compiler == 'gcc4.9' or compiler == 'default': return ('jessie', []) elif compiler == 'gcc4.4': - return ('squeeze', []) + return ('wheezy', self._gcc44_make_options()) elif compiler == 'gcc5.3': return ('ubuntu1604', []) elif compiler == 'clang3.4': @@ -879,7 +883,7 @@ if args.use_docker: sys.exit(1) else: dockerfile_dir = next(iter(dockerfile_dirs)) - + child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index 3840f4d8df..f155c8da45 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -8,3 +8,4 @@ - script: tools/distrib/clang_format_code.sh - script: tools/distrib/check_trailing_newlines.sh - script: tools/distrib/check_nanopb_output.sh +- script: tools/distrib/check_include_guards.py diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 48b7b7c9a2..fd608e0110 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4482,6 +4482,7 @@ "test/core/util/grpc_profiler.h", "test/core/util/parse_hexstring.h", "test/core/util/port.h", + "test/core/util/port_server_client.h", "test/core/util/slice_splitter.h" ], "language": "c", @@ -4505,6 +4506,8 @@ "test/core/util/parse_hexstring.h", "test/core/util/port.h", "test/core/util/port_posix.c", + "test/core/util/port_server_client.c", + "test/core/util/port_server_client.h", "test/core/util/port_windows.c", "test/core/util/slice_splitter.c", "test/core/util/slice_splitter.h" @@ -4525,6 +4528,7 @@ "test/core/util/grpc_profiler.h", "test/core/util/parse_hexstring.h", "test/core/util/port.h", + "test/core/util/port_server_client.h", "test/core/util/slice_splitter.h" ], "language": "c", @@ -4542,6 +4546,8 @@ "test/core/util/parse_hexstring.h", "test/core/util/port.h", "test/core/util/port_posix.c", + "test/core/util/port_server_client.c", + "test/core/util/port_server_client.h", "test/core/util/port_windows.c", "test/core/util/slice_splitter.c", "test/core/util/slice_splitter.h" diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 5daa58f20e..2597d73921 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -1293,7 +1293,7 @@ ], "cpu_cost": 0.1, "exclude_configs": [], - "flaky": false, + "flaky": true, "gtest": false, "language": "c", "name": "lb_policies_test", @@ -2317,7 +2317,9 @@ "posix" ], "cpu_cost": 0.5, - "exclude_configs": [], + "exclude_configs": [ + "tsan" + ], "flaky": false, "gtest": false, "language": "c++", @@ -2336,7 +2338,9 @@ "posix" ], "cpu_cost": 10, - "exclude_configs": [], + "exclude_configs": [ + "tsan" + ], "flaky": false, "gtest": false, "language": "c++", |