diff options
author | Muxi Yan <muxi@users.noreply.github.com> | 2018-05-22 10:12:38 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-05-22 10:12:38 -0700 |
commit | ba221d0423e61451b660b3d551c25cf155709285 (patch) | |
tree | 40e37ad629686a673e97b0dc2fd1f093595c07f4 /templates | |
parent | 92ff0490edfcfdb082659fa13a10ce8857ddd48e (diff) | |
parent | 463e4d56193bbaec3977ce38fb8f0103d006794e (diff) |
Merge branch 'master' into prototype-cfstream
Diffstat (limited to 'templates')
14 files changed, 156 insertions, 88 deletions
diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 8a1581a653..06adb33739 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -82,6 +82,7 @@ set(gRPC_INSTALL_LIBDIR "lib" CACHE STRING "Installation directory for libraries") set(gRPC_INSTALL_INCLUDEDIR "include" CACHE STRING "Installation directory for headers") set(gRPC_INSTALL_CMAKEDIR "lib/cmake/<%text>${PACKAGE_NAME}</%text>" CACHE STRING "Installation directory for cmake config files") + set(gRPC_INSTALL_SHAREDIR "share/grpc" CACHE STRING "Installation directory for root certificates") # Options option(gRPC_BUILD_TESTS "Build tests" OFF) @@ -507,3 +508,6 @@ DESTINATION <%text>${gRPC_INSTALL_CMAKEDIR}</%text> ) endforeach() + + install(FILES <%text>${CMAKE_CURRENT_SOURCE_DIR}/etc/roots.pem</%text> + DESTINATION <%text>${gRPC_INSTALL_SHAREDIR}</%text>) diff --git a/templates/Makefile.template b/templates/Makefile.template index 354f2a2f40..901dbfc9c8 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -1503,13 +1503,9 @@ % endif $(LIBDIR)/$(CONFIG)/lib${lib.name}.a: \ - % if lib.name != 'z': + % if lib.name not in ['z', 'ares', 'address_sorting']: $(ZLIB_DEP) \ - % endif - % if lib.name != 'ares': $(CARES_DEP) \ - % endif - % if lib.name != 'address_sorting': $(ADDRESS_SORTING_DEP) \ % endif % endif diff --git a/templates/README.md b/templates/README.md index 7b7707203f..c837b5b260 100644 --- a/templates/README.md +++ b/templates/README.md @@ -1,6 +1,6 @@ # Regenerating project files -Prerequisites: `python`, `pip install mako` +Prerequisites: `python`, `pip install mako`, `go` ``` # Regenerate the projects files using templates diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include index c29b466125..bc981dc83e 100644 --- a/templates/test/cpp/naming/resolver_component_tests_defs.include +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -1,4 +1,4 @@ -<%def name="resolver_component_tests(tests)">#!/bin/bash +<%def name="resolver_component_tests(tests)">#!/usr/bin/env python # Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,84 +15,127 @@ # This file is auto-generated -set -ex - -# all command args required in this set order -FLAGS_test_bin_path=$(echo "$1" | grep '\--test_bin_path=' | sed 's/^--test_bin_path=//') -FLAGS_dns_server_bin_path=$(echo "$2" | grep '\--dns_server_bin_path=' | sed 's/^--dns_server_bin_path=//') -FLAGS_records_config_path=$(echo "$3" | grep '\--records_config_path=' | sed 's/^--records_config_path=//') -FLAGS_dns_server_port=$(echo "$4" | grep '\--dns_server_port=' | sed 's/^--dns_server_port=//') -FLAGS_dns_resolver_bin_path=$(echo "$5" | grep '\--dns_resolver_bin_path=' | sed 's/^--dns_resolver_bin_path=//') -FLAGS_tcp_connect_bin_path=$(echo "$6" | grep '\--tcp_connect_bin_path=' | sed 's/^--tcp_connect_bin_path=//') - -for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_dns_server_port" "$FLAGS_dns_resolver_bin_path" "$FLAGS_tcp_connect_bin_path"; do - if [[ "$cmd_arg" == "" ]]; then - echo "Missing a CMD arg" && exit 1 - fi -done - -if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then - echo "This test only works under GRPC_DNS_RESOLVER=ares. Have GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER" && exit 1 -fi -export GRPC_DNS_RESOLVER=ares - -DNS_SERVER_LOG="$(mktemp)" -"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_dns_server_port" > "$DNS_SERVER_LOG" 2>&1 & -DNS_SERVER_PID=$! -echo "Local DNS server started. PID: $DNS_SERVER_PID" - -# Health check local DNS server TCP and UDP ports -for ((i=0;i<30;i++)); -do - echo "Retry health-check local DNS server by attempting a DNS query and TCP handshake" - RETRY=0 - $FLAGS_dns_resolver_bin_path -s 127.0.0.1 -p "$FLAGS_dns_server_port" -n health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. -t 1 | grep '123.123.123.123' || RETRY=1 - $FLAGS_tcp_connect_bin_path -s 127.0.0.1 -p "$FLAGS_dns_server_port" -t 1 || RETRY=1 - if [[ "$RETRY" == 0 ]]; then - break - fi; - sleep 0.1 -done - -if [[ $RETRY == 1 ]]; then - echo "FAILED TO START LOCAL DNS SERVER" - kill -SIGTERM "$DNS_SERVER_PID" || true - wait - echo "========== DNS server log (merged stdout and stderr) =========" - cat "$DNS_SERVER_LOG" - echo "========== end DNS server log ================================" - exit 1 -fi - -function terminate_all { - echo "Received signal. Terminating $! and $DNS_SERVER_PID" - kill -SIGTERM "$!" || true - kill -SIGTERM "$DNS_SERVER_PID" || true - wait - exit 1 -} - -trap terminate_all SIGTERM SIGINT - -EXIT_CODE=0 -# TODO: this test should check for GCE residency and skip tests using _grpclb._tcp.* SRV records once GCE residency checks are made -# in the resolver. +import argparse +import sys +import subprocess +import tempfile +import os +import time +import signal + + +argp = argparse.ArgumentParser(description='Run c-ares resolver tests') +argp.add_argument('--test_bin_path', default=None, type=str, + help='Path to gtest test binary to invoke.') +argp.add_argument('--dns_server_bin_path', default=None, type=str, + help='Path to local DNS server python script.') +argp.add_argument('--records_config_path', default=None, type=str, + help=('Path to DNS records yaml file that ' + 'specifies records for the DNS sever. ')) +argp.add_argument('--dns_server_port', default=None, type=int, + help=('Port that local DNS server is listening on.')) +argp.add_argument('--dns_resolver_bin_path', default=None, type=str, + help=('Path to the DNS health check utility.')) +argp.add_argument('--tcp_connect_bin_path', default=None, type=str, + help=('Path to the TCP health check utility.')) +args = argp.parse_args() + +def test_runner_log(msg): + sys.stderr.write('\n%s: %s\n' % (__file__, msg)) + +cur_resolver = os.environ.get('GRPC_DNS_RESOLVER') +if cur_resolver and cur_resolver != 'ares': + test_runner_log(('WARNING: cur resolver set to %s. This set of tests ' + 'needs to use GRPC_DNS_RESOLVER=ares.')) + test_runner_log('Exit 1 without running tests.') + sys.exit(1) +os.environ.update({'GRPC_DNS_RESOLVER': 'ares'}) + +def wait_until_dns_server_is_up(args, + dns_server_subprocess, + dns_server_subprocess_output): + for i in range(0, 30): + test_runner_log('Health check: attempt to connect to DNS server over TCP.') + tcp_connect_subprocess = subprocess.Popen([ + args.tcp_connect_bin_path, + '--server_host', '127.0.0.1', + '--server_port', str(args.dns_server_port), + '--timeout', str(1)]) + tcp_connect_subprocess.communicate() + if tcp_connect_subprocess.returncode == 0: + test_runner_log(('Health check: attempt to make an A-record ' + 'query to DNS server.')) + dns_resolver_subprocess = subprocess.Popen([ + args.dns_resolver_bin_path, + '--qname', 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp', + '--server_host', '127.0.0.1', + '--server_port', str(args.dns_server_port)], + stdout=subprocess.PIPE) + dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() + if dns_resolver_subprocess.returncode == 0: + if '123.123.123.123' in dns_resolver_stdout: + test_runner_log(('DNS server is up! ' + 'Successfully reached it over UDP and TCP.')) + return + time.sleep(0.1) + dns_server_subprocess.kill() + dns_server_subprocess.wait() + test_runner_log(('Failed to reach DNS server over TCP and/or UDP. ' + 'Exitting without running tests.')) + test_runner_log('======= DNS server stdout ' + '(merged stdout and stderr) =============') + with open(dns_server_subprocess_output, 'r') as l: + test_runner_log(l.read()) + test_runner_log('======= end DNS server output=========') + sys.exit(1) + +dns_server_subprocess_output = tempfile.mktemp() +with open(dns_server_subprocess_output, 'w') as l: + dns_server_subprocess = subprocess.Popen([ + args.dns_server_bin_path, + '--port', str(args.dns_server_port), + '--records_config_path', args.records_config_path], + stdin=subprocess.PIPE, + stdout=l, + stderr=l) + +def _quit_on_signal(signum, _frame): + test_runner_log('Received signal: %d' % signum) + dns_server_subprocess.kill() + dns_server_subprocess.wait() + sys.exit(1) + +signal.signal(signal.SIGINT, _quit_on_signal) +signal.signal(signal.SIGTERM, _quit_on_signal) +wait_until_dns_server_is_up(args, + dns_server_subprocess, + dns_server_subprocess_output) +num_test_failures = 0 % for test in tests: -$FLAGS_test_bin_path \\ +test_runner_log('Run test with target: %s' % '${test['test_title']}')\ - --target_name='${test['target_name']}' \\ +current_test_subprocess = subprocess.Popen([\ - --expected_addrs='${test['expected_addrs']}' \\ + args.test_bin_path,\ - --expected_chosen_service_config='${test['expected_chosen_service_config']}' \\ + % for arg_name_and_value in test['arg_names_and_values']: +\ + '--${arg_name_and_value[0]}', '${arg_name_and_value[1]}',\ - --expected_lb_policy='${test['expected_lb_policy']}' \\ +\ + % endfor + '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])\ - --local_dns_server_address="127.0.0.1:$FLAGS_dns_server_port" & -wait "$!" || EXIT_CODE=1 +current_test_subprocess.communicate()\ + +if current_test_subprocess.returncode != 0:\ + + num_test_failures += 1 % endfor -kill -SIGTERM "$DNS_SERVER_PID" || true -wait -exit $EXIT_CODE</%def> +test_runner_log('now kill DNS server') +dns_server_subprocess.kill() +dns_server_subprocess.wait() +test_runner_log('%d tests failed.' % num_test_failures) +sys.exit(num_test_failures)</%def> diff --git a/templates/test/cpp/naming/resolver_component_tests_runner.sh.template b/templates/test/cpp/naming/resolver_component_tests_runner.py.template index 86772dd141..86772dd141 100644 --- a/templates/test/cpp/naming/resolver_component_tests_runner.sh.template +++ b/templates/test/cpp/naming/resolver_component_tests_runner.py.template diff --git a/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include b/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include index 0cb8a1baf3..8a8377e38d 100644 --- a/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include +++ b/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include @@ -48,15 +48,15 @@ echo "Sanity check PASSED. Run resolver tests:" ONE_FAILED=0 bins/$CONFIG/resolver_component_test \\ - --target_name='${test['target_name']}' \\ + % for arg_name_and_value in test['arg_names_and_values'][0:-1]: +\ + --${arg_name_and_value[0]}='${arg_name_and_value[1]}' \\ - --expected_addrs='${test['expected_addrs']}' \\ - - --expected_chosen_service_config='${test['expected_chosen_service_config']}' \\ - - --expected_lb_policy='${test['expected_lb_policy']}' || ONE_FAILED=1 +\ + % endfor + --${test['arg_names_and_values'][-1][0]}='${test['arg_names_and_values'][-1][1]}' || ONE_FAILED=1 if [[ "$ONE_FAILED" != 0 ]]; then - echo "Test based on target record: ${test['target_name']} FAILED" + echo "Test based on target record: ${test['test_title']} FAILED" EXIT_CODE=1 fi diff --git a/templates/tools/dockerfile/clang_update.include b/templates/tools/dockerfile/clang_update.include index 4f827c8dc2..279bb4cf6b 100644 --- a/templates/tools/dockerfile/clang_update.include +++ b/templates/tools/dockerfile/clang_update.include @@ -1,8 +1,6 @@ #================= # Update clang to a version with improved tsan and fuzzing capabilities -RUN apt-get update && apt-get -y install python cmake && apt-get clean - RUN git clone -n -b release_38 http://llvm.org/git/llvm.git && ${'\\'} cd llvm && git checkout ad57503 && cd .. RUN git clone -n -b release_38 http://llvm.org/git/clang.git && ${'\\'} diff --git a/templates/tools/dockerfile/cmake_jessie_backports.include b/templates/tools/dockerfile/cmake_jessie_backports.include new file mode 100644 index 0000000000..2fc49dc8d6 --- /dev/null +++ b/templates/tools/dockerfile/cmake_jessie_backports.include @@ -0,0 +1,6 @@ +#================= +# Use cmake 3.6 from jessie-backports +# should only be used for images based on debian jessie. + +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template index f5a53f045f..bf28796de3 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template @@ -18,6 +18,10 @@ <%include file="../../apt_get_basic.include"/> <%include file="../../python_deps.include"/> + # Install pip and virtualenv for Python 3.4 + RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4 + RUN python3.4 -m pip install virtualenv + <%include file="../../run_tests_addons.include"/> # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template index 1226c19516..9d559dfafa 100644 --- a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template @@ -20,6 +20,7 @@ <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> + <%include file="../../cmake_jessie_backports.include"/> <%include file="../../clang_update.include"/> <%include file="../../run_tests_addons.include"/> <%include file="../../libuv_install.include"/> diff --git a/templates/tools/dockerfile/test/fuzzer/Dockerfile.template b/templates/tools/dockerfile/test/fuzzer/Dockerfile.template index 759b128c13..6dcd7b77de 100644 --- a/templates/tools/dockerfile/test/fuzzer/Dockerfile.template +++ b/templates/tools/dockerfile/test/fuzzer/Dockerfile.template @@ -20,6 +20,7 @@ <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> + <%include file="../../cmake_jessie_backports.include"/> <%include file="../../clang_update.include"/> <%include file="../../run_tests_addons.include"/> RUN clang++ -c -g -O2 -std=c++11 llvm/lib/Fuzzer/*.cpp -IFuzzer diff --git a/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template index 0d47aa91f3..ac687b710f 100644 --- a/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template @@ -25,10 +25,17 @@ <%include file="../../php_deps.include"/> <%include file="../../ruby_deps.include"/> <%include file="../../python_deps.include"/> + # Install pip and virtualenv for Python 3.4 + RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4 + RUN python3.4 -m pip install virtualenv + # Install coverage for Python test coverage reporting RUN pip install coverage ENV PATH ~/.local/bin:$PATH + # Install Mako to generate files in grpc/grpc-node + RUN pip install Mako + <%include file="../../run_tests_addons.include"/> # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/test/python_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/python_jessie_x64/Dockerfile.template index dba6a227f2..e73b839a28 100644 --- a/templates/tools/dockerfile/test/python_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/python_jessie_x64/Dockerfile.template @@ -19,6 +19,10 @@ <%include file="../../apt_get_basic.include"/> <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> + # Install pip and virtualenv for Python 3.4 + RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4 + RUN python3.4 -m pip install virtualenv + <%include file="../../run_tests_addons.include"/> # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/test/python_pyenv_x64/Dockerfile.template b/templates/tools/dockerfile/test/python_pyenv_x64/Dockerfile.template index 0df19f61e1..ba65c06a3b 100644 --- a/templates/tools/dockerfile/test/python_pyenv_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/python_pyenv_x64/Dockerfile.template @@ -20,6 +20,10 @@ <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../apt_get_pyenv.include"/> + # Install pip and virtualenv for Python 3.4 + RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4 + RUN python3.4 -m pip install virtualenv + <%include file="../../run_tests_addons.include"/> # Define the default command. CMD ["bash"] |