aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/doxygen/Doxyfile.core.internal74
-rwxr-xr-xtools/jenkins/run_performance.sh82
-rw-r--r--tools/run_tests/artifact_targets.py41
-rw-r--r--tools/run_tests/build_artifact_python.bat61
-rwxr-xr-xtools/run_tests/run_interop_tests.py24
-rw-r--r--tools/run_tests/sources_and_headers.json24
6 files changed, 232 insertions, 74 deletions
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index a2b83ec711..1f9b9ecc18 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -773,20 +773,6 @@ include/grpc/compression.h \
include/grpc/grpc.h \
include/grpc/status.h \
include/grpc/census.h \
-src/core/security/auth_filters.h \
-src/core/security/base64.h \
-src/core/security/credentials.h \
-src/core/security/handshake.h \
-src/core/security/json_token.h \
-src/core/security/jwt_verifier.h \
-src/core/security/secure_endpoint.h \
-src/core/security/security_connector.h \
-src/core/security/security_context.h \
-src/core/tsi/fake_transport_security.h \
-src/core/tsi/ssl_transport_security.h \
-src/core/tsi/ssl_types.h \
-src/core/tsi/transport_security.h \
-src/core/tsi/transport_security_interface.h \
src/core/census/grpc_filter.h \
src/core/channel/channel_args.h \
src/core/channel/channel_stack.h \
@@ -900,30 +886,23 @@ src/core/transport/metadata_batch.h \
src/core/transport/static_metadata.h \
src/core/transport/transport.h \
src/core/transport/transport_impl.h \
+src/core/security/auth_filters.h \
+src/core/security/base64.h \
+src/core/security/credentials.h \
+src/core/security/handshake.h \
+src/core/security/json_token.h \
+src/core/security/jwt_verifier.h \
+src/core/security/secure_endpoint.h \
+src/core/security/security_connector.h \
+src/core/security/security_context.h \
+src/core/tsi/fake_transport_security.h \
+src/core/tsi/ssl_transport_security.h \
+src/core/tsi/ssl_types.h \
+src/core/tsi/transport_security.h \
+src/core/tsi/transport_security_interface.h \
src/core/census/aggregation.h \
-src/core/census/log.h \
+src/core/census/mlog.h \
src/core/census/rpc_metric_id.h \
-src/core/httpcli/httpcli_security_connector.c \
-src/core/security/base64.c \
-src/core/security/client_auth_filter.c \
-src/core/security/credentials.c \
-src/core/security/credentials_metadata.c \
-src/core/security/credentials_posix.c \
-src/core/security/credentials_win32.c \
-src/core/security/google_default_credentials.c \
-src/core/security/handshake.c \
-src/core/security/json_token.c \
-src/core/security/jwt_verifier.c \
-src/core/security/secure_endpoint.c \
-src/core/security/security_connector.c \
-src/core/security/security_context.c \
-src/core/security/server_auth_filter.c \
-src/core/security/server_secure_chttp2.c \
-src/core/surface/init_secure.c \
-src/core/surface/secure_channel_create.c \
-src/core/tsi/fake_transport_security.c \
-src/core/tsi/ssl_transport_security.c \
-src/core/tsi/transport_security.c \
src/core/census/grpc_context.c \
src/core/census/grpc_filter.c \
src/core/channel/channel_args.c \
@@ -1052,9 +1031,30 @@ src/core/transport/metadata_batch.c \
src/core/transport/static_metadata.c \
src/core/transport/transport.c \
src/core/transport/transport_op_string.c \
+src/core/httpcli/httpcli_security_connector.c \
+src/core/security/base64.c \
+src/core/security/client_auth_filter.c \
+src/core/security/credentials.c \
+src/core/security/credentials_metadata.c \
+src/core/security/credentials_posix.c \
+src/core/security/credentials_win32.c \
+src/core/security/google_default_credentials.c \
+src/core/security/handshake.c \
+src/core/security/json_token.c \
+src/core/security/jwt_verifier.c \
+src/core/security/secure_endpoint.c \
+src/core/security/security_connector.c \
+src/core/security/security_context.c \
+src/core/security/server_auth_filter.c \
+src/core/security/server_secure_chttp2.c \
+src/core/surface/init_secure.c \
+src/core/surface/secure_channel_create.c \
+src/core/tsi/fake_transport_security.c \
+src/core/tsi/ssl_transport_security.c \
+src/core/tsi/transport_security.c \
src/core/census/context.c \
src/core/census/initialize.c \
-src/core/census/log.c \
+src/core/census/mlog.c \
src/core/census/operation.c \
src/core/census/placeholders.c \
src/core/census/tracing.c \
diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh
index d7e73d4272..c80685b23a 100755
--- a/tools/jenkins/run_performance.sh
+++ b/tools/jenkins/run_performance.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -49,7 +49,83 @@ PID2=$!
export QPS_WORKERS="localhost:10000,localhost:10010"
-bins/$config/qps_driver
+# big is the size in bytes of large messages (0 is the size otherwise)
+big=65536
+
+# wide is the number of client channels in multi-channel tests (1 otherwise)
+wide=64
+
+# deep is the number of RPCs outstanding on a channel in non-ping-pong tests
+# (the value used is 1 otherwise)
+deep=100
+
+#
+# Get total core count
+cores=`grep -c ^processor /proc/cpuinfo`
+halfcores=`expr $cores / 2`
+
+for secure in true false; do
+ # Scenario 1: generic async streaming ping-pong (contentionless latency)
+ bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \
+ --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=0 \
+ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
+ --num_servers=1 --num_clients=1 \
+ --server_core_limit=$halfcores --client_core_limit=0
+
+ # Scenario 2: generic async streaming "unconstrained" (QPS)
+ bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
+ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \
+ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
+ --num_servers=1 --num_clients=0 \
+ --server_core_limit=$halfcores --client_core_limit=0 |& tee /tmp/qps-test.$$
+
+ # Scenario 2b: QPS with a single server core
+ bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
+ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \
+ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
+ --num_servers=1 --num_clients=0 --server_core_limit=1 --client_core_limit=0
+
+ # Scenario 2c: protobuf-based QPS
+ bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+ --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \
+ --client_channels=$wide --simple_req_size=0 --simple_resp_size=0 \
+ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
+ --num_servers=1 --num_clients=0 \
+ --server_core_limit=$halfcores --client_core_limit=0
+
+ # Scenario 3: Latency at sub-peak load (all clients equally loaded)
+ for loadfactor in 0.7; do
+ bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
+ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \
+ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
+ --num_servers=1 --num_clients=0 --poisson_load=`awk -v lf=$loadfactor \
+ '$5 == "QPS:" {print int(lf * $6); exit}' /tmp/qps-test.$$` \
+ --server_core_limit=$halfcores --client_core_limit=0
+ done
+
+ rm /tmp/qps-test.$$
+
+ # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM).
+ bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
+ --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \
+ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
+ --num_servers=1 --num_clients=1 \
+ --server_core_limit=$halfcores --client_core_limit=0
+
+ # Scenario 5: Sync unary ping-pong with protobufs
+ bins/$config/qps_driver --rpc_type=UNARY --client_type=SYNC_CLIENT \
+ --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \
+ --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \
+ --secure_test=$secure --num_servers=1 --num_clients=1 \
+ --server_core_limit=$halfcores --client_core_limit=0
+
+done
+
+bins/$config/qps_driver --quit=true
-kill -2 $PID1 $PID2
wait
diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py
index 9cd02c5e43..b565fbb3f0 100644
--- a/tools/run_tests/artifact_targets.py
+++ b/tools/run_tests/artifact_targets.py
@@ -80,6 +80,11 @@ def macos_arch_env(arch):
return {'CFLAGS': arch_arg, 'LDFLAGS': arch_arg}
+python_version_arch_map = {
+ 'x86': 'Python27_32bits',
+ 'x64': 'Python27'
+}
+
class PythonArtifact:
"""Builds Python artifacts."""
@@ -88,27 +93,31 @@ class PythonArtifact:
self.platform = platform
self.arch = arch
self.labels = ['artifact', 'python', platform, arch]
+ self.python_version = python_version_arch_map[arch]
def pre_build_jobspecs(self):
return []
def build_jobspec(self):
- if self.platform == 'windows':
- raise Exception('Not supported yet.')
+ environ = {}
+ if self.platform == 'linux':
+ if self.arch == 'x86':
+ environ['SETARCH_CMD'] = 'linux32'
+ return create_docker_jobspec(self.name,
+ 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
+ 'tools/run_tests/build_artifact_python.sh',
+ environ=environ)
+ elif self.platform == 'windows':
+ return create_jobspec(self.name,
+ ['tools\\run_tests\\build_artifact_python.bat',
+ self.python_version
+ ],
+ shell=True)
else:
- environ = {}
- if self.platform == 'linux':
- if self.arch == 'x86':
- environ['SETARCH_CMD'] = 'linux32'
- return create_docker_jobspec(self.name,
- 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
- 'tools/run_tests/build_artifact_python.sh',
- environ=environ)
- else:
- environ['SKIP_PIP_INSTALL'] = 'TRUE'
- return create_jobspec(self.name,
- ['tools/run_tests/build_artifact_python.sh'],
- environ=environ)
+ environ['SKIP_PIP_INSTALL'] = 'TRUE'
+ return create_jobspec(self.name,
+ ['tools/run_tests/build_artifact_python.sh'],
+ environ=environ)
def __str__(self):
return self.name
@@ -235,6 +244,8 @@ def targets():
[PythonArtifact('linux', 'x86'),
PythonArtifact('linux', 'x64'),
PythonArtifact('macos', 'x64'),
+ PythonArtifact('windows', 'x86'),
+ PythonArtifact('windows', 'x64'),
RubyArtifact('linux', 'x86'),
RubyArtifact('linux', 'x64'),
RubyArtifact('macos', 'x64')])
diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat
new file mode 100644
index 0000000000..023d394549
--- /dev/null
+++ b/tools/run_tests/build_artifact_python.bat
@@ -0,0 +1,61 @@
+@rem Copyright 2016, Google Inc.
+@rem All rights reserved.
+@rem
+@rem Redistribution and use in source and binary forms, with or without
+@rem modification, are permitted provided that the following conditions are
+@rem met:
+@rem
+@rem * Redistributions of source code must retain the above copyright
+@rem notice, this list of conditions and the following disclaimer.
+@rem * Redistributions in binary form must reproduce the above
+@rem copyright notice, this list of conditions and the following disclaimer
+@rem in the documentation and/or other materials provided with the
+@rem distribution.
+@rem * Neither the name of Google Inc. nor the names of its
+@rem contributors may be used to endorse or promote products derived from
+@rem this software without specific prior written permission.
+@rem
+@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+set NUGET=C:\nuget\nuget.exe
+%NUGET% restore vsprojects\grpc.sln || goto :error
+
+
+@call vsprojects\build_vs2013.bat vsprojects\grpc.sln /t:grpc_dll /p:Configuration=Release /p:PlatformToolset=v120 /p:Platform=Win32 || goto :error
+@call vsprojects\build_vs2013.bat vsprojects\grpc.sln /t:grpc_dll /p:Configuration=Release /p:PlatformToolset=v120 /p:Platform=x64 || goto :error
+
+mkdir src\python\grpcio\grpc\_cython\_windows
+
+copy /Y vsprojects\Release\grpc_dll.dll src\python\grpcio\grpc\_cython\_windows\grpc_c.32.python || goto :error
+copy /Y vsprojects\x64\Release\grpc_dll.dll src\python\grpcio\grpc\_cython\_windows\grpc_c.64.python || goto :error
+
+
+set PATH=C:\%1;C:\%1\scripts;%PATH%
+
+pip install --upgrade six
+pip install --upgrade setuptools
+pip install -rrequirements.txt
+
+set GRPC_PYTHON_USE_CUSTOM_BDIST=0
+set GRPC_PYTHON_BUILD_WITH_CYTHON=1
+
+python setup.py bdist_wheel
+
+mkdir artifacts
+xcopy /Y /I /S dist\* artifacts\ || goto :error
+
+goto :EOF
+
+:error
+exit /b 1
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 76be932aef..df3ab90a83 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -422,7 +422,7 @@ def _job_kill_handler(job):
time.sleep(2)
-def cloud_to_prod_jobspec(language, test_case, server_host_name,
+def cloud_to_prod_jobspec(language, test_case, server_host_name,
server_host_detail, docker_image=None, auth=False):
"""Creates jobspec for cloud-to-prod interop test"""
container_name = None
@@ -441,7 +441,7 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name,
cwd = language.client_cwd
if docker_image:
- container_name = dockerjob.random_name('interop_client_%s' %
+ container_name = dockerjob.random_name('interop_client_%s' %
language.safename)
cmdline = docker_run_cmdline(cmdline,
image=docker_image,
@@ -457,7 +457,7 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name,
cmdline=cmdline,
cwd=cwd,
environ=environ,
- shortname='%s:%s:%s:%s' % (suite_name, server_host_name, language,
+ shortname='%s:%s:%s:%s' % (suite_name, server_host_name, language,
test_case),
timeout_seconds=90,
flake_retries=5 if args.allow_flakes else 0,
@@ -575,18 +575,18 @@ def aggregate_http2_results(stdout):
'percent': 1.0 * passed / (passed + failed)
}
-# A dictionary of prod servers to test.
+# A dictionary of prod servers to test.
# Format: server_name: (server_host, server_host_override, errors_allowed)
# TODO(adelez): implement logic for errors_allowed where if the indicated tests
# fail, they don't impact the overall test result.
prod_servers = {
- 'default': ('grpc-test.sandbox.googleapis.com',
+ 'default': ('grpc-test.sandbox.googleapis.com',
'grpc-test.sandbox.googleapis.com', False),
- 'gateway_v2': ('grpc-test2.sandbox.googleapis.com',
+ 'gateway_v2': ('grpc-test2.sandbox.googleapis.com',
'grpc-test2.sandbox.googleapis.com', True),
- 'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com',
+ 'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com',
False),
- 'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com',
+ 'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com',
True)
}
@@ -720,7 +720,7 @@ try:
if not test_case in language.unimplemented_test_cases():
if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION:
test_job = cloud_to_prod_jobspec(
- language, test_case, server_host_name,
+ language, test_case, server_host_name,
prod_servers[server_host_name],
docker_image=docker_images.get(str(language)))
jobs.append(test_job)
@@ -728,7 +728,7 @@ try:
if args.http2_interop:
for test_case in _HTTP2_TEST_CASES:
test_job = cloud_to_prod_jobspec(
- http2Interop, test_case, server_host_name,
+ http2Interop, test_case, server_host_name,
prod_servers[server_host_name],
docker_image=docker_images.get(str(http2Interop)))
jobs.append(test_job)
@@ -739,7 +739,7 @@ try:
for test_case in _AUTH_TEST_CASES:
if not test_case in language.unimplemented_test_cases():
test_job = cloud_to_prod_jobspec(
- language, test_case, server_host_name,
+ language, test_case, server_host_name,
prod_servers[server_host_name],
docker_image=docker_images.get(str(language)), auth=True)
jobs.append(test_job)
@@ -802,7 +802,7 @@ try:
report_utils.render_interop_html_report(
set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES,
_HTTP2_TEST_CASES, resultset, num_failures,
- args.cloud_to_prod_auth or args.cloud_to_prod, args.prod_servers,
+ args.cloud_to_prod_auth or args.cloud_to_prod, args.prod_servers,
args.http2_interop)
finally:
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index a3132dee45..e44a7bc79e 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -92,7 +92,7 @@
"language": "c",
"name": "census_log_test",
"src": [
- "test/core/census/log_test.c"
+ "test/core/census/mlog_test.c"
]
},
{
@@ -2983,7 +2983,7 @@
"include/grpc/status.h",
"src/core/census/aggregation.h",
"src/core/census/grpc_filter.h",
- "src/core/census/log.h",
+ "src/core/census/mlog.h",
"src/core/census/rpc_metric_id.h",
"src/core/channel/channel_args.h",
"src/core/channel/channel_stack.h",
@@ -3134,8 +3134,8 @@
"src/core/census/grpc_filter.c",
"src/core/census/grpc_filter.h",
"src/core/census/initialize.c",
- "src/core/census/log.c",
- "src/core/census/log.h",
+ "src/core/census/mlog.c",
+ "src/core/census/mlog.h",
"src/core/census/operation.c",
"src/core/census/placeholders.c",
"src/core/census/rpc_metric_id.h",
@@ -3418,6 +3418,16 @@
{
"deps": [
"gpr",
+ "grpc"
+ ],
+ "headers": [],
+ "language": "c",
+ "name": "grpc_dll",
+ "src": []
+ },
+ {
+ "deps": [
+ "gpr",
"gpr_test_util",
"grpc"
],
@@ -3512,7 +3522,7 @@
"include/grpc/status.h",
"src/core/census/aggregation.h",
"src/core/census/grpc_filter.h",
- "src/core/census/log.h",
+ "src/core/census/mlog.h",
"src/core/census/rpc_metric_id.h",
"src/core/channel/channel_args.h",
"src/core/channel/channel_stack.h",
@@ -3648,8 +3658,8 @@
"src/core/census/grpc_filter.c",
"src/core/census/grpc_filter.h",
"src/core/census/initialize.c",
- "src/core/census/log.c",
- "src/core/census/log.h",
+ "src/core/census/mlog.c",
+ "src/core/census/mlog.h",
"src/core/census/operation.c",
"src/core/census/placeholders.c",
"src/core/census/rpc_metric_id.h",
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-19 00:14:41 +0000