diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/buildgen/generate_projects.py | 3 | ||||
| -rwxr-xr-x | tools/codegen/core/gen_stats_data.py | 6 | ||||
| -rw-r--r-- | tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile | 2 | ||||
| -rw-r--r-- | tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc | 22 | ||||
| -rw-r--r-- | tools/internal_ci/helper_scripts/prepare_build_linux_rc | 1 | ||||
| -rw-r--r-- | tools/internal_ci/linux/grpc_performance_profile_daily.cfg | 26 | ||||
| -rwxr-xr-x | tools/internal_ci/linux/grpc_performance_profile_daily.sh | 37 | ||||
| -rw-r--r-- | tools/internal_ci/linux/grpc_performance_profile_master.cfg | 26 | ||||
| -rwxr-xr-x | tools/internal_ci/linux/grpc_performance_profile_master.sh | 32 | ||||
| -rwxr-xr-x | tools/profiling/microbenchmarks/bm2bq.py | 43 | ||||
| -rw-r--r-- | tools/run_tests/artifacts/distribtest_targets.py | 12 | ||||
| -rwxr-xr-x | tools/run_tests/run_tests_matrix.py | 50 |
12 files changed, 203 insertions, 57 deletions
diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index f885b26697..559ea1624b 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -34,6 +34,7 @@ argp.add_argument('build_files', nargs='+', default=[]) argp.add_argument('--templates', nargs='+', default=[]) argp.add_argument('--output_merged', default=None, type=str) argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) +argp.add_argument('--base', default='.', type=str) args = argp.parse_args() json = args.build_files @@ -69,7 +70,7 @@ jobs = [] for template in reversed(sorted(templates)): root, f = os.path.split(template) if os.path.splitext(f)[1] == '.template': - out_dir = '.' + root[len('templates'):] + out_dir = args.base + root[len('templates'):] out = out_dir + '/' + os.path.splitext(f)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8e4ef594af..cb01321ed3 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -313,3 +313,9 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % ( len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + +with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: + columns = [] + for counter in inst_map['Counter']: + columns.append(('%s_per_iteration' % counter.name, 'INTEGER')) + print >>S, ',\n'.join('%s:%s' % x for x in columns) diff --git a/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile index ff66bca0ae..0e8186d40c 100644 --- a/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile @@ -27,4 +27,6 @@ RUN apt-get update && apt-get install -y \ pkg-config \ unzip && apt-get clean +RUN apt-get update && apt-get install -y cmake golang && apt-get clean + CMD ["bash"] diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc new file mode 100644 index 0000000000..1b7779caa8 --- /dev/null +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Source this rc script to prepare the environment for linux perf builds + +# Need to increase open files limit and size for perf test +ulimit -n 32768 +ulimit -c unlimited + +git submodule update --init diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc index 91627d60cb..2ade8dac51 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -26,6 +26,7 @@ sudo service docker restart # Populate xdg-cache-home to workaround https://github.com/grpc/grpc/issues/11968 sudo mkdir -p /tmp/xdg-cache-home +PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install setuptools wheel PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install coverage==4.4 pylint==1.6.5 # Download Docker images from DockerHub diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.cfg b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg new file mode 100644 index 0000000000..9831869edb --- /dev/null +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_daily.sh" +timeout_mins: 1440 +action { + define_artifacts { + regex: "**" + regex: "github/grpc/reports/**" + } +} + diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.sh b/tools/internal_ci/linux/grpc_performance_profile_daily.sh new file mode 100755 index 0000000000..25523e21b8 --- /dev/null +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` + +make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS +bins/opt/memory_profile_test +bq load microbenchmarks.memory memory_usage.csv + +tools/run_tests/run_microbenchmark.py --collect summary --bigquery_upload || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.cfg b/tools/internal_ci/linux/grpc_performance_profile_master.cfg new file mode 100644 index 0000000000..e4cefbc3f4 --- /dev/null +++ b/tools/internal_ci/linux/grpc_performance_profile_master.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_master.sh" +timeout_mins: 600 +action { + define_artifacts { + regex: "**" + regex: "github/grpc/reports/**" + } +} + diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.sh b/tools/internal_ci/linux/grpc_performance_profile_master.sh new file mode 100755 index 0000000000..40bbfe89dc --- /dev/null +++ b/tools/internal_ci/linux/grpc_performance_profile_master.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/jenkins/run_performance_profile_hourly.sh || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi + diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 0797f3e21d..6c5040de56 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -22,42 +22,15 @@ import sys import json import csv import bm_json +import json +import subprocess + +columns = [] -columns = [ - ('jenkins_build', 'integer'), - ('jenkins_job', 'string'), - ('date', 'timestamp'), - ('cpu_scaling_enabled', 'boolean'), - ('num_cpus', 'integer'), - ('mhz_per_cpu', 'integer'), - ('library_build_type', 'string'), - ('name', 'string'), - ('fixture', 'string'), - ('client_mutator', 'string'), - ('server_mutator', 'string'), - ('request_size', 'integer'), - ('response_size', 'integer'), - ('request_count', 'integer'), - ('iterations', 'integer'), - ('time_unit', 'string'), - ('real_time', 'integer'), - ('cpu_time', 'integer'), - ('bytes_per_second', 'float'), - ('allocs_per_iteration', 'float'), - ('locks_per_iteration', 'float'), - ('writes_per_iteration', 'float'), - ('bandwidth_kilobits', 'integer'), - ('cli_transport_stalls_per_iteration', 'float'), - ('cli_stream_stalls_per_iteration', 'float'), - ('svr_transport_stalls_per_iteration', 'float'), - ('svr_stream_stalls_per_iteration', 'float'), - ('atm_cas_per_iteration', 'float'), - ('atm_add_per_iteration', 'float'), - ('end_of_stream', 'boolean'), - ('header_bytes_per_iteration', 'float'), - ('framing_bytes_per_iteration', 'float'), - ('nows_per_iteration', 'float'), -] +for row in json.loads( + subprocess.check_output([ + 'bq','--format=json','show','microbenchmarks.microbenchmarks']))['schema']['fields']: + columns.append((row['name'], row['type'])) SANITIZE = { 'integer': int, diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index fa461efa85..fb1be383cd 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -255,12 +255,13 @@ class PHPDistribTest(object): class CppDistribTest(object): """Tests Cpp make intall by building examples.""" - def __init__(self, platform, arch, docker_suffix=None): - self.name = 'cpp_%s_%s_%s' % (platform, arch, docker_suffix) + def __init__(self, platform, arch, docker_suffix=None, testcase=None): + self.name = 'cpp_%s_%s_%s_%s' % (platform, arch, docker_suffix, testcase) self.platform = platform self.arch = arch self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix] + self.testcase = testcase + self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix, testcase] def pre_build_jobspecs(self): return [] @@ -271,7 +272,7 @@ class CppDistribTest(object): 'tools/dockerfile/distribtest/cpp_%s_%s' % ( self.docker_suffix, self.arch), - 'test/distrib/cpp/run_distrib_test.sh') + 'test/distrib/cpp/run_distrib_test_%s.sh' % self.testcase) else: raise Exception("Not supported yet.") @@ -281,7 +282,8 @@ class CppDistribTest(object): def targets(): """Gets list of supported targets""" - return [CppDistribTest('linux', 'x64', 'jessie'), + return [CppDistribTest('linux', 'x64', 'jessie', 'routeguide'), + CppDistribTest('linux', 'x64', 'jessie', 'cmake'), CSharpDistribTest('linux', 'x64', 'wheezy'), CSharpDistribTest('linux', 'x64', 'jessie'), CSharpDistribTest('linux', 'x86', 'jessie'), diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 00680b02d3..957e7b569e 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -29,9 +29,11 @@ from python_utils.filter_pull_request_tests import filter_tests _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) +_DEFAULT_RUNTESTS_TIMEOUT = 1*60*60 + # Set the timeout high to allow enough time for sanitizers and pre-building # clang docker. -_RUNTESTS_TIMEOUT = 4*60*60 +_CPP_RUNTESTS_TIMEOUT = 4*60*60 # Number of jobs assigned to each run_tests.py instance _DEFAULT_INNER_JOBS = 2 @@ -51,8 +53,11 @@ def _report_filename_internal_ci(name): def _docker_jobspec(name, runtests_args=[], runtests_envs={}, - inner_jobs=_DEFAULT_INNER_JOBS): + inner_jobs=_DEFAULT_INNER_JOBS, + timeout_seconds=None): """Run a single instance of run_tests.py in a docker container""" + if not timeout_seconds: + timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT test_job = jobset.JobSpec( cmdline=['python', 'tools/run_tests/run_tests.py', '--use_docker', @@ -62,15 +67,18 @@ def _docker_jobspec(name, runtests_args=[], runtests_envs={}, '--report_suite_name', '%s' % name] + runtests_args, environ=runtests_envs, shortname='run_tests_%s' % name, - timeout_seconds=_RUNTESTS_TIMEOUT) + timeout_seconds=timeout_seconds) return test_job def _workspace_jobspec(name, runtests_args=[], workspace_name=None, - runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS): + runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS, + timeout_seconds=None): """Run a single instance of run_tests.py in a separate workspace""" if not workspace_name: workspace_name = 'workspace_%s' % name + if not timeout_seconds: + timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT env = {'WORKSPACE_NAME': workspace_name} env.update(runtests_envs) test_job = jobset.JobSpec( @@ -82,14 +90,15 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None, '--report_suite_name', '%s' % name] + runtests_args, environ=env, shortname='run_tests_%s' % name, - timeout_seconds=_RUNTESTS_TIMEOUT) + timeout_seconds=timeout_seconds) return test_job def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', arch=None, compiler=None, labels=[], extra_args=[], extra_envs={}, - inner_jobs=_DEFAULT_INNER_JOBS): + inner_jobs=_DEFAULT_INNER_JOBS, + timeout_seconds=None): result = [] for language in languages: for platform in platforms: @@ -110,10 +119,12 @@ def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', runtests_args += extra_args if platform == 'linux': job = _docker_jobspec(name=name, runtests_args=runtests_args, - runtests_envs=extra_envs, inner_jobs=inner_jobs) + runtests_envs=extra_envs, inner_jobs=inner_jobs, + timeout_seconds=timeout_seconds) else: job = _workspace_jobspec(name=name, runtests_args=runtests_args, - runtests_envs=extra_envs, inner_jobs=inner_jobs) + runtests_envs=extra_envs, inner_jobs=inner_jobs, + timeout_seconds=timeout_seconds) job.labels = [platform, config, language, iomgr_platform] + labels result.append(job) @@ -136,7 +147,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): platforms=['linux', 'macos', 'windows'], labels=['basictests', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['csharp', 'node', 'python'], configs=['dbg', 'opt'], @@ -151,7 +163,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): platforms=['linux', 'macos'], labels=['basictests', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['ruby', 'php'], configs=['dbg', 'opt'], @@ -174,13 +187,15 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): platforms=['linux'], labels=['sanitizers', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['c++'], configs=['asan', 'tsan'], platforms=['linux'], labels=['sanitizers', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) return test_jobs @@ -207,7 +222,8 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) compiler=compiler, labels=['portability', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) # portability C on Windows 64-bit (x86 is the default) test_jobs += _generate_jobs(languages=['c'], @@ -246,10 +262,11 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) configs=['dbg'], platforms=['linux'], labels=['portability', 'corelang'], extra_args=extra_args, - extra_envs={'GRPC_DNS_RESOLVER': 'ares'}) + extra_envs={'GRPC_DNS_RESOLVER': 'ares'}, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) # TODO(zyc): Turn on this test after adding c-ares support on windows. - # C with the c-ares DNS resolver on Windonws + # C with the c-ares DNS resolver on Windows # test_jobs += _generate_jobs(languages=['c'], # configs=['dbg'], platforms=['windows'], # labels=['portability', 'corelang'], @@ -292,7 +309,8 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) iomgr_platform='uv', labels=['portability', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['node'], configs=['dbg'], |