diff options
94 files changed, 435 insertions, 217 deletions
diff --git a/.gitignore b/.gitignore index 3cc35ff7cd..1610bd40cd 100644 --- a/.gitignore +++ b/.gitignore @@ -96,7 +96,7 @@ DerivedData Pods/ # Artifacts directory -artifacts/ +/artifacts/ # Git generated files for conflicting *.orig diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c index 42bd89dd0a..03097a57c0 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.c +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c @@ -39,6 +39,7 @@ #include "src/core/lib/http/httpcli.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/security/util/b64.h" +#include "src/core/lib/support/string.h" #include "src/core/lib/tsi/ssl_types.h" #include <grpc/support/alloc.h> @@ -305,6 +306,17 @@ grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims, return GRPC_JWT_VERIFIER_TIME_CONSTRAINT_FAILURE; } + /* This should be probably up to the upper layer to decide but let's harcode + the 99% use case here for email issuers, where the JWT must be self + issued. */ + if (grpc_jwt_issuer_email_domain(claims->iss) != NULL && + claims->sub != NULL && strcmp(claims->iss, claims->sub) != 0) { + gpr_log(GPR_ERROR, + "Email issuer (%s) cannot assert another subject (%s) than itself.", + claims->iss, claims->sub); + return GRPC_JWT_VERIFIER_BAD_SUBJECT; + } + if (audience == NULL) { audience_ok = claims->aud == NULL; } else { @@ -705,10 +717,26 @@ static void verifier_put_mapping(grpc_jwt_verifier *v, const char *email_domain, GPR_ASSERT(v->num_mappings <= v->allocated_mappings); } +/* Very non-sophisticated way to detect an email address. Should be good + enough for now... */ +const char *grpc_jwt_issuer_email_domain(const char *issuer) { + const char *at_sign = strchr(issuer, '@'); + if (at_sign == NULL) return NULL; + const char *email_domain = at_sign + 1; + if (*email_domain == '\0') return NULL; + const char *dot = strrchr(email_domain, '.'); + if (dot == NULL || dot == email_domain) return email_domain; + GPR_ASSERT(dot > email_domain); + /* There may be a subdomain, we just want the domain. */ + dot = gpr_memrchr(email_domain, '.', (size_t)(dot - email_domain)); + if (dot == NULL) return email_domain; + return dot + 1; +} + /* Takes ownership of ctx. */ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx, verifier_cb_ctx *ctx) { - const char *at_sign; + const char *email_domain; grpc_closure *http_cb; char *path_prefix = NULL; const char *iss; @@ -733,13 +761,9 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx, Nobody seems to implement the account/email/webfinger part 2. of the spec so we will rely instead on email/url mappings if we detect such an issuer. Part 4, on the other hand is implemented by both google and salesforce. */ - - /* Very non-sophisticated way to detect an email address. Should be good - enough for now... */ - at_sign = strchr(iss, '@'); - if (at_sign != NULL) { + email_domain = grpc_jwt_issuer_email_domain(iss); + if (email_domain != NULL) { email_key_mapping *mapping; - const char *email_domain = at_sign + 1; GPR_ASSERT(ctx->verifier != NULL); mapping = verifier_get_mapping(ctx->verifier, email_domain); if (mapping == NULL) { diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.h b/src/core/lib/security/credentials/jwt/jwt_verifier.h index f09f9d5d47..54ff9b05e5 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.h +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.h @@ -43,8 +43,7 @@ /* --- Constants. --- */ #define GRPC_OPENID_CONFIG_URL_SUFFIX "/.well-known/openid-configuration" -#define GRPC_GOOGLE_SERVICE_ACCOUNTS_EMAIL_DOMAIN \ - "developer.gserviceaccount.com" +#define GRPC_GOOGLE_SERVICE_ACCOUNTS_EMAIL_DOMAIN "gserviceaccount.com" #define GRPC_GOOGLE_SERVICE_ACCOUNTS_KEY_URL_PREFIX \ "www.googleapis.com/robot/v1/metadata/x509" @@ -57,6 +56,7 @@ typedef enum { GRPC_JWT_VERIFIER_BAD_AUDIENCE, GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR, GRPC_JWT_VERIFIER_TIME_CONSTRAINT_FAILURE, + GRPC_JWT_VERIFIER_BAD_SUBJECT, GRPC_JWT_VERIFIER_GENERIC_ERROR } grpc_jwt_verifier_status; @@ -132,5 +132,6 @@ void grpc_jwt_verifier_verify(grpc_exec_ctx *exec_ctx, grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, grpc_slice buffer); grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims, const char *audience); +const char *grpc_jwt_issuer_email_domain(const char *issuer); #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H */ diff --git a/src/core/lib/support/string.c b/src/core/lib/support/string.c index f10a30f0fd..f263f82baf 100644 --- a/src/core/lib/support/string.c +++ b/src/core/lib/support/string.c @@ -275,3 +275,14 @@ int gpr_stricmp(const char *a, const char *b) { } while (ca == cb && ca && cb); return ca - cb; } + +void *gpr_memrchr(const void *s, int c, size_t n) { + if (s == NULL) return NULL; + char *b = (char *)s; + for (size_t i = 0; i < n; i++) { + if (b[n - i - 1] == c) { + return &b[n - i - 1]; + } + } + return NULL; +} diff --git a/src/core/lib/support/string.h b/src/core/lib/support/string.h index e933e2eb46..6d1f7cc632 100644 --- a/src/core/lib/support/string.h +++ b/src/core/lib/support/string.h @@ -118,6 +118,8 @@ char *gpr_strvec_flatten(gpr_strvec *strs, size_t *total_length); lower(a)==lower(b), >0 if lower(a)>lower(b) */ int gpr_stricmp(const char *a, const char *b); +void *gpr_memrchr(const void *s, int c, size_t n); + #ifdef __cplusplus } #endif diff --git a/src/ruby/lib/grpc/generic/rpc_desc.rb b/src/ruby/lib/grpc/generic/rpc_desc.rb index cd17aed8e7..d46c4a1b5c 100644 --- a/src/ruby/lib/grpc/generic/rpc_desc.rb +++ b/src/ruby/lib/grpc/generic/rpc_desc.rb @@ -119,7 +119,7 @@ module GRPC # Send back a UNKNOWN status to the client GRPC.logger.warn("failed handler: #{active_call}; sending status:UNKNOWN") GRPC.logger.warn(e) - send_status(active_call, UNKNOWN, 'unkown error handling call on server') + send_status(active_call, UNKNOWN, "#{e.class}: #{e.message}") end def assert_arity_matches(mth) diff --git a/src/ruby/spec/generic/rpc_desc_spec.rb b/src/ruby/spec/generic/rpc_desc_spec.rb index a3f0efa603..1ace7211e9 100644 --- a/src/ruby/spec/generic/rpc_desc_spec.rb +++ b/src/ruby/spec/generic/rpc_desc_spec.rb @@ -48,7 +48,6 @@ describe GRPC::RpcDesc do @bidi_streamer = RpcDesc.new('ss', Stream.new(Object.new), Stream.new(Object.new), 'encode', 'decode') @bs_code = INTERNAL - @no_reason = 'unkown error handling call on server' @ok_response = Object.new end @@ -62,8 +61,9 @@ describe GRPC::RpcDesc do it 'sends status UNKNOWN if other StandardErrors are raised' do expect(@call).to receive(:remote_read).once.and_return(Object.new) - expect(@call).to receive(:send_status) .once.with(UNKNOWN, @no_reason, - false, metadata: {}) + expect(@call).to receive(:send_status).once.with(UNKNOWN, + arg_error_msg, + false, metadata: {}) this_desc.run_server_method(@call, method(:other_error)) end @@ -112,7 +112,7 @@ describe GRPC::RpcDesc do end it 'sends status UNKNOWN if other StandardErrors are raised' do - expect(@call).to receive(:send_status).once.with(UNKNOWN, @no_reason, + expect(@call).to receive(:send_status).once.with(UNKNOWN, arg_error_msg, false, metadata: {}) @client_streamer.run_server_method(@call, method(:other_error_alt)) end @@ -174,8 +174,9 @@ describe GRPC::RpcDesc do end it 'sends status UNKNOWN if other StandardErrors are raised' do + error_msg = arg_error_msg(StandardError.new) expect(@call).to receive(:run_server_bidi).and_raise(StandardError) - expect(@call).to receive(:send_status).once.with(UNKNOWN, @no_reason, + expect(@call).to receive(:send_status).once.with(UNKNOWN, error_msg, false, metadata: {}) @bidi_streamer.run_server_method(@call, method(:other_error_alt)) end @@ -342,4 +343,9 @@ describe GRPC::RpcDesc do def other_error_alt(_call) fail(ArgumentError, 'other error') end + + def arg_error_msg(error = nil) + error ||= ArgumentError.new('other error') + "#{error.class}: #{error.message}" + end end diff --git a/templates/tools/dockerfile/clang_format.include b/templates/tools/dockerfile/clang_format.include new file mode 100644 index 0000000000..9a2b60ba8c --- /dev/null +++ b/templates/tools/dockerfile/clang_format.include @@ -0,0 +1,5 @@ +RUN apt-get update && apt-get -y install wget +RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list +RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN apt-get update && apt-get -y install clang-format-3.8 diff --git a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template new file mode 100644 index 0000000000..8360fc121c --- /dev/null +++ b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template @@ -0,0 +1,37 @@ +%YAML 1.2 +--- | + # Copyright 2015, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM ubuntu:15.10 + + <%include file="../clang_format.include"/> + ADD clang_format_all_the_things.sh / + CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"] + diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template index 0168353933..8617666b21 100644 --- a/templates/tools/dockerfile/test/sanity/Dockerfile.template +++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template @@ -52,18 +52,12 @@ # ./compile.sh without a local protoc dependency # TODO(mattkwong): install dependencies to support latest Bazel version if newer # version is needed - RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \ + RUN git clone https://github.com/bazelbuild/bazel.git /bazel && ${"\\"} cd /bazel && git checkout tags/0.4.1 && ./compile.sh RUN ln -s /bazel/output/bazel /bin/ - #=================== - # Docker "inception" - # Note this is quite the ugly hack. - # This makes sure that the docker binary we inject has its dependencies. - RUN curl https://get.docker.com/ | sh - RUN apt-get remove --purge -y docker-engine - - RUN mkdir /var/local/jenkins + <%include file="../../clang_format.include"/> + <%include file="../../run_tests_addons.include"/> # Define the default command. CMD ["bash"] diff --git a/templates/tools/run_tests/configs.json.template b/templates/tools/run_tests/generated/configs.json.template index 5c82dfb347..5c82dfb347 100644 --- a/templates/tools/run_tests/configs.json.template +++ b/templates/tools/run_tests/generated/configs.json.template diff --git a/templates/tools/run_tests/sources_and_headers.json.template b/templates/tools/run_tests/generated/sources_and_headers.json.template index 1c5c9747d6..1c5c9747d6 100644 --- a/templates/tools/run_tests/sources_and_headers.json.template +++ b/templates/tools/run_tests/generated/sources_and_headers.json.template diff --git a/templates/tools/run_tests/tests.json.template b/templates/tools/run_tests/generated/tests.json.template index 1e21465dd2..1e21465dd2 100644 --- a/templates/tools/run_tests/tests.json.template +++ b/templates/tools/run_tests/generated/tests.json.template diff --git a/test/core/security/create_jwt.c b/test/core/security/create_jwt.c index 741ace9bdd..ac795f29d2 100644 --- a/test/core/security/create_jwt.c +++ b/test/core/security/create_jwt.c @@ -72,6 +72,7 @@ int main(int argc, char **argv) { char *scope = NULL; char *json_key_file_path = NULL; char *service_url = NULL; + grpc_init(); gpr_cmdline *cl = gpr_cmdline_create("create_jwt"); gpr_cmdline_add_string(cl, "json_key", "File path of the json key.", &json_key_file_path); @@ -102,5 +103,6 @@ int main(int argc, char **argv) { create_jwt(json_key_file_path, service_url, scope); gpr_cmdline_destroy(cl); + grpc_shutdown(); return 0; } diff --git a/test/core/security/jwt_verifier_test.c b/test/core/security/jwt_verifier_test.c index f8afba8d6d..9a21814adc 100644 --- a/test/core/security/jwt_verifier_test.c +++ b/test/core/security/jwt_verifier_test.c @@ -166,6 +166,13 @@ static const char claims_without_time_constraint[] = " \"jti\": \"jwtuniqueid\"," " \"foo\": \"bar\"}"; +static const char claims_with_bad_subject[] = + "{ \"aud\": \"https://foo.com\"," + " \"iss\": \"evil@blah.foo.com\"," + " \"sub\": \"juju@blah.foo.com\"," + " \"jti\": \"jwtuniqueid\"," + " \"foo\": \"bar\"}"; + static const char invalid_claims[] = "{ \"aud\": \"https://foo.com\"," " \"iss\": 46," /* Issuer cannot be a number. */ @@ -179,6 +186,38 @@ typedef struct { const char *expected_subject; } verifier_test_config; +static void test_jwt_issuer_email_domain(void) { + const char *d = grpc_jwt_issuer_email_domain("https://foo.com"); + GPR_ASSERT(d == NULL); + d = grpc_jwt_issuer_email_domain("foo.com"); + GPR_ASSERT(d == NULL); + d = grpc_jwt_issuer_email_domain(""); + GPR_ASSERT(d == NULL); + d = grpc_jwt_issuer_email_domain("@"); + GPR_ASSERT(d == NULL); + d = grpc_jwt_issuer_email_domain("bar@foo"); + GPR_ASSERT(strcmp(d, "foo") == 0); + d = grpc_jwt_issuer_email_domain("bar@foo.com"); + GPR_ASSERT(strcmp(d, "foo.com") == 0); + d = grpc_jwt_issuer_email_domain("bar@blah.foo.com"); + GPR_ASSERT(strcmp(d, "foo.com") == 0); + d = grpc_jwt_issuer_email_domain("bar.blah@blah.foo.com"); + GPR_ASSERT(strcmp(d, "foo.com") == 0); + d = grpc_jwt_issuer_email_domain("bar.blah@baz.blah.foo.com"); + GPR_ASSERT(strcmp(d, "foo.com") == 0); + + /* This is not a very good parser but make sure we do not crash on these weird + inputs. */ + d = grpc_jwt_issuer_email_domain("@foo"); + GPR_ASSERT(strcmp(d, "foo") == 0); + d = grpc_jwt_issuer_email_domain("bar@."); + GPR_ASSERT(d != NULL); + d = grpc_jwt_issuer_email_domain("bar@.."); + GPR_ASSERT(d != NULL); + d = grpc_jwt_issuer_email_domain("bar@..."); + GPR_ASSERT(d != NULL); +} + static void test_claims_success(void) { grpc_jwt_claims *claims; grpc_slice s = grpc_slice_from_copied_string(claims_without_time_constraint); @@ -242,6 +281,19 @@ static void test_bad_audience_claims_failure(void) { grpc_jwt_claims_destroy(claims); } +static void test_bad_subject_claims_failure(void) { + grpc_jwt_claims *claims; + grpc_slice s = grpc_slice_from_copied_string(claims_with_bad_subject); + grpc_json *json = grpc_json_parse_string_with_len( + (char *)GRPC_SLICE_START_PTR(s), GRPC_SLICE_LENGTH(s)); + GPR_ASSERT(json != NULL); + claims = grpc_jwt_claims_from_json(json, s); + GPR_ASSERT(claims != NULL); + GPR_ASSERT(grpc_jwt_claims_check(claims, "https://foo.com") == + GRPC_JWT_VERIFIER_BAD_SUBJECT); + grpc_jwt_claims_destroy(claims); +} + static char *json_key_str(const char *last_part) { size_t result_len = strlen(json_key_str_part1) + strlen(json_key_str_part2) + strlen(last_part); @@ -563,10 +615,12 @@ static void test_jwt_verifier_bad_format(void) { int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); + test_jwt_issuer_email_domain(); test_claims_success(); test_expired_claims_failure(); test_invalid_claims_failure(); test_bad_audience_claims_failure(); + test_bad_subject_claims_failure(); test_jwt_verifier_google_email_issuer_success(); test_jwt_verifier_custom_email_issuer_success(); test_jwt_verifier_url_issuer_success(); diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index 043d29e6bb..ccc85c9f32 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -93,6 +93,7 @@ int main(int argc, char **argv) { char *aud = NULL; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_init(); cl = gpr_cmdline_create("JWT verifier tool"); gpr_cmdline_add_string(cl, "jwt", "JSON web token to verify", &jwt); gpr_cmdline_add_string(cl, "aud", "Audience for the JWT", &aud); @@ -131,5 +132,6 @@ int main(int argc, char **argv) { grpc_jwt_verifier_destroy(verifier); gpr_cmdline_destroy(cl); + grpc_shutdown(); return !sync.success; } diff --git a/test/core/support/string_test.c b/test/core/support/string_test.c index 78b77fad8e..af232db350 100644 --- a/test/core/support/string_test.c +++ b/test/core/support/string_test.c @@ -243,6 +243,8 @@ static void test_int64toa() { static void test_leftpad() { char *padded; + LOG_TEST_NAME("test_leftpad"); + padded = gpr_leftpad("foo", ' ', 5); GPR_ASSERT(0 == strcmp(" foo", padded)); gpr_free(padded); @@ -273,12 +275,25 @@ static void test_leftpad() { } static void test_stricmp(void) { + LOG_TEST_NAME("test_stricmp"); + GPR_ASSERT(0 == gpr_stricmp("hello", "hello")); GPR_ASSERT(0 == gpr_stricmp("HELLO", "hello")); GPR_ASSERT(gpr_stricmp("a", "b") < 0); GPR_ASSERT(gpr_stricmp("b", "a") > 0); } +static void test_memrchr(void) { + LOG_TEST_NAME("test_memrchr"); + + GPR_ASSERT(NULL == gpr_memrchr(NULL, 'a', 0)); + GPR_ASSERT(NULL == gpr_memrchr("", 'a', 0)); + GPR_ASSERT(NULL == gpr_memrchr("hello", 'b', 5)); + GPR_ASSERT(0 == strcmp((const char *)gpr_memrchr("hello", 'h', 5), "hello")); + GPR_ASSERT(0 == strcmp((const char *)gpr_memrchr("hello", 'o', 5), "o")); + GPR_ASSERT(0 == strcmp((const char *)gpr_memrchr("hello", 'l', 5), "lo")); +} + int main(int argc, char **argv) { grpc_test_init(argc, argv); test_strdup(); @@ -291,5 +306,6 @@ int main(int argc, char **argv) { test_int64toa(); test_leftpad(); test_stricmp(); + test_memrchr(); return 0; } diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 3e509e2abd..93ef32db77 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -44,6 +44,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/host_port.h> #include <grpc/support/log.h> +#include <grpc/support/string_util.h> #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/env.h" @@ -99,23 +100,36 @@ static std::unordered_map<string, std::deque<int>> get_hosts_and_cores( return hosts; } -static deque<string> get_workers(const string& name) { - char* env = gpr_getenv(name.c_str()); - if (!env || strlen(env) == 0) return deque<string>(); - +static deque<string> get_workers(const string& env_name) { + char* env = gpr_getenv(env_name.c_str()); + if (!env) { + env = gpr_strdup(""); + } deque<string> out; char* p = env; - for (;;) { - char* comma = strchr(p, ','); - if (comma) { - out.emplace_back(p, comma); - p = comma + 1; - } else { - out.emplace_back(p); - gpr_free(env); - return out; + if (strlen(env) != 0) { + for (;;) { + char* comma = strchr(p, ','); + if (comma) { + out.emplace_back(p, comma); + p = comma + 1; + } else { + out.emplace_back(p); + break; + } } } + if (out.size() == 0) { + gpr_log(GPR_ERROR, + "Environment variable \"%s\" does not contain a list of QPS " + "workers to use. Set it to a comma-separated list of " + "hostname:port pairs, starting with hosts that should act as " + "servers. E.g. export " + "%s=\"serverhost1:1234,clienthost1:1234,clienthost2:1234\"", + env_name.c_str(), env_name.c_str()); + } + gpr_free(env); + return out; } // helpers for postprocess_scenario_result @@ -241,6 +255,7 @@ std::unique_ptr<ScenarioResult> RunScenario( workers.push_back(addr); } } + GPR_ASSERT(workers.size() != 0); // if num_clients is set to <=0, do dynamic sizing: all workers // except for servers are clients @@ -560,6 +575,9 @@ bool RunQuit() { // Get client, server lists bool result = true; auto workers = get_workers("QPS_WORKERS"); + if (workers.size() == 0) { + return false; + } for (size_t i = 0; i < workers.size(); i++) { auto stub = WorkerService::NewStub( CreateChannel(workers[i], InsecureChannelCredentials())); diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 5e78ad52d6..f8ddaf4963 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -36,7 +36,7 @@ import shutil import sys import tempfile import multiprocessing -sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests')) +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests', 'python_utils')) assert sys.argv[1:], 'run generate_projects.sh instead of this directly' diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index c0707051a6..eb64e23daf 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -37,7 +37,7 @@ readonly PROTOBUF_INSTALL_PREFIX="$(mktemp -d)" pushd third_party/protobuf ./autogen.sh ./configure --prefix="$PROTOBUF_INSTALL_PREFIX" -make +make -j 8 make install #ldconfig popd @@ -51,7 +51,7 @@ fi # stack up and change to nanopb's proto generator directory pushd third_party/nanopb/generator/proto export PATH="$PROTOC_BIN_PATH:$PATH" -make +make -j 8 # back to the root directory popd diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh index 858e074898..13e018709f 100755 --- a/tools/distrib/clang_format_code.sh +++ b/tools/distrib/clang_format_code.sh @@ -32,9 +32,15 @@ set -ex # change to root directory cd $(dirname $0)/../.. +REPO_ROOT=$(pwd) -# build clang-format docker image -docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format +if [ "$CLANG_FORMAT_SKIP_DOCKER" == "" ] +then + # build clang-format docker image + docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format -# run clang-format against the checked out codebase -docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" --rm=true -v ${HOST_GIT_ROOT:-`pwd`}:/local-code -t grpc_clang_format /clang_format_all_the_things.sh + # run clang-format against the checked out codebase + docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_FORMAT_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_format /clang_format_all_the_things.sh +else + CLANG_FORMAT_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +fi diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 581ecc40c0..502d7ef27b 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -184,7 +184,7 @@ def extension_modules(): return extensions setuptools.setup( - name='grpcio_tools', + name='grpcio-tools', version=grpc_version.VERSION, license='3-clause BSD', ext_modules=extension_modules(), diff --git a/tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile index 1d4e8e1a4a..69e624aa41 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile @@ -34,6 +34,19 @@ FROM quay.io/pypa/manylinux1_x86_64 # Update the package manager RUN yum update -y +############################################################# +# Update Git to allow cloning submodules with --reference arg +RUN yum remove -y git +RUN yum install -y curl-devel expat-devel gettext-devel openssl-devel zlib-devel gcc +RUN cd /usr/src && \ + wget https://kernel.org/pub/software/scm/git/git-2.0.5.tar.gz && \ + tar xzf git-2.0.5.tar.gz +RUN cd /usr/src/git-2.0.5 && \ + make prefix=/usr/local/git all && \ + make prefix=/usr/local/git install +ENV PATH /usr/local/git/bin:$PATH +RUN source /etc/bashrc + ################################### # Install Python build requirements RUN /opt/python/cp27-cp27m/bin/pip install cython diff --git a/tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile index 810499695e..9af80078ed 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile @@ -34,6 +34,19 @@ FROM quay.io/pypa/manylinux1_i686 # Update the package manager RUN yum update -y +############################################################# +# Update Git to allow cloning submodules with --reference arg +RUN yum remove -y git +RUN yum install -y curl-devel expat-devel gettext-devel openssl-devel zlib-devel gcc +RUN cd /usr/src && \ + wget https://kernel.org/pub/software/scm/git/git-2.0.5.tar.gz && \ + tar xzf git-2.0.5.tar.gz +RUN cd /usr/src/git-2.0.5 && \ + make prefix=/usr/local/git all && \ + make prefix=/usr/local/git install +ENV PATH /usr/local/git/bin:$PATH +RUN source /etc/bashrc + ################################### # Install Python build requirements RUN /opt/python/cp27-cp27m/bin/pip install cython diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile index ab58017a02..85f5e4db74 100644 --- a/tools/dockerfile/grpc_clang_format/Dockerfile +++ b/tools/dockerfile/grpc_clang_format/Dockerfile @@ -27,13 +27,13 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM ubuntu:wily -RUN apt-get update -RUN apt-get -y install wget +FROM ubuntu:15.10 + +RUN apt-get update && apt-get -y install wget RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - -RUN apt-get update -RUN apt-get -y install clang-format-3.8 +RUN apt-get update && apt-get -y install clang-format-3.8 + ADD clang_format_all_the_things.sh / CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"] diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh index 462c65ab5e..c6e4aabfe6 100755 --- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh @@ -44,7 +44,7 @@ for dir in $DIRS do for glob in $GLOB do - files="$files `find /local-code/$dir -name $glob -and -not -name *.generated.* -and -not -name *.pb.h -and -not -name *.pb.c -and -not -name *.pb.cc`" + files="$files `find ${CLANG_FORMAT_ROOT}/$dir -name $glob -and -not -name *.generated.* -and -not -name *.pb.h -and -not -name *.pb.c -and -not -name *.pb.cc`" done done @@ -54,7 +54,7 @@ if [ -n "$CHANGED_FILES" ]; then files=$(comm -12 <(echo $files | tr ' ' '\n' | sort -u) <(echo $CHANGED_FILES | tr ' ' '\n' | sort -u)) fi -if [ "x$TEST" = "x" ] +if [ "$TEST" == "" ] then echo $files | xargs $CLANG_FORMAT -i else diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 6b19ac845b..811384fda1 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -97,17 +97,27 @@ RUN apt-get install -y openjdk-8-jdk # ./compile.sh without a local protoc dependency # TODO(mattkwong): install dependencies to support latest Bazel version if newer # version is needed -RUN git clone https://github.com/bazelbuild/bazel.git /bazel && cd /bazel && git checkout tags/0.4.1 && ./compile.sh +RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \ + cd /bazel && git checkout tags/0.4.1 && ./compile.sh RUN ln -s /bazel/output/bazel /bin/ -#=================== -# Docker "inception" -# Note this is quite the ugly hack. -# This makes sure that the docker binary we inject has its dependencies. -RUN curl https://get.docker.com/ | sh -RUN apt-get remove --purge -y docker-engine +RUN apt-get update && apt-get -y install wget +RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list +RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN apt-get update && apt-get -y install clang-format-3.8 + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + RUN mkdir /var/local/jenkins + # Define the default command. CMD ["bash"] diff --git a/tools/run_tests/prepare_travis.sh b/tools/run_tests/artifacts/__init__.py index 10546535e8..100a624dc9 100755..100644 --- a/tools/run_tests/prepare_travis.sh +++ b/tools/run_tests/artifacts/__init__.py @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -27,41 +26,3 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -cd `dirname $0`/../.. -grpc_dir=`pwd` - -distrib=`md5sum /etc/issue | cut -f1 -d\ ` -echo "Configuring for distribution $distrib" -git submodule | while read sha path extra ; do - cd /tmp - name=`basename $path` - file=$name-$sha-$CONFIG-prebuilt-$distrib.tar.gz - echo -n "Looking for $file ..." - url=http://storage.googleapis.com/grpc-prebuilt-packages/$file - wget -q $url && ( - echo " Found." - tar xfz $file - ) || echo " Not found." -done - -mkdir -p bins/$CONFIG/protobuf -mkdir -p libs/$CONFIG/protobuf -mkdir -p libs/$CONFIG/openssl - -function cpt { - cp /tmp/prebuilt/$1 $2/$CONFIG/$3 - touch $2/$CONFIG/$3/`basename $1` -} - -if [ -e /tmp/prebuilt/bin/protoc ] ; then - touch third_party/protobuf/configure - cpt bin/protoc bins protobuf - cpt lib/libprotoc.a libs protobuf - cpt lib/libprotobuf.a libs protobuf -fi - -if [ -e /tmp/prebuilt/lib/libssl.a ] ; then - cpt lib/libcrypto.a libs openssl - cpt lib/libssl.a libs openssl -fi diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 65d34e17e1..005d99790a 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -35,7 +35,8 @@ import random import string import sys -import jobset +sys.path.insert(0, os.path.abspath('..')) +import python_utils.jobset as jobset def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, @@ -113,7 +114,7 @@ class PythonArtifact: environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, - 'tools/run_tests/build_artifact_python.sh', + 'tools/run_tests/artifacts/build_artifact_python.sh', environ=environ, timeout_seconds=60*60) elif self.platform == 'windows': @@ -125,7 +126,7 @@ class PythonArtifact: # seed. We create a random temp-dir here dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10)) return create_jobspec(self.name, - ['tools\\run_tests\\build_artifact_python.bat', + ['tools\\run_tests\\artifacts\\build_artifact_python.bat', self.py_version, '32' if self.arch == 'x86' else '64', dir @@ -136,7 +137,7 @@ class PythonArtifact: environ['PYTHON'] = self.py_version environ['SKIP_PIP_INSTALL'] = 'TRUE' return create_jobspec(self.name, - ['tools/run_tests/build_artifact_python.sh'], + ['tools/run_tests/artifacts/build_artifact_python.sh'], environ=environ) def __str__(self): @@ -165,11 +166,11 @@ class RubyArtifact: environ['SETARCH_CMD'] = 'linux32' return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/build_artifact_ruby.sh', + 'tools/run_tests/artifacts/build_artifact_ruby.sh', environ=environ) else: return create_jobspec(self.name, - ['tools/run_tests/build_artifact_ruby.sh']) + ['tools/run_tests/artifacts/build_artifact_ruby.sh']) class CSharpExtArtifact: @@ -184,7 +185,7 @@ class CSharpExtArtifact: def pre_build_jobspecs(self): if self.platform == 'windows': return [create_jobspec('prebuild_%s' % self.name, - ['tools\\run_tests\\pre_build_c.bat'], + ['tools\\run_tests\\helper_scripts\\pre_build_c.bat'], shell=True, flake_retries=5, timeout_retries=2)] @@ -195,7 +196,7 @@ class CSharpExtArtifact: if self.platform == 'windows': msbuild_platform = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec(self.name, - ['tools\\run_tests\\build_artifact_csharp.bat', + ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat', 'vsprojects\\grpc_csharp_ext.sln', '/p:Configuration=Release', '/p:PlatformToolset=v120', @@ -210,14 +211,14 @@ class CSharpExtArtifact: if self.platform == 'linux': return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/build_artifact_csharp.sh', + 'tools/run_tests/artifacts/build_artifact_csharp.sh', environ=environ) else: archflag = _ARCH_FLAG_MAP[self.arch] environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) environ['LDFLAGS'] += ' %s' % archflag return create_jobspec(self.name, - ['tools/run_tests/build_artifact_csharp.sh'], + ['tools/run_tests/artifacts/build_artifact_csharp.sh'], environ=environ) def __str__(self): @@ -245,7 +246,7 @@ class NodeExtArtifact: def build_jobspec(self): if self.platform == 'windows': return create_jobspec(self.name, - ['tools\\run_tests\\build_artifact_node.bat', + ['tools\\run_tests\\artifacts\\build_artifact_node.bat', self.gyp_arch], shell=True) else: @@ -253,10 +254,10 @@ class NodeExtArtifact: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), - 'tools/run_tests/build_artifact_node.sh {}'.format(self.gyp_arch)) + 'tools/run_tests/artifacts/build_artifact_node.sh {}'.format(self.gyp_arch)) else: return create_jobspec(self.name, - ['tools/run_tests/build_artifact_node.sh', + ['tools/run_tests/artifacts/build_artifact_node.sh', self.gyp_arch]) class PHPArtifact: @@ -276,10 +277,10 @@ class PHPArtifact: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), - 'tools/run_tests/build_artifact_php.sh') + 'tools/run_tests/artifacts/build_artifact_php.sh') else: return create_jobspec(self.name, - ['tools/run_tests/build_artifact_php.sh']) + ['tools/run_tests/artifacts/build_artifact_php.sh']) class ProtocArtifact: """Builds protoc and protoc-plugin artifacts""" @@ -306,18 +307,18 @@ class ProtocArtifact: if self.platform == 'linux': return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_protoc', - 'tools/run_tests/build_artifact_protoc.sh', + 'tools/run_tests/artifacts/build_artifact_protoc.sh', environ=environ) else: environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG return create_jobspec(self.name, - ['tools/run_tests/build_artifact_protoc.sh'], + ['tools/run_tests/artifacts/build_artifact_protoc.sh'], environ=environ) else: generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' vcplatform = 'x64' if self.arch == 'x64' else 'Win32' return create_jobspec(self.name, - ['tools\\run_tests\\build_artifact_protoc.bat'], + ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], environ={'generator': generator, 'Platform': vcplatform}) diff --git a/tools/run_tests/build_artifact_csharp.bat b/tools/run_tests/artifacts/build_artifact_csharp.bat index 24c8d485f9..24c8d485f9 100644 --- a/tools/run_tests/build_artifact_csharp.bat +++ b/tools/run_tests/artifacts/build_artifact_csharp.bat diff --git a/tools/run_tests/build_artifact_csharp.sh b/tools/run_tests/artifacts/build_artifact_csharp.sh index 7438713f5c..aed04b2745 100755 --- a/tools/run_tests/build_artifact_csharp.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. make grpc_csharp_ext diff --git a/tools/run_tests/build_artifact_node.bat b/tools/run_tests/artifacts/build_artifact_node.bat index 2e0ecd21d0..2e0ecd21d0 100644 --- a/tools/run_tests/build_artifact_node.bat +++ b/tools/run_tests/artifacts/build_artifact_node.bat diff --git a/tools/run_tests/build_artifact_node.sh b/tools/run_tests/artifacts/build_artifact_node.sh index 778a5c95d4..1066ebde19 100755 --- a/tools/run_tests/build_artifact_node.sh +++ b/tools/run_tests/artifacts/build_artifact_node.sh @@ -34,7 +34,7 @@ source ~/.nvm/nvm.sh nvm use 4 set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. rm -rf build || true diff --git a/tools/run_tests/build_artifact_php.sh b/tools/run_tests/artifacts/build_artifact_php.sh index 669447fa9a..c8d55860c1 100755 --- a/tools/run_tests/build_artifact_php.sh +++ b/tools/run_tests/artifacts/build_artifact_php.sh @@ -31,7 +31,7 @@ PHP_TARGET_ARCH=$1 set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. mkdir -p artifacts diff --git a/tools/run_tests/build_artifact_protoc.bat b/tools/run_tests/artifacts/build_artifact_protoc.bat index b2bf86da40..fd93318833 100644 --- a/tools/run_tests/build_artifact_protoc.bat +++ b/tools/run_tests/artifacts/build_artifact_protoc.bat @@ -34,7 +34,7 @@ cd third_party/protobuf/cmake mkdir build & cd build mkdir solution & cd solution -cmake -G "%generator%" -Dprotobuf_BUILD_TESTS=OFF ../.. || goto :error +cmake -G "%generator%" -Dprotobuf_BUILD_TESTS=OFF ../../.. || goto :error endlocal call vsprojects/build_plugins.bat || goto :error diff --git a/tools/run_tests/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh index 161d3a84d6..26c2280eff 100755 --- a/tools/run_tests/build_artifact_protoc.sh +++ b/tools/run_tests/artifacts/build_artifact_protoc.sh @@ -33,7 +33,7 @@ source scl_source enable devtoolset-1.1 set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. make plugins diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index 246713a6ce..246713a6ce 100644 --- a/tools/run_tests/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 2a1d41fd68..5a5506029a 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. export GRPC_PYTHON_USE_CUSTOM_BDIST=0 export GRPC_PYTHON_BUILD_WITH_CYTHON=1 diff --git a/tools/run_tests/build_artifact_ruby.sh b/tools/run_tests/artifacts/build_artifact_ruby.sh index 2d97b4068b..019efb01fd 100755 --- a/tools/run_tests/build_artifact_ruby.sh +++ b/tools/run_tests/artifacts/build_artifact_ruby.sh @@ -31,7 +31,7 @@ set -ex SYSTEM=`uname | cut -f 1 -d_` -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. set +ex [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh [[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/artifacts/build_package_node.sh index a5636cf87a..8b5e8c0bc1 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/artifacts/build_package_node.sh @@ -33,7 +33,7 @@ source ~/.nvm/nvm.sh nvm use 4 set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. base=$(pwd) diff --git a/tools/run_tests/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh index 56e3319ed9..42a8d9f8df 100755 --- a/tools/run_tests/build_package_php.sh +++ b/tools/run_tests/artifacts/build_package_php.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. mkdir -p artifacts/ cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=php,platform={windows,linux,macos}/artifacts/* artifacts/ || true diff --git a/tools/run_tests/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 2511a6ae46..4a1c15ceee 100755 --- a/tools/run_tests/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. mkdir -p artifacts/ diff --git a/tools/run_tests/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh index 0a755bddb0..b4d20d8a4c 100755 --- a/tools/run_tests/build_package_ruby.sh +++ b/tools/run_tests/artifacts/build_package_ruby.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. base=$(pwd) diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index a16daac4fe..a7535b3852 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -30,7 +30,11 @@ """Definition of targets run distribution package tests.""" -import jobset +import os.path +import sys + +sys.path.insert(0, os.path.abspath('..')) +import python_utils.jobset as jobset def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/artifacts/package_targets.py index 673affeac0..d490f571c3 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -30,7 +30,12 @@ """Definition of targets to build distribution packages.""" -import jobset +import os.path +import sys + +sys.path.insert(0, os.path.abspath('..')) +import python_utils.jobset as jobset + def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, flake_retries=0, timeout_retries=0): @@ -114,7 +119,7 @@ class NodePackage: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/build_package_node.sh') + 'tools/run_tests/artifacts/build_package_node.sh') class RubyPackage: @@ -131,7 +136,7 @@ class RubyPackage: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/build_package_ruby.sh') + 'tools/run_tests/artifacts/build_package_ruby.sh') class PythonPackage: @@ -148,7 +153,7 @@ class PythonPackage: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/build_package_python.sh') + 'tools/run_tests/artifacts/build_package_python.sh') class PHPPackage: @@ -165,7 +170,7 @@ class PHPPackage: return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/build_package_php.sh') + 'tools/run_tests/artifacts/build_package_php.sh') def targets(): diff --git a/tools/run_tests/build_stats_schema.json b/tools/run_tests/build_stats/build_stats_schema.json index 021a349545..021a349545 100644 --- a/tools/run_tests/build_stats_schema.json +++ b/tools/run_tests/build_stats/build_stats_schema.json diff --git a/tools/run_tests/build_stats_schema_no_matrix.json b/tools/run_tests/build_stats/build_stats_schema_no_matrix.json index eeb067d7a5..eeb067d7a5 100644 --- a/tools/run_tests/build_stats_schema_no_matrix.json +++ b/tools/run_tests/build_stats/build_stats_schema_no_matrix.json diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index c3219c533d..b68ac89121 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -77,8 +77,6 @@ docker run \ -v /tmp/ccache:/tmp/ccache \ -v /tmp/npm-cache:/tmp/npm-cache \ -v /tmp/xdg-cache-home:/tmp/xdg-cache-home \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v $(which docker):/bin/docker \ -w /var/local/git/grpc \ --name=$CONTAINER_NAME \ $DOCKER_IMAGE_NAME \ diff --git a/tools/run_tests/configs.json b/tools/run_tests/generated/configs.json index b0839ef026..b0839ef026 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/generated/configs.json diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 6ae269cc20..6ae269cc20 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json diff --git a/tools/run_tests/tests.json b/tools/run_tests/generated/tests.json index b76263b8b9..b76263b8b9 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/generated/tests.json diff --git a/tools/run_tests/build_csharp.sh b/tools/run_tests/helper_scripts/build_csharp.sh index 48ce11a10b..84c5b1c777 100755 --- a/tools/run_tests/build_csharp.sh +++ b/tools/run_tests/helper_scripts/build_csharp.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../../src/csharp +cd $(dirname $0)/../../../src/csharp # overriding NativeDependenciesConfigurationUnix is needed to make gcov code coverage work. xbuild /p:Configuration=$MSBUILD_CONFIG /p:NativeDependenciesConfigurationUnix=$CONFIG Grpc.sln diff --git a/tools/run_tests/build_csharp_coreclr.bat b/tools/run_tests/helper_scripts/build_csharp_coreclr.bat index b6e3ccbd2b..78e5f5998b 100644 --- a/tools/run_tests/build_csharp_coreclr.bat +++ b/tools/run_tests/helper_scripts/build_csharp_coreclr.bat @@ -29,7 +29,7 @@ setlocal -cd /d %~dp0\..\..\src\csharp +cd /d %~dp0\..\..\..\src\csharp dotnet restore . || goto :error diff --git a/tools/run_tests/build_csharp_coreclr.sh b/tools/run_tests/helper_scripts/build_csharp_coreclr.sh index 02cf0d39cb..dd5fd31c75 100755 --- a/tools/run_tests/build_csharp_coreclr.sh +++ b/tools/run_tests/helper_scripts/build_csharp_coreclr.sh @@ -30,7 +30,7 @@ set -ex -cd $(dirname $0)/../../src/csharp +cd $(dirname $0)/../../../src/csharp # TODO(jtattermusch): introduce caching dotnet restore . diff --git a/tools/run_tests/build_node.bat b/tools/run_tests/helper_scripts/build_node.bat index 82e8208348..82e8208348 100644 --- a/tools/run_tests/build_node.bat +++ b/tools/run_tests/helper_scripts/build_node.bat diff --git a/tools/run_tests/build_node.sh b/tools/run_tests/helper_scripts/build_node.sh index d9292fd8aa..8a928bb762 100755 --- a/tools/run_tests/build_node.sh +++ b/tools/run_tests/helper_scripts/build_node.sh @@ -38,6 +38,6 @@ set -ex CONFIG=${CONFIG:-opt} # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. npm install --unsafe-perm --build-from-source diff --git a/tools/run_tests/build_php.sh b/tools/run_tests/helper_scripts/build_php.sh index 77a8abcfe7..acaaa23adf 100755 --- a/tools/run_tests/build_php.sh +++ b/tools/run_tests/helper_scripts/build_php.sh @@ -33,7 +33,7 @@ set -ex CONFIG=${CONFIG:-opt} # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. root=`pwd` export GRPC_LIB_SUBDIR=libs/$CONFIG diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 7cac394960..0e88e96765 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -31,7 +31,7 @@ set -ex # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. ########################## # Portability operations # diff --git a/tools/run_tests/build_python_msys2.sh b/tools/run_tests/helper_scripts/build_python_msys2.sh index 6e9d369018..6e9d369018 100644 --- a/tools/run_tests/build_python_msys2.sh +++ b/tools/run_tests/helper_scripts/build_python_msys2.sh diff --git a/tools/run_tests/build_ruby.sh b/tools/run_tests/helper_scripts/build_ruby.sh index 10343fce69..32638dede9 100755 --- a/tools/run_tests/build_ruby.sh +++ b/tools/run_tests/helper_scripts/build_ruby.sh @@ -34,7 +34,7 @@ set -ex export GRPC_CONFIG=${CONFIG:-opt} # change to grpc's ruby directory -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. rm -rf ./tmp rake compile diff --git a/tools/run_tests/post_tests_c.sh b/tools/run_tests/helper_scripts/post_tests_c.sh index 4409526dab..a83a59e23b 100755 --- a/tools/run_tests/post_tests_c.sh +++ b/tools/run_tests/helper_scripts/post_tests_c.sh @@ -32,7 +32,7 @@ set -ex if [ "$CONFIG" != "gcov" ] ; then exit ; fi -root=$(readlink -f $(dirname $0)/../..) +root=$(readlink -f $(dirname $0)/../../..) out=$root/reports/c_cxx_coverage tmp1=$(mktemp) tmp2=$(mktemp) diff --git a/tools/run_tests/post_tests_csharp.bat b/tools/run_tests/helper_scripts/post_tests_csharp.bat index 0d49a00b2a..2359f148ce 100644 --- a/tools/run_tests/post_tests_csharp.bat +++ b/tools/run_tests/helper_scripts/post_tests_csharp.bat @@ -36,7 +36,7 @@ if not "%CONFIG%" == "gcov" ( ) @rem enter src/csharp directory -cd /d %~dp0\..\..\src\csharp +cd /d %~dp0\..\..\..\src\csharp @rem Generate code coverage report @rem TODO(jtattermusch): currently the report list is hardcoded diff --git a/tools/run_tests/post_tests_csharp.sh b/tools/run_tests/helper_scripts/post_tests_csharp.sh index bb6f5c6e18..762c1f8827 100755 --- a/tools/run_tests/post_tests_csharp.sh +++ b/tools/run_tests/helper_scripts/post_tests_csharp.sh @@ -33,7 +33,7 @@ set -ex if [ "$CONFIG" != "gcov" ] ; then exit ; fi # change to gRPC repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. # Generate the csharp extension coverage report gcov objs/gcov/src/csharp/ext/*.o diff --git a/tools/run_tests/post_tests_php.sh b/tools/run_tests/helper_scripts/post_tests_php.sh index b4098066ea..23dc202322 100755 --- a/tools/run_tests/post_tests_php.sh +++ b/tools/run_tests/helper_scripts/post_tests_php.sh @@ -32,7 +32,7 @@ set -ex if [ "$CONFIG" != "gcov" ] ; then exit ; fi -root=$(readlink -f $(dirname $0)/../..) +root=$(readlink -f $(dirname $0)/../../..) out=$root/reports/php_ext_coverage tmp1=$(mktemp) tmp2=$(mktemp) diff --git a/tools/run_tests/post_tests_ruby.sh b/tools/run_tests/helper_scripts/post_tests_ruby.sh index 0877e44805..300edfe8a3 100755 --- a/tools/run_tests/post_tests_ruby.sh +++ b/tools/run_tests/helper_scripts/post_tests_ruby.sh @@ -32,7 +32,7 @@ set -ex if [ "$CONFIG" != "gcov" ] ; then exit ; fi -root=$(readlink -f $(dirname $0)/../..) +root=$(readlink -f $(dirname $0)/../../..) out=$root/reports/ruby_ext_coverage tmp1=$(mktemp) tmp2=$(mktemp) diff --git a/tools/run_tests/pre_build_c.bat b/tools/run_tests/helper_scripts/pre_build_c.bat index e4ab69384c..75b90f85b2 100644 --- a/tools/run_tests/pre_build_c.bat +++ b/tools/run_tests/helper_scripts/pre_build_c.bat @@ -32,7 +32,7 @@ setlocal @rem enter repo root -cd /d %~dp0\..\.. +cd /d %~dp0\..\..\.. @rem Location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/tools/run_tests/pre_build_csharp.bat b/tools/run_tests/helper_scripts/pre_build_csharp.bat index f15979a96b..139955d4da 100644 --- a/tools/run_tests/pre_build_csharp.bat +++ b/tools/run_tests/helper_scripts/pre_build_csharp.bat @@ -32,7 +32,7 @@ setlocal @rem enter repo root -cd /d %~dp0\..\.. +cd /d %~dp0\..\..\.. @rem Location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/tools/run_tests/pre_build_csharp.sh b/tools/run_tests/helper_scripts/pre_build_csharp.sh index ee678ddce5..1f808556f4 100755 --- a/tools/run_tests/pre_build_csharp.sh +++ b/tools/run_tests/helper_scripts/pre_build_csharp.sh @@ -31,7 +31,7 @@ set -ex # cd to gRPC csharp directory -cd $(dirname $0)/../../src/csharp +cd $(dirname $0)/../../../src/csharp root=`pwd` diff --git a/tools/run_tests/pre_build_node.bat b/tools/run_tests/helper_scripts/pre_build_node.bat index addb01a2a4..addb01a2a4 100644 --- a/tools/run_tests/pre_build_node.bat +++ b/tools/run_tests/helper_scripts/pre_build_node.bat diff --git a/tools/run_tests/pre_build_node.sh b/tools/run_tests/helper_scripts/pre_build_node.sh index e63be9da52..e63be9da52 100755 --- a/tools/run_tests/pre_build_node.sh +++ b/tools/run_tests/helper_scripts/pre_build_node.sh diff --git a/tools/run_tests/pre_build_ruby.sh b/tools/run_tests/helper_scripts/pre_build_ruby.sh index e7074c45c2..56b58df544 100755 --- a/tools/run_tests/pre_build_ruby.sh +++ b/tools/run_tests/helper_scripts/pre_build_ruby.sh @@ -34,6 +34,6 @@ set -ex export GRPC_CONFIG=${CONFIG:-opt} # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. bundle install diff --git a/tools/run_tests/run_lcov.sh b/tools/run_tests/helper_scripts/run_lcov.sh index 796a0b5ceb..bc7b44cd3e 100755 --- a/tools/run_tests/run_lcov.sh +++ b/tools/run_tests/helper_scripts/run_lcov.sh @@ -32,7 +32,7 @@ set -ex out=$(readlink -f ${1:-coverage}) -root=$(readlink -f $(dirname $0)/../..) +root=$(readlink -f $(dirname $0)/../../..) shift || true tmp=$(mktemp) cd $root diff --git a/tools/run_tests/run_node.bat b/tools/run_tests/helper_scripts/run_node.bat index 0987fbee55..0987fbee55 100644 --- a/tools/run_tests/run_node.bat +++ b/tools/run_tests/helper_scripts/run_node.bat diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/helper_scripts/run_node.sh index 44f75645f5..0fafe9481a 100755 --- a/tools/run_tests/run_node.sh +++ b/tools/run_tests/helper_scripts/run_node.sh @@ -37,7 +37,7 @@ set -ex CONFIG=${CONFIG:-opt} # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. root=`pwd` diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/helper_scripts/run_python.sh index 17e0186f2a..7be473428f 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/helper_scripts/run_python.sh @@ -31,7 +31,7 @@ set -ex # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. PYTHON=`realpath -s "${1:-py27/bin/python}"` diff --git a/tools/run_tests/run_ruby.sh b/tools/run_tests/helper_scripts/run_ruby.sh index 73a84ac361..ab153b7e25 100755 --- a/tools/run_tests/run_ruby.sh +++ b/tools/run_tests/helper_scripts/run_ruby.sh @@ -31,6 +31,6 @@ set -ex # change to grpc repo root -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. rake diff --git a/tools/run_tests/run_tests_in_workspace.sh b/tools/run_tests/helper_scripts/run_tests_in_workspace.sh index 9c6c5b76e0..002c8d6de2 100755 --- a/tools/run_tests/run_tests_in_workspace.sh +++ b/tools/run_tests/helper_scripts/run_tests_in_workspace.sh @@ -34,7 +34,7 @@ # newly created workspace) set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. export repo_root=$(pwd) rm -rf "${WORKSPACE_NAME}" diff --git a/tools/run_tests/interop_html_report.template b/tools/run_tests/interop/interop_html_report.template index 46cce426b7..46cce426b7 100644 --- a/tools/run_tests/interop_html_report.template +++ b/tools/run_tests/interop/interop_html_report.template diff --git a/tools/run_tests/python_utils/__init__.py b/tools/run_tests/python_utils/__init__.py new file mode 100644 index 0000000000..100a624dc9 --- /dev/null +++ b/tools/run_tests/python_utils/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/run_tests/antagonist.py b/tools/run_tests/python_utils/antagonist.py index 857addfb38..857addfb38 100755 --- a/tools/run_tests/antagonist.py +++ b/tools/run_tests/python_utils/antagonist.py diff --git a/tools/run_tests/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index 4a7e61b3c4..0869c5cee9 100755 --- a/tools/run_tests/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -31,13 +31,14 @@ from __future__ import print_function -import jobset import tempfile import time import uuid import os import subprocess +import jobset + _DEVNULL = open(os.devnull, 'w') diff --git a/tools/run_tests/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index ca1d6d4eb5..ca1d6d4eb5 100644 --- a/tools/run_tests/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py diff --git a/tools/run_tests/jobset.py b/tools/run_tests/python_utils/jobset.py index 7b2c62d1a2..7b2c62d1a2 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/python_utils/jobset.py diff --git a/tools/run_tests/port_server.py b/tools/run_tests/python_utils/port_server.py index e9b3f7ff79..e9b3f7ff79 100755 --- a/tools/run_tests/port_server.py +++ b/tools/run_tests/python_utils/port_server.py diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/python_utils/report_utils.py index 5ce2a87cfa..352cf7abe7 100644 --- a/tools/run_tests/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py @@ -84,7 +84,7 @@ def render_interop_html_report( client_langs, server_langs, test_cases, auth_test_cases, http2_cases, resultset, num_failures, cloud_to_prod, prod_servers, http2_interop): """Generate HTML report for interop tests.""" - template_file = 'tools/run_tests/interop_html_report.template' + template_file = 'tools/run_tests/interop/interop_html_report.template' try: mytemplate = Template(filename=template_file, format_exceptions=True) except NameError: diff --git a/tools/run_tests/watch_dirs.py b/tools/run_tests/python_utils/watch_dirs.py index 21ef23e158..21ef23e158 100755 --- a/tools/run_tests/watch_dirs.py +++ b/tools/run_tests/python_utils/watch_dirs.py diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 83cfc429f9..c14f18af81 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -34,20 +34,21 @@ from __future__ import print_function import argparse import atexit -import dockerjob import itertools -import jobset import json import multiprocessing import os import re -import report_utils import subprocess import sys import tempfile import time import uuid +import python_utils.dockerjob as dockerjob +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils + # Docker doesn't clean up after itself, so we do it on exit. atexit.register(lambda: subprocess.call(['stty', 'echo'])) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 69ccff85cf..b7b742d7af 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -35,21 +35,21 @@ from __future__ import print_function import argparse import collections import itertools -import jobset import json import multiprocessing import os -import performance.scenario_config as scenario_config import pipes import re -import report_utils import subprocess import sys import tempfile import time import traceback import uuid -import report_utils + +import performance.scenario_config as scenario_config +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py index de4a22877c..a94a615b88 100755 --- a/tools/run_tests/run_stress_tests.py +++ b/tools/run_tests/run_stress_tests.py @@ -33,9 +33,7 @@ from __future__ import print_function import argparse import atexit -import dockerjob import itertools -import jobset import json import multiprocessing import os @@ -46,6 +44,9 @@ import tempfile import time import uuid +import python_utils.dockerjob as dockerjob +import python_utils.jobset as jobset + # Docker doesn't clean up after itself, so we do it on exit. atexit.register(lambda: subprocess.call(['stty', 'echo'])) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index fe56f4a175..924274191e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -54,9 +54,9 @@ import time from six.moves import urllib import uuid -import jobset -import report_utils -import watch_dirs +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils +import python_utils.watch_dirs as watch_dirs _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) @@ -116,7 +116,7 @@ class Config(object): def get_c_tests(travis, test_lang) : out = [] platforms_str = 'ci_platforms' if travis else 'platforms' - with open('tools/run_tests/tests.json') as f: + with open('tools/run_tests/generated/tests.json') as f: js = json.load(f) return [tgt for tgt in js @@ -300,7 +300,7 @@ class CLanguage(object): def pre_build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\pre_build_c.bat']] + return [['tools\\run_tests\\helper_scripts\\pre_build_c.bat']] else: return [] @@ -311,7 +311,7 @@ class CLanguage(object): if self.platform == 'windows': return [] else: - return [['tools/run_tests/post_tests_c.sh']] + return [['tools/run_tests/helper_scripts/post_tests_c.sh']] def makefile_name(self): return 'Makefile' @@ -382,16 +382,16 @@ class NodeLanguage(object): def test_specs(self): if self.platform == 'windows': - return [self.config.job_spec(['tools\\run_tests\\run_node.bat'])] + return [self.config.job_spec(['tools\\run_tests\\helper_scripts\\run_node.bat'])] else: - return [self.config.job_spec(['tools/run_tests/run_node.sh', self.node_version], + return [self.config.job_spec(['tools/run_tests/helper_scripts/run_node.sh', self.node_version], environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\pre_build_node.bat']] + return [['tools\\run_tests\\helper_scripts\\pre_build_node.bat']] else: - return [['tools/run_tests/pre_build_node.sh', self.node_version]] + return [['tools/run_tests/helper_scripts/pre_build_node.sh', self.node_version]] def make_targets(self): return [] @@ -401,9 +401,9 @@ class NodeLanguage(object): def build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\build_node.bat']] + return [['tools\\run_tests\\helper_scripts\\build_node.bat']] else: - return [['tools/run_tests/build_node.sh', self.node_version]] + return [['tools/run_tests/helper_scripts/build_node.sh', self.node_version]] def post_tests_steps(self): return [] @@ -439,10 +439,10 @@ class PhpLanguage(object): return [] def build_steps(self): - return [['tools/run_tests/build_php.sh']] + return [['tools/run_tests/helper_scripts/build_php.sh']] def post_tests_steps(self): - return [['tools/run_tests/post_tests_php.sh']] + return [['tools/run_tests/helper_scripts/post_tests_php.sh']] def makefile_name(self): return 'Makefile' @@ -475,10 +475,10 @@ class Php7Language(object): return [] def build_steps(self): - return [['tools/run_tests/build_php.sh']] + return [['tools/run_tests/helper_scripts/build_php.sh']] def post_tests_steps(self): - return [['tools/run_tests/post_tests_php.sh']] + return [['tools/run_tests/helper_scripts/post_tests_php.sh']] def makefile_name(self): return 'Makefile' @@ -547,18 +547,18 @@ class PythonLanguage(object): if os.name == 'nt': shell = ['bash'] - builder = [os.path.abspath('tools/run_tests/build_python_msys2.sh')] + builder = [os.path.abspath('tools/run_tests/helper_scripts/build_python_msys2.sh')] builder_prefix_arguments = ['MINGW{}'.format(bits)] venv_relative_python = ['Scripts/python.exe'] toolchain = ['mingw32'] else: shell = [] - builder = [os.path.abspath('tools/run_tests/build_python.sh')] + builder = [os.path.abspath('tools/run_tests/helper_scripts/build_python.sh')] builder_prefix_arguments = [] venv_relative_python = ['bin/python'] toolchain = ['unix'] - runner = [os.path.abspath('tools/run_tests/run_python.sh')] + runner = [os.path.abspath('tools/run_tests/helper_scripts/run_python.sh')] config_vars = _PythonConfigVars(shell, builder, builder_prefix_arguments, venv_relative_python, toolchain, runner) python27_config = _python_config_generator(name='py27', major='2', @@ -610,12 +610,12 @@ class RubyLanguage(object): _check_compiler(self.args.compiler, ['default']) def test_specs(self): - return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], + return [self.config.job_spec(['tools/run_tests/helper_scripts/run_ruby.sh'], timeout_seconds=10*60, environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): - return [['tools/run_tests/pre_build_ruby.sh']] + return [['tools/run_tests/helper_scripts/pre_build_ruby.sh']] def make_targets(self): return [] @@ -624,10 +624,10 @@ class RubyLanguage(object): return [] def build_steps(self): - return [['tools/run_tests/build_ruby.sh']] + return [['tools/run_tests/helper_scripts/build_ruby.sh']] def post_tests_steps(self): - return [['tools/run_tests/post_tests_ruby.sh']] + return [['tools/run_tests/helper_scripts/post_tests_ruby.sh']] def makefile_name(self): return 'Makefile' @@ -725,9 +725,9 @@ class CSharpLanguage(object): def pre_build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\pre_build_csharp.bat']] + return [['tools\\run_tests\\helper_scripts\\pre_build_csharp.bat']] else: - return [['tools/run_tests/pre_build_csharp.sh']] + return [['tools/run_tests/helper_scripts/pre_build_csharp.sh']] def make_targets(self): return ['grpc_csharp_ext'] @@ -738,22 +738,22 @@ class CSharpLanguage(object): def build_steps(self): if self.args.compiler == 'coreclr': if self.platform == 'windows': - return [['tools\\run_tests\\build_csharp_coreclr.bat']] + return [['tools\\run_tests\\helper_scripts\\build_csharp_coreclr.bat']] else: - return [['tools/run_tests/build_csharp_coreclr.sh']] + return [['tools/run_tests/helper_scripts/build_csharp_coreclr.sh']] else: if self.platform == 'windows': return [[_windows_build_bat(self.args.compiler), 'src/csharp/Grpc.sln', '/p:Configuration=%s' % _MSBUILD_CONFIG[self.config.build_config]]] else: - return [['tools/run_tests/build_csharp.sh']] + return [['tools/run_tests/helper_scripts/build_csharp.sh']] def post_tests_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\post_tests_csharp.bat']] + return [['tools\\run_tests\\helper_scripts\\post_tests_csharp.bat']] else: - return [['tools/run_tests/post_tests_csharp.sh']] + return [['tools/run_tests/helper_scripts/post_tests_csharp.sh']] def makefile_name(self): return 'Makefile' @@ -820,8 +820,12 @@ class Sanity(object): def test_specs(self): import yaml with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f: + environ={'TEST': 'true'} + if _is_use_docker_child(): + environ['CLANG_FORMAT_SKIP_DOCKER'] = 'true' return [self.config.job_spec(cmd['script'].split(), - timeout_seconds=30*60, environ={'TEST': 'true'}, + timeout_seconds=30*60, + environ=environ, cpu_cost=cmd.get('cpu_cost', 1)) for cmd in yaml.load(f)] @@ -872,9 +876,9 @@ class NodeExpressLanguage(object): def pre_build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\pre_build_node.bat']] + return [['tools\\run_tests\\helper_scripts\\pre_build_node.bat']] else: - return [['tools/run_tests/pre_build_node.sh', self.node_version]] + return [['tools/run_tests/helper_scripts/pre_build_node.sh', self.node_version]] def make_targets(self): return [] @@ -898,7 +902,7 @@ class NodeExpressLanguage(object): return 'node_express' # different configurations we can run under -with open('tools/run_tests/configs.json') as f: +with open('tools/run_tests/generated/configs.json') as f: _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read())) @@ -1299,7 +1303,7 @@ def _start_port_server(port_server_port): running = False if running: current_version = int(subprocess.check_output( - [sys.executable, os.path.abspath('tools/run_tests/port_server.py'), + [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), 'dump_version'])) print('my port server is version %d' % current_version) running = (version >= current_version) @@ -1311,7 +1315,7 @@ def _start_port_server(port_server_port): fd, logfile = tempfile.mkstemp() os.close(fd) print('starting port_server, with log file %s' % logfile) - args = [sys.executable, os.path.abspath('tools/run_tests/port_server.py'), + args = [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), '-p', '%d' % port_server_port, '-l', logfile] env = dict(os.environ) env['BUILD_ID'] = 'pleaseDontKillMeJenkins' @@ -1417,7 +1421,7 @@ def _build_and_run( return [] # start antagonists - antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) + antagonists = [subprocess.Popen(['tools/run_tests/python_utils/antagonist.py']) for _ in range(0, args.antagonists)] port_server_port = 32766 _start_port_server(port_server_port) diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index df48099971..6e83180c66 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -31,12 +31,13 @@ """Run test matrix.""" import argparse -import jobset import multiprocessing import os -import report_utils import sys -from filter_pull_request_tests import filter_tests + +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils +from python_utils.filter_pull_request_tests import filter_tests _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) @@ -69,7 +70,7 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None, inner_jobs=_ workspace_name = 'workspace_%s' % name env = {'WORKSPACE_NAME': workspace_name} test_job = jobset.JobSpec( - cmdline=['tools/run_tests/run_tests_in_workspace.sh', + cmdline=['tools/run_tests/helper_scripts/run_tests_in_workspace.sh', '-t', '-j', str(inner_jobs), '-x', '../report_%s.xml' % name, diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index b733ba173f..a86db02b80 100755 --- a/tools/run_tests/sanity/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -34,7 +34,7 @@ import re import sys root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) -with open(os.path.join(root, 'tools', 'run_tests', 'sources_and_headers.json')) as f: +with open(os.path.join(root, 'tools', 'run_tests', 'generated', 'sources_and_headers.json')) as f: js = json.loads(f.read()) re_inc1 = re.compile(r'^#\s*include\s*"([^"]*)"') diff --git a/tools/run_tests/sanity/check_test_filtering.py b/tools/run_tests/sanity/check_test_filtering.py index b522cdeb49..290a6e2ddf 100755 --- a/tools/run_tests/sanity/check_test_filtering.py +++ b/tools/run_tests/sanity/check_test_filtering.py @@ -38,7 +38,7 @@ import re # hack import paths to pick up extra code sys.path.insert(0, os.path.abspath('tools/run_tests/')) from run_tests_matrix import _create_test_jobs, _create_portability_test_jobs -import filter_pull_request_tests +import python_utils.filter_pull_request_tests as filter_pull_request_tests _LIST_OF_LANGUAGE_LABELS = ['c', 'c++', 'csharp', 'node', 'objc', 'php', 'php7', 'python', 'ruby'] _LIST_OF_PLATFORM_LABELS = ['linux', 'macos', 'windows'] diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py index 2e3fa443b9..fdc4668222 100755 --- a/tools/run_tests/task_runner.py +++ b/tools/run_tests/task_runner.py @@ -33,14 +33,13 @@ from __future__ import print_function import argparse -import atexit -import jobset import multiprocessing import sys -import artifact_targets -import distribtest_targets -import package_targets +import artifacts.artifact_targets as artifact_targets +import artifacts.distribtest_targets as distribtest_targets +import artifacts.package_targets as package_targets +import python_utils.jobset as jobset _TARGETS = [] _TARGETS += artifact_targets.targets() |