diff options
| author |  ncteisen <ncteisen@gmail.com> | 2018-01-09 13:49:37 -0800 |
|---|---|---|
| committer | ncteisen <ncteisen@gmail.com> | 2018-01-09 13:49:37 -0800 |
| commit | b04efac6a5ac0842bc2b6c3e72b786b646644ac0 (patch) | |
| tree | ca70be08a55ea3aeade97e226b232b1786c63656 | |
| parent | b928fd496efbe2265169077bcfdf684f7ad6aea3 (diff) | |
| parent | 0ea629c61ec70a35075e800bc3f85651f00e746f (diff) | |
Merge branch 'master' of https://github.com/grpc/grpc into channel-tracing
232 files changed, 4655 insertions, 4197 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index c819a09e60..f9a3f83afe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4628,6 +4628,7 @@ add_library(end2end_tests test/core/end2end/tests/filter_call_init_fails.cc test/core/end2end/tests/filter_causes_close.cc test/core/end2end/tests/filter_latency.cc + test/core/end2end/tests/filter_status_code.cc test/core/end2end/tests/graceful_server_shutdown.cc test/core/end2end/tests/high_initial_seqno.cc test/core/end2end/tests/hpack_size.cc @@ -4729,6 +4730,7 @@ add_library(end2end_nosec_tests test/core/end2end/tests/filter_call_init_fails.cc test/core/end2end/tests/filter_causes_close.cc test/core/end2end/tests/filter_latency.cc + test/core/end2end/tests/filter_status_code.cc test/core/end2end/tests/graceful_server_shutdown.cc test/core/end2end/tests/high_initial_seqno.cc test/core/end2end/tests/hpack_size.cc @@ -327,7 +327,7 @@ CXXFLAGS += -std=c++11 ifeq ($(SYSTEM),Darwin) CXXFLAGS += -stdlib=libc++ endif -CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 +CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 -Wno-deprecated-declarations COREFLAGS += -fno-rtti -fno-exceptions LDFLAGS += -g @@ -8570,6 +8570,7 @@ LIBEND2END_TESTS_SRC = \ test/core/end2end/tests/filter_call_init_fails.cc \ test/core/end2end/tests/filter_causes_close.cc \ test/core/end2end/tests/filter_latency.cc \ + test/core/end2end/tests/filter_status_code.cc \ test/core/end2end/tests/graceful_server_shutdown.cc \ test/core/end2end/tests/high_initial_seqno.cc \ test/core/end2end/tests/hpack_size.cc \ @@ -8668,6 +8669,7 @@ LIBEND2END_NOSEC_TESTS_SRC = \ test/core/end2end/tests/filter_call_init_fails.cc \ test/core/end2end/tests/filter_causes_close.cc \ test/core/end2end/tests/filter_latency.cc \ + test/core/end2end/tests/filter_status_code.cc \ test/core/end2end/tests/graceful_server_shutdown.cc \ test/core/end2end/tests/high_initial_seqno.cc \ test/core/end2end/tests/hpack_size.cc \ @@ -113,10 +113,10 @@ task 'gem:native' do if RUBY_PLATFORM =~ /darwin/ FileUtils.touch 'grpc_c.32.ruby' FileUtils.touch 'grpc_c.64.ruby' - system "rake cross native gem RUBY_CC_VERSION=2.4.0:2.3.0:2.2.2:2.1.5:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}" + system "rake cross native gem RUBY_CC_VERSION=2.5.0:2.4.0:2.3.0:2.2.2:2.1.6:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}" else Rake::Task['dlls'].execute - docker_for_windows "gem update --system && bundle && rake cross native gem RUBY_CC_VERSION=2.4.0:2.3.0:2.2.2:2.1.5:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}" + docker_for_windows "gem update --system && bundle && rake cross native gem RUBY_CC_VERSION=2.5.0:2.4.0:2.3.0:2.2.2:2.1.6:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}" end end diff --git a/binding.gyp b/binding.gyp deleted file mode 100644 index cb0a2fbab1..0000000000 --- a/binding.gyp +++ /dev/null @@ -1,994 +0,0 @@ -# GRPC Node gyp file -# This currently builds the Node extension and dependencies -# This file has been automatically generated from a template file. -# Please look at the templates directory instead. -# This file can be regenerated from the template by running -# tools/buildgen/generate_projects.sh - -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Some of this file is built with the help of -# https://n8.io/converting-a-c-library-to-gyp/ -{ - 'variables': { - 'runtime%': 'node', - # Some Node installations use the system installation of OpenSSL, and on - # some systems, the system OpenSSL still does not have ALPN support. This - # will let users recompile gRPC to work without ALPN. - 'grpc_alpn%': 'true', - # Indicates that the library should be built with gcov. - 'grpc_gcov%': 'false', - # Indicates that the library should be built with compatibility for musl - # libc, so that it can run on Alpine Linux. This is only necessary if not - # building on Alpine Linux - 'grpc_alpine%': 'false' - }, - 'target_defaults': { - 'configurations': { - 'Release': { - 'cflags': [ - '-O2', - ], - 'defines': [ - 'NDEBUG', - ], - }, - 'Debug': { - 'cflags': [ - '-O0', - ], - 'defines': [ - '_DEBUG', - 'DEBUG', - ], - }, - }, - 'cflags': [ - '-g', - '-Wall', - '-Wextra', - '-Werror', - '-Wno-long-long', - '-Wno-unused-parameter', - '-DOSATOMIC_USE_INLINED=1', - ], - 'ldflags': [ - '-g', - ], - 'cflags_c': [ - '-Werror', - '-std=c99' - ], - 'cflags_cc': [ - '-Werror', - '-std=c++11' - ], - 'include_dirs': [ - '.', - 'include' - ], - 'defines': [ - 'GPR_BACKWARDS_COMPATIBILITY_MODE', - 'GRPC_ARES=0', - 'GRPC_UV' - ], - 'conditions': [ - ['grpc_gcov=="true"', { - 'cflags': [ - '-O0', - '-fprofile-arcs', - '-ftest-coverage', - '-Wno-return-type', - ], - 'defines': [ - '_DEBUG', - 'DEBUG', - 'GPR_GCOV', - ], - 'ldflags': [ - '-fprofile-arcs', - '-ftest-coverage', - '-rdynamic', - ], - }], - ['grpc_alpine=="true"', { - 'defines': [ - 'GPR_MUSL_LIBC_COMPAT' - ] - }], - ['OS!="win" and runtime=="electron"', { - "defines": [ - 'OPENSSL_NO_THREADS' - ] - }], - # This is the condition for using boringssl - ['OS=="win" or runtime=="electron"', { - "include_dirs": [ - "third_party/boringssl/include" - ], - "defines": [ - 'OPENSSL_NO_ASM' - ] - }, { - 'conditions': [ - ["target_arch=='ia32'", { - "include_dirs": [ "<(node_root_dir)/deps/openssl/config/piii" ] - }], - ["target_arch=='x64'", { - "include_dirs": [ "<(node_root_dir)/deps/openssl/config/k8" ] - }], - ["target_arch=='arm'", { - "include_dirs": [ "<(node_root_dir)/deps/openssl/config/arm" ] - }], - ['grpc_alpn=="true"', { - 'defines': [ - 'TSI_OPENSSL_ALPN_SUPPORT=1' - ], - }, { - 'defines': [ - 'TSI_OPENSSL_ALPN_SUPPORT=0' - ], - }] - ], - 'include_dirs': [ - '<(node_root_dir)/deps/openssl/openssl/include', - ] - }], - ['OS == "win"', { - "include_dirs": [ - "third_party/zlib", - "third_party/cares/cares" - ], - "defines": [ - '_WIN32_WINNT=0x0600', - 'WIN32_LEAN_AND_MEAN', - '_HAS_EXCEPTIONS=0', - 'UNICODE', - '_UNICODE', - 'NOMINMAX', - ], - "msvs_settings": { - 'VCCLCompilerTool': { - 'RuntimeLibrary': 1, # static debug - } - }, - "libraries": [ - "ws2_32" - ] - }, { # OS != "win" - 'include_dirs': [ - '<(node_root_dir)/deps/zlib', - '<(node_root_dir)/deps/cares/include' - ] - }], - ['OS == "mac"', { - 'xcode_settings': { - 'OTHER_CFLAGS': [ - '-g', - '-Wall', - '-Wextra', - '-Werror', - '-Wno-long-long', - '-Wno-unused-parameter', - '-DOSATOMIC_USE_INLINED=1', - ], - 'OTHER_CPLUSPLUSFLAGS': [ - '-g', - '-Wall', - '-Wextra', - '-Werror', - '-Wno-long-long', - '-Wno-unused-parameter', - '-DOSATOMIC_USE_INLINED=1', - '-stdlib=libc++', - '-std=c++11', - '-Wno-error=deprecated-declarations' - ], - }, - }] - ] - }, - 'conditions': [ - ['OS=="win" or runtime=="electron"', { - 'targets': [ - { - 'target_name': 'boringssl', - 'product_prefix': 'lib', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'src/boringssl/err_data.c', - 'third_party/boringssl/crypto/aes/aes.c', - 'third_party/boringssl/crypto/aes/key_wrap.c', - 'third_party/boringssl/crypto/aes/mode_wrappers.c', - 'third_party/boringssl/crypto/asn1/a_bitstr.c', - 'third_party/boringssl/crypto/asn1/a_bool.c', - 'third_party/boringssl/crypto/asn1/a_d2i_fp.c', - 'third_party/boringssl/crypto/asn1/a_dup.c', - 'third_party/boringssl/crypto/asn1/a_enum.c', - 'third_party/boringssl/crypto/asn1/a_gentm.c', - 'third_party/boringssl/crypto/asn1/a_i2d_fp.c', - 'third_party/boringssl/crypto/asn1/a_int.c', - 'third_party/boringssl/crypto/asn1/a_mbstr.c', - 'third_party/boringssl/crypto/asn1/a_object.c', - 'third_party/boringssl/crypto/asn1/a_octet.c', - 'third_party/boringssl/crypto/asn1/a_print.c', - 'third_party/boringssl/crypto/asn1/a_strnid.c', - 'third_party/boringssl/crypto/asn1/a_time.c', - 'third_party/boringssl/crypto/asn1/a_type.c', - 'third_party/boringssl/crypto/asn1/a_utctm.c', - 'third_party/boringssl/crypto/asn1/a_utf8.c', - 'third_party/boringssl/crypto/asn1/asn1_lib.c', - 'third_party/boringssl/crypto/asn1/asn1_par.c', - 'third_party/boringssl/crypto/asn1/asn_pack.c', - 'third_party/boringssl/crypto/asn1/f_enum.c', - 'third_party/boringssl/crypto/asn1/f_int.c', - 'third_party/boringssl/crypto/asn1/f_string.c', - 'third_party/boringssl/crypto/asn1/t_bitst.c', - 'third_party/boringssl/crypto/asn1/tasn_dec.c', - 'third_party/boringssl/crypto/asn1/tasn_enc.c', - 'third_party/boringssl/crypto/asn1/tasn_fre.c', - 'third_party/boringssl/crypto/asn1/tasn_new.c', - 'third_party/boringssl/crypto/asn1/tasn_typ.c', - 'third_party/boringssl/crypto/asn1/tasn_utl.c', - 'third_party/boringssl/crypto/asn1/time_support.c', - 'third_party/boringssl/crypto/asn1/x_bignum.c', - 'third_party/boringssl/crypto/asn1/x_long.c', - 'third_party/boringssl/crypto/base64/base64.c', - 'third_party/boringssl/crypto/bio/bio.c', - 'third_party/boringssl/crypto/bio/bio_mem.c', - 'third_party/boringssl/crypto/bio/connect.c', - 'third_party/boringssl/crypto/bio/fd.c', - 'third_party/boringssl/crypto/bio/file.c', - 'third_party/boringssl/crypto/bio/hexdump.c', - 'third_party/boringssl/crypto/bio/pair.c', - 'third_party/boringssl/crypto/bio/printf.c', - 'third_party/boringssl/crypto/bio/socket.c', - 'third_party/boringssl/crypto/bio/socket_helper.c', - 'third_party/boringssl/crypto/bn/add.c', - 'third_party/boringssl/crypto/bn/asm/x86_64-gcc.c', - 'third_party/boringssl/crypto/bn/bn.c', - 'third_party/boringssl/crypto/bn/bn_asn1.c', - 'third_party/boringssl/crypto/bn/cmp.c', - 'third_party/boringssl/crypto/bn/convert.c', - 'third_party/boringssl/crypto/bn/ctx.c', - 'third_party/boringssl/crypto/bn/div.c', - 'third_party/boringssl/crypto/bn/exponentiation.c', - 'third_party/boringssl/crypto/bn/gcd.c', - 'third_party/boringssl/crypto/bn/generic.c', - 'third_party/boringssl/crypto/bn/kronecker.c', - 'third_party/boringssl/crypto/bn/montgomery.c', - 'third_party/boringssl/crypto/bn/montgomery_inv.c', - 'third_party/boringssl/crypto/bn/mul.c', - 'third_party/boringssl/crypto/bn/prime.c', - 'third_party/boringssl/crypto/bn/random.c', - 'third_party/boringssl/crypto/bn/rsaz_exp.c', - 'third_party/boringssl/crypto/bn/shift.c', - 'third_party/boringssl/crypto/bn/sqrt.c', - 'third_party/boringssl/crypto/buf/buf.c', - 'third_party/boringssl/crypto/bytestring/asn1_compat.c', - 'third_party/boringssl/crypto/bytestring/ber.c', - 'third_party/boringssl/crypto/bytestring/cbb.c', - 'third_party/boringssl/crypto/bytestring/cbs.c', - 'third_party/boringssl/crypto/chacha/chacha.c', - 'third_party/boringssl/crypto/cipher/aead.c', - 'third_party/boringssl/crypto/cipher/cipher.c', - 'third_party/boringssl/crypto/cipher/derive_key.c', - 'third_party/boringssl/crypto/cipher/e_aes.c', - 'third_party/boringssl/crypto/cipher/e_chacha20poly1305.c', - 'third_party/boringssl/crypto/cipher/e_des.c', - 'third_party/boringssl/crypto/cipher/e_null.c', - 'third_party/boringssl/crypto/cipher/e_rc2.c', - 'third_party/boringssl/crypto/cipher/e_rc4.c', - 'third_party/boringssl/crypto/cipher/e_ssl3.c', - 'third_party/boringssl/crypto/cipher/e_tls.c', - 'third_party/boringssl/crypto/cipher/tls_cbc.c', - 'third_party/boringssl/crypto/cmac/cmac.c', - 'third_party/boringssl/crypto/conf/conf.c', - 'third_party/boringssl/crypto/cpu-aarch64-linux.c', - 'third_party/boringssl/crypto/cpu-arm-linux.c', - 'third_party/boringssl/crypto/cpu-arm.c', - 'third_party/boringssl/crypto/cpu-intel.c', - 'third_party/boringssl/crypto/cpu-ppc64le.c', - 'third_party/boringssl/crypto/crypto.c', - 'third_party/boringssl/crypto/curve25519/curve25519.c', - 'third_party/boringssl/crypto/curve25519/spake25519.c', - 'third_party/boringssl/crypto/curve25519/x25519-x86_64.c', - 'third_party/boringssl/crypto/des/des.c', - 'third_party/boringssl/crypto/dh/check.c', - 'third_party/boringssl/crypto/dh/dh.c', - 'third_party/boringssl/crypto/dh/dh_asn1.c', - 'third_party/boringssl/crypto/dh/params.c', - 'third_party/boringssl/crypto/digest/digest.c', - 'third_party/boringssl/crypto/digest/digests.c', - 'third_party/boringssl/crypto/dsa/dsa.c', - 'third_party/boringssl/crypto/dsa/dsa_asn1.c', - 'third_party/boringssl/crypto/ec/ec.c', - 'third_party/boringssl/crypto/ec/ec_asn1.c', - 'third_party/boringssl/crypto/ec/ec_key.c', - 'third_party/boringssl/crypto/ec/ec_montgomery.c', - 'third_party/boringssl/crypto/ec/oct.c', - 'third_party/boringssl/crypto/ec/p224-64.c', - 'third_party/boringssl/crypto/ec/p256-64.c', - 'third_party/boringssl/crypto/ec/p256-x86_64.c', - 'third_party/boringssl/crypto/ec/simple.c', - 'third_party/boringssl/crypto/ec/util-64.c', - 'third_party/boringssl/crypto/ec/wnaf.c', - 'third_party/boringssl/crypto/ecdh/ecdh.c', - 'third_party/boringssl/crypto/ecdsa/ecdsa.c', - 'third_party/boringssl/crypto/ecdsa/ecdsa_asn1.c', - 'third_party/boringssl/crypto/engine/engine.c', - 'third_party/boringssl/crypto/err/err.c', - 'third_party/boringssl/crypto/evp/digestsign.c', - 'third_party/boringssl/crypto/evp/evp.c', - 'third_party/boringssl/crypto/evp/evp_asn1.c', - 'third_party/boringssl/crypto/evp/evp_ctx.c', - 'third_party/boringssl/crypto/evp/p_dsa_asn1.c', - 'third_party/boringssl/crypto/evp/p_ec.c', - 'third_party/boringssl/crypto/evp/p_ec_asn1.c', - 'third_party/boringssl/crypto/evp/p_rsa.c', - 'third_party/boringssl/crypto/evp/p_rsa_asn1.c', - 'third_party/boringssl/crypto/evp/pbkdf.c', - 'third_party/boringssl/crypto/evp/print.c', - 'third_party/boringssl/crypto/evp/sign.c', - 'third_party/boringssl/crypto/ex_data.c', - 'third_party/boringssl/crypto/hkdf/hkdf.c', - 'third_party/boringssl/crypto/hmac/hmac.c', - 'third_party/boringssl/crypto/lhash/lhash.c', - 'third_party/boringssl/crypto/md4/md4.c', - 'third_party/boringssl/crypto/md5/md5.c', - 'third_party/boringssl/crypto/mem.c', - 'third_party/boringssl/crypto/modes/cbc.c', - 'third_party/boringssl/crypto/modes/cfb.c', - 'third_party/boringssl/crypto/modes/ctr.c', - 'third_party/boringssl/crypto/modes/gcm.c', - 'third_party/boringssl/crypto/modes/ofb.c', - 'third_party/boringssl/crypto/modes/polyval.c', - 'third_party/boringssl/crypto/obj/obj.c', - 'third_party/boringssl/crypto/obj/obj_xref.c', - 'third_party/boringssl/crypto/pem/pem_all.c', - 'third_party/boringssl/crypto/pem/pem_info.c', - 'third_party/boringssl/crypto/pem/pem_lib.c', - 'third_party/boringssl/crypto/pem/pem_oth.c', - 'third_party/boringssl/crypto/pem/pem_pk8.c', - 'third_party/boringssl/crypto/pem/pem_pkey.c', - 'third_party/boringssl/crypto/pem/pem_x509.c', - 'third_party/boringssl/crypto/pem/pem_xaux.c', - 'third_party/boringssl/crypto/pkcs8/p5_pbev2.c', - 'third_party/boringssl/crypto/pkcs8/p8_pkey.c', - 'third_party/boringssl/crypto/pkcs8/pkcs8.c', - 'third_party/boringssl/crypto/poly1305/poly1305.c', - 'third_party/boringssl/crypto/poly1305/poly1305_arm.c', - 'third_party/boringssl/crypto/poly1305/poly1305_vec.c', - 'third_party/boringssl/crypto/pool/pool.c', - 'third_party/boringssl/crypto/rand/deterministic.c', - 'third_party/boringssl/crypto/rand/fuchsia.c', - 'third_party/boringssl/crypto/rand/rand.c', - 'third_party/boringssl/crypto/rand/urandom.c', - 'third_party/boringssl/crypto/rand/windows.c', - 'third_party/boringssl/crypto/rc4/rc4.c', - 'third_party/boringssl/crypto/refcount_c11.c', - 'third_party/boringssl/crypto/refcount_lock.c', - 'third_party/boringssl/crypto/rsa/blinding.c', - 'third_party/boringssl/crypto/rsa/padding.c', - 'third_party/boringssl/crypto/rsa/rsa.c', - 'third_party/boringssl/crypto/rsa/rsa_asn1.c', - 'third_party/boringssl/crypto/rsa/rsa_impl.c', - 'third_party/boringssl/crypto/sha/sha1-altivec.c', - 'third_party/boringssl/crypto/sha/sha1.c', - 'third_party/boringssl/crypto/sha/sha256.c', - 'third_party/boringssl/crypto/sha/sha512.c', - 'third_party/boringssl/crypto/stack/stack.c', - 'third_party/boringssl/crypto/thread.c', - 'third_party/boringssl/crypto/thread_none.c', - 'third_party/boringssl/crypto/thread_pthread.c', - 'third_party/boringssl/crypto/thread_win.c', - 'third_party/boringssl/crypto/x509/a_digest.c', - 'third_party/boringssl/crypto/x509/a_sign.c', - 'third_party/boringssl/crypto/x509/a_strex.c', - 'third_party/boringssl/crypto/x509/a_verify.c', - 'third_party/boringssl/crypto/x509/algorithm.c', - 'third_party/boringssl/crypto/x509/asn1_gen.c', - 'third_party/boringssl/crypto/x509/by_dir.c', - 'third_party/boringssl/crypto/x509/by_file.c', - 'third_party/boringssl/crypto/x509/i2d_pr.c', - 'third_party/boringssl/crypto/x509/pkcs7.c', - 'third_party/boringssl/crypto/x509/rsa_pss.c', - 'third_party/boringssl/crypto/x509/t_crl.c', - 'third_party/boringssl/crypto/x509/t_req.c', - 'third_party/boringssl/crypto/x509/t_x509.c', - 'third_party/boringssl/crypto/x509/t_x509a.c', - 'third_party/boringssl/crypto/x509/x509.c', - 'third_party/boringssl/crypto/x509/x509_att.c', - 'third_party/boringssl/crypto/x509/x509_cmp.c', - 'third_party/boringssl/crypto/x509/x509_d2.c', - 'third_party/boringssl/crypto/x509/x509_def.c', - 'third_party/boringssl/crypto/x509/x509_ext.c', - 'third_party/boringssl/crypto/x509/x509_lu.c', - 'third_party/boringssl/crypto/x509/x509_obj.c', - 'third_party/boringssl/crypto/x509/x509_r2x.c', - 'third_party/boringssl/crypto/x509/x509_req.c', - 'third_party/boringssl/crypto/x509/x509_set.c', - 'third_party/boringssl/crypto/x509/x509_trs.c', - 'third_party/boringssl/crypto/x509/x509_txt.c', - 'third_party/boringssl/crypto/x509/x509_v3.c', - 'third_party/boringssl/crypto/x509/x509_vfy.c', - 'third_party/boringssl/crypto/x509/x509_vpm.c', - 'third_party/boringssl/crypto/x509/x509cset.c', - 'third_party/boringssl/crypto/x509/x509name.c', - 'third_party/boringssl/crypto/x509/x509rset.c', - 'third_party/boringssl/crypto/x509/x509spki.c', - 'third_party/boringssl/crypto/x509/x509type.c', - 'third_party/boringssl/crypto/x509/x_algor.c', - 'third_party/boringssl/crypto/x509/x_all.c', - 'third_party/boringssl/crypto/x509/x_attrib.c', - 'third_party/boringssl/crypto/x509/x_crl.c', - 'third_party/boringssl/crypto/x509/x_exten.c', - 'third_party/boringssl/crypto/x509/x_info.c', - 'third_party/boringssl/crypto/x509/x_name.c', - 'third_party/boringssl/crypto/x509/x_pkey.c', - 'third_party/boringssl/crypto/x509/x_pubkey.c', - 'third_party/boringssl/crypto/x509/x_req.c', - 'third_party/boringssl/crypto/x509/x_sig.c', - 'third_party/boringssl/crypto/x509/x_spki.c', - 'third_party/boringssl/crypto/x509/x_val.c', - 'third_party/boringssl/crypto/x509/x_x509.c', - 'third_party/boringssl/crypto/x509/x_x509a.c', - 'third_party/boringssl/crypto/x509v3/pcy_cache.c', - 'third_party/boringssl/crypto/x509v3/pcy_data.c', - 'third_party/boringssl/crypto/x509v3/pcy_lib.c', - 'third_party/boringssl/crypto/x509v3/pcy_map.c', - 'third_party/boringssl/crypto/x509v3/pcy_node.c', - 'third_party/boringssl/crypto/x509v3/pcy_tree.c', - 'third_party/boringssl/crypto/x509v3/v3_akey.c', - 'third_party/boringssl/crypto/x509v3/v3_akeya.c', - 'third_party/boringssl/crypto/x509v3/v3_alt.c', - 'third_party/boringssl/crypto/x509v3/v3_bcons.c', - 'third_party/boringssl/crypto/x509v3/v3_bitst.c', - 'third_party/boringssl/crypto/x509v3/v3_conf.c', - 'third_party/boringssl/crypto/x509v3/v3_cpols.c', - 'third_party/boringssl/crypto/x509v3/v3_crld.c', - 'third_party/boringssl/crypto/x509v3/v3_enum.c', - 'third_party/boringssl/crypto/x509v3/v3_extku.c', - 'third_party/boringssl/crypto/x509v3/v3_genn.c', - 'third_party/boringssl/crypto/x509v3/v3_ia5.c', - 'third_party/boringssl/crypto/x509v3/v3_info.c', - 'third_party/boringssl/crypto/x509v3/v3_int.c', - 'third_party/boringssl/crypto/x509v3/v3_lib.c', - 'third_party/boringssl/crypto/x509v3/v3_ncons.c', - 'third_party/boringssl/crypto/x509v3/v3_pci.c', - 'third_party/boringssl/crypto/x509v3/v3_pcia.c', - 'third_party/boringssl/crypto/x509v3/v3_pcons.c', - 'third_party/boringssl/crypto/x509v3/v3_pku.c', - 'third_party/boringssl/crypto/x509v3/v3_pmaps.c', - 'third_party/boringssl/crypto/x509v3/v3_prn.c', - 'third_party/boringssl/crypto/x509v3/v3_purp.c', - 'third_party/boringssl/crypto/x509v3/v3_skey.c', - 'third_party/boringssl/crypto/x509v3/v3_sxnet.c', - 'third_party/boringssl/crypto/x509v3/v3_utl.c', - 'third_party/boringssl/ssl/bio_ssl.c', - 'third_party/boringssl/ssl/custom_extensions.c', - 'third_party/boringssl/ssl/d1_both.c', - 'third_party/boringssl/ssl/d1_lib.c', - 'third_party/boringssl/ssl/d1_pkt.c', - 'third_party/boringssl/ssl/d1_srtp.c', - 'third_party/boringssl/ssl/dtls_method.c', - 'third_party/boringssl/ssl/dtls_record.c', - 'third_party/boringssl/ssl/handshake_client.c', - 'third_party/boringssl/ssl/handshake_server.c', - 'third_party/boringssl/ssl/s3_both.c', - 'third_party/boringssl/ssl/s3_lib.c', - 'third_party/boringssl/ssl/s3_pkt.c', - 'third_party/boringssl/ssl/ssl_aead_ctx.c', - 'third_party/boringssl/ssl/ssl_asn1.c', - 'third_party/boringssl/ssl/ssl_buffer.c', - 'third_party/boringssl/ssl/ssl_cert.c', - 'third_party/boringssl/ssl/ssl_cipher.c', - 'third_party/boringssl/ssl/ssl_ecdh.c', - 'third_party/boringssl/ssl/ssl_file.c', - 'third_party/boringssl/ssl/ssl_lib.c', - 'third_party/boringssl/ssl/ssl_privkey.c', - 'third_party/boringssl/ssl/ssl_privkey_cc.cc', - 'third_party/boringssl/ssl/ssl_session.c', - 'third_party/boringssl/ssl/ssl_stat.c', - 'third_party/boringssl/ssl/ssl_transcript.c', - 'third_party/boringssl/ssl/ssl_x509.c', - 'third_party/boringssl/ssl/t1_enc.c', - 'third_party/boringssl/ssl/t1_lib.c', - 'third_party/boringssl/ssl/tls13_both.c', - 'third_party/boringssl/ssl/tls13_client.c', - 'third_party/boringssl/ssl/tls13_enc.c', - 'third_party/boringssl/ssl/tls13_server.c', - 'third_party/boringssl/ssl/tls_method.c', - 'third_party/boringssl/ssl/tls_record.c', - ], - 'conditions': [ - ['OS == "mac"', { - 'xcode_settings': { - 'MACOSX_DEPLOYMENT_TARGET': '10.9' - } - }] - ] - }, - ], - }], - ['OS == "win" and runtime!="electron"', { - 'targets': [ - { - # IMPORTANT WINDOWS BUILD INFORMATION - # This library does not build on Windows without modifying the Node - # development packages that node-gyp downloads in order to build. - # Due to https://github.com/nodejs/node/issues/4932, the headers for - # BoringSSL conflict with the OpenSSL headers included by default - # when including the Node headers. The remedy for this is to remove - # the OpenSSL headers, from the downloaded Node development package, - # which is typically located in `.node-gyp` in your home directory. - # - # This is not true of Electron, which does not have OpenSSL headers. - 'target_name': 'WINDOWS_BUILD_WARNING', - 'rules': [ - { - 'rule_name': 'WINDOWS_BUILD_WARNING', - 'extension': 'S', - 'inputs': [ - 'package.json' - ], - 'outputs': [ - 'ignore_this_part' - ], - 'action': ['echo', 'IMPORTANT: Due to https://github.com/nodejs/node/issues/4932, to build this library on Windows, you must first remove <(node_root_dir)/include/node/openssl/'] - } - ] - }, - ] - }], - ['OS == "win"', { - 'targets': [ - # Only want to compile zlib under Windows - { - 'target_name': 'z', - 'product_prefix': 'lib', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/zlib/adler32.c', - 'third_party/zlib/compress.c', - 'third_party/zlib/crc32.c', - 'third_party/zlib/deflate.c', - 'third_party/zlib/gzclose.c', - 'third_party/zlib/gzlib.c', - 'third_party/zlib/gzread.c', - 'third_party/zlib/gzwrite.c', - 'third_party/zlib/infback.c', - 'third_party/zlib/inffast.c', - 'third_party/zlib/inflate.c', - 'third_party/zlib/inftrees.c', - 'third_party/zlib/trees.c', - 'third_party/zlib/uncompr.c', - 'third_party/zlib/zutil.c', - ] - }, - ] - }] - ], - 'targets': [ - { - 'target_name': 'gpr', - 'product_prefix': 'lib', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'src/core/lib/profiling/basic_timers.c', - 'src/core/lib/profiling/stap_timers.c', - 'src/core/lib/support/alloc.c', - 'src/core/lib/support/arena.c', - 'src/core/lib/support/atm.c', - 'src/core/lib/support/avl.c', - 'src/core/lib/support/backoff.c', - 'src/core/lib/support/cmdline.c', - 'src/core/lib/support/cpu_iphone.c', - 'src/core/lib/support/cpu_linux.c', - 'src/core/lib/support/cpu_posix.c', - 'src/core/lib/support/cpu_windows.c', - 'src/core/lib/support/env_linux.c', - 'src/core/lib/support/env_posix.c', - 'src/core/lib/support/env_windows.c', - 'src/core/lib/support/fork.c', - 'src/core/lib/support/histogram.c', - 'src/core/lib/support/host_port.c', - 'src/core/lib/support/log.c', - 'src/core/lib/support/log_android.c', - 'src/core/lib/support/log_linux.c', - 'src/core/lib/support/log_posix.c', - 'src/core/lib/support/log_windows.c', - 'src/core/lib/support/mpscq.c', - 'src/core/lib/support/murmur_hash.c', - 'src/core/lib/support/stack_lockfree.c', - 'src/core/lib/support/string.c', - 'src/core/lib/support/string_posix.c', - 'src/core/lib/support/string_util_windows.c', - 'src/core/lib/support/string_windows.c', - 'src/core/lib/support/subprocess_posix.c', - 'src/core/lib/support/subprocess_windows.c', - 'src/core/lib/support/sync.c', - 'src/core/lib/support/sync_posix.c', - 'src/core/lib/support/sync_windows.c', - 'src/core/lib/support/thd.c', - 'src/core/lib/support/thd_posix.c', - 'src/core/lib/support/thd_windows.c', - 'src/core/lib/support/time.c', - 'src/core/lib/support/time_posix.c', - 'src/core/lib/support/time_precise.c', - 'src/core/lib/support/time_windows.c', - 'src/core/lib/support/tls_pthread.c', - 'src/core/lib/support/tmpfile_msys.c', - 'src/core/lib/support/tmpfile_posix.c', - 'src/core/lib/support/tmpfile_windows.c', - 'src/core/lib/support/wrap_memcpy.c', - ], - 'conditions': [ - ['OS == "mac"', { - 'xcode_settings': { - 'MACOSX_DEPLOYMENT_TARGET': '10.9' - } - }] - ] - }, - { - 'target_name': 'grpc', - 'product_prefix': 'lib', - 'type': 'static_library', - 'dependencies': [ - 'gpr', - ], - 'sources': [ - 'src/core/lib/surface/init.c', - 'src/core/lib/channel/channel_args.c', - 'src/core/lib/channel/channel_stack.c', - 'src/core/lib/channel/channel_stack_builder.c', - 'src/core/lib/channel/connected_channel.c', - 'src/core/lib/channel/handshaker.c', - 'src/core/lib/channel/handshaker_factory.c', - 'src/core/lib/channel/handshaker_registry.c', - 'src/core/lib/compression/compression.c', - 'src/core/lib/compression/message_compress.c', - 'src/core/lib/compression/stream_compression.c', - 'src/core/lib/compression/stream_compression_gzip.c', - 'src/core/lib/compression/stream_compression_identity.c', - 'src/core/lib/debug/stats.c', - 'src/core/lib/debug/stats_data.c', - 'src/core/lib/http/format_request.c', - 'src/core/lib/http/httpcli.c', - 'src/core/lib/http/parser.c', - 'src/core/lib/iomgr/call_combiner.c', - 'src/core/lib/iomgr/closure.c', - 'src/core/lib/iomgr/combiner.c', - 'src/core/lib/iomgr/endpoint.c', - 'src/core/lib/iomgr/endpoint_pair_posix.c', - 'src/core/lib/iomgr/endpoint_pair_uv.c', - 'src/core/lib/iomgr/endpoint_pair_windows.c', - 'src/core/lib/iomgr/error.c', - 'src/core/lib/iomgr/ev_epoll1_linux.c', - 'src/core/lib/iomgr/ev_epollex_linux.c', - 'src/core/lib/iomgr/ev_epollsig_linux.c', - 'src/core/lib/iomgr/ev_poll_posix.c', - 'src/core/lib/iomgr/ev_posix.c', - 'src/core/lib/iomgr/ev_windows.c', - 'src/core/lib/iomgr/exec_ctx.c', - 'src/core/lib/iomgr/executor.c', - 'src/core/lib/iomgr/fork_posix.c', - 'src/core/lib/iomgr/fork_windows.c', - 'src/core/lib/iomgr/gethostname_fallback.c', - 'src/core/lib/iomgr/gethostname_host_name_max.c', - 'src/core/lib/iomgr/gethostname_sysconf.c', - 'src/core/lib/iomgr/iocp_windows.c', - 'src/core/lib/iomgr/iomgr.c', - 'src/core/lib/iomgr/iomgr_posix.c', - 'src/core/lib/iomgr/iomgr_uv.c', - 'src/core/lib/iomgr/iomgr_windows.c', - 'src/core/lib/iomgr/is_epollexclusive_available.c', - 'src/core/lib/iomgr/load_file.c', - 'src/core/lib/iomgr/lockfree_event.c', - 'src/core/lib/iomgr/network_status_tracker.c', - 'src/core/lib/iomgr/polling_entity.c', - 'src/core/lib/iomgr/pollset_set_uv.c', - 'src/core/lib/iomgr/pollset_set_windows.c', - 'src/core/lib/iomgr/pollset_uv.c', - 'src/core/lib/iomgr/pollset_windows.c', - 'src/core/lib/iomgr/resolve_address_posix.c', - 'src/core/lib/iomgr/resolve_address_uv.c', - 'src/core/lib/iomgr/resolve_address_windows.c', - 'src/core/lib/iomgr/resource_quota.c', - 'src/core/lib/iomgr/sockaddr_utils.c', - 'src/core/lib/iomgr/socket_factory_posix.c', - 'src/core/lib/iomgr/socket_mutator.c', - 'src/core/lib/iomgr/socket_utils_common_posix.c', - 'src/core/lib/iomgr/socket_utils_linux.c', - 'src/core/lib/iomgr/socket_utils_posix.c', - 'src/core/lib/iomgr/socket_utils_uv.c', - 'src/core/lib/iomgr/socket_utils_windows.c', - 'src/core/lib/iomgr/socket_windows.c', - 'src/core/lib/iomgr/tcp_client_posix.c', - 'src/core/lib/iomgr/tcp_client_uv.c', - 'src/core/lib/iomgr/tcp_client_windows.c', - 'src/core/lib/iomgr/tcp_posix.c', - 'src/core/lib/iomgr/tcp_server_posix.c', - 'src/core/lib/iomgr/tcp_server_utils_posix_common.c', - 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c', - 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c', - 'src/core/lib/iomgr/tcp_server_uv.c', - 'src/core/lib/iomgr/tcp_server_windows.c', - 'src/core/lib/iomgr/tcp_uv.c', - 'src/core/lib/iomgr/tcp_windows.c', - 'src/core/lib/iomgr/time_averaged_stats.c', - 'src/core/lib/iomgr/timer_generic.c', - 'src/core/lib/iomgr/timer_heap.c', - 'src/core/lib/iomgr/timer_manager.c', - 'src/core/lib/iomgr/timer_uv.c', - 'src/core/lib/iomgr/udp_server.c', - 'src/core/lib/iomgr/unix_sockets_posix.c', - 'src/core/lib/iomgr/unix_sockets_posix_noop.c', - 'src/core/lib/iomgr/wakeup_fd_cv.c', - 'src/core/lib/iomgr/wakeup_fd_eventfd.c', - 'src/core/lib/iomgr/wakeup_fd_nospecial.c', - 'src/core/lib/iomgr/wakeup_fd_pipe.c', - 'src/core/lib/iomgr/wakeup_fd_posix.c', - 'src/core/lib/json/json.c', - 'src/core/lib/json/json_reader.c', - 'src/core/lib/json/json_string.c', - 'src/core/lib/json/json_writer.c', - 'src/core/lib/slice/b64.c', - 'src/core/lib/slice/percent_encoding.c', - 'src/core/lib/slice/slice.c', - 'src/core/lib/slice/slice_buffer.c', - 'src/core/lib/slice/slice_hash_table.c', - 'src/core/lib/slice/slice_intern.c', - 'src/core/lib/slice/slice_string_helpers.c', - 'src/core/lib/surface/alarm.c', - 'src/core/lib/surface/api_trace.c', - 'src/core/lib/surface/byte_buffer.c', - 'src/core/lib/surface/byte_buffer_reader.c', - 'src/core/lib/surface/call.c', - 'src/core/lib/surface/call_details.c', - 'src/core/lib/surface/call_log_batch.c', - 'src/core/lib/surface/channel.c', - 'src/core/lib/surface/channel_init.c', - 'src/core/lib/surface/channel_ping.c', - 'src/core/lib/surface/channel_stack_type.c', - 'src/core/lib/surface/completion_queue.c', - 'src/core/lib/surface/completion_queue_factory.c', - 'src/core/lib/surface/event_string.c', - 'src/core/lib/surface/lame_client.cc', - 'src/core/lib/surface/metadata_array.c', - 'src/core/lib/surface/server.c', - 'src/core/lib/surface/validate_metadata.c', - 'src/core/lib/surface/version.c', - 'src/core/lib/transport/bdp_estimator.c', - 'src/core/lib/transport/byte_stream.c', - 'src/core/lib/transport/connectivity_state.c', - 'src/core/lib/transport/error_utils.c', - 'src/core/lib/transport/metadata.c', - 'src/core/lib/transport/metadata_batch.c', - 'src/core/lib/transport/pid_controller.c', - 'src/core/lib/transport/service_config.c', - 'src/core/lib/transport/static_metadata.c', - 'src/core/lib/transport/status_conversion.c', - 'src/core/lib/transport/timeout_encoding.c', - 'src/core/lib/transport/transport.c', - 'src/core/lib/transport/transport_op_string.c', - 'src/core/lib/debug/trace.c', - 'src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c', - 'src/core/ext/transport/chttp2/transport/bin_decoder.c', - 'src/core/ext/transport/chttp2/transport/bin_encoder.c', - 'src/core/ext/transport/chttp2/transport/chttp2_plugin.c', - 'src/core/ext/transport/chttp2/transport/chttp2_transport.c', - 'src/core/ext/transport/chttp2/transport/flow_control.c', - 'src/core/ext/transport/chttp2/transport/frame_data.c', - 'src/core/ext/transport/chttp2/transport/frame_goaway.c', - 'src/core/ext/transport/chttp2/transport/frame_ping.c', - 'src/core/ext/transport/chttp2/transport/frame_rst_stream.c', - 'src/core/ext/transport/chttp2/transport/frame_settings.c', - 'src/core/ext/transport/chttp2/transport/frame_window_update.c', - 'src/core/ext/transport/chttp2/transport/hpack_encoder.c', - 'src/core/ext/transport/chttp2/transport/hpack_parser.c', - 'src/core/ext/transport/chttp2/transport/hpack_table.c', - 'src/core/ext/transport/chttp2/transport/http2_settings.c', - 'src/core/ext/transport/chttp2/transport/huffsyms.c', - 'src/core/ext/transport/chttp2/transport/incoming_metadata.c', - 'src/core/ext/transport/chttp2/transport/parsing.c', - 'src/core/ext/transport/chttp2/transport/stream_lists.c', - 'src/core/ext/transport/chttp2/transport/stream_map.c', - 'src/core/ext/transport/chttp2/transport/varint.c', - 'src/core/ext/transport/chttp2/transport/writing.c', - 'src/core/ext/transport/chttp2/alpn/alpn.c', - 'src/core/ext/filters/http/client/http_client_filter.c', - 'src/core/ext/filters/http/http_filters_plugin.c', - 'src/core/ext/filters/http/message_compress/message_compress_filter.c', - 'src/core/ext/filters/http/server/http_server_filter.c', - 'src/core/lib/http/httpcli_security_connector.c', - 'src/core/lib/security/context/security_context.c', - 'src/core/lib/security/credentials/composite/composite_credentials.c', - 'src/core/lib/security/credentials/credentials.c', - 'src/core/lib/security/credentials/credentials_metadata.c', - 'src/core/lib/security/credentials/fake/fake_credentials.c', - 'src/core/lib/security/credentials/google_default/credentials_generic.c', - 'src/core/lib/security/credentials/google_default/google_default_credentials.c', - 'src/core/lib/security/credentials/iam/iam_credentials.c', - 'src/core/lib/security/credentials/jwt/json_token.c', - 'src/core/lib/security/credentials/jwt/jwt_credentials.c', - 'src/core/lib/security/credentials/jwt/jwt_verifier.c', - 'src/core/lib/security/credentials/oauth2/oauth2_credentials.c', - 'src/core/lib/security/credentials/plugin/plugin_credentials.c', - 'src/core/lib/security/credentials/ssl/ssl_credentials.c', - 'src/core/lib/security/transport/client_auth_filter.c', - 'src/core/lib/security/transport/lb_targets_info.c', - 'src/core/lib/security/transport/secure_endpoint.c', - 'src/core/lib/security/transport/security_connector.c', - 'src/core/lib/security/transport/security_handshaker.c', - 'src/core/lib/security/transport/server_auth_filter.c', - 'src/core/lib/security/transport/tsi_error.c', - 'src/core/lib/security/util/json_util.c', - 'src/core/lib/surface/init_secure.c', - 'src/core/tsi/fake_transport_security.c', - 'src/core/tsi/gts_transport_security.c', - 'src/core/tsi/ssl_transport_security.c', - 'src/core/tsi/transport_security_grpc.c', - 'src/core/tsi/transport_security.c', - 'src/core/tsi/transport_security_adapter.c', - 'src/core/ext/transport/chttp2/server/chttp2_server.c', - 'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c', - 'src/core/ext/filters/client_channel/channel_connectivity.c', - 'src/core/ext/filters/client_channel/client_channel.c', - 'src/core/ext/filters/client_channel/client_channel_factory.c', - 'src/core/ext/filters/client_channel/client_channel_plugin.c', - 'src/core/ext/filters/client_channel/connector.c', - 'src/core/ext/filters/client_channel/http_connect_handshaker.c', - 'src/core/ext/filters/client_channel/http_proxy.c', - 'src/core/ext/filters/client_channel/lb_policy.c', - 'src/core/ext/filters/client_channel/lb_policy_factory.c', - 'src/core/ext/filters/client_channel/lb_policy_registry.c', - 'src/core/ext/filters/client_channel/parse_address.c', - 'src/core/ext/filters/client_channel/proxy_mapper.c', - 'src/core/ext/filters/client_channel/proxy_mapper_registry.c', - 'src/core/ext/filters/client_channel/resolver.c', - 'src/core/ext/filters/client_channel/resolver_factory.c', - 'src/core/ext/filters/client_channel/resolver_registry.c', - 'src/core/ext/filters/client_channel/retry_throttle.c', - 'src/core/ext/filters/client_channel/subchannel.c', - 'src/core/ext/filters/client_channel/subchannel_index.c', - 'src/core/ext/filters/client_channel/uri_parser.c', - 'src/core/ext/filters/deadline/deadline_filter.c', - 'src/core/ext/transport/chttp2/client/chttp2_connector.c', - 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', - 'src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c', - 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', - 'src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c', - 'src/core/ext/transport/inproc/inproc_plugin.c', - 'src/core/ext/transport/inproc/inproc_transport.c', - 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c', - 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c', - 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.c', - 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.c', - 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c', - 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', - 'third_party/nanopb/pb_common.c', - 'third_party/nanopb/pb_decode.c', - 'third_party/nanopb/pb_encode.c', - 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c', - 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c', - 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c', - 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c', - 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c', - 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c', - 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c', - 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c', - 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c', - 'src/core/ext/filters/load_reporting/server_load_reporting_filter.c', - 'src/core/ext/filters/load_reporting/server_load_reporting_plugin.c', - 'src/core/ext/census/base_resources.c', - 'src/core/ext/census/context.c', - 'src/core/ext/census/gen/census.pb.c', - 'src/core/ext/census/gen/trace_context.pb.c', - 'src/core/ext/census/grpc_context.c', - 'src/core/ext/census/grpc_filter.c', - 'src/core/ext/census/grpc_plugin.c', - 'src/core/ext/census/initialize.c', - 'src/core/ext/census/intrusive_hash_map.c', - 'src/core/ext/census/mlog.c', - 'src/core/ext/census/operation.c', - 'src/core/ext/census/placeholders.c', - 'src/core/ext/census/resource.c', - 'src/core/ext/census/trace_context.c', - 'src/core/ext/census/tracing.c', - 'src/core/ext/filters/max_age/max_age_filter.c', - 'src/core/ext/filters/message_size/message_size_filter.c', - 'src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c', - 'src/core/ext/filters/workarounds/workaround_utils.c', - 'src/core/plugin_registry/grpc_plugin_registry.c', - ], - 'conditions': [ - ['OS == "mac"', { - 'xcode_settings': { - 'MACOSX_DEPLOYMENT_TARGET': '10.9' - } - }] - ] - }, - { - 'include_dirs': [ - "<!(node -e \"require('nan')\")" - ], - 'cflags': [ - '-pthread', - '-zdefs', - '-Wno-error=deprecated-declarations' - ], - "conditions": [ - ['OS=="win" or runtime=="electron"', { - 'dependencies': [ - "boringssl", - ] - }], - ['OS=="win"', { - 'dependencies': [ - "z", - ] - }], - ['OS=="linux"', { - 'ldflags': [ - '-Wl,-wrap,memcpy' - ] - }], - ['OS == "mac"', { - 'xcode_settings': { - 'MACOSX_DEPLOYMENT_TARGET': '10.9' - } - }] - ], - "target_name": "grpc_node", - "sources": [ - "src/node/ext/byte_buffer.cc", - "src/node/ext/call.cc", - "src/node/ext/call_credentials.cc", - "src/node/ext/channel.cc", - "src/node/ext/channel_credentials.cc", - "src/node/ext/completion_queue.cc", - "src/node/ext/node_grpc.cc", - "src/node/ext/server.cc", - "src/node/ext/server_credentials.cc", - "src/node/ext/slice.cc", - "src/node/ext/timeval.cc", - ], - "dependencies": [ - "grpc", - "gpr", - ] - }, - { - "target_name": "action_after_build", - "type": "none", - "dependencies": [ "<(module_name)" ], - "copies": [ - { - "files": [ "<(PRODUCT_DIR)/<(module_name).node"], - "destination": "<(module_path)" - } - ] - } - ] -} diff --git a/build.yaml b/build.yaml index 67513d8445..fa65e96759 100644 --- a/build.yaml +++ b/build.yaml @@ -5021,6 +5021,7 @@ defaults: global: COREFLAGS: -fno-rtti -fno-exceptions CPPFLAGS: -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 + -Wno-deprecated-declarations LDFLAGS: -g zlib: CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-implicit-function-declaration diff --git a/doc/PROTOCOL-WEB.md b/doc/PROTOCOL-WEB.md index 226871d7ae..c31a048a7c 100644 --- a/doc/PROTOCOL-WEB.md +++ b/doc/PROTOCOL-WEB.md @@ -3,14 +3,14 @@ gRPC-Web provides a JS client library that supports the same API as gRPC-Node to access a gRPC service. Due to browser limitation, the Web client library implements a different protocol than the -[native gRPC protocol](https://grpc.io/docs/guides/wire.html). +[native gRPC protocol](PROTOCOL-HTTP2.md). This protocol is designed to make it easy for a proxy to translate between the protocols as this is the most likely deployment model. This document lists the differences between the two protocols. To help tracking future revisions, this document describes a delta with the protocol details specified in the -[native gRPC protocol](https://grpc.io/docs/guides/wire.html). +[native gRPC protocol](PROTOCOL-HTTP2.md). # Design goals @@ -31,7 +31,7 @@ web-specific features such as CORS, XSRF * become optional (in 1-2 years) when browsers are able to speak the native gRPC protocol via the new [whatwg fetch/streams API](https://github.com/whatwg/fetch) -# Protocol differences vs [gRPC over HTTP2](https://grpc.io/docs/guides/wire.html) +# Protocol differences vs [gRPC over HTTP2](PROTOCOL-HTTP2.md) Content-Type @@ -53,14 +53,14 @@ HTTP wire protocols --- -HTTP/2 related behavior (specified in [gRPC over HTTP2](https://grpc.io/docs/guides/wire.html)) +HTTP/2 related behavior (specified in [gRPC over HTTP2](PROTOCOL-HTTP2.md)) 1. stream-id is not supported or used 2. go-away is not supported or used --- -Message framing (vs. [http2-transport-mapping](https://grpc.io/docs/guides/wire.html#http2-transport-mapping)) +Message framing (vs. [http2-transport-mapping](PROTOCOL-HTTP2.md#http2-transport-mapping)) 1. Response status encoded as part of the response body * Key-value pairs encoded as a HTTP/1 headers block (without the terminating newline), per https://tools.ietf.org/html/rfc7230#section-3.2 @@ -86,7 +86,7 @@ in the body. User Agent * Do NOT use User-Agent header (which is to be set by browsers, by default) -* Use X-User-Agent: grpc-web-javascript/0.1 (follow the same format as specified in [gRPC over HTTP2](https://grpc.io/docs/guides/wire.html)) +* Use X-User-Agent: grpc-web-javascript/0.1 (follow the same format as specified in [gRPC over HTTP2](PROTOCOL-HTTP2.md)) --- diff --git a/examples/python/interceptors/headers/header_manipulator_client_interceptor.py b/examples/python/interceptors/headers/header_manipulator_client_interceptor.py index ac7c605144..89b1bef05f 100644 --- a/examples/python/interceptors/headers/header_manipulator_client_interceptor.py +++ b/examples/python/interceptors/headers/header_manipulator_client_interceptor.py @@ -20,9 +20,10 @@ import generic_client_interceptor class _ClientCallDetails( - collections.namedtuple('_ClientCallDetails', - ('method', 'timeout', 'metadata', - 'credentials')), grpc.ClientCallDetails): + collections.namedtuple( + '_ClientCallDetails', + ('method', 'timeout', 'metadata', 'credentials')), + grpc.ClientCallDetails): pass @@ -33,7 +34,10 @@ def header_adder_interceptor(header, value): metadata = [] if client_call_details.metadata is not None: metadata = list(client_call_details.metadata) - metadata.append((header, value,)) + metadata.append(( + header, + value, + )) client_call_details = _ClientCallDetails( client_call_details.method, client_call_details.timeout, metadata, client_call_details.credentials) diff --git a/examples/python/multiplex/multiplex_client.py b/examples/python/multiplex/multiplex_client.py index 49713f35b7..9baa10247d 100644 --- a/examples/python/multiplex/multiplex_client.py +++ b/examples/python/multiplex/multiplex_client.py @@ -46,9 +46,9 @@ def guide_get_one_feature(route_guide_stub, point): def guide_get_feature(route_guide_stub): - guide_get_one_feature( - route_guide_stub, - route_guide_pb2.Point(latitude=409146138, longitude=-746188906)) + guide_get_one_feature(route_guide_stub, + route_guide_pb2.Point( + latitude=409146138, longitude=-746188906)) guide_get_one_feature(route_guide_stub, route_guide_pb2.Point(latitude=0, longitude=0)) @@ -101,8 +101,8 @@ def generate_messages(): def guide_route_chat(route_guide_stub): responses = route_guide_stub.RouteChat(generate_messages()) for response in responses: - print("Received message %s at %s" % - (response.message, response.location)) + print("Received message %s at %s" % (response.message, + response.location)) def run(): diff --git a/examples/python/multiplex/multiplex_server.py b/examples/python/multiplex/multiplex_server.py index e2ff671f97..70dec3c939 100644 --- a/examples/python/multiplex/multiplex_server.py +++ b/examples/python/multiplex/multiplex_server.py @@ -124,8 +124,8 @@ def serve(): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) helloworld_pb2_grpc.add_GreeterServicer_to_server(_GreeterServicer(), server) - route_guide_pb2_grpc.add_RouteGuideServicer_to_server(_RouteGuideServicer(), - server) + route_guide_pb2_grpc.add_RouteGuideServicer_to_server( + _RouteGuideServicer(), server) server.add_insecure_port('[::]:50051') server.start() try: diff --git a/examples/python/multiplex/run_codegen.py b/examples/python/multiplex/run_codegen.py index d960c3cf16..be8915fe20 100644 --- a/examples/python/multiplex/run_codegen.py +++ b/examples/python/multiplex/run_codegen.py @@ -15,7 +15,17 @@ from grpc_tools import protoc -protoc.main(('', '-I../../protos', '--python_out=.', '--grpc_python_out=.', - '../../protos/helloworld.proto',)) -protoc.main(('', '-I../../protos', '--python_out=.', '--grpc_python_out=.', - '../../protos/route_guide.proto',)) +protoc.main(( + '', + '-I../../protos', + '--python_out=.', + '--grpc_python_out=.', + '../../protos/helloworld.proto', +)) +protoc.main(( + '', + '-I../../protos', + '--python_out=.', + '--grpc_python_out=.', + '../../protos/route_guide.proto', +)) diff --git a/examples/python/route_guide/route_guide_client.py b/examples/python/route_guide/route_guide_client.py index c9d0e96ad6..f2d4317ef1 100644 --- a/examples/python/route_guide/route_guide_client.py +++ b/examples/python/route_guide/route_guide_client.py @@ -43,8 +43,9 @@ def guide_get_one_feature(stub, point): def guide_get_feature(stub): - guide_get_one_feature( - stub, route_guide_pb2.Point(latitude=409146138, longitude=-746188906)) + guide_get_one_feature(stub, + route_guide_pb2.Point( + latitude=409146138, longitude=-746188906)) guide_get_one_feature(stub, route_guide_pb2.Point(latitude=0, longitude=0)) @@ -94,8 +95,8 @@ def generate_messages(): def guide_route_chat(stub): responses = stub.RouteChat(generate_messages()) for response in responses: - print("Received message %s at %s" % - (response.message, response.location)) + print("Received message %s at %s" % (response.message, + response.location)) def run(): diff --git a/examples/python/route_guide/route_guide_server.py b/examples/python/route_guide/route_guide_server.py index 46f3322a39..f10008fdec 100644 --- a/examples/python/route_guide/route_guide_server.py +++ b/examples/python/route_guide/route_guide_server.py @@ -113,8 +113,8 @@ class RouteGuideServicer(route_guide_pb2_grpc.RouteGuideServicer): def serve(): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - route_guide_pb2_grpc.add_RouteGuideServicer_to_server(RouteGuideServicer(), - server) + route_guide_pb2_grpc.add_RouteGuideServicer_to_server( + RouteGuideServicer(), server) server.add_insecure_port('[::]:50051') server.start() try: diff --git a/examples/python/route_guide/run_codegen.py b/examples/python/route_guide/run_codegen.py index 1ec7fcd51f..8df562d349 100644 --- a/examples/python/route_guide/run_codegen.py +++ b/examples/python/route_guide/run_codegen.py @@ -15,5 +15,10 @@ from grpc_tools import protoc -protoc.main(('', '-I../../protos', '--python_out=.', '--grpc_python_out=.', - '../../protos/route_guide.proto',)) +protoc.main(( + '', + '-I../../protos', + '--python_out=.', + '--grpc_python_out=.', + '../../protos/route_guide.proto', +)) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 1cae08d088..bcbc90e2f3 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1046,6 +1046,7 @@ Pod::Spec.new do |s| 'test/core/end2end/tests/filter_call_init_fails.cc', 'test/core/end2end/tests/filter_causes_close.cc', 'test/core/end2end/tests/filter_latency.cc', + 'test/core/end2end/tests/filter_status_code.cc', 'test/core/end2end/tests/graceful_server_shutdown.cc', 'test/core/end2end/tests/high_initial_seqno.cc', 'test/core/end2end/tests/hpack_size.cc', @@ -57,6 +57,7 @@ '-Wno-long-long', '-Wno-unused-parameter', '-DOSATOMIC_USE_INLINED=1', + '-Wno-deprecated-declarations', ], 'ldflags': [ '-g', @@ -134,6 +135,7 @@ '-Wno-long-long', '-Wno-unused-parameter', '-DOSATOMIC_USE_INLINED=1', + '-Wno-deprecated-declarations', ], 'OTHER_CPLUSPLUSFLAGS': [ '-g', @@ -143,6 +145,7 @@ '-Wno-long-long', '-Wno-unused-parameter', '-DOSATOMIC_USE_INLINED=1', + '-Wno-deprecated-declarations', '-stdlib=libc++', '-std=c++11', '-Wno-error=deprecated-declarations' @@ -2385,6 +2388,7 @@ 'test/core/end2end/tests/filter_call_init_fails.cc', 'test/core/end2end/tests/filter_causes_close.cc', 'test/core/end2end/tests/filter_latency.cc', + 'test/core/end2end/tests/filter_status_code.cc', 'test/core/end2end/tests/graceful_server_shutdown.cc', 'test/core/end2end/tests/high_initial_seqno.cc', 'test/core/end2end/tests/hpack_size.cc', @@ -2457,6 +2461,7 @@ 'test/core/end2end/tests/filter_call_init_fails.cc', 'test/core/end2end/tests/filter_causes_close.cc', 'test/core/end2end/tests/filter_latency.cc', + 'test/core/end2end/tests/filter_status_code.cc', 'test/core/end2end/tests/graceful_server_shutdown.cc', 'test/core/end2end/tests/high_initial_seqno.cc', 'test/core/end2end/tests/hpack_size.cc', diff --git a/include/grpc++/impl/codegen/client_unary_call.h b/include/grpc++/impl/codegen/client_unary_call.h index 256dd859d3..543e54b972 100644 --- a/include/grpc++/impl/codegen/client_unary_call.h +++ b/include/grpc++/impl/codegen/client_unary_call.h @@ -65,6 +65,7 @@ class BlockingUnaryCallImpl { context->initial_metadata_flags()); ops.RecvInitialMetadata(context); ops.RecvMessage(result); + ops.AllowNoMessage(); ops.ClientSendClose(); ops.ClientRecvStatus(context, &status_); call.PerformOps(&ops); diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 5a0ed85111..ae1f951b49 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -195,12 +195,25 @@ #define GPR_PTHREAD_TLS 1 #else /* __MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_7 */ #define GPR_CPU_POSIX 1 +/* TODO(vjpai): there is a reported issue in bazel build for Mac where __thread + in a header is currently not working (bazelbuild/bazel#4341). Remove + the following conditional and use GPR_GCC_TLS when that is fixed */ +#ifndef GRPC_BAZEL_BUILD #define GPR_GCC_TLS 1 +#else /* GRPC_BAZEL_BUILD */ +#define GPR_PTHREAD_TLS 1 +#endif /* GRPC_BAZEL_BUILD */ #define GPR_APPLE_PTHREAD_NAME 1 #endif #else /* __MAC_OS_X_VERSION_MIN_REQUIRED */ #define GPR_CPU_POSIX 1 +/* TODO(vjpai): Remove the following conditional and use only GPR_GCC_TLS + when bazelbuild/bazel#4341 is fixed */ +#ifndef GRPC_BAZEL_BUILD #define GPR_GCC_TLS 1 +#else /* GRPC_BAZEL_BUILD */ +#define GPR_PTHREAD_TLS 1 +#endif /* GRPC_BAZEL_BUILD */ #endif #define GPR_POSIX_CRASH_HANDLER 1 #endif @@ -305,20 +318,23 @@ * This is primarily because of linker problems and toolchain misconfiguration: * TLS isn't supported until NDK r12b per * https://developer.android.com/ndk/downloads/revision_history.html + * TLS also does not work with Android NDK if GCC is being used as the compiler + * instead of Clang. * Since NDK r16, `__NDK_MAJOR__` and `__NDK_MINOR__` are defined in * <android/ndk-version.h>. For NDK < r16, users should define these macros, * e.g. `-D__NDK_MAJOR__=11 -D__NKD_MINOR__=0` for NDK r11. */ -#if defined(__ANDROID__) && defined(__clang__) && defined(GPR_GCC_TLS) +#if defined(__ANDROID__) && defined(GPR_GCC_TLS) #if __has_include(<android/ndk-version.h>) #include <android/ndk-version.h> #endif /* __has_include(<android/ndk-version.h>) */ -#if defined(__ANDROID__) && defined(__clang__) && defined(__NDK_MAJOR__) && \ - defined(__NDK_MINOR__) && \ - ((__NDK_MAJOR__ < 12) || ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1))) +#if (defined(__clang__) && defined(__NDK_MAJOR__) && defined(__NDK_MINOR__) && \ + ((__NDK_MAJOR__ < 12) || \ + ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1)))) || \ + (defined(__GNUC__) && !defined(__clang__)) #undef GPR_GCC_TLS #define GPR_PTHREAD_TLS 1 #endif -#endif /*defined(__ANDROID__) && defined(__clang__) && defined(GPR_GCC_TLS) */ +#endif /*defined(__ANDROID__) && defined(GPR_GCC_TLS) */ #if defined(__has_include) #if __has_include(<atomic>) @@ -418,6 +434,14 @@ typedef unsigned __int64 uint64_t; #endif #endif +#ifndef GRPC_UNUSED +#if defined(__GNUC__) && !defined(__MINGW32__) +#define GRPC_UNUSED __attribute__((unused)) +#else +#define GRPC_UNUSED +#endif +#endif + #ifndef GPR_PRINT_FORMAT_CHECK #ifdef __GNUC__ #define GPR_PRINT_FORMAT_CHECK(FORMAT_STR, ARGS) \ diff --git a/package.json b/package.json deleted file mode 100644 index 2b7930ce70..0000000000 --- a/package.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "name": "grpc", - "version": "1.7.2", - "author": "Google Inc.", - "description": "gRPC Library for Node", - "homepage": "https://grpc.io/", - "repository": { - "type": "git", - "url": "https://github.com/grpc/grpc.git" - }, - "bugs": "https://github.com/grpc/grpc/issues", - "contributors": [ - { - "name": "Michael Lumish", - "email": "mlumish@google.com" - } - ], - "directories": { - "lib": "src/node/src" - }, - "scripts": { - "lint": "node ./node_modules/jshint/bin/jshint src/node/src src/node/test src/node/interop src/node/index.js --exclude-path=src/node/.jshintignore", - "test": "./node_modules/.bin/mocha src/node/test && npm run-script lint", - "electron-build": "./node_modules/.bin/node-pre-gyp configure build --runtime=electron --disturl=https://atom.io/download/atom-shell", - "gen_docs": "./node_modules/.bin/jsdoc -c src/node/jsdoc_conf.json", - "coverage": "./node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha src/node/test", - "install": "./node_modules/.bin/node-pre-gyp install --fallback-to-build --library=static_library" - }, - "bundledDependencies": [ - "node-pre-gyp" - ], - "dependencies": { - "arguejs": "^0.2.3", - "lodash": "^4.15.0", - "nan": "^2.0.0", - "node-pre-gyp": "^0.6.35", - "protobufjs": "^5.0.0" - }, - "devDependencies": { - "async": "^2.0.1", - "body-parser": "^1.15.2", - "electron-mocha": "^3.1.1", - "express": "^4.14.0", - "google-auth-library": "^0.9.2", - "google-protobuf": "^3.0.0", - "istanbul": "^0.4.4", - "jsdoc": "^3.3.2", - "jshint": "^2.5.0", - "minimist": "^1.1.0", - "mocha": "^3.0.2", - "mocha-jenkins-reporter": "^0.2.3", - "poisson-process": "^0.2.1" - }, - "engines": { - "node": ">=4" - }, - "binary": { - "module_name": "grpc_node", - "module_path": "src/node/extension_binary/{node_abi}-{platform}-{arch}", - "host": "https://storage.googleapis.com/", - "remote_path": "grpc-precompiled-binaries/node/{name}/v{version}", - "package_name": "{node_abi}-{platform}-{arch}.tar.gz" - }, - "files": [ - "LICENSE", - "src/node/README.md", - "src/proto", - "etc", - "src/node/index.js", - "src/node/src", - "src/node/ext", - "include/grpc", - "src/core", - "src/boringssl", - "src/zlib", - "third_party/nanopb", - "third_party/zlib", - "third_party/boringssl", - "binding.gyp" - ], - "main": "src/node/index.js", - "license": "Apache-2.0", - "jshintConfig": { - "bitwise": true, - "curly": true, - "eqeqeq": true, - "esnext": true, - "freeze": true, - "immed": true, - "indent": 2, - "latedef": "nofunc", - "maxlen": 80, - "mocha": true, - "newcap": true, - "node": true, - "noarg": true, - "quotmark": "single", - "strict": true, - "trailing": true, - "undef": true, - "unused": "vars" - } -} diff --git a/src/core/ext/filters/client_channel/backup_poller.cc b/src/core/ext/filters/client_channel/backup_poller.cc index bfc549e709..4ee5e9c109 100644 --- a/src/core/ext/filters/client_channel/backup_poller.cc +++ b/src/core/ext/filters/client_channel/backup_poller.cc @@ -33,7 +33,8 @@ #define DEFAULT_POLL_INTERVAL_MS 5000 -typedef struct backup_poller { +namespace { +struct backup_poller { grpc_timer polling_timer; grpc_closure run_poller_closure; grpc_closure shutdown_closure; @@ -42,7 +43,8 @@ typedef struct backup_poller { bool shutting_down; // guarded by pollset_mu gpr_refcount refs; gpr_refcount shutdown_refs; -} backup_poller; +}; +} // namespace static gpr_once g_once = GPR_ONCE_INIT; static gpr_mu g_poller_mu; diff --git a/src/core/ext/filters/client_channel/channel_connectivity.cc b/src/core/ext/filters/client_channel/channel_connectivity.cc index 20693ba419..a827aa30ec 100644 --- a/src/core/ext/filters/client_channel/channel_connectivity.cc +++ b/src/core/ext/filters/client_channel/channel_connectivity.cc @@ -58,7 +58,8 @@ typedef enum { CALLING_BACK_AND_FINISHED, } callback_phase; -typedef struct { +namespace { +struct state_watcher { gpr_mu mu; callback_phase phase; grpc_closure on_complete; @@ -71,7 +72,8 @@ typedef struct { grpc_channel* channel; grpc_error* error; void* tag; -} state_watcher; +}; +} // namespace static void delete_state_watcher(state_watcher* w) { grpc_channel_element* client_channel_elem = grpc_channel_stack_last_element( diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index e99022a91b..3f3334d44a 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -553,6 +553,7 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) { } grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties, chand->interested_parties); + grpc_lb_policy_shutdown_locked(chand->lb_policy, new_lb_policy); GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel"); } chand->lb_policy = new_lb_policy; @@ -658,6 +659,7 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) { if (chand->lb_policy != nullptr) { grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties, chand->interested_parties); + grpc_lb_policy_shutdown_locked(chand->lb_policy, nullptr); GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel"); chand->lb_policy = nullptr; } @@ -792,6 +794,7 @@ static void cc_destroy_channel_elem(grpc_channel_element* elem) { if (chand->lb_policy != nullptr) { grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties, chand->interested_parties); + grpc_lb_policy_shutdown_locked(chand->lb_policy, nullptr); GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel"); } gpr_free(chand->info_lb_policy_name); @@ -852,12 +855,10 @@ typedef struct client_channel_call_data { grpc_subchannel_call* subchannel_call; grpc_error* error; - grpc_lb_policy* lb_policy; // Holds ref while LB pick is pending. + grpc_lb_policy_pick_state pick; grpc_closure lb_pick_closure; grpc_closure lb_pick_cancel_closure; - grpc_connected_subchannel* connected_subchannel; - grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT]; grpc_polling_entity* pollent; grpc_transport_stream_op_batch* waiting_for_pick_batches[MAX_WAITING_BATCHES]; @@ -866,8 +867,6 @@ typedef struct client_channel_call_data { grpc_transport_stream_op_batch* initial_metadata_batch; - grpc_linked_mdelem lb_token_mdelem; - grpc_closure on_complete; grpc_closure* original_on_complete; } call_data; @@ -1005,16 +1004,16 @@ static void create_subchannel_call_locked(grpc_call_element* elem, channel_data* chand = (channel_data*)elem->channel_data; call_data* calld = (call_data*)elem->call_data; const grpc_connected_subchannel_call_args call_args = { - calld->pollent, // pollent - calld->path, // path - calld->call_start_time, // start_time - calld->deadline, // deadline - calld->arena, // arena - calld->subchannel_call_context, // context - calld->call_combiner // call_combiner + calld->pollent, // pollent + calld->path, // path + calld->call_start_time, // start_time + calld->deadline, // deadline + calld->arena, // arena + calld->pick.subchannel_call_context, // context + calld->call_combiner // call_combiner }; grpc_error* new_error = grpc_connected_subchannel_create_call( - calld->connected_subchannel, &call_args, &calld->subchannel_call); + calld->pick.connected_subchannel, &call_args, &calld->subchannel_call); if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_DEBUG, "chand=%p calld=%p: create subchannel_call=%p: error=%s", chand, calld, calld->subchannel_call, grpc_error_string(new_error)); @@ -1032,7 +1031,7 @@ static void create_subchannel_call_locked(grpc_call_element* elem, static void pick_done_locked(grpc_call_element* elem, grpc_error* error) { call_data* calld = (call_data*)elem->call_data; channel_data* chand = (channel_data*)elem->channel_data; - if (calld->connected_subchannel == nullptr) { + if (calld->pick.connected_subchannel == nullptr) { // Failed to create subchannel. GRPC_ERROR_UNREF(calld->error); calld->error = error == GRPC_ERROR_NONE @@ -1071,13 +1070,16 @@ static void pick_callback_cancel_locked(void* arg, grpc_error* error) { grpc_call_element* elem = (grpc_call_element*)arg; channel_data* chand = (channel_data*)elem->channel_data; call_data* calld = (call_data*)elem->call_data; - if (calld->lb_policy != nullptr) { + // Note: chand->lb_policy may have changed since we started our pick, + // in which case we will be cancelling the pick on a policy other than + // the one we started it on. However, this will just be a no-op. + if (error != GRPC_ERROR_NONE && chand->lb_policy != nullptr) { if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p", - chand, calld, calld->lb_policy); + chand, calld, chand->lb_policy); } - grpc_lb_policy_cancel_pick_locked( - calld->lb_policy, &calld->connected_subchannel, GRPC_ERROR_REF(error)); + grpc_lb_policy_cancel_pick_locked(chand->lb_policy, &calld->pick, + GRPC_ERROR_REF(error)); } GRPC_CALL_STACK_UNREF(calld->owning_call, "pick_callback_cancel"); } @@ -1092,9 +1094,6 @@ static void pick_callback_done_locked(void* arg, grpc_error* error) { gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously", chand, calld); } - GPR_ASSERT(calld->lb_policy != nullptr); - GRPC_LB_POLICY_UNREF(calld->lb_policy, "pick_subchannel"); - calld->lb_policy = nullptr; async_pick_done_locked(elem, GRPC_ERROR_REF(error)); } @@ -1128,26 +1127,21 @@ static bool pick_callback_start_locked(grpc_call_element* elem) { initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY; } } - const grpc_lb_policy_pick_args inputs = { + calld->pick.initial_metadata = calld->initial_metadata_batch->payload->send_initial_metadata - .send_initial_metadata, - initial_metadata_flags, &calld->lb_token_mdelem}; - // Keep a ref to the LB policy in calld while the pick is pending. - GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel"); - calld->lb_policy = chand->lb_policy; + .send_initial_metadata; + calld->pick.initial_metadata_flags = initial_metadata_flags; GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem, grpc_combiner_scheduler(chand->combiner)); - const bool pick_done = grpc_lb_policy_pick_locked( - chand->lb_policy, &inputs, &calld->connected_subchannel, - calld->subchannel_call_context, nullptr, &calld->lb_pick_closure); + calld->pick.on_complete = &calld->lb_pick_closure; + const bool pick_done = + grpc_lb_policy_pick_locked(chand->lb_policy, &calld->pick); if (pick_done) { /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */ if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously", chand, calld); } - GRPC_LB_POLICY_UNREF(calld->lb_policy, "pick_subchannel"); - calld->lb_policy = nullptr; } else { GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel"); grpc_call_combiner_set_notify_on_cancel( @@ -1289,7 +1283,7 @@ static void start_pick_locked(void* arg, grpc_error* ignored) { grpc_call_element* elem = (grpc_call_element*)arg; call_data* calld = (call_data*)elem->call_data; channel_data* chand = (channel_data*)elem->channel_data; - GPR_ASSERT(calld->connected_subchannel == nullptr); + GPR_ASSERT(calld->pick.connected_subchannel == nullptr); if (chand->lb_policy != nullptr) { // We already have an LB policy, so ask it for a pick. if (pick_callback_start_locked(elem)) { @@ -1467,15 +1461,14 @@ static void cc_destroy_call_elem(grpc_call_element* elem, GRPC_SUBCHANNEL_CALL_UNREF(calld->subchannel_call, "client_channel_destroy_call"); } - GPR_ASSERT(calld->lb_policy == nullptr); GPR_ASSERT(calld->waiting_for_pick_batches_count == 0); - if (calld->connected_subchannel != nullptr) { - GRPC_CONNECTED_SUBCHANNEL_UNREF(calld->connected_subchannel, "picked"); + if (calld->pick.connected_subchannel != nullptr) { + GRPC_CONNECTED_SUBCHANNEL_UNREF(calld->pick.connected_subchannel, "picked"); } for (size_t i = 0; i < GRPC_CONTEXT_COUNT; ++i) { - if (calld->subchannel_call_context[i].value != nullptr) { - calld->subchannel_call_context[i].destroy( - calld->subchannel_call_context[i].value); + if (calld->pick.subchannel_call_context[i].value != nullptr) { + calld->pick.subchannel_call_context[i].destroy( + calld->pick.subchannel_call_context[i].value); } } GRPC_CLOSURE_SCHED(then_schedule_closure, GRPC_ERROR_NONE); diff --git a/src/core/ext/filters/client_channel/lb_policy.cc b/src/core/ext/filters/client_channel/lb_policy.cc index 7a5a8dec34..cc4fe7ec62 100644 --- a/src/core/ext/filters/client_channel/lb_policy.cc +++ b/src/core/ext/filters/client_channel/lb_policy.cc @@ -19,8 +19,6 @@ #include "src/core/ext/filters/client_channel/lb_policy.h" #include "src/core/lib/iomgr/combiner.h" -#define WEAK_REF_BITS 16 - grpc_core::DebugOnlyTraceFlag grpc_trace_lb_policy_refcount( false, "lb_policy_refcount"); @@ -28,91 +26,60 @@ void grpc_lb_policy_init(grpc_lb_policy* policy, const grpc_lb_policy_vtable* vtable, grpc_combiner* combiner) { policy->vtable = vtable; - gpr_atm_no_barrier_store(&policy->ref_pair, 1 << WEAK_REF_BITS); + gpr_ref_init(&policy->refs, 1); policy->interested_parties = grpc_pollset_set_create(); policy->combiner = GRPC_COMBINER_REF(combiner, "lb_policy"); } #ifndef NDEBUG -#define REF_FUNC_EXTRA_ARGS , const char *file, int line, const char *reason -#define REF_MUTATE_EXTRA_ARGS REF_FUNC_EXTRA_ARGS, const char* purpose -#define REF_FUNC_PASS_ARGS(new_reason) , file, line, new_reason -#define REF_MUTATE_PASS_ARGS(purpose) , file, line, reason, purpose +void grpc_lb_policy_ref(grpc_lb_policy* lb_policy, const char* file, int line, + const char* reason) { + if (grpc_trace_lb_policy_refcount.enabled()) { + gpr_atm old_refs = gpr_atm_no_barrier_load(&lb_policy->refs.count); + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, + "LB_POLICY:%p ref %" PRIdPTR " -> %" PRIdPTR " %s", lb_policy, + old_refs, old_refs + 1, reason); + } #else -#define REF_FUNC_EXTRA_ARGS -#define REF_MUTATE_EXTRA_ARGS -#define REF_FUNC_PASS_ARGS(new_reason) -#define REF_MUTATE_PASS_ARGS(x) +void grpc_lb_policy_ref(grpc_lb_policy* lb_policy) { #endif + gpr_ref(&lb_policy->refs); +} -static gpr_atm ref_mutate(grpc_lb_policy* c, gpr_atm delta, - int barrier REF_MUTATE_EXTRA_ARGS) { - gpr_atm old_val = barrier ? gpr_atm_full_fetch_add(&c->ref_pair, delta) - : gpr_atm_no_barrier_fetch_add(&c->ref_pair, delta); #ifndef NDEBUG +void grpc_lb_policy_unref(grpc_lb_policy* lb_policy, const char* file, int line, + const char* reason) { if (grpc_trace_lb_policy_refcount.enabled()) { + gpr_atm old_refs = gpr_atm_no_barrier_load(&lb_policy->refs.count); gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, - "LB_POLICY: %p %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR " [%s]", c, - purpose, old_val, old_val + delta, reason); + "LB_POLICY:%p unref %" PRIdPTR " -> %" PRIdPTR " %s", lb_policy, + old_refs, old_refs - 1, reason); } +#else +void grpc_lb_policy_unref(grpc_lb_policy* lb_policy) { #endif - return old_val; -} - -void grpc_lb_policy_ref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) { - ref_mutate(policy, 1 << WEAK_REF_BITS, 0 REF_MUTATE_PASS_ARGS("STRONG_REF")); -} - -static void shutdown_locked(void* arg, grpc_error* error) { - grpc_lb_policy* policy = (grpc_lb_policy*)arg; - policy->vtable->shutdown_locked(policy); - GRPC_LB_POLICY_WEAK_UNREF(policy, "strong-unref"); -} - -void grpc_lb_policy_unref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) { - gpr_atm old_val = - ref_mutate(policy, (gpr_atm)1 - (gpr_atm)(1 << WEAK_REF_BITS), - 1 REF_MUTATE_PASS_ARGS("STRONG_UNREF")); - gpr_atm mask = ~(gpr_atm)((1 << WEAK_REF_BITS) - 1); - gpr_atm check = 1 << WEAK_REF_BITS; - if ((old_val & mask) == check) { - GRPC_CLOSURE_SCHED( - GRPC_CLOSURE_CREATE(shutdown_locked, policy, - grpc_combiner_scheduler(policy->combiner)), - GRPC_ERROR_NONE); - } else { - grpc_lb_policy_weak_unref(policy REF_FUNC_PASS_ARGS("strong-unref")); + if (gpr_unref(&lb_policy->refs)) { + grpc_pollset_set_destroy(lb_policy->interested_parties); + grpc_combiner* combiner = lb_policy->combiner; + lb_policy->vtable->destroy(lb_policy); + GRPC_COMBINER_UNREF(combiner, "lb_policy"); } } -void grpc_lb_policy_weak_ref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) { - ref_mutate(policy, 1, 0 REF_MUTATE_PASS_ARGS("WEAK_REF")); -} - -void grpc_lb_policy_weak_unref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) { - gpr_atm old_val = - ref_mutate(policy, -(gpr_atm)1, 1 REF_MUTATE_PASS_ARGS("WEAK_UNREF")); - if (old_val == 1) { - grpc_pollset_set_destroy(policy->interested_parties); - grpc_combiner* combiner = policy->combiner; - policy->vtable->destroy(policy); - GRPC_COMBINER_UNREF(combiner, "lb_policy"); - } +void grpc_lb_policy_shutdown_locked(grpc_lb_policy* policy, + grpc_lb_policy* new_policy) { + policy->vtable->shutdown_locked(policy, new_policy); } int grpc_lb_policy_pick_locked(grpc_lb_policy* policy, - const grpc_lb_policy_pick_args* pick_args, - grpc_connected_subchannel** target, - grpc_call_context_element* context, - void** user_data, grpc_closure* on_complete) { - return policy->vtable->pick_locked(policy, pick_args, target, context, - user_data, on_complete); + grpc_lb_policy_pick_state* pick) { + return policy->vtable->pick_locked(policy, pick); } void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy, - grpc_connected_subchannel** target, + grpc_lb_policy_pick_state* pick, grpc_error* error) { - policy->vtable->cancel_pick_locked(policy, target, error); + policy->vtable->cancel_pick_locked(policy, pick, error); } void grpc_lb_policy_cancel_picks_locked(grpc_lb_policy* policy, diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 3572c97ed1..1176a05b78 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -33,7 +33,7 @@ extern grpc_core::DebugOnlyTraceFlag grpc_trace_lb_policy_refcount; struct grpc_lb_policy { const grpc_lb_policy_vtable* vtable; - gpr_atm ref_pair; + gpr_refcount refs; /* owned pointer to interested parties in load balancing decisions */ grpc_pollset_set* interested_parties; /* combiner under which lb_policy actions take place */ @@ -42,32 +42,42 @@ struct grpc_lb_policy { grpc_closure* request_reresolution; }; -/** Extra arguments for an LB pick */ -typedef struct grpc_lb_policy_pick_args { - /** Initial metadata associated with the picking call. */ +/// State used for an LB pick. +typedef struct grpc_lb_policy_pick_state { + /// Initial metadata associated with the picking call. grpc_metadata_batch* initial_metadata; - /** Bitmask used for selective cancelling. See \a - * grpc_lb_policy_cancel_picks() and \a GRPC_INITIAL_METADATA_* in - * grpc_types.h */ + /// Bitmask used for selective cancelling. See \a + /// grpc_lb_policy_cancel_picks() and \a GRPC_INITIAL_METADATA_* in + /// grpc_types.h. uint32_t initial_metadata_flags; - /** Storage for LB token in \a initial_metadata, or NULL if not used */ - grpc_linked_mdelem* lb_token_mdelem_storage; -} grpc_lb_policy_pick_args; + /// Storage for LB token in \a initial_metadata, or NULL if not used. + grpc_linked_mdelem lb_token_mdelem_storage; + /// Closure to run when pick is complete, if not completed synchronously. + grpc_closure* on_complete; + /// Will be set to the selected subchannel, or NULL on failure or when + /// the LB policy decides to drop the call. + grpc_connected_subchannel* connected_subchannel; + /// Will be populated with context to pass to the subchannel call, if needed. + grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT]; + /// Upon success, \a *user_data will be set to whatever opaque information + /// may need to be propagated from the LB policy, or NULL if not needed. + void** user_data; + /// Next pointer. For internal use by LB policy. + struct grpc_lb_policy_pick_state* next; +} grpc_lb_policy_pick_state; struct grpc_lb_policy_vtable { void (*destroy)(grpc_lb_policy* policy); - void (*shutdown_locked)(grpc_lb_policy* policy); + + /// \see grpc_lb_policy_shutdown_locked(). + void (*shutdown_locked)(grpc_lb_policy* policy, grpc_lb_policy* new_policy); /** \see grpc_lb_policy_pick */ - int (*pick_locked)(grpc_lb_policy* policy, - const grpc_lb_policy_pick_args* pick_args, - grpc_connected_subchannel** target, - grpc_call_context_element* context, void** user_data, - grpc_closure* on_complete); + int (*pick_locked)(grpc_lb_policy* policy, grpc_lb_policy_pick_state* pick); /** \see grpc_lb_policy_cancel_pick */ void (*cancel_pick_locked)(grpc_lb_policy* policy, - grpc_connected_subchannel** target, + grpc_lb_policy_pick_state* pick, grpc_error* error); /** \see grpc_lb_policy_cancel_picks */ @@ -103,37 +113,19 @@ struct grpc_lb_policy_vtable { }; #ifndef NDEBUG - -/* Strong references: the policy will shutdown when they reach zero */ #define GRPC_LB_POLICY_REF(p, r) \ grpc_lb_policy_ref((p), __FILE__, __LINE__, (r)) #define GRPC_LB_POLICY_UNREF(p, r) \ grpc_lb_policy_unref((p), __FILE__, __LINE__, (r)) - -/* Weak references: they don't prevent the shutdown of the LB policy. When no - * strong references are left but there are still weak ones, shutdown is called. - * Once the weak reference also reaches zero, the LB policy is destroyed. */ -#define GRPC_LB_POLICY_WEAK_REF(p, r) \ - grpc_lb_policy_weak_ref((p), __FILE__, __LINE__, (r)) -#define GRPC_LB_POLICY_WEAK_UNREF(p, r) \ - grpc_lb_policy_weak_unref((p), __FILE__, __LINE__, (r)) void grpc_lb_policy_ref(grpc_lb_policy* policy, const char* file, int line, const char* reason); void grpc_lb_policy_unref(grpc_lb_policy* policy, const char* file, int line, const char* reason); -void grpc_lb_policy_weak_ref(grpc_lb_policy* policy, const char* file, int line, - const char* reason); -void grpc_lb_policy_weak_unref(grpc_lb_policy* policy, const char* file, - int line, const char* reason); -#else +#else // !NDEBUG #define GRPC_LB_POLICY_REF(p, r) grpc_lb_policy_ref((p)) #define GRPC_LB_POLICY_UNREF(p, r) grpc_lb_policy_unref((p)) -#define GRPC_LB_POLICY_WEAK_REF(p, r) grpc_lb_policy_weak_ref((p)) -#define GRPC_LB_POLICY_WEAK_UNREF(p, r) grpc_lb_policy_weak_unref((p)) void grpc_lb_policy_ref(grpc_lb_policy* policy); void grpc_lb_policy_unref(grpc_lb_policy* policy); -void grpc_lb_policy_weak_ref(grpc_lb_policy* policy); -void grpc_lb_policy_weak_unref(grpc_lb_policy* policy); #endif /** called by concrete implementations to initialize the base struct */ @@ -141,28 +133,24 @@ void grpc_lb_policy_init(grpc_lb_policy* policy, const grpc_lb_policy_vtable* vtable, grpc_combiner* combiner); -/** Finds an appropriate subchannel for a call, based on \a pick_args. - - \a target will be set to the selected subchannel, or NULL on failure - or when the LB policy decides to drop the call. +/// Shuts down \a policy. +/// If \a new_policy is non-null, any pending picks will be restarted +/// on that policy; otherwise, they will be failed. +void grpc_lb_policy_shutdown_locked(grpc_lb_policy* policy, + grpc_lb_policy* new_policy); - Upon success, \a user_data will be set to whatever opaque information - may need to be propagated from the LB policy, or NULL if not needed. - \a context will be populated with context to pass to the subchannel - call, if needed. +/** Finds an appropriate subchannel for a call, based on data in \a pick. + \a pick must remain alive until the pick is complete. If the pick succeeds and a result is known immediately, a non-zero - value will be returned. Otherwise, \a on_complete will be invoked + value will be returned. Otherwise, \a pick->on_complete will be invoked once the pick is complete with its error argument set to indicate success or failure. Any IO should be done under the \a interested_parties \a grpc_pollset_set in the \a grpc_lb_policy struct. */ int grpc_lb_policy_pick_locked(grpc_lb_policy* policy, - const grpc_lb_policy_pick_args* pick_args, - grpc_connected_subchannel** target, - grpc_call_context_element* context, - void** user_data, grpc_closure* on_complete); + grpc_lb_policy_pick_state* pick); /** Perform a connected subchannel ping (see \a grpc_connected_subchannel_ping) against one of the connected subchannels managed by \a policy. */ @@ -170,11 +158,11 @@ void grpc_lb_policy_ping_one_locked(grpc_lb_policy* policy, grpc_closure* on_initiate, grpc_closure* on_ack); -/** Cancel picks for \a target. +/** Cancel picks for \a pick. The \a on_complete callback of the pending picks will be invoked with \a *target set to NULL. */ void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy, - grpc_connected_subchannel** target, + grpc_lb_policy_pick_state* pick, grpc_error* error); /** Cancel all pending picks for which their \a initial_metadata_flags (as given diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc index 3eedb08ecc..1708d81e61 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc @@ -32,7 +32,8 @@ static grpc_error* init_channel_elem(grpc_channel_element* elem, static void destroy_channel_elem(grpc_channel_element* elem) {} -typedef struct { +namespace { +struct call_data { // Stats object to update. grpc_grpclb_client_stats* client_stats; // State for intercepting send_initial_metadata. @@ -43,7 +44,8 @@ typedef struct { grpc_closure recv_initial_metadata_ready; grpc_closure* original_recv_initial_metadata_ready; bool recv_initial_metadata_succeeded; -} call_data; +}; +} // namespace static void on_complete_for_send(void* arg, grpc_error* error) { call_data* calld = (call_data*)arg; diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index dd6fc602ab..5849ac9d2d 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -54,7 +54,7 @@ * operations in progress over the old RR instance. This is done by * decreasing the reference count on the old policy. The moment no more * references are held on the old RR policy, it'll be destroyed and \a - * glb_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN + * on_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN * state. At this point we can transition to a new RR instance safely, which * is done once again via \a rr_handover_locked(). * @@ -128,185 +128,48 @@ grpc_core::TraceFlag grpc_lb_glb_trace(false, "glb"); -/* add lb_token of selected subchannel (address) to the call's initial - * metadata */ -static grpc_error* initial_metadata_add_lb_token( - grpc_metadata_batch* initial_metadata, - grpc_linked_mdelem* lb_token_mdelem_storage, grpc_mdelem lb_token) { - GPR_ASSERT(lb_token_mdelem_storage != nullptr); - GPR_ASSERT(!GRPC_MDISNULL(lb_token)); - return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage, - lb_token); -} - -static void destroy_client_stats(void* arg) { - grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats*)arg); -} - -typedef struct wrapped_rr_closure_arg { - /* the closure instance using this struct as argument */ - grpc_closure wrapper_closure; - - /* the original closure. Usually a on_complete/notify cb for pick() and ping() - * calls against the internal RR instance, respectively. */ - grpc_closure* wrapped_closure; - - /* the pick's initial metadata, kept in order to append the LB token for the - * pick */ - grpc_metadata_batch* initial_metadata; - - /* the picked target, used to determine which LB token to add to the pick's - * initial metadata */ - grpc_connected_subchannel** target; - - /* the context to be populated for the subchannel call */ - grpc_call_context_element* context; - - /* Stats for client-side load reporting. Note that this holds a - * reference, which must be either passed on via context or unreffed. */ +struct glb_lb_policy; + +namespace { + +/// Linked list of pending pick requests. It stores all information needed to +/// eventually call (Round Robin's) pick() on them. They mainly stay pending +/// waiting for the RR policy to be created. +/// +/// Note that when a pick is sent to the RR policy, we inject our own +/// on_complete callback, so that we can intercept the result before +/// invoking the original on_complete callback. This allows us to set the +/// LB token metadata and add client_stats to the call context. +/// See \a pending_pick_complete() for details. +struct pending_pick { + // Our on_complete closure and the original one. + grpc_closure on_complete; + grpc_closure* original_on_complete; + // The original pick. + grpc_lb_policy_pick_state* pick; + // Stats for client-side load reporting. Note that this holds a + // reference, which must be either passed on via context or unreffed. grpc_grpclb_client_stats* client_stats; - - /* the LB token associated with the pick */ + // The LB token associated with the pick. This is set via user_data in + // the pick. grpc_mdelem lb_token; - - /* storage for the lb token initial metadata mdelem */ - grpc_linked_mdelem* lb_token_mdelem_storage; - - /* The RR instance related to the closure */ - grpc_lb_policy* rr_policy; - - /* The grpclb instance that created the wrapping. This instance is not owned, - * reference counts are untouched. It's used only for logging purposes. */ - grpc_lb_policy* glb_policy; - - /* heap memory to be freed upon closure execution. */ - void* free_when_done; -} wrapped_rr_closure_arg; - -/* The \a on_complete closure passed as part of the pick requires keeping a - * reference to its associated round robin instance. We wrap this closure in - * order to unref the round robin instance upon its invocation */ -static void wrapped_rr_closure(void* arg, grpc_error* error) { - wrapped_rr_closure_arg* wc_arg = (wrapped_rr_closure_arg*)arg; - - GPR_ASSERT(wc_arg->wrapped_closure != nullptr); - GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_REF(error)); - - if (wc_arg->rr_policy != nullptr) { - /* if *target is nullptr, no pick has been made by the RR policy (eg, all - * addresses failed to connect). There won't be any user_data/token - * available */ - if (*wc_arg->target != nullptr) { - if (!GRPC_MDISNULL(wc_arg->lb_token)) { - initial_metadata_add_lb_token(wc_arg->initial_metadata, - wc_arg->lb_token_mdelem_storage, - GRPC_MDELEM_REF(wc_arg->lb_token)); - } else { - gpr_log( - GPR_ERROR, - "[grpclb %p] No LB token for connected subchannel pick %p (from RR " - "instance %p).", - wc_arg->glb_policy, *wc_arg->target, wc_arg->rr_policy); - abort(); - } - // Pass on client stats via context. Passes ownership of the reference. - GPR_ASSERT(wc_arg->client_stats != nullptr); - wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].value = wc_arg->client_stats; - wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].destroy = destroy_client_stats; - } else { - grpc_grpclb_client_stats_unref(wc_arg->client_stats); - } - if (grpc_lb_glb_trace.enabled()) { - gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p", wc_arg->glb_policy, - wc_arg->rr_policy); - } - GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "wrapped_rr_closure"); - } - GPR_ASSERT(wc_arg->free_when_done != nullptr); - gpr_free(wc_arg->free_when_done); -} - -/* Linked list of pending pick requests. It stores all information needed to - * eventually call (Round Robin's) pick() on them. They mainly stay pending - * waiting for the RR policy to be created/updated. - * - * One particularity is the wrapping of the user-provided \a on_complete closure - * (in \a wrapped_on_complete and \a wrapped_on_complete_arg). This is needed in - * order to correctly unref the RR policy instance upon completion of the pick. - * See \a wrapped_rr_closure for details. */ -typedef struct pending_pick { + // The grpclb instance that created the wrapping. This instance is not owned, + // reference counts are untouched. It's used only for logging purposes. + glb_lb_policy* glb_policy; + // Next pending pick. struct pending_pick* next; +}; - /* original pick()'s arguments */ - grpc_lb_policy_pick_args pick_args; - - /* output argument where to store the pick()ed connected subchannel, or - * nullptr upon error. */ - grpc_connected_subchannel** target; - - /* args for wrapped_on_complete */ - wrapped_rr_closure_arg wrapped_on_complete_arg; -} pending_pick; - -static void add_pending_pick(pending_pick** root, - const grpc_lb_policy_pick_args* pick_args, - grpc_connected_subchannel** target, - grpc_call_context_element* context, - grpc_closure* on_complete) { - pending_pick* pp = (pending_pick*)gpr_zalloc(sizeof(*pp)); - pp->next = *root; - pp->pick_args = *pick_args; - pp->target = target; - pp->wrapped_on_complete_arg.wrapped_closure = on_complete; - pp->wrapped_on_complete_arg.target = target; - pp->wrapped_on_complete_arg.context = context; - pp->wrapped_on_complete_arg.initial_metadata = pick_args->initial_metadata; - pp->wrapped_on_complete_arg.lb_token_mdelem_storage = - pick_args->lb_token_mdelem_storage; - pp->wrapped_on_complete_arg.free_when_done = pp; - GRPC_CLOSURE_INIT(&pp->wrapped_on_complete_arg.wrapper_closure, - wrapped_rr_closure, &pp->wrapped_on_complete_arg, - grpc_schedule_on_exec_ctx); - *root = pp; -} - -/* Same as the \a pending_pick struct but for ping operations */ -typedef struct pending_ping { +/// A linked list of pending pings waiting for the RR policy to be created. +struct pending_ping { + grpc_closure* on_initiate; + grpc_closure* on_ack; struct pending_ping* next; +}; - /* args for sending the ping */ - wrapped_rr_closure_arg* on_initiate; - wrapped_rr_closure_arg* on_ack; -} pending_ping; - -static void add_pending_ping(pending_ping** root, grpc_closure* on_initiate, - grpc_closure* on_ack) { - pending_ping* pping = (pending_ping*)gpr_zalloc(sizeof(*pping)); - if (on_initiate != nullptr) { - pping->on_initiate = - (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(*pping->on_initiate)); - pping->on_initiate->wrapped_closure = on_initiate; - pping->on_initiate->free_when_done = pping->on_initiate; - GRPC_CLOSURE_INIT(&pping->on_initiate->wrapper_closure, wrapped_rr_closure, - &pping->on_initiate, grpc_schedule_on_exec_ctx); - } - if (on_ack != nullptr) { - pping->on_ack = (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(*pping->on_ack)); - pping->on_ack->wrapped_closure = on_ack; - pping->on_ack->free_when_done = pping->on_ack; - GRPC_CLOSURE_INIT(&pping->on_ack->wrapper_closure, wrapped_rr_closure, - &pping->on_ack, grpc_schedule_on_exec_ctx); - } - pping->next = *root; - *root = pping; -} - -/* - * glb_lb_policy - */ -typedef struct rr_connectivity_data rr_connectivity_data; +} // namespace -typedef struct glb_lb_policy { +struct glb_lb_policy { /** base policy: must be first */ grpc_lb_policy base; @@ -331,6 +194,9 @@ typedef struct glb_lb_policy { /** the RR policy to use of the backend servers returned by the LB server */ grpc_lb_policy* rr_policy; + grpc_closure on_rr_connectivity_changed; + grpc_connectivity_state rr_connectivity_state; + bool started_picking; /** our connectivity state tracker */ @@ -365,11 +231,11 @@ typedef struct glb_lb_policy { /** are we already watching the LB channel's connectivity? */ bool watching_lb_channel; - /** is \a lb_call_retry_timer active? */ - bool retry_timer_active; + /** is the callback associated with \a lb_call_retry_timer pending? */ + bool retry_timer_callback_pending; - /** is \a lb_fallback_timer active? */ - bool fallback_timer_active; + /** is the callback associated with \a lb_fallback_timer pending? */ + bool fallback_timer_callback_pending; /** called upon changes to the LB channel's connectivity. */ grpc_closure lb_channel_on_connectivity_changed; @@ -377,6 +243,9 @@ typedef struct glb_lb_policy { /************************************************************/ /* client data associated with the LB server communication */ /************************************************************/ + /* Finished sending initial request. */ + grpc_closure lb_on_sent_initial_request; + /* Status from the LB server has been received. This signals the end of the LB * call. */ grpc_closure lb_on_server_status_received; @@ -416,6 +285,7 @@ typedef struct glb_lb_policy { /** LB fallback timer */ grpc_timer lb_fallback_timer; + bool initial_request_sent; bool seen_initial_response; /* Stats for client-side load reporting. Should be unreffed and @@ -424,22 +294,92 @@ typedef struct glb_lb_policy { /* Interval and timer for next client load report. */ grpc_millis client_stats_report_interval; grpc_timer client_load_report_timer; - bool client_load_report_timer_pending; + bool client_load_report_timer_callback_pending; bool last_client_load_report_counters_were_zero; /* Closure used for either the load report timer or the callback for * completion of sending the load report. */ grpc_closure client_load_report_closure; /* Client load report message payload. */ grpc_byte_buffer* client_load_report_payload; -} glb_lb_policy; - -/* Keeps track and reacts to changes in connectivity of the RR instance */ -struct rr_connectivity_data { - grpc_closure on_change; - grpc_connectivity_state state; - glb_lb_policy* glb_policy; }; +/* add lb_token of selected subchannel (address) to the call's initial + * metadata */ +static grpc_error* initial_metadata_add_lb_token( + grpc_metadata_batch* initial_metadata, + grpc_linked_mdelem* lb_token_mdelem_storage, grpc_mdelem lb_token) { + GPR_ASSERT(lb_token_mdelem_storage != nullptr); + GPR_ASSERT(!GRPC_MDISNULL(lb_token)); + return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage, + lb_token); +} + +static void destroy_client_stats(void* arg) { + grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats*)arg); +} + +static void pending_pick_set_metadata_and_context(pending_pick* pp) { + /* if connected_subchannel is nullptr, no pick has been made by the RR + * policy (e.g., all addresses failed to connect). There won't be any + * user_data/token available */ + if (pp->pick->connected_subchannel != nullptr) { + if (!GRPC_MDISNULL(pp->lb_token)) { + initial_metadata_add_lb_token(pp->pick->initial_metadata, + &pp->pick->lb_token_mdelem_storage, + GRPC_MDELEM_REF(pp->lb_token)); + } else { + gpr_log(GPR_ERROR, + "[grpclb %p] No LB token for connected subchannel pick %p", + pp->glb_policy, pp->pick); + abort(); + } + // Pass on client stats via context. Passes ownership of the reference. + GPR_ASSERT(pp->client_stats != nullptr); + pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].value = + pp->client_stats; + pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy = + destroy_client_stats; + } else { + grpc_grpclb_client_stats_unref(pp->client_stats); + } +} + +/* The \a on_complete closure passed as part of the pick requires keeping a + * reference to its associated round robin instance. We wrap this closure in + * order to unref the round robin instance upon its invocation */ +static void pending_pick_complete(void* arg, grpc_error* error) { + pending_pick* pp = (pending_pick*)arg; + pending_pick_set_metadata_and_context(pp); + GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_REF(error)); + gpr_free(pp); +} + +static pending_pick* pending_pick_create(glb_lb_policy* glb_policy, + grpc_lb_policy_pick_state* pick) { + pending_pick* pp = (pending_pick*)gpr_zalloc(sizeof(*pp)); + pp->pick = pick; + pp->glb_policy = glb_policy; + GRPC_CLOSURE_INIT(&pp->on_complete, pending_pick_complete, pp, + grpc_schedule_on_exec_ctx); + pp->original_on_complete = pick->on_complete; + pp->pick->on_complete = &pp->on_complete; + return pp; +} + +static void pending_pick_add(pending_pick** root, pending_pick* new_pp) { + new_pp->next = *root; + *root = new_pp; +} + +static void pending_ping_add(pending_ping** root, grpc_closure* on_initiate, + grpc_closure* on_ack) { + pending_ping* pping = (pending_ping*)gpr_zalloc(sizeof(*pping)); + pping->on_initiate = on_initiate; + pping->on_ack = on_ack; + pping->next = *root; + *root = pping; +} + static bool is_server_valid(const grpc_grpclb_server* server, size_t idx, bool log) { if (server->drop) return false; @@ -551,7 +491,6 @@ static grpc_lb_addresses* process_serverlist_locked( gpr_free(uri); user_data = (void*)GRPC_MDELEM_LB_TOKEN_EMPTY.payload; } - grpc_lb_addresses_set_address(lb_addresses, addr_idx, &addr.addr, addr.len, false /* is_balancer */, nullptr /* balancer_name */, user_data); @@ -592,7 +531,6 @@ static void update_lb_connectivity_status_locked( grpc_error* rr_state_error) { const grpc_connectivity_state curr_glb_state = grpc_connectivity_state_check(&glb_policy->state_tracker); - /* The new connectivity status is a function of the previous one and the new * input coming from the status of the RR policy. * @@ -622,7 +560,6 @@ static void update_lb_connectivity_status_locked( * * (*) This function mustn't be called during shutting down. */ GPR_ASSERT(curr_glb_state != GRPC_CHANNEL_SHUTDOWN); - switch (rr_state) { case GRPC_CHANNEL_TRANSIENT_FAILURE: case GRPC_CHANNEL_SHUTDOWN: @@ -633,7 +570,6 @@ static void update_lb_connectivity_status_locked( case GRPC_CHANNEL_READY: GPR_ASSERT(rr_state_error == GRPC_ERROR_NONE); } - if (grpc_lb_glb_trace.enabled()) { gpr_log( GPR_INFO, @@ -651,10 +587,8 @@ static void update_lb_connectivity_status_locked( * cleanups this callback would otherwise be responsible for. * If \a force_async is true, then we will manually schedule the * completion callback even if the pick is available immediately. */ -static bool pick_from_internal_rr_locked( - glb_lb_policy* glb_policy, const grpc_lb_policy_pick_args* pick_args, - bool force_async, grpc_connected_subchannel** target, - wrapped_rr_closure_arg* wc_arg) { +static bool pick_from_internal_rr_locked(glb_lb_policy* glb_policy, + bool force_async, pending_pick* pp) { // Check for drops if we are not using fallback backend addresses. if (glb_policy->serverlist != nullptr) { // Look at the index into the serverlist to see if we should drop this call. @@ -664,57 +598,36 @@ static bool pick_from_internal_rr_locked( glb_policy->serverlist_index = 0; // Wrap-around. } if (server->drop) { - // Not using the RR policy, so unref it. - if (grpc_lb_glb_trace.enabled()) { - gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p for drop", glb_policy, - wc_arg->rr_policy); - } - GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "glb_pick_sync"); // Update client load reporting stats to indicate the number of // dropped calls. Note that we have to do this here instead of in // the client_load_reporting filter, because we do not create a // subchannel call (and therefore no client_load_reporting filter) // for dropped calls. - GPR_ASSERT(wc_arg->client_stats != nullptr); + GPR_ASSERT(glb_policy->client_stats != nullptr); grpc_grpclb_client_stats_add_call_dropped_locked( - server->load_balance_token, wc_arg->client_stats); - grpc_grpclb_client_stats_unref(wc_arg->client_stats); + server->load_balance_token, glb_policy->client_stats); if (force_async) { - GPR_ASSERT(wc_arg->wrapped_closure != nullptr); - GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_NONE); - gpr_free(wc_arg->free_when_done); + GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE); + gpr_free(pp); return false; } - gpr_free(wc_arg->free_when_done); + gpr_free(pp); return true; } } + // Set client_stats and user_data. + pp->client_stats = grpc_grpclb_client_stats_ref(glb_policy->client_stats); + GPR_ASSERT(pp->pick->user_data == nullptr); + pp->pick->user_data = (void**)&pp->lb_token; // Pick via the RR policy. - const bool pick_done = grpc_lb_policy_pick_locked( - wc_arg->rr_policy, pick_args, target, wc_arg->context, - (void**)&wc_arg->lb_token, &wc_arg->wrapper_closure); + bool pick_done = grpc_lb_policy_pick_locked(glb_policy->rr_policy, pp->pick); if (pick_done) { - /* synchronous grpc_lb_policy_pick call. Unref the RR policy. */ - if (grpc_lb_glb_trace.enabled()) { - gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p", glb_policy, - wc_arg->rr_policy); - } - GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "glb_pick_sync"); - /* add the load reporting initial metadata */ - initial_metadata_add_lb_token(pick_args->initial_metadata, - pick_args->lb_token_mdelem_storage, - GRPC_MDELEM_REF(wc_arg->lb_token)); - // Pass on client stats via context. Passes ownership of the reference. - GPR_ASSERT(wc_arg->client_stats != nullptr); - wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].value = wc_arg->client_stats; - wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].destroy = destroy_client_stats; + pending_pick_set_metadata_and_context(pp); if (force_async) { - GPR_ASSERT(wc_arg->wrapped_closure != nullptr); - GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_NONE); - gpr_free(wc_arg->free_when_done); - return false; + GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE); + pick_done = false; } - gpr_free(wc_arg->free_when_done); + gpr_free(pp); } /* else, the pending pick will be registered and taken care of by the * pending pick list inside the RR policy (glb_policy->rr_policy). @@ -756,7 +669,7 @@ static void lb_policy_args_destroy(grpc_lb_policy_args* args) { gpr_free(args); } -static void glb_rr_connectivity_changed_locked(void* arg, grpc_error* error); +static void on_rr_connectivity_changed_locked(void* arg, grpc_error* error); static void create_rr_locked(glb_lb_policy* glb_policy, grpc_lb_policy_args* args) { GPR_ASSERT(glb_policy->rr_policy == nullptr); @@ -778,72 +691,46 @@ static void create_rr_locked(glb_lb_policy* glb_policy, glb_policy->base.request_reresolution = nullptr; glb_policy->rr_policy = new_rr_policy; grpc_error* rr_state_error = nullptr; - const grpc_connectivity_state rr_state = - grpc_lb_policy_check_connectivity_locked(glb_policy->rr_policy, - &rr_state_error); + glb_policy->rr_connectivity_state = grpc_lb_policy_check_connectivity_locked( + glb_policy->rr_policy, &rr_state_error); /* Connectivity state is a function of the RR policy updated/created */ - update_lb_connectivity_status_locked(glb_policy, rr_state, rr_state_error); + update_lb_connectivity_status_locked( + glb_policy, glb_policy->rr_connectivity_state, rr_state_error); /* Add the gRPC LB's interested_parties pollset_set to that of the newly * created RR policy. This will make the RR policy progress upon activity on * gRPC LB, which in turn is tied to the application's call */ grpc_pollset_set_add_pollset_set(glb_policy->rr_policy->interested_parties, glb_policy->base.interested_parties); - - /* Allocate the data for the tracking of the new RR policy's connectivity. - * It'll be deallocated in glb_rr_connectivity_changed() */ - rr_connectivity_data* rr_connectivity = - (rr_connectivity_data*)gpr_zalloc(sizeof(rr_connectivity_data)); - GRPC_CLOSURE_INIT(&rr_connectivity->on_change, - glb_rr_connectivity_changed_locked, rr_connectivity, + GRPC_CLOSURE_INIT(&glb_policy->on_rr_connectivity_changed, + on_rr_connectivity_changed_locked, glb_policy, grpc_combiner_scheduler(glb_policy->base.combiner)); - rr_connectivity->glb_policy = glb_policy; - rr_connectivity->state = rr_state; - /* Subscribe to changes to the connectivity of the new RR */ - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "glb_rr_connectivity_cb"); - grpc_lb_policy_notify_on_state_change_locked(glb_policy->rr_policy, - &rr_connectivity->state, - &rr_connectivity->on_change); + GRPC_LB_POLICY_REF(&glb_policy->base, "glb_rr_connectivity_cb"); + grpc_lb_policy_notify_on_state_change_locked( + glb_policy->rr_policy, &glb_policy->rr_connectivity_state, + &glb_policy->on_rr_connectivity_changed); grpc_lb_policy_exit_idle_locked(glb_policy->rr_policy); - - /* Update picks and pings in wait */ + // Send pending picks to RR policy. pending_pick* pp; while ((pp = glb_policy->pending_picks)) { glb_policy->pending_picks = pp->next; - GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_pick"); - pp->wrapped_on_complete_arg.rr_policy = glb_policy->rr_policy; - pp->wrapped_on_complete_arg.client_stats = - grpc_grpclb_client_stats_ref(glb_policy->client_stats); if (grpc_lb_glb_trace.enabled()) { gpr_log(GPR_INFO, "[grpclb %p] Pending pick about to (async) PICK from RR %p", glb_policy, glb_policy->rr_policy); } - pick_from_internal_rr_locked(glb_policy, &pp->pick_args, - true /* force_async */, pp->target, - &pp->wrapped_on_complete_arg); + pick_from_internal_rr_locked(glb_policy, true /* force_async */, pp); } - + // Send pending pings to RR policy. pending_ping* pping; while ((pping = glb_policy->pending_pings)) { glb_policy->pending_pings = pping->next; - grpc_closure* on_initiate = nullptr; - grpc_closure* on_ack = nullptr; - if (pping->on_initiate != nullptr) { - GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping"); - pping->on_initiate->rr_policy = glb_policy->rr_policy; - on_initiate = &pping->on_initiate->wrapper_closure; - } - if (pping->on_ack != nullptr) { - GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping"); - pping->on_ack->rr_policy = glb_policy->rr_policy; - on_ack = &pping->on_ack->wrapper_closure; - } if (grpc_lb_glb_trace.enabled()) { gpr_log(GPR_INFO, "[grpclb %p] Pending ping about to PING from RR %p", glb_policy, glb_policy->rr_policy); } - grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, on_initiate, on_ack); + grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, pping->on_initiate, + pping->on_ack); gpr_free(pping); } } @@ -869,31 +756,28 @@ static void rr_handover_locked(glb_lb_policy* glb_policy) { lb_policy_args_destroy(args); } -static void glb_rr_connectivity_changed_locked(void* arg, grpc_error* error) { - rr_connectivity_data* rr_connectivity = (rr_connectivity_data*)arg; - glb_lb_policy* glb_policy = rr_connectivity->glb_policy; +static void on_rr_connectivity_changed_locked(void* arg, grpc_error* error) { + glb_lb_policy* glb_policy = (glb_lb_policy*)arg; if (glb_policy->shutting_down) { - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "glb_rr_connectivity_cb"); - gpr_free(rr_connectivity); + GRPC_LB_POLICY_UNREF(&glb_policy->base, "glb_rr_connectivity_cb"); return; } - if (rr_connectivity->state == GRPC_CHANNEL_SHUTDOWN) { + if (glb_policy->rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) { /* An RR policy that has transitioned into the SHUTDOWN connectivity state * should not be considered for picks or updates: the SHUTDOWN state is a * sink, policies can't transition back from it. .*/ GRPC_LB_POLICY_UNREF(glb_policy->rr_policy, "rr_connectivity_shutdown"); glb_policy->rr_policy = nullptr; - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "glb_rr_connectivity_cb"); - gpr_free(rr_connectivity); + GRPC_LB_POLICY_UNREF(&glb_policy->base, "glb_rr_connectivity_cb"); return; } /* rr state != SHUTDOWN && !glb_policy->shutting down: biz as usual */ - update_lb_connectivity_status_locked(glb_policy, rr_connectivity->state, - GRPC_ERROR_REF(error)); - /* Resubscribe. Reuse the "glb_rr_connectivity_cb" weak ref. */ - grpc_lb_policy_notify_on_state_change_locked(glb_policy->rr_policy, - &rr_connectivity->state, - &rr_connectivity->on_change); + update_lb_connectivity_status_locked( + glb_policy, glb_policy->rr_connectivity_state, GRPC_ERROR_REF(error)); + /* Resubscribe. Reuse the "glb_rr_connectivity_cb" ref. */ + grpc_lb_policy_notify_on_state_change_locked( + glb_policy->rr_policy, &glb_policy->rr_connectivity_state, + &glb_policy->on_rr_connectivity_changed); } static void destroy_balancer_name(void* balancer_name) { @@ -1001,38 +885,27 @@ static void glb_destroy(grpc_lb_policy* pol) { gpr_free(glb_policy); } -static void glb_shutdown_locked(grpc_lb_policy* pol) { +static void glb_shutdown_locked(grpc_lb_policy* pol, + grpc_lb_policy* new_policy) { glb_lb_policy* glb_policy = (glb_lb_policy*)pol; grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown"); glb_policy->shutting_down = true; - - /* We need a copy of the lb_call pointer because we can't cancell the call - * while holding glb_policy->mu: lb_on_server_status_received, invoked due to - * the cancel, needs to acquire that same lock */ - grpc_call* lb_call = glb_policy->lb_call; - /* glb_policy->lb_call and this local lb_call must be consistent at this point * because glb_policy->lb_call is only assigned in lb_call_init_locked as part * of query_for_backends_locked, which can only be invoked while * glb_policy->shutting_down is false. */ - if (lb_call != nullptr) { - grpc_call_cancel(lb_call, nullptr); + if (glb_policy->lb_call != nullptr) { + grpc_call_cancel(glb_policy->lb_call, nullptr); /* lb_on_server_status_received will pick up the cancel and clean up */ } - if (glb_policy->retry_timer_active) { + if (glb_policy->retry_timer_callback_pending) { grpc_timer_cancel(&glb_policy->lb_call_retry_timer); - glb_policy->retry_timer_active = false; } - if (glb_policy->fallback_timer_active) { + if (glb_policy->fallback_timer_callback_pending) { grpc_timer_cancel(&glb_policy->lb_fallback_timer); - glb_policy->fallback_timer_active = false; } - - pending_pick* pp = glb_policy->pending_picks; - glb_policy->pending_picks = nullptr; - pending_ping* pping = glb_policy->pending_pings; - glb_policy->pending_pings = nullptr; if (glb_policy->rr_policy != nullptr) { + grpc_lb_policy_shutdown_locked(glb_policy->rr_policy, nullptr); GRPC_LB_POLICY_UNREF(glb_policy->rr_policy, "glb_shutdown"); } else { grpc_lb_policy_try_reresolve(pol, &grpc_lb_glb_trace, GRPC_ERROR_CANCELLED); @@ -1047,28 +920,33 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) { } grpc_connectivity_state_set(&glb_policy->state_tracker, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error), "glb_shutdown"); - + // Clear pending picks. + pending_pick* pp = glb_policy->pending_picks; + glb_policy->pending_picks = nullptr; while (pp != nullptr) { pending_pick* next = pp->next; - *pp->target = nullptr; - GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure, - GRPC_ERROR_REF(error)); - gpr_free(pp); + if (new_policy != nullptr) { + // Hand pick over to new policy. + grpc_grpclb_client_stats_unref(pp->client_stats); + pp->pick->on_complete = pp->original_on_complete; + if (grpc_lb_policy_pick_locked(new_policy, pp->pick)) { + // Synchronous return; schedule callback. + GRPC_CLOSURE_SCHED(pp->pick->on_complete, GRPC_ERROR_NONE); + } + gpr_free(pp); + } else { + pp->pick->connected_subchannel = nullptr; + GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error)); + } pp = next; } - + // Clear pending pings. + pending_ping* pping = glb_policy->pending_pings; + glb_policy->pending_pings = nullptr; while (pping != nullptr) { pending_ping* next = pping->next; - if (pping->on_initiate != nullptr) { - GRPC_CLOSURE_SCHED(&pping->on_initiate->wrapper_closure, - GRPC_ERROR_REF(error)); - gpr_free(pping->on_initiate); - } - if (pping->on_ack != nullptr) { - GRPC_CLOSURE_SCHED(&pping->on_ack->wrapper_closure, - GRPC_ERROR_REF(error)); - gpr_free(pping->on_ack); - } + GRPC_CLOSURE_SCHED(pping->on_initiate, GRPC_ERROR_REF(error)); + GRPC_CLOSURE_SCHED(pping->on_ack, GRPC_ERROR_REF(error)); gpr_free(pping); pping = next; } @@ -1086,16 +964,16 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) { // level (grpclb), inside the glb_policy->pending_picks list. To cancel these, // we invoke the completion closure and set *target to nullptr right here. static void glb_cancel_pick_locked(grpc_lb_policy* pol, - grpc_connected_subchannel** target, + grpc_lb_policy_pick_state* pick, grpc_error* error) { glb_lb_policy* glb_policy = (glb_lb_policy*)pol; pending_pick* pp = glb_policy->pending_picks; glb_policy->pending_picks = nullptr; while (pp != nullptr) { pending_pick* next = pp->next; - if (pp->target == target) { - *target = nullptr; - GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure, + if (pp->pick == pick) { + pick->connected_subchannel = nullptr; + GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( "Pick Cancelled", &error, 1)); } else { @@ -1105,7 +983,7 @@ static void glb_cancel_pick_locked(grpc_lb_policy* pol, pp = next; } if (glb_policy->rr_policy != nullptr) { - grpc_lb_policy_cancel_pick_locked(glb_policy->rr_policy, target, + grpc_lb_policy_cancel_pick_locked(glb_policy->rr_policy, pick, GRPC_ERROR_REF(error)); } GRPC_ERROR_UNREF(error); @@ -1130,9 +1008,9 @@ static void glb_cancel_picks_locked(grpc_lb_policy* pol, glb_policy->pending_picks = nullptr; while (pp != nullptr) { pending_pick* next = pp->next; - if ((pp->pick_args.initial_metadata_flags & initial_metadata_flags_mask) == + if ((pp->pick->initial_metadata_flags & initial_metadata_flags_mask) == initial_metadata_flags_eq) { - GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure, + GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( "Pick Cancelled", &error, 1)); } else { @@ -1154,14 +1032,15 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy); static void start_picking_locked(glb_lb_policy* glb_policy) { /* start a timer to fall back */ if (glb_policy->lb_fallback_timeout_ms > 0 && - glb_policy->serverlist == nullptr && !glb_policy->fallback_timer_active) { + glb_policy->serverlist == nullptr && + !glb_policy->fallback_timer_callback_pending) { grpc_millis deadline = grpc_core::ExecCtx::Get()->Now() + glb_policy->lb_fallback_timeout_ms; - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer"); + GRPC_LB_POLICY_REF(&glb_policy->base, "grpclb_fallback_timer"); GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked, glb_policy, grpc_combiner_scheduler(glb_policy->base.combiner)); - glb_policy->fallback_timer_active = true; + glb_policy->fallback_timer_callback_pending = true; grpc_timer_init(&glb_policy->lb_fallback_timer, deadline, &glb_policy->lb_on_fallback); } @@ -1179,19 +1058,9 @@ static void glb_exit_idle_locked(grpc_lb_policy* pol) { } static int glb_pick_locked(grpc_lb_policy* pol, - const grpc_lb_policy_pick_args* pick_args, - grpc_connected_subchannel** target, - grpc_call_context_element* context, void** user_data, - grpc_closure* on_complete) { - if (pick_args->lb_token_mdelem_storage == nullptr) { - *target = nullptr; - GRPC_CLOSURE_SCHED(on_complete, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "No mdelem storage for the LB token. Load reporting " - "won't work without it. Failing")); - return 0; - } + grpc_lb_policy_pick_state* pick) { glb_lb_policy* glb_policy = (glb_lb_policy*)pol; + pending_pick* pp = pending_pick_create(glb_policy, pick); bool pick_done = false; if (glb_policy->rr_policy != nullptr) { const grpc_connectivity_state rr_connectivity_state = @@ -1199,7 +1068,7 @@ static int glb_pick_locked(grpc_lb_policy* pol, nullptr); // The glb_policy->rr_policy may have transitioned to SHUTDOWN but the // callback registered to capture this event - // (glb_rr_connectivity_changed_locked) may not have been invoked yet. We + // (on_rr_connectivity_changed_locked) may not have been invoked yet. We // need to make sure we aren't trying to pick from a RR policy instance // that's in shutdown. if (rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) { @@ -1209,32 +1078,16 @@ static int glb_pick_locked(grpc_lb_policy* pol, glb_policy, glb_policy->rr_policy, grpc_connectivity_state_name(rr_connectivity_state)); } - add_pending_pick(&glb_policy->pending_picks, pick_args, target, context, - on_complete); + pending_pick_add(&glb_policy->pending_picks, pp); pick_done = false; } else { // RR not in shutdown if (grpc_lb_glb_trace.enabled()) { gpr_log(GPR_INFO, "[grpclb %p] about to PICK from RR %p", glb_policy, glb_policy->rr_policy); } - GRPC_LB_POLICY_REF(glb_policy->rr_policy, "glb_pick"); - wrapped_rr_closure_arg* wc_arg = - (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(wrapped_rr_closure_arg)); - GRPC_CLOSURE_INIT(&wc_arg->wrapper_closure, wrapped_rr_closure, wc_arg, - grpc_schedule_on_exec_ctx); - wc_arg->rr_policy = glb_policy->rr_policy; - wc_arg->target = target; - wc_arg->context = context; GPR_ASSERT(glb_policy->client_stats != nullptr); - wc_arg->client_stats = - grpc_grpclb_client_stats_ref(glb_policy->client_stats); - wc_arg->wrapped_closure = on_complete; - wc_arg->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage; - wc_arg->initial_metadata = pick_args->initial_metadata; - wc_arg->free_when_done = wc_arg; - wc_arg->glb_policy = pol; - pick_done = pick_from_internal_rr_locked( - glb_policy, pick_args, false /* force_async */, target, wc_arg); + pick_done = + pick_from_internal_rr_locked(glb_policy, false /* force_async */, pp); } } else { // glb_policy->rr_policy == NULL if (grpc_lb_glb_trace.enabled()) { @@ -1242,8 +1095,7 @@ static int glb_pick_locked(grpc_lb_policy* pol, "[grpclb %p] No RR policy. Adding to grpclb's pending picks", glb_policy); } - add_pending_pick(&glb_policy->pending_picks, pick_args, target, context, - on_complete); + pending_pick_add(&glb_policy->pending_picks, pp); if (!glb_policy->started_picking) { start_picking_locked(glb_policy); } @@ -1265,7 +1117,7 @@ static void glb_ping_one_locked(grpc_lb_policy* pol, grpc_closure* on_initiate, if (glb_policy->rr_policy) { grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, on_initiate, on_ack); } else { - add_pending_ping(&glb_policy->pending_pings, on_initiate, on_ack); + pending_ping_add(&glb_policy->pending_pings, on_initiate, on_ack); if (!glb_policy->started_picking) { start_picking_locked(glb_policy); } @@ -1282,7 +1134,7 @@ static void glb_notify_on_state_change_locked(grpc_lb_policy* pol, static void lb_call_on_retry_timer_locked(void* arg, grpc_error* error) { glb_lb_policy* glb_policy = (glb_lb_policy*)arg; - glb_policy->retry_timer_active = false; + glb_policy->retry_timer_callback_pending = false; if (!glb_policy->shutting_down && glb_policy->lb_call == nullptr && error == GRPC_ERROR_NONE) { if (grpc_lb_glb_trace.enabled()) { @@ -1290,12 +1142,12 @@ static void lb_call_on_retry_timer_locked(void* arg, grpc_error* error) { } query_for_backends_locked(glb_policy); } - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "grpclb_retry_timer"); + GRPC_LB_POLICY_UNREF(&glb_policy->base, "grpclb_retry_timer"); } static void maybe_restart_lb_call(glb_lb_policy* glb_policy) { if (glb_policy->started_picking && glb_policy->updating_lb_call) { - if (glb_policy->retry_timer_active) { + if (glb_policy->retry_timer_callback_pending) { grpc_timer_cancel(&glb_policy->lb_call_retry_timer); } if (!glb_policy->shutting_down) start_picking_locked(glb_policy); @@ -1309,23 +1161,23 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) { grpc_millis timeout = next_try - grpc_core::ExecCtx::Get()->Now(); if (timeout > 0) { gpr_log(GPR_DEBUG, - "[grpclb %p] ... retry_timer_active in %" PRIuPTR "ms.", + "[grpclb %p] ... retry LB call after %" PRIuPTR "ms.", glb_policy, timeout); } else { - gpr_log(GPR_DEBUG, "[grpclb %p] ... retry_timer_active immediately.", + gpr_log(GPR_DEBUG, "[grpclb %p] ... retry LB call immediately.", glb_policy); } } - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_retry_timer"); + GRPC_LB_POLICY_REF(&glb_policy->base, "grpclb_retry_timer"); GRPC_CLOSURE_INIT(&glb_policy->lb_on_call_retry, lb_call_on_retry_timer_locked, glb_policy, grpc_combiner_scheduler(glb_policy->base.combiner)); - glb_policy->retry_timer_active = true; + glb_policy->retry_timer_callback_pending = true; grpc_timer_init(&glb_policy->lb_call_retry_timer, next_try, &glb_policy->lb_on_call_retry); } - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, - "lb_on_server_status_received_locked"); + GRPC_LB_POLICY_UNREF(&glb_policy->base, + "lb_on_server_status_received_locked"); } static void send_client_load_report_locked(void* arg, grpc_error* error); @@ -1347,8 +1199,8 @@ static void client_load_report_done_locked(void* arg, grpc_error* error) { grpc_byte_buffer_destroy(glb_policy->client_load_report_payload); glb_policy->client_load_report_payload = nullptr; if (error != GRPC_ERROR_NONE || glb_policy->lb_call == nullptr) { - glb_policy->client_load_report_timer_pending = false; - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report"); + glb_policy->client_load_report_timer_callback_pending = false; + GRPC_LB_POLICY_UNREF(&glb_policy->base, "client_load_report"); if (glb_policy->lb_call == nullptr) { maybe_restart_lb_call(glb_policy); } @@ -1357,6 +1209,22 @@ static void client_load_report_done_locked(void* arg, grpc_error* error) { schedule_next_client_load_report(glb_policy); } +static void do_send_client_load_report_locked(glb_lb_policy* glb_policy) { + grpc_op op; + memset(&op, 0, sizeof(op)); + op.op = GRPC_OP_SEND_MESSAGE; + op.data.send_message.send_message = glb_policy->client_load_report_payload; + GRPC_CLOSURE_INIT(&glb_policy->client_load_report_closure, + client_load_report_done_locked, glb_policy, + grpc_combiner_scheduler(glb_policy->base.combiner)); + grpc_call_error call_error = grpc_call_start_batch_and_execute( + glb_policy->lb_call, &op, 1, &glb_policy->client_load_report_closure); + if (call_error != GRPC_CALL_OK) { + gpr_log(GPR_ERROR, "[grpclb %p] call_error=%d", glb_policy, call_error); + GPR_ASSERT(GRPC_CALL_OK == call_error); + } +} + static bool load_report_counters_are_zero(grpc_grpclb_request* request) { grpc_grpclb_dropped_call_counts* drop_entries = (grpc_grpclb_dropped_call_counts*) @@ -1372,8 +1240,8 @@ static bool load_report_counters_are_zero(grpc_grpclb_request* request) { static void send_client_load_report_locked(void* arg, grpc_error* error) { glb_lb_policy* glb_policy = (glb_lb_policy*)arg; if (error == GRPC_ERROR_CANCELLED || glb_policy->lb_call == nullptr) { - glb_policy->client_load_report_timer_pending = false; - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report"); + glb_policy->client_load_report_timer_callback_pending = false; + GRPC_LB_POLICY_UNREF(&glb_policy->base, "client_load_report"); if (glb_policy->lb_call == nullptr) { maybe_restart_lb_call(glb_policy); } @@ -1400,22 +1268,15 @@ static void send_client_load_report_locked(void* arg, grpc_error* error) { grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_slice_unref_internal(request_payload_slice); grpc_grpclb_request_destroy(request); - // Send load report message. - grpc_op op; - memset(&op, 0, sizeof(op)); - op.op = GRPC_OP_SEND_MESSAGE; - op.data.send_message.send_message = glb_policy->client_load_report_payload; - GRPC_CLOSURE_INIT(&glb_policy->client_load_report_closure, - client_load_report_done_locked, glb_policy, - grpc_combiner_scheduler(glb_policy->base.combiner)); - grpc_call_error call_error = grpc_call_start_batch_and_execute( - glb_policy->lb_call, &op, 1, &glb_policy->client_load_report_closure); - if (call_error != GRPC_CALL_OK) { - gpr_log(GPR_ERROR, "[grpclb %p] call_error=%d", glb_policy, call_error); - GPR_ASSERT(GRPC_CALL_OK == call_error); + // If we've already sent the initial request, then we can go ahead and send + // the load report. Otherwise, we need to wait until the initial request has + // been sent to send this (see lb_on_sent_initial_request_locked() below). + if (glb_policy->initial_request_sent) { + do_send_client_load_report_locked(glb_policy); } } +static void lb_on_sent_initial_request_locked(void* arg, grpc_error* error); static void lb_on_server_status_received_locked(void* arg, grpc_error* error); static void lb_on_response_received_locked(void* arg, grpc_error* error); static void lb_call_init_locked(glb_lb_policy* glb_policy) { @@ -1455,6 +1316,9 @@ static void lb_call_init_locked(glb_lb_policy* glb_policy) { grpc_slice_unref_internal(request_payload_slice); grpc_grpclb_request_destroy(request); + GRPC_CLOSURE_INIT(&glb_policy->lb_on_sent_initial_request, + lb_on_sent_initial_request_locked, glb_policy, + grpc_combiner_scheduler(glb_policy->base.combiner)); GRPC_CLOSURE_INIT(&glb_policy->lb_on_server_status_received, lb_on_server_status_received_locked, glb_policy, grpc_combiner_scheduler(glb_policy->base.combiner)); @@ -1471,6 +1335,7 @@ static void lb_call_init_locked(glb_lb_policy* glb_policy) { glb_policy->lb_call_backoff.Init(backoff_options); + glb_policy->initial_request_sent = false; glb_policy->seen_initial_response = false; glb_policy->last_client_load_report_counters_were_zero = false; } @@ -1486,7 +1351,7 @@ static void lb_call_destroy_locked(glb_lb_policy* glb_policy) { grpc_byte_buffer_destroy(glb_policy->lb_request_payload); grpc_slice_unref_internal(glb_policy->lb_call_status_details); - if (glb_policy->client_load_report_timer_pending) { + if (glb_policy->client_load_report_timer_callback_pending) { grpc_timer_cancel(&glb_policy->client_load_report_timer); } } @@ -1529,8 +1394,11 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) { op->flags = 0; op->reserved = nullptr; op++; - call_error = grpc_call_start_batch_and_execute(glb_policy->lb_call, ops, - (size_t)(op - ops), nullptr); + /* take a ref to be released in lb_on_sent_initial_request_locked() */ + GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_sent_initial_request_locked"); + call_error = grpc_call_start_batch_and_execute( + glb_policy->lb_call, ops, (size_t)(op - ops), + &glb_policy->lb_on_sent_initial_request); GPR_ASSERT(GRPC_CALL_OK == call_error); op = ops; @@ -1543,10 +1411,8 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) { op->flags = 0; op->reserved = nullptr; op++; - /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref - * count goes to zero) to be unref'd in lb_on_server_status_received_locked */ - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, - "lb_on_server_status_received_locked"); + /* take a ref to be released in lb_on_server_status_received_locked() */ + GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_server_status_received_locked"); call_error = grpc_call_start_batch_and_execute( glb_policy->lb_call, ops, (size_t)(op - ops), &glb_policy->lb_on_server_status_received); @@ -1558,15 +1424,25 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) { op->flags = 0; op->reserved = nullptr; op++; - /* take another weak ref to be unref'd/reused in - * lb_on_response_received_locked */ - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "lb_on_response_received_locked"); + /* take a ref to be unref'd/reused in lb_on_response_received_locked() */ + GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_response_received_locked"); call_error = grpc_call_start_batch_and_execute( glb_policy->lb_call, ops, (size_t)(op - ops), &glb_policy->lb_on_response_received); GPR_ASSERT(GRPC_CALL_OK == call_error); } +static void lb_on_sent_initial_request_locked(void* arg, grpc_error* error) { + glb_lb_policy* glb_policy = (glb_lb_policy*)arg; + glb_policy->initial_request_sent = true; + // If we attempted to send a client load report before the initial request was + // sent, send the load report now. + if (glb_policy->client_load_report_payload != nullptr) { + do_send_client_load_report_locked(glb_policy); + } + GRPC_LB_POLICY_UNREF(&glb_policy->base, "lb_on_sent_initial_request_locked"); +} + static void lb_on_response_received_locked(void* arg, grpc_error* error) { glb_lb_policy* glb_policy = (glb_lb_policy*)arg; grpc_op ops[2]; @@ -1596,11 +1472,9 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) { "client load reporting interval = %" PRIdPTR " milliseconds", glb_policy, glb_policy->client_stats_report_interval); } - /* take a weak ref (won't prevent calling of \a glb_shutdown() if the - * strong ref count goes to zero) to be unref'd in - * send_client_load_report_locked() */ - glb_policy->client_load_report_timer_pending = true; - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "client_load_report"); + /* take a ref to be unref'd in send_client_load_report_locked() */ + glb_policy->client_load_report_timer_callback_pending = true; + GRPC_LB_POLICY_REF(&glb_policy->base, "client_load_report"); schedule_next_client_load_report(glb_policy); } else if (grpc_lb_glb_trace.enabled()) { gpr_log(GPR_INFO, @@ -1648,9 +1522,8 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) { /* or dispose of the fallback */ grpc_lb_addresses_destroy(glb_policy->fallback_backend_addresses); glb_policy->fallback_backend_addresses = nullptr; - if (glb_policy->fallback_timer_active) { + if (glb_policy->fallback_timer_callback_pending) { grpc_timer_cancel(&glb_policy->lb_fallback_timer); - glb_policy->fallback_timer_active = false; } } /* and update the copy in the glb_lb_policy instance. This @@ -1683,27 +1556,27 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) { op->flags = 0; op->reserved = nullptr; op++; - /* reuse the "lb_on_response_received_locked" weak ref taken in + /* reuse the "lb_on_response_received_locked" ref taken in * query_for_backends_locked() */ const grpc_call_error call_error = grpc_call_start_batch_and_execute( glb_policy->lb_call, ops, (size_t)(op - ops), &glb_policy->lb_on_response_received); /* loop */ GPR_ASSERT(GRPC_CALL_OK == call_error); } else { - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, - "lb_on_response_received_locked_shutdown"); + GRPC_LB_POLICY_UNREF(&glb_policy->base, + "lb_on_response_received_locked_shutdown"); } } else { /* empty payload: call cancelled. */ - /* dispose of the "lb_on_response_received_locked" weak ref taken in + /* dispose of the "lb_on_response_received_locked" ref taken in * query_for_backends_locked() and reused in every reception loop */ - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, - "lb_on_response_received_locked_empty_payload"); + GRPC_LB_POLICY_UNREF(&glb_policy->base, + "lb_on_response_received_locked_empty_payload"); } } static void lb_on_fallback_timer_locked(void* arg, grpc_error* error) { glb_lb_policy* glb_policy = (glb_lb_policy*)arg; - glb_policy->fallback_timer_active = false; + glb_policy->fallback_timer_callback_pending = false; /* If we receive a serverlist after the timer fires but before this callback * actually runs, don't fall back. */ if (glb_policy->serverlist == nullptr) { @@ -1717,7 +1590,7 @@ static void lb_on_fallback_timer_locked(void* arg, grpc_error* error) { rr_handover_locked(glb_policy); } } - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "grpclb_fallback_timer"); + GRPC_LB_POLICY_UNREF(&glb_policy->base, "grpclb_fallback_timer"); } static void lb_on_server_status_received_locked(void* arg, grpc_error* error) { @@ -1738,7 +1611,7 @@ static void lb_on_server_status_received_locked(void* arg, grpc_error* error) { // If the load report timer is still pending, we wait for it to be // called before restarting the call. Otherwise, we restart the call // here. - if (!glb_policy->client_load_report_timer_pending) { + if (!glb_policy->client_load_report_timer_callback_pending) { maybe_restart_lb_call(glb_policy); } } @@ -1801,7 +1674,7 @@ static void glb_update_locked(grpc_lb_policy* policy, grpc_channel_get_channel_stack(glb_policy->lb_channel)); GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter); glb_policy->watching_lb_channel = true; - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "watch_lb_channel_connectivity"); + GRPC_LB_POLICY_REF(&glb_policy->base, "watch_lb_channel_connectivity"); grpc_client_channel_watch_connectivity_state( client_channel_elem, grpc_polling_entity_create_from_pollset_set( @@ -1848,9 +1721,8 @@ static void glb_lb_channel_on_connectivity_changed_cb(void* arg, // lb_on_server_status_received() will pick up the cancel and reinit // lb_call. } else if (glb_policy->started_picking) { - if (glb_policy->retry_timer_active) { + if (glb_policy->retry_timer_callback_pending) { grpc_timer_cancel(&glb_policy->lb_call_retry_timer); - glb_policy->retry_timer_active = false; } start_picking_locked(glb_policy); } @@ -1858,8 +1730,8 @@ static void glb_lb_channel_on_connectivity_changed_cb(void* arg, case GRPC_CHANNEL_SHUTDOWN: done: glb_policy->watching_lb_channel = false; - GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, - "watch_lb_channel_connectivity_cb_shutdown"); + GRPC_LB_POLICY_UNREF(&glb_policy->base, + "watch_lb_channel_connectivity_cb_shutdown"); break; } } diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 0861261359..60385272cf 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -31,13 +31,6 @@ grpc_core::TraceFlag grpc_lb_pick_first_trace(false, "pick_first"); -typedef struct pending_pick { - struct pending_pick* next; - uint32_t initial_metadata_flags; - grpc_connected_subchannel** target; - grpc_closure* on_complete; -} pending_pick; - typedef struct { /** base policy: must be first */ grpc_lb_policy base; @@ -52,7 +45,7 @@ typedef struct { /** are we shut down? */ bool shutdown; /** list of picks that are waiting on connectivity */ - pending_pick* pending_picks; + grpc_lb_policy_pick_state* pending_picks; /** our connectivity state tracker */ grpc_connectivity_state_tracker state_tracker; } pick_first_lb_policy; @@ -70,19 +63,27 @@ static void pf_destroy(grpc_lb_policy* pol) { } } -static void pf_shutdown_locked(grpc_lb_policy* pol) { +static void pf_shutdown_locked(grpc_lb_policy* pol, + grpc_lb_policy* new_policy) { pick_first_lb_policy* p = (pick_first_lb_policy*)pol; grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown"); if (grpc_lb_pick_first_trace.enabled()) { gpr_log(GPR_DEBUG, "Pick First %p Shutting down", p); } p->shutdown = true; - pending_pick* pp; - while ((pp = p->pending_picks) != nullptr) { - p->pending_picks = pp->next; - *pp->target = nullptr; - GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_REF(error)); - gpr_free(pp); + grpc_lb_policy_pick_state* pick; + while ((pick = p->pending_picks) != nullptr) { + p->pending_picks = pick->next; + if (new_policy != nullptr) { + // Hand off to new LB policy. + if (grpc_lb_policy_pick_locked(new_policy, pick)) { + // Synchronous return, schedule closure. + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); + } + } else { + pick->connected_subchannel = nullptr; + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error)); + } } grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error), "shutdown"); @@ -102,19 +103,18 @@ static void pf_shutdown_locked(grpc_lb_policy* pol) { } static void pf_cancel_pick_locked(grpc_lb_policy* pol, - grpc_connected_subchannel** target, + grpc_lb_policy_pick_state* pick, grpc_error* error) { pick_first_lb_policy* p = (pick_first_lb_policy*)pol; - pending_pick* pp = p->pending_picks; + grpc_lb_policy_pick_state* pp = p->pending_picks; p->pending_picks = nullptr; while (pp != nullptr) { - pending_pick* next = pp->next; - if (pp->target == target) { - *target = nullptr; - GRPC_CLOSURE_SCHED(pp->on_complete, + grpc_lb_policy_pick_state* next = pp->next; + if (pp == pick) { + pick->connected_subchannel = nullptr; + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( "Pick Cancelled", &error, 1)); - gpr_free(pp); } else { pp->next = p->pending_picks; p->pending_picks = pp; @@ -129,21 +129,20 @@ static void pf_cancel_picks_locked(grpc_lb_policy* pol, uint32_t initial_metadata_flags_eq, grpc_error* error) { pick_first_lb_policy* p = (pick_first_lb_policy*)pol; - pending_pick* pp = p->pending_picks; + grpc_lb_policy_pick_state* pick = p->pending_picks; p->pending_picks = nullptr; - while (pp != nullptr) { - pending_pick* next = pp->next; - if ((pp->initial_metadata_flags & initial_metadata_flags_mask) == + while (pick != nullptr) { + grpc_lb_policy_pick_state* next = pick->next; + if ((pick->initial_metadata_flags & initial_metadata_flags_mask) == initial_metadata_flags_eq) { - GRPC_CLOSURE_SCHED(pp->on_complete, + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( "Pick Cancelled", &error, 1)); - gpr_free(pp); } else { - pp->next = p->pending_picks; - p->pending_picks = pp; + pick->next = p->pending_picks; + p->pending_picks = pick; } - pp = next; + pick = next; } GRPC_ERROR_UNREF(error); } @@ -173,27 +172,20 @@ static void pf_exit_idle_locked(grpc_lb_policy* pol) { } static int pf_pick_locked(grpc_lb_policy* pol, - const grpc_lb_policy_pick_args* pick_args, - grpc_connected_subchannel** target, - grpc_call_context_element* context, void** user_data, - grpc_closure* on_complete) { + grpc_lb_policy_pick_state* pick) { pick_first_lb_policy* p = (pick_first_lb_policy*)pol; // If we have a selected subchannel already, return synchronously. if (p->selected != nullptr) { - *target = GRPC_CONNECTED_SUBCHANNEL_REF(p->selected->connected_subchannel, - "picked"); + pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF( + p->selected->connected_subchannel, "picked"); return 1; } // No subchannel selected yet, so handle asynchronously. if (!p->started_picking) { start_picking_locked(p); } - pending_pick* pp = (pending_pick*)gpr_malloc(sizeof(*pp)); - pp->next = p->pending_picks; - pp->target = target; - pp->initial_metadata_flags = pick_args->initial_metadata_flags; - pp->on_complete = on_complete; - p->pending_picks = pp; + pick->next = p->pending_picks; + p->pending_picks = pick; return 0; } @@ -479,18 +471,17 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) { // Drop all other subchannels, since we are now connected. destroy_unselected_subchannels_locked(p); // Update any calls that were waiting for a pick. - pending_pick* pp; - while ((pp = p->pending_picks)) { - p->pending_picks = pp->next; - *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF( + grpc_lb_policy_pick_state* pick; + while ((pick = p->pending_picks)) { + p->pending_picks = pick->next; + pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF( p->selected->connected_subchannel, "picked"); if (grpc_lb_pick_first_trace.enabled()) { gpr_log(GPR_INFO, "Servicing pending pick with selected subchannel %p", (void*)p->selected); } - GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_NONE); - gpr_free(pp); + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); } // Renew notification. grpc_lb_subchannel_data_start_connectivity_watch(sd); diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index b0c84017df..92c7d5bd5d 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -41,29 +41,6 @@ grpc_core::TraceFlag grpc_lb_round_robin_trace(false, "round_robin"); -/** List of entities waiting for a pick. - * - * Once a pick is available, \a target is updated and \a on_complete called. */ -typedef struct pending_pick { - struct pending_pick* next; - - /* output argument where to store the pick()ed user_data. It'll be NULL if no - * such data is present or there's an error (the definite test for errors is - * \a target being NULL). */ - void** user_data; - - /* bitmask passed to pick() and used for selective cancelling. See - * grpc_lb_policy_cancel_picks() */ - uint32_t initial_metadata_flags; - - /* output argument where to store the pick()ed connected subchannel, or NULL - * upon error. */ - grpc_connected_subchannel** target; - - /* to be invoked once the pick() has completed (regardless of success) */ - grpc_closure* on_complete; -} pending_pick; - typedef struct round_robin_lb_policy { /** base policy: must be first */ grpc_lb_policy base; @@ -75,7 +52,7 @@ typedef struct round_robin_lb_policy { /** are we shutting down? */ bool shutdown; /** List of picks that are waiting on connectivity */ - pending_pick* pending_picks; + grpc_lb_policy_pick_state* pending_picks; /** our connectivity state tracker */ grpc_connectivity_state_tracker state_tracker; @@ -167,19 +144,27 @@ static void rr_destroy(grpc_lb_policy* pol) { gpr_free(p); } -static void rr_shutdown_locked(grpc_lb_policy* pol) { +static void rr_shutdown_locked(grpc_lb_policy* pol, + grpc_lb_policy* new_policy) { round_robin_lb_policy* p = (round_robin_lb_policy*)pol; grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown"); if (grpc_lb_round_robin_trace.enabled()) { gpr_log(GPR_DEBUG, "[RR %p] Shutting down", p); } p->shutdown = true; - pending_pick* pp; - while ((pp = p->pending_picks) != nullptr) { - p->pending_picks = pp->next; - *pp->target = nullptr; - GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_REF(error)); - gpr_free(pp); + grpc_lb_policy_pick_state* pick; + while ((pick = p->pending_picks) != nullptr) { + p->pending_picks = pick->next; + if (new_policy != nullptr) { + // Hand off to new LB policy. + if (grpc_lb_policy_pick_locked(new_policy, pick)) { + // Synchronous return; schedule callback. + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); + } + } else { + pick->connected_subchannel = nullptr; + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error)); + } } grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error), "rr_shutdown"); @@ -199,19 +184,18 @@ static void rr_shutdown_locked(grpc_lb_policy* pol) { } static void rr_cancel_pick_locked(grpc_lb_policy* pol, - grpc_connected_subchannel** target, + grpc_lb_policy_pick_state* pick, grpc_error* error) { round_robin_lb_policy* p = (round_robin_lb_policy*)pol; - pending_pick* pp = p->pending_picks; + grpc_lb_policy_pick_state* pp = p->pending_picks; p->pending_picks = nullptr; while (pp != nullptr) { - pending_pick* next = pp->next; - if (pp->target == target) { - *target = nullptr; - GRPC_CLOSURE_SCHED(pp->on_complete, + grpc_lb_policy_pick_state* next = pp->next; + if (pp == pick) { + pick->connected_subchannel = nullptr; + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( "Pick cancelled", &error, 1)); - gpr_free(pp); } else { pp->next = p->pending_picks; p->pending_picks = pp; @@ -226,22 +210,21 @@ static void rr_cancel_picks_locked(grpc_lb_policy* pol, uint32_t initial_metadata_flags_eq, grpc_error* error) { round_robin_lb_policy* p = (round_robin_lb_policy*)pol; - pending_pick* pp = p->pending_picks; + grpc_lb_policy_pick_state* pick = p->pending_picks; p->pending_picks = nullptr; - while (pp != nullptr) { - pending_pick* next = pp->next; - if ((pp->initial_metadata_flags & initial_metadata_flags_mask) == + while (pick != nullptr) { + grpc_lb_policy_pick_state* next = pick->next; + if ((pick->initial_metadata_flags & initial_metadata_flags_mask) == initial_metadata_flags_eq) { - *pp->target = nullptr; - GRPC_CLOSURE_SCHED(pp->on_complete, + pick->connected_subchannel = nullptr; + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( "Pick cancelled", &error, 1)); - gpr_free(pp); } else { - pp->next = p->pending_picks; - p->pending_picks = pp; + pick->next = p->pending_picks; + p->pending_picks = pick; } - pp = next; + pick = next; } GRPC_ERROR_UNREF(error); } @@ -266,13 +249,10 @@ static void rr_exit_idle_locked(grpc_lb_policy* pol) { } static int rr_pick_locked(grpc_lb_policy* pol, - const grpc_lb_policy_pick_args* pick_args, - grpc_connected_subchannel** target, - grpc_call_context_element* context, void** user_data, - grpc_closure* on_complete) { + grpc_lb_policy_pick_state* pick) { round_robin_lb_policy* p = (round_robin_lb_policy*)pol; if (grpc_lb_round_robin_trace.enabled()) { - gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", (void*)pol, + gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", pol, p->shutdown); } GPR_ASSERT(!p->shutdown); @@ -282,18 +262,18 @@ static int rr_pick_locked(grpc_lb_policy* pol, /* readily available, report right away */ grpc_lb_subchannel_data* sd = &p->subchannel_list->subchannels[next_ready_index]; - *target = + pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(sd->connected_subchannel, "rr_picked"); - if (user_data != nullptr) { - *user_data = sd->user_data; + if (pick->user_data != nullptr) { + *pick->user_data = sd->user_data; } if (grpc_lb_round_robin_trace.enabled()) { gpr_log( GPR_DEBUG, "[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, " - "index %lu)", - (void*)p, (void*)sd->subchannel, (void*)*target, - (void*)sd->subchannel_list, (unsigned long)next_ready_index); + "index %" PRIuPTR ")", + p, sd->subchannel, pick->connected_subchannel, sd->subchannel_list, + next_ready_index); } /* only advance the last picked pointer if the selection was used */ update_last_ready_subchannel_index_locked(p, next_ready_index); @@ -304,13 +284,8 @@ static int rr_pick_locked(grpc_lb_policy* pol, if (!p->started_picking) { start_picking_locked(p); } - pending_pick* pp = (pending_pick*)gpr_malloc(sizeof(*pp)); - pp->next = p->pending_picks; - pp->target = target; - pp->on_complete = on_complete; - pp->initial_metadata_flags = pick_args->initial_metadata_flags; - pp->user_data = user_data; - p->pending_picks = pp; + pick->next = p->pending_picks; + p->pending_picks = pick; return 0; } @@ -493,13 +468,13 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) { // picks, update the last picked pointer update_last_ready_subchannel_index_locked(p, next_ready_index); } - pending_pick* pp; - while ((pp = p->pending_picks)) { - p->pending_picks = pp->next; - *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF( + grpc_lb_policy_pick_state* pick; + while ((pick = p->pending_picks)) { + p->pending_picks = pick->next; + pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF( selected->connected_subchannel, "rr_picked"); - if (pp->user_data != nullptr) { - *pp->user_data = selected->user_data; + if (pick->user_data != nullptr) { + *pick->user_data = selected->user_data; } if (grpc_lb_round_robin_trace.enabled()) { gpr_log(GPR_DEBUG, @@ -508,8 +483,7 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) { (void*)p, (void*)selected->subchannel, (void*)p->subchannel_list, (unsigned long)next_ready_index); } - GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_NONE); - gpr_free(pp); + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); } } // Renew notification. diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc index a3b4c8e524..5ce1298afc 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc @@ -213,13 +213,13 @@ void grpc_lb_subchannel_list_unref(grpc_lb_subchannel_list* subchannel_list, void grpc_lb_subchannel_list_ref_for_connectivity_watch( grpc_lb_subchannel_list* subchannel_list, const char* reason) { - GRPC_LB_POLICY_WEAK_REF(subchannel_list->policy, reason); + GRPC_LB_POLICY_REF(subchannel_list->policy, reason); grpc_lb_subchannel_list_ref(subchannel_list, reason); } void grpc_lb_subchannel_list_unref_for_connectivity_watch( grpc_lb_subchannel_list* subchannel_list, const char* reason) { - GRPC_LB_POLICY_WEAK_UNREF(subchannel_list->policy, reason); + GRPC_LB_POLICY_UNREF(subchannel_list->policy, reason); grpc_lb_subchannel_list_unref(subchannel_list, reason); } diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 02bcba0e73..f4b6c2a161 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -61,11 +61,13 @@ ((grpc_connected_subchannel*)(gpr_atm_##barrier##_load( \ &(subchannel)->connected_subchannel))) -typedef struct { +namespace { +struct state_watcher { grpc_closure closure; grpc_subchannel* subchannel; grpc_connectivity_state connectivity_state; -} state_watcher; +}; +} // namespace typedef struct external_state_watcher { grpc_subchannel* subchannel; diff --git a/src/core/ext/filters/http/client/http_client_filter.cc b/src/core/ext/filters/http/client/http_client_filter.cc index a1fb10f5b8..6dbd8c2a6d 100644 --- a/src/core/ext/filters/http/client/http_client_filter.cc +++ b/src/core/ext/filters/http/client/http_client_filter.cc @@ -35,7 +35,8 @@ /* default maximum size of payload eligable for GET request */ static const size_t kMaxPayloadSizeForGet = 2048; -typedef struct call_data { +namespace { +struct call_data { grpc_call_combiner* call_combiner; // State for handling send_initial_metadata ops. grpc_linked_mdelem method; @@ -60,13 +61,14 @@ typedef struct call_data { grpc_closure on_send_message_next_done; grpc_closure* original_send_message_on_complete; grpc_closure send_message_on_complete; -} call_data; +}; -typedef struct channel_data { +struct channel_data { grpc_mdelem static_scheme; grpc_mdelem user_agent; size_t max_payload_size_for_get; -} channel_data; +}; +} // namespace static grpc_error* client_filter_incoming_metadata(grpc_call_element* elem, grpc_metadata_batch* b) { diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.cc b/src/core/ext/filters/http/message_compress/message_compress_filter.cc index 9ae13d2ed2..92d1716200 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.cc @@ -35,16 +35,17 @@ #include "src/core/lib/surface/call.h" #include "src/core/lib/transport/static_metadata.h" -typedef enum { +namespace { +enum initial_metadata_state { // Initial metadata not yet seen. INITIAL_METADATA_UNSEEN = 0, // Initial metadata seen; compression algorithm set. HAS_COMPRESSION_ALGORITHM, // Initial metadata seen; no compression algorithm set. NO_COMPRESSION_ALGORITHM, -} initial_metadata_state; +}; -typedef struct call_data { +struct call_data { grpc_call_combiner* call_combiner; grpc_linked_mdelem compression_algorithm_storage; grpc_linked_mdelem stream_compression_algorithm_storage; @@ -62,9 +63,9 @@ typedef struct call_data { grpc_closure* original_send_message_on_complete; grpc_closure send_message_on_complete; grpc_closure on_send_message_next_done; -} call_data; +}; -typedef struct channel_data { +struct channel_data { /** The default, channel-level, compression algorithm */ grpc_compression_algorithm default_compression_algorithm; /** Bitset of enabled algorithms */ @@ -78,7 +79,8 @@ typedef struct channel_data { uint32_t enabled_stream_compression_algorithms_bitset; /** Supported stream compression algorithms */ uint32_t supported_stream_compression_algorithms; -} channel_data; +}; +} // namespace static bool skip_compression(grpc_call_element* elem, uint32_t flags, bool has_compression_algorithm) { diff --git a/src/core/ext/filters/http/server/http_server_filter.cc b/src/core/ext/filters/http/server/http_server_filter.cc index b872dc98f5..508a3bf9fc 100644 --- a/src/core/ext/filters/http/server/http_server_filter.cc +++ b/src/core/ext/filters/http/server/http_server_filter.cc @@ -31,7 +31,8 @@ #define EXPECTED_CONTENT_TYPE "application/grpc" #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1 -typedef struct call_data { +namespace { +struct call_data { grpc_call_combiner* call_combiner; grpc_linked_mdelem status; @@ -60,11 +61,12 @@ typedef struct call_data { grpc_closure hs_on_recv; grpc_closure hs_on_complete; grpc_closure hs_recv_message_ready; -} call_data; +}; -typedef struct channel_data { +struct channel_data { uint8_t unused; -} channel_data; +}; +} // namespace static grpc_error* server_filter_outgoing_metadata(grpc_call_element* elem, grpc_metadata_batch* b) { diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc index f50a928fcd..a414229768 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc +++ b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc @@ -31,7 +31,8 @@ #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/transport/static_metadata.h" -typedef struct call_data { +namespace { +struct call_data { intptr_t id; /**< an id unique to the call */ bool have_trailing_md_string; grpc_slice trailing_md_string; @@ -48,11 +49,12 @@ typedef struct call_data { /* to get notified of the availability of the incoming initial metadata. */ grpc_closure on_initial_md_ready; grpc_metadata_batch* recv_initial_metadata; -} call_data; +}; -typedef struct channel_data { +struct channel_data { intptr_t id; /**< an id unique to the channel */ -} channel_data; +}; +} // namespace static void on_initial_md_ready(void* user_data, grpc_error* err) { grpc_call_element* elem = (grpc_call_element*)user_data; diff --git a/src/core/ext/filters/max_age/max_age_filter.cc b/src/core/ext/filters/max_age/max_age_filter.cc index 0499c6ecfc..7b86e4cd6c 100644 --- a/src/core/ext/filters/max_age/max_age_filter.cc +++ b/src/core/ext/filters/max_age/max_age_filter.cc @@ -37,7 +37,8 @@ #define MAX_CONNECTION_IDLE_INTEGER_OPTIONS \ { DEFAULT_MAX_CONNECTION_IDLE_MS, 1, INT_MAX } -typedef struct channel_data { +namespace { +struct channel_data { /* We take a reference to the channel stack for the timer callback */ grpc_channel_stack* channel_stack; /* Guards access to max_age_timer, max_age_timer_pending, max_age_grace_timer @@ -84,7 +85,8 @@ typedef struct channel_data { grpc_connectivity_state connectivity_state; /* Number of active calls */ gpr_atm call_count; -} channel_data; +}; +} // namespace /* Increase the nubmer of active calls. Before the increasement, if there are no calls, the max_idle_timer should be cancelled. */ diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index f8487f9a9e..3cb7b136c0 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -86,7 +86,8 @@ static void* refcounted_message_size_limits_create_from_json( return value; } -typedef struct call_data { +namespace { +struct call_data { grpc_call_combiner* call_combiner; message_size_limits limits; // Receive closures are chained: we inject this closure as the @@ -97,13 +98,14 @@ typedef struct call_data { grpc_byte_stream** recv_message; // Original recv_message_ready callback, invoked after our own. grpc_closure* next_recv_message_ready; -} call_data; +}; -typedef struct channel_data { +struct channel_data { message_size_limits limits; // Maps path names to refcounted_message_size_limits structs. grpc_slice_hash_table* method_limit_table; -} channel_data; +}; +} // namespace // Callback invoked when we receive a message. Here we check the max // receive message size. diff --git a/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc b/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc index 555a9134a2..88bb8c71cc 100644 --- a/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc +++ b/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc @@ -25,7 +25,8 @@ #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/transport/metadata.h" -typedef struct call_data { +namespace { +struct call_data { // Receive closures are chained: we inject this closure as the // recv_initial_metadata_ready up-call on transport_stream_op, and remember to // call our next_recv_initial_metadata_ready member after handling it. @@ -37,7 +38,8 @@ typedef struct call_data { // Marks whether the workaround is active bool workaround_active; -} call_data; +}; +} // namespace // Find the user agent metadata element in the batch static bool get_user_agent_mdelem(const grpc_metadata_batch* batch, diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc index 043ca9bb83..9a6f5e9bcf 100644 --- a/src/core/ext/transport/chttp2/transport/writing.cc +++ b/src/core/ext/transport/chttp2/transport/writing.cc @@ -138,10 +138,11 @@ static void report_stall(grpc_chttp2_transport* t, grpc_chttp2_stream* s, const char* staller) { gpr_log( GPR_DEBUG, - "%s:%p stream %d stalled by %s [fc:pending=%" PRIdPTR ":flowed=%" PRId64 + "%s:%p stream %d stalled by %s [fc:pending=%" PRIdPTR + ":pending-compressed=%" PRIdPTR ":flowed=%" PRId64 ":peer_initwin=%d:t_win=%" PRId64 ":s_win=%d:s_delta=%" PRId64 "]", t->peer_string, t, s->id, staller, s->flow_controlled_buffer.length, - s->flow_controlled_bytes_flowed, + s->compressed_data_buffer.length, s->flow_controlled_bytes_flowed, t->settings[GRPC_ACKED_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], t->flow_control->remote_window(), diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc index 92b4b8b90b..1ab7e516de 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.cc +++ b/src/core/lib/iomgr/ev_epoll1_linux.cc @@ -1232,8 +1232,6 @@ const grpc_event_engine_vtable* grpc_init_epoll1_linux(bool explicit_request) { /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ const grpc_event_engine_vtable* grpc_init_epoll1_linux(bool explicit_request) { - gpr_log(GPR_ERROR, - "Skipping epoll1 because GRPC_LINUX_EPOLL is not defined."); return nullptr; } #endif /* defined(GRPC_POSIX_SOCKET) */ diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc index df2f629703..5f5f45a7a5 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.cc +++ b/src/core/lib/iomgr/ev_epollex_linux.cc @@ -1449,8 +1449,6 @@ const grpc_event_engine_vtable* grpc_init_epollex_linux( * NULL */ const grpc_event_engine_vtable* grpc_init_epollex_linux( bool explicitly_requested) { - gpr_log(GPR_ERROR, - "Skipping epollex because GRPC_LINUX_EPOLL is not defined."); return nullptr; } #endif /* defined(GRPC_POSIX_SOCKET) */ diff --git a/src/core/lib/iomgr/ev_epollsig_linux.cc b/src/core/lib/iomgr/ev_epollsig_linux.cc index bc548a1fda..8072a6cbed 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.cc +++ b/src/core/lib/iomgr/ev_epollsig_linux.cc @@ -1732,8 +1732,6 @@ const grpc_event_engine_vtable* grpc_init_epollsig_linux( * NULL */ const grpc_event_engine_vtable* grpc_init_epollsig_linux( bool explicit_request) { - gpr_log(GPR_ERROR, - "Skipping epollsig because GRPC_LINUX_EPOLL is not defined."); return nullptr; } #endif /* defined(GRPC_POSIX_SOCKET) */ diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc index 53de94fb6e..08edff5159 100644 --- a/src/core/lib/iomgr/ev_poll_posix.cc +++ b/src/core/lib/iomgr/ev_poll_posix.cc @@ -71,6 +71,7 @@ struct grpc_fd { int shutdown; int closed; int released; + gpr_atm pollhup; grpc_error* shutdown_error; /* The watcher list. @@ -242,7 +243,7 @@ struct grpc_pollset_set { typedef struct poll_result { gpr_refcount refcount; - cv_node* watchers; + grpc_cv_node* watchers; int watchcount; struct pollfd* fds; nfds_t nfds; @@ -273,7 +274,7 @@ typedef struct poll_hash_table { } poll_hash_table; poll_hash_table poll_cache; -cv_fd_table g_cvfds; +grpc_cv_fd_table g_cvfds; /******************************************************************************* * fd_posix.c @@ -335,6 +336,7 @@ static grpc_fd* fd_create(int fd, const char* name) { r->on_done_closure = nullptr; r->closed = 0; r->released = 0; + gpr_atm_no_barrier_store(&r->pollhup, 0); r->read_notifier_pollset = nullptr; char* name2; @@ -462,7 +464,7 @@ static grpc_error* fd_shutdown_error(grpc_fd* fd) { static void notify_on_locked(grpc_fd* fd, grpc_closure** st, grpc_closure* closure) { - if (fd->shutdown) { + if (fd->shutdown || gpr_atm_no_barrier_load(&fd->pollhup)) { GRPC_CLOSURE_SCHED(closure, GRPC_ERROR_CREATE_FROM_STATIC_STRING("FD shutdown")); } else if (*st == CLOSURE_NOT_READY) { @@ -950,7 +952,8 @@ static grpc_error* pollset_work(grpc_pollset* pollset, pfds[0].events = POLLIN; pfds[0].revents = 0; for (i = 0; i < pollset->fd_count; i++) { - if (fd_is_orphaned(pollset->fds[i])) { + if (fd_is_orphaned(pollset->fds[i]) || + gpr_atm_no_barrier_load(&pollset->fds[i]->pollhup) == 1) { GRPC_FD_UNREF(pollset->fds[i], "multipoller"); } else { pollset->fds[fd_count++] = pollset->fds[i]; @@ -1017,6 +1020,12 @@ static grpc_error* pollset_work(grpc_pollset* pollset, pfds[i].fd, (pfds[i].revents & POLLIN_CHECK) != 0, (pfds[i].revents & POLLOUT_CHECK) != 0, pfds[i].revents); } + /* This is a mitigation to prevent poll() from spinning on a + ** POLLHUP https://github.com/grpc/grpc/pull/13665 + */ + if (pfds[i].revents & POLLHUP) { + gpr_atm_no_barrier_store(&watchers[i].fd->pollhup, 1); + } fd_end_poll(&watchers[i], pfds[i].revents & POLLIN_CHECK, pfds[i].revents & POLLOUT_CHECK, pollset); } @@ -1435,7 +1444,7 @@ static void decref_poll_result(poll_result* res) { } } -void remove_cvn(cv_node** head, cv_node* target) { +void remove_cvn(grpc_cv_node** head, grpc_cv_node* target) { if (target->next) { target->next->prev = target->prev; } @@ -1460,7 +1469,7 @@ static void run_poll(void* args) { result->completed = 1; result->retval = retval; result->err = errno; - cv_node* watcher = result->watchers; + grpc_cv_node* watcher = result->watchers; while (watcher) { gpr_cv_signal(watcher->cv); watcher = watcher->next; @@ -1494,17 +1503,17 @@ static void run_poll(void* args) { static int cvfd_poll(struct pollfd* fds, nfds_t nfds, int timeout) { unsigned int i; int res, idx; - cv_node* pollcv; + grpc_cv_node* pollcv; int skip_poll = 0; nfds_t nsockfds = 0; poll_result* result = nullptr; gpr_mu_lock(&g_cvfds.mu); - pollcv = (cv_node*)gpr_malloc(sizeof(cv_node)); + pollcv = (grpc_cv_node*)gpr_malloc(sizeof(grpc_cv_node)); pollcv->next = nullptr; gpr_cv pollcv_cv; gpr_cv_init(&pollcv_cv); pollcv->cv = &pollcv_cv; - cv_node* fd_cvs = (cv_node*)gpr_malloc(nfds * sizeof(cv_node)); + grpc_cv_node* fd_cvs = (grpc_cv_node*)gpr_malloc(nfds * sizeof(grpc_cv_node)); for (i = 0; i < nfds; i++) { fds[i].revents = 0; @@ -1600,7 +1609,8 @@ static void global_cv_fd_table_init() { gpr_cv_init(&g_cvfds.shutdown_cv); gpr_ref_init(&g_cvfds.pollcount, 1); g_cvfds.size = CV_DEFAULT_TABLE_SIZE; - g_cvfds.cvfds = (fd_node*)gpr_malloc(sizeof(fd_node) * CV_DEFAULT_TABLE_SIZE); + g_cvfds.cvfds = + (grpc_fd_node*)gpr_malloc(sizeof(grpc_fd_node) * CV_DEFAULT_TABLE_SIZE); g_cvfds.free_fds = nullptr; thread_grace = gpr_time_from_millis(POLLCV_THREAD_GRACE_MS, GPR_TIMESPAN); for (int i = 0; i < CV_DEFAULT_TABLE_SIZE; i++) { diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 816acf2a23..d47a077251 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -63,7 +63,8 @@ typedef size_t msg_iovlen_type; grpc_core::TraceFlag grpc_tcp_trace(false, "tcp"); -typedef struct { +namespace { +struct grpc_tcp { grpc_endpoint base; grpc_fd* em_fd; int fd; @@ -96,12 +97,13 @@ typedef struct { grpc_resource_user* resource_user; grpc_resource_user_slice_allocator slice_allocator; -} grpc_tcp; +}; -typedef struct backup_poller { +struct backup_poller { gpr_mu* pollset_mu; grpc_closure run_poller; -} backup_poller; +}; +} // namespace #define BACKUP_POLLER_POLLSET(b) ((grpc_pollset*)((b) + 1)) diff --git a/src/core/lib/iomgr/tcp_uv.cc b/src/core/lib/iomgr/tcp_uv.cc index 2c26b60511..baa49d5cc5 100644 --- a/src/core/lib/iomgr/tcp_uv.cc +++ b/src/core/lib/iomgr/tcp_uv.cc @@ -65,6 +65,17 @@ typedef struct { grpc_pollset* pollset; } grpc_tcp; +static grpc_error* tcp_annotate_error(grpc_error* src_error, grpc_tcp* tcp) { + return grpc_error_set_str( + grpc_error_set_int( + src_error, + /* All tcp errors are marked with UNAVAILABLE so that application may + * choose to retry. */ + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE), + GRPC_ERROR_STR_TARGET_ADDRESS, + grpc_slice_from_copied_string(tcp->peer_string)); +} + static void tcp_free(grpc_tcp* tcp) { grpc_resource_user_unref(tcp->resource_user); gpr_free(tcp->handle); @@ -162,7 +173,8 @@ static void read_callback(uv_stream_t* stream, ssize_t nread, // TODO(murgatroid99): figure out what the return value here means uv_read_stop(stream); if (nread == UV_EOF) { - error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("EOF"); + error = + tcp_annotate_error(GRPC_ERROR_CREATE_FROM_STATIC_STRING("EOF"), tcp); grpc_slice_buffer_reset_and_unref_internal(tcp->read_slices); } else if (nread > 0) { // Successful read @@ -177,7 +189,8 @@ static void read_callback(uv_stream_t* stream, ssize_t nread, } } else { // nread < 0: Error - error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Read failed"); + error = tcp_annotate_error( + GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Read failed"), tcp); grpc_slice_buffer_reset_and_unref_internal(tcp->read_slices); } call_read_cb(tcp, error); @@ -194,7 +207,9 @@ static void tcp_read_allocation_done(void* tcpp, grpc_error* error) { status = uv_read_start((uv_stream_t*)tcp->handle, alloc_uv_buf, read_callback); if (status != 0) { - error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Read failed at start"); + error = tcp_annotate_error( + GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Read failed at start"), + tcp); error = grpc_error_set_str( error, GRPC_ERROR_STR_OS_ERROR, grpc_slice_from_static_string(uv_strerror(status))); @@ -235,7 +250,8 @@ static void write_callback(uv_write_t* req, int status) { if (status == 0) { error = GRPC_ERROR_NONE; } else { - error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Write failed"); + error = tcp_annotate_error( + GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Write failed"), tcp); } if (grpc_tcp_trace.enabled()) { const char* str = grpc_error_string(error); @@ -268,8 +284,10 @@ static void uv_endpoint_write(grpc_endpoint* ep, } if (tcp->shutting_down) { - GRPC_CLOSURE_SCHED(cb, GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "TCP socket is shutting down")); + GRPC_CLOSURE_SCHED(cb, + tcp_annotate_error(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "TCP socket is shutting down"), + tcp)); return; } diff --git a/src/core/lib/iomgr/wakeup_fd_cv.cc b/src/core/lib/iomgr/wakeup_fd_cv.cc index 5c1f16d3fc..c785114212 100644 --- a/src/core/lib/iomgr/wakeup_fd_cv.cc +++ b/src/core/lib/iomgr/wakeup_fd_cv.cc @@ -34,7 +34,7 @@ #define MAX_TABLE_RESIZE 256 -extern cv_fd_table g_cvfds; +extern grpc_cv_fd_table g_cvfds; static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) { unsigned int i, newsize; @@ -42,8 +42,8 @@ static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) { gpr_mu_lock(&g_cvfds.mu); if (!g_cvfds.free_fds) { newsize = GPR_MIN(g_cvfds.size * 2, g_cvfds.size + MAX_TABLE_RESIZE); - g_cvfds.cvfds = - (fd_node*)gpr_realloc(g_cvfds.cvfds, sizeof(fd_node) * newsize); + g_cvfds.cvfds = (grpc_fd_node*)gpr_realloc(g_cvfds.cvfds, + sizeof(grpc_fd_node) * newsize); for (i = g_cvfds.size; i < newsize; i++) { g_cvfds.cvfds[i].is_set = 0; g_cvfds.cvfds[i].cvs = nullptr; @@ -64,7 +64,7 @@ static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) { } static grpc_error* cv_fd_wakeup(grpc_wakeup_fd* fd_info) { - cv_node* cvn; + grpc_cv_node* cvn; gpr_mu_lock(&g_cvfds.mu); g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].is_set = 1; cvn = g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].cvs; diff --git a/src/core/lib/iomgr/wakeup_fd_cv.h b/src/core/lib/iomgr/wakeup_fd_cv.h index 017e41bfa8..399620af76 100644 --- a/src/core/lib/iomgr/wakeup_fd_cv.h +++ b/src/core/lib/iomgr/wakeup_fd_cv.h @@ -40,27 +40,27 @@ #define GRPC_FD_TO_IDX(fd) (-(fd)-1) #define GRPC_IDX_TO_FD(idx) (-(idx)-1) -typedef struct cv_node { +typedef struct grpc_cv_node { gpr_cv* cv; - struct cv_node* next; - struct cv_node* prev; -} cv_node; + struct grpc_cv_node* next; + struct grpc_cv_node* prev; +} grpc_cv_node; -typedef struct fd_node { +typedef struct grpc_fd_node { int is_set; - cv_node* cvs; - struct fd_node* next_free; -} fd_node; + grpc_cv_node* cvs; + struct grpc_fd_node* next_free; +} grpc_fd_node; -typedef struct cv_fd_table { +typedef struct grpc_cv_fd_table { gpr_mu mu; gpr_refcount pollcount; gpr_cv shutdown_cv; - fd_node* cvfds; - fd_node* free_fds; + grpc_fd_node* cvfds; + grpc_fd_node* free_fds; unsigned int size; grpc_poll_function_type poll; -} cv_fd_table; +} grpc_cv_fd_table; extern const grpc_wakeup_fd_vtable grpc_cv_wakeup_fd_vtable; diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc index cd3c2e3f19..6a3641f112 100644 --- a/src/core/lib/security/transport/client_auth_filter.cc +++ b/src/core/lib/security/transport/client_auth_filter.cc @@ -37,8 +37,9 @@ #define MAX_CREDENTIALS_METADATA_COUNT 4 +namespace { /* We can have a per-call credentials. */ -typedef struct { +struct call_data { grpc_call_stack* owning_call; grpc_call_combiner* call_combiner; grpc_call_credentials* creds; @@ -57,13 +58,14 @@ typedef struct { grpc_closure async_result_closure; grpc_closure check_call_host_cancel_closure; grpc_closure get_request_metadata_cancel_closure; -} call_data; +}; /* We can have a per-channel credentials. */ -typedef struct { +struct channel_data { grpc_channel_security_connector* security_connector; grpc_auth_context* auth_context; -} channel_data; +}; +} // namespace void grpc_auth_metadata_context_reset( grpc_auth_metadata_context* auth_md_context) { diff --git a/src/core/lib/security/transport/server_auth_filter.cc b/src/core/lib/security/transport/server_auth_filter.cc index 73653f2a66..f82971dc56 100644 --- a/src/core/lib/security/transport/server_auth_filter.cc +++ b/src/core/lib/security/transport/server_auth_filter.cc @@ -26,13 +26,14 @@ #include "src/core/lib/security/transport/auth_filters.h" #include "src/core/lib/slice/slice_internal.h" -typedef enum { +namespace { +enum async_state { STATE_INIT = 0, STATE_DONE, STATE_CANCELLED, -} async_state; +}; -typedef struct call_data { +struct call_data { grpc_call_combiner* call_combiner; grpc_call_stack* owning_call; grpc_transport_stream_op_batch* recv_initial_metadata_batch; @@ -44,12 +45,13 @@ typedef struct call_data { grpc_auth_context* auth_context; grpc_closure cancel_closure; gpr_atm state; // async_state -} call_data; +}; -typedef struct channel_data { +struct channel_data { grpc_auth_context* auth_context; grpc_server_credentials* creds; -} channel_data; +}; +} // namespace static grpc_metadata_array metadata_batch_to_md_array( const grpc_metadata_batch* batch) { diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index a457aaa7a2..d677576c14 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -1851,8 +1851,9 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops, { grpc_error* override_error = GRPC_ERROR_NONE; if (op->data.send_status_from_server.status != GRPC_STATUS_OK) { - override_error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Error from server send status"); + override_error = + error_from_status(op->data.send_status_from_server.status, + "Returned non-ok status"); } if (op->data.send_status_from_server.status_details != nullptr) { call->send_extra_metadata[1].md = grpc_mdelem_from_slices( diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index f1d428f0a1..ee98cf2693 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -44,24 +44,23 @@ #include "src/core/lib/transport/metadata.h" #include "src/core/lib/transport/static_metadata.h" -typedef struct listener { +grpc_core::TraceFlag grpc_server_channel_trace(false, "server_channel"); + +namespace { +struct listener { void* arg; void (*start)(grpc_server* server, void* arg, grpc_pollset** pollsets, size_t pollset_count); void (*destroy)(grpc_server* server, void* arg, grpc_closure* closure); struct listener* next; grpc_closure destroy_done; -} listener; +}; -typedef struct call_data call_data; -typedef struct channel_data channel_data; -typedef struct registered_method registered_method; +enum requested_call_type { BATCH_CALL, REGISTERED_CALL }; -typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; +struct registered_method; -grpc_core::TraceFlag grpc_server_channel_trace(false, "server_channel"); - -typedef struct requested_call { +struct requested_call { gpr_mpscq_node request_link; /* must be first */ requested_call_type type; size_t cq_idx; @@ -81,15 +80,15 @@ typedef struct requested_call { grpc_byte_buffer** optional_payload; } registered; } data; -} requested_call; +}; -typedef struct channel_registered_method { +struct channel_registered_method { registered_method* server_registered_method; uint32_t flags; bool has_host; grpc_slice method; grpc_slice host; -} channel_registered_method; +}; struct channel_data { grpc_server* server; @@ -176,6 +175,7 @@ typedef struct { grpc_channel** channels; size_t num_channels; } channel_broadcaster; +} // namespace struct grpc_server { grpc_channel_args* channel_args; diff --git a/src/core/tsi/ssl_transport_security.cc b/src/core/tsi/ssl_transport_security.cc index f35caef640..229f7efd37 100644 --- a/src/core/tsi/ssl_transport_security.cc +++ b/src/core/tsi/ssl_transport_security.cc @@ -116,6 +116,9 @@ typedef struct { static gpr_once init_openssl_once = GPR_ONCE_INIT; static gpr_mu* openssl_mutexes = nullptr; +static void openssl_locking_cb(int mode, int type, const char* file, + int line) GRPC_UNUSED; +static unsigned long openssl_thread_id_cb(void) GRPC_UNUSED; static void openssl_locking_cb(int mode, int type, const char* file, int line) { if (mode & CRYPTO_LOCK) { diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj index bbcbd95be5..5bbff38948 100755 --- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj +++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj @@ -15,12 +15,12 @@ <PackageTags>gRPC RPC Protocol HTTP/2 Auth OAuth2</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <IncludeSymbols>true</IncludeSymbols> - <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> + <Import Project="..\Grpc.Core\SourceLink.csproj.include" /> + <ItemGroup> <Compile Include="..\Grpc.Core\Version.cs" /> </ItemGroup> diff --git a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj index 4d6767fa98..40840d4da3 100755 --- a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj +++ b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj @@ -15,12 +15,12 @@ <PackageTags>gRPC test testing</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <IncludeSymbols>true</IncludeSymbols> - <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> + <Import Project="..\Grpc.Core\SourceLink.csproj.include" /> + <ItemGroup> <Compile Include="..\Grpc.Core\Version.cs" /> </ItemGroup> diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index d9950b2f20..6d44be7ddd 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -14,12 +14,12 @@ <PackageTags>gRPC RPC Protocol HTTP/2</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <IncludeSymbols>true</IncludeSymbols> - <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> + <Import Project="SourceLink.csproj.include" /> + <ItemGroup> <EmbeddedResource Include="..\..\..\etc\roots.pem" /> <Content Include="..\nativelibs\csharp_ext_macos_x64\libgrpc_csharp_ext.dylib"> diff --git a/src/csharp/Grpc.Core/SourceLink.csproj.include b/src/csharp/Grpc.Core/SourceLink.csproj.include new file mode 100755 index 0000000000..02ae79fb89 --- /dev/null +++ b/src/csharp/Grpc.Core/SourceLink.csproj.include @@ -0,0 +1,19 @@ +<!-- Ensure that debugging of the resulting NuGet packages work (we're using SourceLink). --> +<Project> + + <ItemGroup Label="dotnet pack instructions"> + <Content Include="$(OutputPath)netstandard1.5\$(PackageId).pdb"> + <Pack>true</Pack> + <PackagePath>lib/netstandard1.5</PackagePath> + </Content> + <Content Include="$(OutputPath)net45\$(PackageId).pdb"> + <Pack>true</Pack> + <PackagePath>lib/net45</PackagePath> + </Content> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="SourceLink.Embed.AllSourceFiles" Version="2.7.3" PrivateAssets="all" /> + </ItemGroup> + +</Project> diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj index 681719d124..da61253455 100755 --- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj +++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj @@ -14,12 +14,12 @@ <PackageTags>gRPC health check</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <IncludeSymbols>true</IncludeSymbols> - <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> + <Import Project="..\Grpc.Core\SourceLink.csproj.include" /> + <ItemGroup> <Compile Include="..\Grpc.Core\Version.cs" /> </ItemGroup> diff --git a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj index 704eea5c17..862ecda5fd 100755 --- a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj +++ b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj @@ -14,12 +14,12 @@ <PackageTags>gRPC reflection</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <IncludeSymbols>true</IncludeSymbols> - <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> + <Import Project="..\Grpc.Core\SourceLink.csproj.include" /> + <ItemGroup> <Compile Include="..\Grpc.Core\Version.cs" /> </ItemGroup> diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json deleted file mode 100644 index fca3a2a7a6..0000000000 --- a/src/node/health_check/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "grpc-health-check", - "version": "1.7.2", - "author": "Google Inc.", - "description": "Health check service for use with gRPC", - "repository": { - "type": "git", - "url": "https://github.com/grpc/grpc.git" - }, - "bugs": "https://github.com/grpc/grpc/issues", - "contributors": [ - { - "name": "Michael Lumish", - "email": "mlumish@google.com" - } - ], - "dependencies": { - "grpc": "^1.7.2", - "lodash": "^3.9.3", - "google-protobuf": "^3.0.0" - }, - "files": [ - "LICENSE", - "health.js", - "v1" - ], - "main": "src/node/index.js", - "license": "Apache-2.0" -} diff --git a/src/node/tools/package.json b/src/node/tools/package.json deleted file mode 100644 index 99fd854067..0000000000 --- a/src/node/tools/package.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "name": "grpc-tools", - "version": "1.7.2", - "author": "Google Inc.", - "description": "Tools for developing with gRPC on Node.js", - "homepage": "https://grpc.io/", - "repository": { - "type": "git", - "url": "https://github.com/grpc/grpc.git" - }, - "bugs": "https://github.com/grpc/grpc/issues", - "contributors": [ - { - "name": "Michael Lumish", - "email": "mlumish@google.com" - } - ], - "bin": { - "grpc_tools_node_protoc": "./bin/protoc.js", - "grpc_tools_node_protoc_plugin": "./bin/protoc_plugin.js" - }, - "scripts": { - "install": "./node_modules/.bin/node-pre-gyp install" - }, - "bundledDependencies": ["node-pre-gyp"], - "binary": { - "module_name": "grpc_tools", - "host": "https://storage.googleapis.com/", - "remote_path": "grpc-precompiled-binaries/node/{name}/v{version}", - "package_name": "{platform}-{arch}.tar.gz", - "module_path": "bin" - }, - "files": [ - "index.js", - "bin/protoc.js", - "bin/protoc_plugin.js", - "bin/google/protobuf", - "LICENSE" - ], - "main": "index.js" -} diff --git a/src/objective-c/GRPCClient/GRPCCall+ChannelArg.h b/src/objective-c/GRPCClient/GRPCCall+ChannelArg.h index 9f2361bd22..18d4597098 100644 --- a/src/objective-c/GRPCClient/GRPCCall+ChannelArg.h +++ b/src/objective-c/GRPCClient/GRPCCall+ChannelArg.h @@ -19,6 +19,12 @@ #include <AvailabilityMacros.h> +typedef NS_ENUM(NSInteger, GRPCCompressAlgorithm) { + GRPCCompressNone, + GRPCCompressDeflate, + GRPCCompressGzip, +}; + /** * Methods to configure GRPC channel options. */ @@ -36,4 +42,8 @@ + (void)closeOpenConnections DEPRECATED_MSG_ATTRIBUTE("The API for this feature is experimental, " "and might be removed or modified at any " "time."); + ++ (void)setDefaultCompressMethod:(GRPCCompressAlgorithm)algorithm + forhost:(nonnull NSString *)host; + @end diff --git a/src/objective-c/GRPCClient/GRPCCall+ChannelArg.m b/src/objective-c/GRPCClient/GRPCCall+ChannelArg.m index 398d98fbc7..805e54b890 100644 --- a/src/objective-c/GRPCClient/GRPCCall+ChannelArg.m +++ b/src/objective-c/GRPCClient/GRPCCall+ChannelArg.m @@ -20,6 +20,8 @@ #import "private/GRPCHost.h" +#import <grpc/impl/codegen/compression_types.h> + @implementation GRPCCall (ChannelArg) + (void)setUserAgentPrefix:(nonnull NSString *)userAgentPrefix forHost:(nonnull NSString *)host { @@ -36,4 +38,23 @@ [GRPCHost flushChannelCache]; } ++ (void)setDefaultCompressMethod:(GRPCCompressAlgorithm)algorithm + forhost:(nonnull NSString *)host { + GRPCHost *hostConfig = [GRPCHost hostWithAddress:host]; + switch (algorithm) { + case GRPCCompressNone: + hostConfig.compressAlgorithm = GRPC_COMPRESS_NONE; + break; + case GRPCCompressDeflate: + hostConfig.compressAlgorithm = GRPC_COMPRESS_DEFLATE; + break; + case GRPCCompressGzip: + hostConfig.compressAlgorithm = GRPC_COMPRESS_GZIP; + break; + default: + NSLog(@"Invalid compression algorithm"); + abort(); + } +} + @end diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index 58171211b0..0215db8531 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -18,6 +18,8 @@ #import <Foundation/Foundation.h> +#import <grpc/impl/codegen/compression_types.h> + NS_ASSUME_NONNULL_BEGIN @class GRPCCompletionQueue; @@ -32,6 +34,7 @@ struct grpc_channel_credentials; @property(nonatomic, readonly) NSString *address; @property(nonatomic, copy, nullable) NSString *userAgentPrefix; @property(nonatomic, nullable) struct grpc_channel_credentials *channelCreds; +@property(nonatomic) grpc_compression_algorithm compressAlgorithm; /** The following properties should only be modified for testing: */ diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index a0f4118740..665943f181 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -87,6 +87,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; _address = address; _secure = YES; kHostCache[address] = self; + _compressAlgorithm = GRPC_COMPRESS_NONE; } // Keep a single monitor to flush the cache if the connectivity status changes // Thread safety guarded by @synchronized(kHostCache) @@ -226,6 +227,12 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; } // Use 10000ms initial backoff time for correct behavior on bad/slow networks args[@GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS] = @10000; + + if (_compressAlgorithm != GRPC_COMPRESS_NONE) { + args[@GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM] = + [NSNumber numberWithInt:_compressAlgorithm]; + } + return args; } diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index 0be8669aa2..dfa874adab 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -68,6 +68,10 @@ } @end +BOOL isRemoteInteropTest(NSString *host) { + return [host isEqualToString:@"grpc-test.sandbox.googleapis.com"]; +} + #pragma mark Tests @implementation InteropTests { @@ -452,4 +456,34 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +- (void)testCompressedUnaryRPC { + // This test needs to be disabled for remote test because interop server grpc-test + // does not support compression. + if (isRemoteInteropTest(self.class.host)) { + return; + } + XCTAssertNotNil(self.class.host); + __weak XCTestExpectation *expectation = [self expectationWithDescription:@"LargeUnary"]; + + RMTSimpleRequest *request = [RMTSimpleRequest message]; + request.responseType = RMTPayloadType_Compressable; + request.responseSize = 314159; + request.payload.body = [NSMutableData dataWithLength:271828]; + request.expectCompressed.value = YES; + [GRPCCall setDefaultCompressMethod:GRPCCompressGzip forhost:self.class.host]; + + [_service unaryCallWithRequest:request handler:^(RMTSimpleResponse *response, NSError *error) { + XCTAssertNil(error, @"Finished with unexpected error: %@", error); + + RMTSimpleResponse *expectedResponse = [RMTSimpleResponse message]; + expectedResponse.payload.type = RMTPayloadType_Compressable; + expectedResponse.payload.body = [NSMutableData dataWithLength:314159]; + XCTAssertEqualObjects(response, expectedResponse); + + [expectation fulfill]; + }]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; +} + @end diff --git a/src/objective-c/tests/RemoteTestClient/messages.proto b/src/objective-c/tests/RemoteTestClient/messages.proto index 128efd9337..342e91d5aa 100644 --- a/src/objective-c/tests/RemoteTestClient/messages.proto +++ b/src/objective-c/tests/RemoteTestClient/messages.proto @@ -20,34 +20,45 @@ package grpc.testing; option objc_class_prefix = "RMT"; +// TODO(dgq): Go back to using well-known types once +// https://github.com/grpc/grpc/issues/6980 has been fixed. +// import "google/protobuf/wrappers.proto"; +message BoolValue { + // The bool value. + bool value = 1; +} + +// DEPRECATED, don't use. To be removed shortly. // The type of payload that should be returned. enum PayloadType { // Compressable text format. COMPRESSABLE = 0; - - // Uncompressable binary format. - UNCOMPRESSABLE = 1; - - // Randomly chosen from all other formats defined in this enum. - RANDOM = 2; } // A block of data, to simply increase gRPC message size. message Payload { + // DEPRECATED, don't use. To be removed shortly. // The type of data in body. PayloadType type = 1; // Primary contents of payload. bytes body = 2; } +// A protobuf representation for grpc status. This is used by test +// clients to specify a status that the server should attempt to return. +message EchoStatus { + int32 code = 1; + string message = 2; +} + // Unary request. message SimpleRequest { + // DEPRECATED, don't use. To be removed shortly. // Desired payload type in the response from the server. // If response_type is RANDOM, server randomly chooses one from other formats. PayloadType response_type = 1; // Desired payload size in the response from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. int32 response_size = 2; // Optional input payload sent along with the request. @@ -58,6 +69,18 @@ message SimpleRequest { // Whether SimpleResponse should include OAuth scope. bool fill_oauth_scope = 5; + + // Whether to request the server to compress the response. This field is + // "nullable" in order to interoperate seamlessly with clients not able to + // implement the full compression tests by introspecting the call to verify + // the response's compression status. + BoolValue response_compressed = 6; + + // Whether server should return a given status + EchoStatus response_status = 7; + + // Whether the server should expect this request to be compressed. + BoolValue expect_compressed = 8; } // Unary response, as configured by the request. @@ -76,6 +99,12 @@ message StreamingInputCallRequest { // Optional input payload sent along with the request. Payload payload = 1; + // Whether the server should expect this request to be compressed. This field + // is "nullable" in order to interoperate seamlessly with servers not able to + // implement the full compression tests by introspecting the call to verify + // the request's compression status. + BoolValue expect_compressed = 2; + // Not expecting any payload from the response. } @@ -88,16 +117,22 @@ message StreamingInputCallResponse { // Configuration for a particular response. message ResponseParameters { // Desired payload sizes in responses from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. int32 size = 1; // Desired interval between consecutive responses in the response stream in // microseconds. int32 interval_us = 2; + + // Whether to request the server to compress the response. This field is + // "nullable" in order to interoperate seamlessly with clients not able to + // implement the full compression tests by introspecting the call to verify + // the response's compression status. + BoolValue compressed = 3; } // Server-streaming request. message StreamingOutputCallRequest { + // DEPRECATED, don't use. To be removed shortly. // Desired payload type in the response from the server. // If response_type is RANDOM, the payload from each response in the stream // might be of different types. This is to simulate a mixed type of payload @@ -109,6 +144,9 @@ message StreamingOutputCallRequest { // Optional input payload sent along with the request. Payload payload = 3; + + // Whether server should return a given status + EchoStatus response_status = 7; } // Server-streaming response, as configured by the request and parameters. @@ -116,3 +154,17 @@ message StreamingOutputCallResponse { // Payload to increase response size. Payload payload = 1; } + +// For reconnect interop test only. +// Client tells server what reconnection parameters it used. +message ReconnectParams { + int32 max_reconnect_backoff_ms = 1; +} + +// For reconnect interop test only. +// Server tells client whether its reconnects are following the spec and the +// reconnect backoffs it saw. +message ReconnectInfo { + bool passed = 1; + repeated int32 backoff_ms = 2; +} diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 5f28e9101f..4c2ebaeaea 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -104,8 +104,8 @@ def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename): with open(bdist_path, 'w') as bdist_file: bdist_file.write(bdist_data) except IOError as error: - raise CommandError('{}\n\nCould not write grpcio bdist: {}' - .format(traceback.format_exc(), error.message)) + raise CommandError('{}\n\nCould not write grpcio bdist: {}'.format( + traceback.format_exc(), error.message)) return bdist_path @@ -141,7 +141,8 @@ class SphinxDocumentation(setuptools.Command): with open(glossary_filepath, 'a') as glossary_filepath: glossary_filepath.write(API_GLOSSARY) sphinx.main( - ['', os.path.join('doc', 'src'), os.path.join('doc', 'build')]) + ['', os.path.join('doc', 'src'), + os.path.join('doc', 'build')]) class BuildProjectMetadata(setuptools.Command): @@ -189,10 +190,11 @@ def check_and_update_cythonization(extensions): for source in extension.sources: base, file_ext = os.path.splitext(source) if file_ext == '.pyx': - generated_pyx_source = next((base + gen_ext - for gen_ext in ('.c', '.cpp',) - if os.path.isfile(base + gen_ext)), - None) + generated_pyx_source = next( + (base + gen_ext for gen_ext in ( + '.c', + '.cpp', + ) if os.path.isfile(base + gen_ext)), None) if generated_pyx_source: generated_pyx_sources.append(generated_pyx_source) else: @@ -299,10 +301,10 @@ class Gather(setuptools.Command): """Command to gather project dependencies.""" description = 'gather dependencies for grpcio' - user_options = [ - ('test', 't', 'flag indicating to gather test dependencies'), - ('install', 'i', 'flag indicating to gather install dependencies') - ] + user_options = [('test', 't', + 'flag indicating to gather test dependencies'), + ('install', 'i', + 'flag indicating to gather install dependencies')] def initialize_options(self): self.test = False diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 8b913ac949..db410d307b 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1376,8 +1376,8 @@ def metadata_call_credentials(metadata_plugin, name=None): A CallCredentials. """ from grpc import _plugin_wrapping # pylint: disable=cyclic-import - return _plugin_wrapping.metadata_plugin_call_credentials(metadata_plugin, - name) + return _plugin_wrapping.metadata_plugin_call_credentials( + metadata_plugin, name) def access_token_call_credentials(access_token): @@ -1631,25 +1631,57 @@ def server(thread_pool, ################################### __all__ ################################# __all__ = ( - 'FutureTimeoutError', 'FutureCancelledError', 'Future', - 'ChannelConnectivity', 'StatusCode', 'RpcError', 'RpcContext', 'Call', - 'ChannelCredentials', 'CallCredentials', 'AuthMetadataContext', - 'AuthMetadataPluginCallback', 'AuthMetadataPlugin', 'ClientCallDetails', - 'ServerCertificateConfiguration', 'ServerCredentials', - 'UnaryUnaryMultiCallable', 'UnaryStreamMultiCallable', - 'StreamUnaryMultiCallable', 'StreamStreamMultiCallable', - 'UnaryUnaryClientInterceptor', 'UnaryStreamClientInterceptor', - 'StreamUnaryClientInterceptor', 'StreamStreamClientInterceptor', 'Channel', - 'ServicerContext', 'RpcMethodHandler', 'HandlerCallDetails', - 'GenericRpcHandler', 'ServiceRpcHandler', 'Server', 'ServerInterceptor', - 'unary_unary_rpc_method_handler', 'unary_stream_rpc_method_handler', - 'stream_unary_rpc_method_handler', 'stream_stream_rpc_method_handler', - 'method_handlers_generic_handler', 'ssl_channel_credentials', - 'metadata_call_credentials', 'access_token_call_credentials', - 'composite_call_credentials', 'composite_channel_credentials', - 'ssl_server_credentials', 'ssl_server_certificate_configuration', - 'dynamic_ssl_server_credentials', 'channel_ready_future', - 'insecure_channel', 'secure_channel', 'intercept_channel', 'server',) + 'FutureTimeoutError', + 'FutureCancelledError', + 'Future', + 'ChannelConnectivity', + 'StatusCode', + 'RpcError', + 'RpcContext', + 'Call', + 'ChannelCredentials', + 'CallCredentials', + 'AuthMetadataContext', + 'AuthMetadataPluginCallback', + 'AuthMetadataPlugin', + 'ClientCallDetails', + 'ServerCertificateConfiguration', + 'ServerCredentials', + 'UnaryUnaryMultiCallable', + 'UnaryStreamMultiCallable', + 'StreamUnaryMultiCallable', + 'StreamStreamMultiCallable', + 'UnaryUnaryClientInterceptor', + 'UnaryStreamClientInterceptor', + 'StreamUnaryClientInterceptor', + 'StreamStreamClientInterceptor', + 'Channel', + 'ServicerContext', + 'RpcMethodHandler', + 'HandlerCallDetails', + 'GenericRpcHandler', + 'ServiceRpcHandler', + 'Server', + 'ServerInterceptor', + 'unary_unary_rpc_method_handler', + 'unary_stream_rpc_method_handler', + 'stream_unary_rpc_method_handler', + 'stream_stream_rpc_method_handler', + 'method_handlers_generic_handler', + 'ssl_channel_credentials', + 'metadata_call_credentials', + 'access_token_call_credentials', + 'composite_call_credentials', + 'composite_channel_credentials', + 'ssl_server_credentials', + 'ssl_server_certificate_configuration', + 'dynamic_ssl_server_credentials', + 'channel_ready_future', + 'insecure_channel', + 'secure_channel', + 'intercept_channel', + 'server', +) ############################### Extension Shims ################################ diff --git a/src/python/grpcio/grpc/_auth.py b/src/python/grpcio/grpc/_auth.py index 9a339b5900..c17824563d 100644 --- a/src/python/grpcio/grpc/_auth.py +++ b/src/python/grpcio/grpc/_auth.py @@ -54,7 +54,9 @@ class GoogleCallCredentials(grpc.AuthMetadataPlugin): if self._is_jwt: future = self._pool.submit( self._credentials.get_access_token, - additional_claims={'aud': context.service_url}) + additional_claims={ + 'aud': context.service_url + }) else: future = self._pool.submit(self._credentials.get_access_token) future.add_done_callback(_create_get_token_callback(callback)) diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index d7456a3dd1..24be042f61 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -29,24 +29,32 @@ _USER_AGENT = 'grpc-python/{}'.format(_grpcio_metadata.__version__) _EMPTY_FLAGS = 0 _INFINITE_FUTURE = cygrpc.Timespec(float('+inf')) -_UNARY_UNARY_INITIAL_DUE = (cygrpc.OperationType.send_initial_metadata, - cygrpc.OperationType.send_message, - cygrpc.OperationType.send_close_from_client, - cygrpc.OperationType.receive_initial_metadata, - cygrpc.OperationType.receive_message, - cygrpc.OperationType.receive_status_on_client,) -_UNARY_STREAM_INITIAL_DUE = (cygrpc.OperationType.send_initial_metadata, - cygrpc.OperationType.send_message, - cygrpc.OperationType.send_close_from_client, - cygrpc.OperationType.receive_initial_metadata, - cygrpc.OperationType.receive_status_on_client,) -_STREAM_UNARY_INITIAL_DUE = (cygrpc.OperationType.send_initial_metadata, - cygrpc.OperationType.receive_initial_metadata, - cygrpc.OperationType.receive_message, - cygrpc.OperationType.receive_status_on_client,) -_STREAM_STREAM_INITIAL_DUE = (cygrpc.OperationType.send_initial_metadata, - cygrpc.OperationType.receive_initial_metadata, - cygrpc.OperationType.receive_status_on_client,) +_UNARY_UNARY_INITIAL_DUE = ( + cygrpc.OperationType.send_initial_metadata, + cygrpc.OperationType.send_message, + cygrpc.OperationType.send_close_from_client, + cygrpc.OperationType.receive_initial_metadata, + cygrpc.OperationType.receive_message, + cygrpc.OperationType.receive_status_on_client, +) +_UNARY_STREAM_INITIAL_DUE = ( + cygrpc.OperationType.send_initial_metadata, + cygrpc.OperationType.send_message, + cygrpc.OperationType.send_close_from_client, + cygrpc.OperationType.receive_initial_metadata, + cygrpc.OperationType.receive_status_on_client, +) +_STREAM_UNARY_INITIAL_DUE = ( + cygrpc.OperationType.send_initial_metadata, + cygrpc.OperationType.receive_initial_metadata, + cygrpc.OperationType.receive_message, + cygrpc.OperationType.receive_status_on_client, +) +_STREAM_STREAM_INITIAL_DUE = ( + cygrpc.OperationType.send_initial_metadata, + cygrpc.OperationType.receive_initial_metadata, + cygrpc.OperationType.receive_status_on_client, +) _CHANNEL_SUBSCRIPTION_CALLBACK_ERROR_LOG_MESSAGE = ( 'Exception calling channel subscription callback!') @@ -129,12 +137,12 @@ def _abort(state, code, details): def _handle_event(event, state, response_deserializer): callbacks = [] for batch_operation in event.batch_operations: - operation_type = batch_operation.type + operation_type = batch_operation.type() state.due.remove(operation_type) if operation_type == cygrpc.OperationType.receive_initial_metadata: - state.initial_metadata = batch_operation.received_metadata + state.initial_metadata = batch_operation.initial_metadata() elif operation_type == cygrpc.OperationType.receive_message: - serialized_response = batch_operation.received_message.bytes() + serialized_response = batch_operation.message() if serialized_response is not None: response = _common.deserialize(serialized_response, response_deserializer) @@ -144,18 +152,17 @@ def _handle_event(event, state, response_deserializer): else: state.response = response elif operation_type == cygrpc.OperationType.receive_status_on_client: - state.trailing_metadata = batch_operation.received_metadata + state.trailing_metadata = batch_operation.trailing_metadata() if state.code is None: code = _common.CYGRPC_STATUS_CODE_TO_STATUS_CODE.get( - batch_operation.received_status_code) + batch_operation.code()) if code is None: state.code = grpc.StatusCode.UNKNOWN state.details = _unknown_code_details( - batch_operation.received_status_code, - batch_operation.received_status_details) + code, batch_operation.details()) else: state.code = code - state.details = batch_operation.received_status_details + state.details = batch_operation.details() callbacks.extend(state.callbacks) state.callbacks = None return callbacks @@ -200,7 +207,7 @@ def _consume_request_iterator(request_iterator, state, call, _abort(state, grpc.StatusCode.INTERNAL, details) return else: - operations = (cygrpc.operation_send_message( + operations = (cygrpc.SendMessageOperation( serialized_request, _EMPTY_FLAGS),) call.start_client_batch(operations, event_handler) state.due.add(cygrpc.OperationType.send_message) @@ -216,7 +223,7 @@ def _consume_request_iterator(request_iterator, state, call, with state.condition: if state.code is None: operations = ( - cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),) + cygrpc.SendCloseFromClientOperation(_EMPTY_FLAGS),) call.start_client_batch(operations, event_handler) state.due.add(cygrpc.OperationType.send_close_from_client) @@ -319,7 +326,7 @@ class _Rendezvous(grpc.RpcError, grpc.Future, grpc.Call): event_handler = _event_handler(self._state, self._call, self._response_deserializer) self._call.start_client_batch( - (cygrpc.operation_receive_message(_EMPTY_FLAGS),), + (cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS),), event_handler) self._state.due.add(cygrpc.OperationType.receive_message) elif self._state.code is grpc.StatusCode.OK: @@ -453,12 +460,13 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): else: state = _RPCState(_UNARY_UNARY_INITIAL_DUE, None, None, None, None) operations = ( - cygrpc.operation_send_initial_metadata(metadata, _EMPTY_FLAGS), - cygrpc.operation_send_message(serialized_request, _EMPTY_FLAGS), - cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), - cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS), - cygrpc.operation_receive_message(_EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) + cygrpc.SendInitialMetadataOperation(metadata, _EMPTY_FLAGS), + cygrpc.SendMessageOperation(serialized_request, _EMPTY_FLAGS), + cygrpc.SendCloseFromClientOperation(_EMPTY_FLAGS), + cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS), + cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS), + ) return state, operations, deadline, deadline_timespec, None def _blocking(self, request, timeout, metadata, credentials): @@ -536,14 +544,15 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): self._response_deserializer) with state.condition: call.start_client_batch( - (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),), + (cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS),), event_handler) operations = ( - cygrpc.operation_send_initial_metadata( - metadata, _EMPTY_FLAGS), cygrpc.operation_send_message( - serialized_request, _EMPTY_FLAGS), - cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) + cygrpc.SendInitialMetadataOperation(metadata, _EMPTY_FLAGS), + cygrpc.SendMessageOperation(serialized_request, + _EMPTY_FLAGS), + cygrpc.SendCloseFromClientOperation(_EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS), + ) call_error = call.start_client_batch(operations, event_handler) if call_error != cygrpc.CallError.ok: _call_error_set_RPCstate(state, call_error, metadata) @@ -573,12 +582,12 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): call.set_credentials(credentials._credentials) with state.condition: call.start_client_batch( - (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),), - None) + (cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS),), None) operations = ( - cygrpc.operation_send_initial_metadata(metadata, _EMPTY_FLAGS), - cygrpc.operation_receive_message(_EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) + cygrpc.SendInitialMetadataOperation(metadata, _EMPTY_FLAGS), + cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS), + ) call_error = call.start_client_batch(operations, None) _check_call_error(call_error, metadata) _consume_request_iterator(request_iterator, state, call, @@ -624,12 +633,13 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): event_handler = _event_handler(state, call, self._response_deserializer) with state.condition: call.start_client_batch( - (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),), + (cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS),), event_handler) operations = ( - cygrpc.operation_send_initial_metadata(metadata, _EMPTY_FLAGS), - cygrpc.operation_receive_message(_EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) + cygrpc.SendInitialMetadataOperation(metadata, _EMPTY_FLAGS), + cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS), + ) call_error = call.start_client_batch(operations, event_handler) if call_error != cygrpc.CallError.ok: _call_error_set_RPCstate(state, call_error, metadata) @@ -664,11 +674,12 @@ class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable): event_handler = _event_handler(state, call, self._response_deserializer) with state.condition: call.start_client_batch( - (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),), + (cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS),), event_handler) operations = ( - cygrpc.operation_send_initial_metadata(metadata, _EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) + cygrpc.SendInitialMetadataOperation(metadata, _EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS), + ) call_error = call.start_client_batch(operations, event_handler) if call_error != cygrpc.CallError.ok: _call_error_set_RPCstate(state, call_error, metadata) @@ -789,7 +800,11 @@ def _deliver(state, initial_connectivity, initial_callbacks): def _spawn_delivery(state, callbacks): delivering_thread = threading.Thread( - target=_deliver, args=(state, state.connectivity, callbacks,)) + target=_deliver, args=( + state, + state.connectivity, + callbacks, + )) delivering_thread.start() state.delivering = True @@ -864,17 +879,16 @@ def _subscribe(state, callback, try_to_connect): def _unsubscribe(state, callback): with state.lock: - for index, (subscribed_callback, unused_connectivity - ) in enumerate(state.callbacks_and_connectivities): + for index, (subscribed_callback, unused_connectivity) in enumerate( + state.callbacks_and_connectivities): if callback == subscribed_callback: state.callbacks_and_connectivities.pop(index) break def _options(options): - return list(options) + [ - (cygrpc.ChannelArgKey.primary_user_agent_string, _USER_AGENT) - ] + return list(options) + [(cygrpc.ChannelArgKey.primary_user_agent_string, + _USER_AGENT)] class Channel(grpc.Channel): @@ -889,8 +903,8 @@ class Channel(grpc.Channel): credentials: A cygrpc.ChannelCredentials or None. """ self._channel = cygrpc.Channel( - _common.encode(target), - _common.channel_args(_options(options)), credentials) + _common.encode(target), _common.channel_args(_options(options)), + credentials) self._call_state = _ChannelCallState(self._channel) self._connectivity_state = _ChannelConnectivityState(self._channel) @@ -910,8 +924,7 @@ class Channel(grpc.Channel): request_serializer=None, response_deserializer=None): return _UnaryUnaryMultiCallable( - self._channel, - _channel_managed_call_management(self._call_state), + self._channel, _channel_managed_call_management(self._call_state), _common.encode(method), request_serializer, response_deserializer) def unary_stream(self, @@ -919,8 +932,7 @@ class Channel(grpc.Channel): request_serializer=None, response_deserializer=None): return _UnaryStreamMultiCallable( - self._channel, - _channel_managed_call_management(self._call_state), + self._channel, _channel_managed_call_management(self._call_state), _common.encode(method), request_serializer, response_deserializer) def stream_unary(self, @@ -928,8 +940,7 @@ class Channel(grpc.Channel): request_serializer=None, response_deserializer=None): return _StreamUnaryMultiCallable( - self._channel, - _channel_managed_call_management(self._call_state), + self._channel, _channel_managed_call_management(self._call_state), _common.encode(method), request_serializer, response_deserializer) def stream_stream(self, @@ -937,8 +948,7 @@ class Channel(grpc.Channel): request_serializer=None, response_deserializer=None): return _StreamStreamMultiCallable( - self._channel, - _channel_managed_call_management(self._call_state), + self._channel, _channel_managed_call_management(self._call_state), _common.encode(method), request_serializer, response_deserializer) def __del__(self): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi index 6361669757..0892215b6d 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi @@ -26,16 +26,13 @@ cdef class Call: def _start_batch(self, operations, tag, retain_self): if not self.is_valid: raise ValueError("invalid call object cannot be used from Python") - cdef OperationTag operation_tag = OperationTag(tag, operations) - if retain_self: - operation_tag.operation_call = self - else: - operation_tag.operation_call = None - operation_tag.store_ops() - cpython.Py_INCREF(operation_tag) + cdef _BatchOperationTag batch_operation_tag = _BatchOperationTag( + tag, operations, self if retain_self else None) + batch_operation_tag.prepare() + cpython.Py_INCREF(batch_operation_tag) return grpc_call_start_batch( - self.c_call, operation_tag.c_ops, operation_tag.c_nops, - <cpython.PyObject *>operation_tag, NULL) + self.c_call, batch_operation_tag.c_ops, batch_operation_tag.c_nops, + <cpython.PyObject *>batch_operation_tag, NULL) def start_client_batch(self, operations, tag): # We don't reference this call in the operations tag because diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index 644df674cc..443d534d7e 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -76,12 +76,12 @@ cdef class Channel: def watch_connectivity_state( self, grpc_connectivity_state last_observed_state, Timespec deadline not None, CompletionQueue queue not None, tag): - cdef OperationTag operation_tag = OperationTag(tag, None) - cpython.Py_INCREF(operation_tag) + cdef _ConnectivityTag connectivity_tag = _ConnectivityTag(tag) + cpython.Py_INCREF(connectivity_tag) with nogil: grpc_channel_watch_connectivity_state( self.c_channel, last_observed_state, deadline.c_time, - queue.c_completion_queue, <cpython.PyObject *>operation_tag) + queue.c_completion_queue, <cpython.PyObject *>connectivity_tag) def target(self): cdef char *target = NULL diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index 140fc357b9..e259789b35 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -37,42 +37,20 @@ cdef class CompletionQueue: self.is_shutdown = False cdef _interpret_event(self, grpc_event event): - cdef OperationTag tag = None - cdef object user_tag = None - cdef Call operation_call = None - cdef CallDetails request_call_details = None - cdef object request_metadata = None - cdef object batch_operations = None + cdef _Tag tag = None if event.type == GRPC_QUEUE_TIMEOUT: - return Event( - event.type, False, None, None, None, None, False, None) + # NOTE(nathaniel): For now we coopt ConnectivityEvent here. + return ConnectivityEvent(GRPC_QUEUE_TIMEOUT, False, None) elif event.type == GRPC_QUEUE_SHUTDOWN: self.is_shutdown = True - return Event( - event.type, True, None, None, None, None, False, None) + # NOTE(nathaniel): For now we coopt ConnectivityEvent here. + return ConnectivityEvent(GRPC_QUEUE_TIMEOUT, True, None) else: - if event.tag != NULL: - tag = <OperationTag>event.tag - # We receive event tags only after they've been inc-ref'd elsewhere in - # the code. - cpython.Py_DECREF(tag) - if tag.shutting_down_server is not None: - tag.shutting_down_server.notify_shutdown_complete() - user_tag = tag.user_tag - operation_call = tag.operation_call - request_call_details = tag.request_call_details - if tag.is_new_request: - request_metadata = _metadata(&tag._c_request_metadata) - grpc_metadata_array_destroy(&tag._c_request_metadata) - batch_operations = tag.release_ops() - if tag.is_new_request: - # Stuff in the tag not explicitly handled by us needs to live through - # the life of the call - operation_call.references.extend(tag.references) - return Event( - event.type, event.success, user_tag, operation_call, - request_call_details, request_metadata, tag.is_new_request, - batch_operations) + tag = <_Tag>event.tag + # We receive event tags only after they've been inc-ref'd elsewhere in + # the code. + cpython.Py_DECREF(tag) + return tag.event(event) def poll(self, Timespec deadline=None): # We name this 'poll' to avoid problems with CPython's expectations for diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/event.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/event.pxd.pxi new file mode 100644 index 0000000000..686199ecf4 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/event.pxd.pxi @@ -0,0 +1,45 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef class ConnectivityEvent: + + cdef readonly grpc_completion_type completion_type + cdef readonly bint success + cdef readonly object tag + + +cdef class RequestCallEvent: + + cdef readonly grpc_completion_type completion_type + cdef readonly bint success + cdef readonly object tag + cdef readonly Call call + cdef readonly CallDetails call_details + cdef readonly tuple invocation_metadata + + +cdef class BatchOperationEvent: + + cdef readonly grpc_completion_type completion_type + cdef readonly bint success + cdef readonly object tag + cdef readonly object batch_operations + + +cdef class ServerShutdownEvent: + + cdef readonly grpc_completion_type completion_type + cdef readonly bint success + cdef readonly object tag diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/event.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/event.pyx.pxi new file mode 100644 index 0000000000..af26d27318 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/event.pyx.pxi @@ -0,0 +1,55 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef class ConnectivityEvent: + + def __cinit__( + self, grpc_completion_type completion_type, bint success, object tag): + self.completion_type = completion_type + self.success = success + self.tag = tag + + +cdef class RequestCallEvent: + + def __cinit__( + self, grpc_completion_type completion_type, bint success, object tag, + Call call, CallDetails call_details, tuple invocation_metadata): + self.completion_type = completion_type + self.success = success + self.tag = tag + self.call = call + self.call_details = call_details + self.invocation_metadata = invocation_metadata + + +cdef class BatchOperationEvent: + + def __cinit__( + self, grpc_completion_type completion_type, bint success, object tag, + object batch_operations): + self.completion_type = completion_type + self.success = success + self.tag = tag + self.batch_operations = batch_operations + + +cdef class ServerShutdownEvent: + + def __cinit__( + self, grpc_completion_type completion_type, bint success, object tag): + self.completion_type = completion_type + self.success = success + self.tag = tag diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 6a72bbf693..6ee833697d 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -17,6 +17,7 @@ cimport libc.time # Typedef types with approximately the same semantics to provide their names to # Cython +ctypedef unsigned char uint8_t ctypedef int int32_t ctypedef unsigned uint32_t ctypedef long int64_t @@ -25,6 +26,7 @@ ctypedef long int64_t cdef extern from "grpc/support/alloc.h": void *gpr_malloc(size_t size) nogil + void *gpr_zalloc(size_t size) nogil void gpr_free(void *ptr) nogil void *gpr_realloc(void *p, size_t size) nogil @@ -183,6 +185,18 @@ cdef extern from "grpc/grpc.h": size_t arguments_length "num_args" grpc_arg *arguments "args" + ctypedef enum grpc_compression_level: + GRPC_COMPRESS_LEVEL_NONE + GRPC_COMPRESS_LEVEL_LOW + GRPC_COMPRESS_LEVEL_MED + GRPC_COMPRESS_LEVEL_HIGH + + ctypedef enum grpc_stream_compression_level: + GRPC_STREAM_COMPRESS_LEVEL_NONE + GRPC_STREAM_COMPRESS_LEVEL_LOW + GRPC_STREAM_COMPRESS_LEVEL_MED + GRPC_STREAM_COMPRESS_LEVEL_HIGH + ctypedef enum grpc_call_error: GRPC_CALL_OK GRPC_CALL_ERROR @@ -258,9 +272,19 @@ cdef extern from "grpc/grpc.h": GRPC_OP_RECV_STATUS_ON_CLIENT GRPC_OP_RECV_CLOSE_ON_SERVER + ctypedef struct grpc_op_send_initial_metadata_maybe_compression_level: + uint8_t is_set + grpc_compression_level level + + ctypedef struct grpc_op_send_initial_metadata_maybe_stream_compression_level: + uint8_t is_set + grpc_stream_compression_level level + ctypedef struct grpc_op_data_send_initial_metadata: size_t count grpc_metadata *metadata + grpc_op_send_initial_metadata_maybe_compression_level maybe_compression_level + grpc_op_send_initial_metadata_maybe_stream_compression_level maybe_stream_compression_level ctypedef struct grpc_op_data_send_status_from_server: size_t trailing_metadata_count diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/operation.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/operation.pxd.pxi new file mode 100644 index 0000000000..bfbe27785b --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/operation.pxd.pxi @@ -0,0 +1,109 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef class Operation: + + cdef void c(self) + cdef void un_c(self) + + # TODO(https://github.com/grpc/grpc/issues/7950): Eliminate this! + cdef grpc_op c_op + + +cdef class SendInitialMetadataOperation(Operation): + + cdef readonly object _initial_metadata; + cdef readonly int _flags + cdef grpc_metadata *_c_initial_metadata + cdef size_t _c_initial_metadata_count + + cdef void c(self) + cdef void un_c(self) + + +cdef class SendMessageOperation(Operation): + + cdef readonly bytes _message + cdef readonly int _flags + cdef grpc_byte_buffer *_c_message_byte_buffer + + cdef void c(self) + cdef void un_c(self) + + +cdef class SendCloseFromClientOperation(Operation): + + cdef readonly int _flags + + cdef void c(self) + cdef void un_c(self) + + +cdef class SendStatusFromServerOperation(Operation): + + cdef readonly object _trailing_metadata + cdef readonly object _code + cdef readonly object _details + cdef readonly int _flags + cdef grpc_metadata *_c_trailing_metadata + cdef size_t _c_trailing_metadata_count + cdef grpc_slice _c_details + + cdef void c(self) + cdef void un_c(self) + + +cdef class ReceiveInitialMetadataOperation(Operation): + + cdef readonly int _flags + cdef tuple _initial_metadata + cdef grpc_metadata_array _c_initial_metadata + + cdef void c(self) + cdef void un_c(self) + + +cdef class ReceiveMessageOperation(Operation): + + cdef readonly int _flags + cdef grpc_byte_buffer *_c_message_byte_buffer + cdef bytes _message + + cdef void c(self) + cdef void un_c(self) + + +cdef class ReceiveStatusOnClientOperation(Operation): + + cdef readonly int _flags + cdef grpc_metadata_array _c_trailing_metadata + cdef grpc_status_code _c_code + cdef grpc_slice _c_details + cdef tuple _trailing_metadata + cdef object _code + cdef str _details + + cdef void c(self) + cdef void un_c(self) + + +cdef class ReceiveCloseOnServerOperation(Operation): + + cdef readonly int _flags + cdef object _cancelled + cdef int _c_cancelled + + cdef void c(self) + cdef void un_c(self) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/operation.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/operation.pyx.pxi new file mode 100644 index 0000000000..3c91abf722 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/operation.pyx.pxi @@ -0,0 +1,238 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef class Operation: + + cdef void c(self): + raise NotImplementedError() + + cdef void un_c(self): + raise NotImplementedError() + + +cdef class SendInitialMetadataOperation(Operation): + + def __cinit__(self, initial_metadata, flags): + self._initial_metadata = initial_metadata + self._flags = flags + + def type(self): + return GRPC_OP_SEND_INITIAL_METADATA + + cdef void c(self): + self.c_op.type = GRPC_OP_SEND_INITIAL_METADATA + self.c_op.flags = self._flags + _store_c_metadata( + self._initial_metadata, &self._c_initial_metadata, + &self._c_initial_metadata_count) + self.c_op.data.send_initial_metadata.metadata = self._c_initial_metadata + self.c_op.data.send_initial_metadata.count = self._c_initial_metadata_count + self.c_op.data.send_initial_metadata.maybe_compression_level.is_set = 0 + self.c_op.data.send_initial_metadata.maybe_stream_compression_level.is_set = 0 + + cdef void un_c(self): + _release_c_metadata( + self._c_initial_metadata, self._c_initial_metadata_count) + + +cdef class SendMessageOperation(Operation): + + def __cinit__(self, bytes message, int flags): + self._message = message + self._flags = flags + + def type(self): + return GRPC_OP_SEND_MESSAGE + + cdef void c(self): + self.c_op.type = GRPC_OP_SEND_MESSAGE + self.c_op.flags = self._flags + cdef grpc_slice message_slice = grpc_slice_from_copied_buffer( + self._message, len(self._message)) + self._c_message_byte_buffer = grpc_raw_byte_buffer_create( + &message_slice, 1) + grpc_slice_unref(message_slice) + self.c_op.data.send_message.send_message = self._c_message_byte_buffer + + cdef void un_c(self): + grpc_byte_buffer_destroy(self._c_message_byte_buffer) + + +cdef class SendCloseFromClientOperation(Operation): + + def __cinit__(self, int flags): + self._flags = flags + + def type(self): + return GRPC_OP_SEND_CLOSE_FROM_CLIENT + + cdef void c(self): + self.c_op.type = GRPC_OP_SEND_CLOSE_FROM_CLIENT + self.c_op.flags = self._flags + + cdef void un_c(self): + pass + + +cdef class SendStatusFromServerOperation(Operation): + + def __cinit__(self, trailing_metadata, code, object details, int flags): + self._trailing_metadata = trailing_metadata + self._code = code + self._details = details + self._flags = flags + + def type(self): + return GRPC_OP_SEND_STATUS_FROM_SERVER + + cdef void c(self): + self.c_op.type = GRPC_OP_SEND_STATUS_FROM_SERVER + self.c_op.flags = self._flags + _store_c_metadata( + self._trailing_metadata, &self._c_trailing_metadata, + &self._c_trailing_metadata_count) + self.c_op.data.send_status_from_server.trailing_metadata = ( + self._c_trailing_metadata) + self.c_op.data.send_status_from_server.trailing_metadata_count = ( + self._c_trailing_metadata_count) + self.c_op.data.send_status_from_server.status = self._code + self._c_details = _slice_from_bytes(_encode(self._details)) + self.c_op.data.send_status_from_server.status_details = &self._c_details + + cdef void un_c(self): + grpc_slice_unref(self._c_details) + _release_c_metadata( + self._c_trailing_metadata, self._c_trailing_metadata_count) + + +cdef class ReceiveInitialMetadataOperation(Operation): + + def __cinit__(self, flags): + self._flags = flags + + def type(self): + return GRPC_OP_RECV_INITIAL_METADATA + + cdef void c(self): + self.c_op.type = GRPC_OP_RECV_INITIAL_METADATA + self.c_op.flags = self._flags + grpc_metadata_array_init(&self._c_initial_metadata) + self.c_op.data.receive_initial_metadata.receive_initial_metadata = ( + &self._c_initial_metadata) + + cdef void un_c(self): + self._initial_metadata = _metadata(&self._c_initial_metadata) + grpc_metadata_array_destroy(&self._c_initial_metadata) + + def initial_metadata(self): + return self._initial_metadata + + +cdef class ReceiveMessageOperation(Operation): + + def __cinit__(self, flags): + self._flags = flags + + def type(self): + return GRPC_OP_RECV_MESSAGE + + cdef void c(self): + self.c_op.type = GRPC_OP_RECV_MESSAGE + self.c_op.flags = self._flags + self.c_op.data.receive_message.receive_message = ( + &self._c_message_byte_buffer) + + cdef void un_c(self): + cdef grpc_byte_buffer_reader message_reader + cdef bint message_reader_status + cdef grpc_slice message_slice + cdef size_t message_slice_length + cdef void *message_slice_pointer + if self._c_message_byte_buffer != NULL: + message_reader_status = grpc_byte_buffer_reader_init( + &message_reader, self._c_message_byte_buffer) + if message_reader_status: + message = bytearray() + while grpc_byte_buffer_reader_next(&message_reader, &message_slice): + message_slice_pointer = grpc_slice_start_ptr(message_slice) + message_slice_length = grpc_slice_length(message_slice) + message += (<char *>message_slice_pointer)[:message_slice_length] + grpc_slice_unref(message_slice) + grpc_byte_buffer_reader_destroy(&message_reader) + self._message = bytes(message) + else: + self._message = None + grpc_byte_buffer_destroy(self._c_message_byte_buffer) + else: + self._message = None + + def message(self): + return self._message + + +cdef class ReceiveStatusOnClientOperation(Operation): + + def __cinit__(self, flags): + self._flags = flags + + def type(self): + return GRPC_OP_RECV_STATUS_ON_CLIENT + + cdef void c(self): + self.c_op.type = GRPC_OP_RECV_STATUS_ON_CLIENT + self.c_op.flags = self._flags + grpc_metadata_array_init(&self._c_trailing_metadata) + self.c_op.data.receive_status_on_client.trailing_metadata = ( + &self._c_trailing_metadata) + self.c_op.data.receive_status_on_client.status = ( + &self._c_code) + self.c_op.data.receive_status_on_client.status_details = ( + &self._c_details) + + cdef void un_c(self): + self._trailing_metadata = _metadata(&self._c_trailing_metadata) + grpc_metadata_array_destroy(&self._c_trailing_metadata) + self._code = self._c_code + self._details = _decode(_slice_bytes(self._c_details)) + grpc_slice_unref(self._c_details) + + def trailing_metadata(self): + return self._trailing_metadata + + def code(self): + return self._code + + def details(self): + return self._details + + +cdef class ReceiveCloseOnServerOperation(Operation): + + def __cinit__(self, flags): + self._flags = flags + + def type(self): + return GRPC_OP_RECV_CLOSE_ON_SERVER + + cdef void c(self): + self.c_op.type = GRPC_OP_RECV_CLOSE_ON_SERVER + self.c_op.flags = self._flags + self.c_op.data.receive_close_on_server.cancelled = &self._c_cancelled + + cdef void un_c(self): + self._cancelled = bool(self._c_cancelled) + + def cancelled(self): + return self._cancelled diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi index 594fdb1a8b..7b2482d947 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi @@ -28,48 +28,6 @@ cdef class CallDetails: cdef grpc_call_details c_details -cdef class OperationTag: - - cdef object user_tag - cdef list references - # This allows CompletionQueue to notify the Python Server object that the - # underlying GRPC core server has shutdown - cdef Server shutting_down_server - cdef Call operation_call - cdef CallDetails request_call_details - cdef grpc_metadata_array _c_request_metadata - cdef grpc_op *c_ops - cdef size_t c_nops - cdef readonly object _operations - cdef bint is_new_request - - cdef void store_ops(self) - cdef object release_ops(self) - - -cdef class Event: - - cdef readonly grpc_completion_type type - cdef readonly bint success - cdef readonly object tag - - # For Server.request_call - cdef readonly bint is_new_request - cdef readonly CallDetails request_call_details - cdef readonly object request_metadata - - # For server calls - cdef readonly Call operation_call - - # For Call.start_batch - cdef readonly object batch_operations - - -cdef class ByteBuffer: - - cdef grpc_byte_buffer *c_byte_buffer - - cdef class SslPemKeyCertPair: cdef grpc_ssl_pem_key_cert_pair c_pair @@ -89,22 +47,6 @@ cdef class ChannelArgs: cdef list args -cdef class Operation: - - cdef grpc_op c_op - cdef bint _c_metadata_needs_release - cdef size_t _c_metadata_count - cdef grpc_metadata *_c_metadata - cdef ByteBuffer _received_message - cdef bint _c_metadata_array_needs_destruction - cdef grpc_metadata_array _c_metadata_array - cdef grpc_status_code _received_status_code - cdef grpc_slice _status_details - cdef int _received_cancelled - cdef readonly bint is_valid - cdef object references - - cdef class CompressionOptions: cdef grpc_compression_options c_options diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi index 26eaf50eb4..bc2cd0338e 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi @@ -218,111 +218,6 @@ cdef class CallDetails: return timespec -cdef class OperationTag: - - def __cinit__(self, user_tag, operations): - self.user_tag = user_tag - self.references = [] - self._operations = operations - - cdef void store_ops(self): - self.c_nops = 0 if self._operations is None else len(self._operations) - if 0 < self.c_nops: - self.c_ops = <grpc_op *>gpr_malloc(sizeof(grpc_op) * self.c_nops) - for index in range(self.c_nops): - self.c_ops[index] = (<Operation>(self._operations[index])).c_op - - cdef object release_ops(self): - if 0 < self.c_nops: - for index, operation in enumerate(self._operations): - (<Operation>operation).c_op = self.c_ops[index] - gpr_free(self.c_ops) - return self._operations - else: - return () - - -cdef class Event: - - def __cinit__(self, grpc_completion_type type, bint success, - object tag, Call operation_call, - CallDetails request_call_details, - object request_metadata, - bint is_new_request, - object batch_operations): - self.type = type - self.success = success - self.tag = tag - self.operation_call = operation_call - self.request_call_details = request_call_details - self.request_metadata = request_metadata - self.batch_operations = batch_operations - self.is_new_request = is_new_request - - -cdef class ByteBuffer: - - def __cinit__(self, bytes data): - grpc_init() - if data is None: - self.c_byte_buffer = NULL - return - - cdef char *c_data = data - cdef grpc_slice data_slice - cdef size_t data_length = len(data) - with nogil: - data_slice = grpc_slice_from_copied_buffer(c_data, data_length) - with nogil: - self.c_byte_buffer = grpc_raw_byte_buffer_create( - &data_slice, 1) - with nogil: - grpc_slice_unref(data_slice) - - def bytes(self): - cdef grpc_byte_buffer_reader reader - cdef grpc_slice data_slice - cdef size_t data_slice_length - cdef void *data_slice_pointer - cdef bint reader_status - if self.c_byte_buffer != NULL: - with nogil: - reader_status = grpc_byte_buffer_reader_init( - &reader, self.c_byte_buffer) - if not reader_status: - return None - result = bytearray() - with nogil: - while grpc_byte_buffer_reader_next(&reader, &data_slice): - data_slice_pointer = grpc_slice_start_ptr(data_slice) - data_slice_length = grpc_slice_length(data_slice) - with gil: - result += (<char *>data_slice_pointer)[:data_slice_length] - grpc_slice_unref(data_slice) - with nogil: - grpc_byte_buffer_reader_destroy(&reader) - return bytes(result) - else: - return None - - def __len__(self): - cdef size_t result - if self.c_byte_buffer != NULL: - with nogil: - result = grpc_byte_buffer_length(self.c_byte_buffer) - return result - else: - return 0 - - def __str__(self): - return self.bytes() - - def __dealloc__(self): - if self.c_byte_buffer != NULL: - grpc_byte_buffer_destroy(self.c_byte_buffer) - grpc_shutdown() - - cdef class SslPemKeyCertPair: def __cinit__(self, bytes private_key, bytes certificate_chain): @@ -407,185 +302,6 @@ cdef class ChannelArgs: return self.args[i] -cdef class Operation: - - def __cinit__(self): - grpc_init() - self.references = [] - self._c_metadata_needs_release = False - self._c_metadata_array_needs_destruction = False - self._status_details = grpc_empty_slice() - self.is_valid = False - - @property - def type(self): - return self.c_op.type - - @property - def flags(self): - return self.c_op.flags - - @property - def has_status(self): - return self.c_op.type == GRPC_OP_RECV_STATUS_ON_CLIENT - - @property - def received_message(self): - if self.c_op.type != GRPC_OP_RECV_MESSAGE: - raise TypeError("self must be an operation receiving a message") - return self._received_message - - @property - def received_message_or_none(self): - if self.c_op.type != GRPC_OP_RECV_MESSAGE: - return None - return self._received_message - - @property - def received_metadata(self): - if (self.c_op.type != GRPC_OP_RECV_INITIAL_METADATA and - self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT): - raise TypeError("self must be an operation receiving metadata") - return _metadata(&self._c_metadata_array) - - @property - def received_status_code(self): - if self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT: - raise TypeError("self must be an operation receiving a status code") - return self._received_status_code - - @property - def received_status_code_or_none(self): - if self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT: - return None - return self._received_status_code - - @property - def received_status_details(self): - if self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT: - raise TypeError("self must be an operation receiving status details") - return _slice_bytes(self._status_details) - - @property - def received_status_details_or_none(self): - if self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT: - return None - return _slice_bytes(self._status_details) - - @property - def received_cancelled(self): - if self.c_op.type != GRPC_OP_RECV_CLOSE_ON_SERVER: - raise TypeError("self must be an operation receiving cancellation " - "information") - return False if self._received_cancelled == 0 else True - - @property - def received_cancelled_or_none(self): - if self.c_op.type != GRPC_OP_RECV_CLOSE_ON_SERVER: - return None - return False if self._received_cancelled == 0 else True - - def __dealloc__(self): - if self._c_metadata_needs_release: - _release_c_metadata(self._c_metadata, self._c_metadata_count) - if self._c_metadata_array_needs_destruction: - grpc_metadata_array_destroy(&self._c_metadata_array) - grpc_slice_unref(self._status_details) - grpc_shutdown() - -def operation_send_initial_metadata(metadata, int flags): - cdef Operation op = Operation() - op.c_op.type = GRPC_OP_SEND_INITIAL_METADATA - op.c_op.flags = flags - _store_c_metadata(metadata, &op._c_metadata, &op._c_metadata_count) - op._c_metadata_needs_release = True - op.c_op.data.send_initial_metadata.count = op._c_metadata_count - op.c_op.data.send_initial_metadata.metadata = op._c_metadata - op.is_valid = True - return op - -def operation_send_message(data, int flags): - cdef Operation op = Operation() - op.c_op.type = GRPC_OP_SEND_MESSAGE - op.c_op.flags = flags - byte_buffer = ByteBuffer(data) - op.c_op.data.send_message.send_message = byte_buffer.c_byte_buffer - op.references.append(byte_buffer) - op.is_valid = True - return op - -def operation_send_close_from_client(int flags): - cdef Operation op = Operation() - op.c_op.type = GRPC_OP_SEND_CLOSE_FROM_CLIENT - op.c_op.flags = flags - op.is_valid = True - return op - -def operation_send_status_from_server( - metadata, grpc_status_code code, bytes details, int flags): - cdef Operation op = Operation() - op.c_op.type = GRPC_OP_SEND_STATUS_FROM_SERVER - op.c_op.flags = flags - _store_c_metadata(metadata, &op._c_metadata, &op._c_metadata_count) - op._c_metadata_needs_release = True - op.c_op.data.send_status_from_server.trailing_metadata_count = ( - op._c_metadata_count) - op.c_op.data.send_status_from_server.trailing_metadata = op._c_metadata - op.c_op.data.send_status_from_server.status = code - grpc_slice_unref(op._status_details) - op._status_details = _slice_from_bytes(details) - op.c_op.data.send_status_from_server.status_details = &op._status_details - op.is_valid = True - return op - -def operation_receive_initial_metadata(int flags): - cdef Operation op = Operation() - op.c_op.type = GRPC_OP_RECV_INITIAL_METADATA - op.c_op.flags = flags - grpc_metadata_array_init(&op._c_metadata_array) - op.c_op.data.receive_initial_metadata.receive_initial_metadata = ( - &op._c_metadata_array) - op._c_metadata_array_needs_destruction = True - op.is_valid = True - return op - -def operation_receive_message(int flags): - cdef Operation op = Operation() - op.c_op.type = GRPC_OP_RECV_MESSAGE - op.c_op.flags = flags - op._received_message = ByteBuffer(None) - # n.b. the c_op.data.receive_message field needs to be deleted by us, - # anyway, so we just let that be handled by the ByteBuffer() we allocated - # the line before. - op.c_op.data.receive_message.receive_message = ( - &op._received_message.c_byte_buffer) - op.is_valid = True - return op - -def operation_receive_status_on_client(int flags): - cdef Operation op = Operation() - op.c_op.type = GRPC_OP_RECV_STATUS_ON_CLIENT - op.c_op.flags = flags - grpc_metadata_array_init(&op._c_metadata_array) - op.c_op.data.receive_status_on_client.trailing_metadata = ( - &op._c_metadata_array) - op._c_metadata_array_needs_destruction = True - op.c_op.data.receive_status_on_client.status = ( - &op._received_status_code) - op.c_op.data.receive_status_on_client.status_details = ( - &op._status_details) - op.is_valid = True - return op - -def operation_receive_close_on_server(int flags): - cdef Operation op = Operation() - op.c_op.type = GRPC_OP_RECV_CLOSE_ON_SERVER - op.c_op.flags = flags - op.c_op.data.receive_close_on_server.cancelled = &op._received_cancelled - op.is_valid = True - return op - - cdef class CompressionOptions: def __cinit__(self): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index f8d7892858..c19beccde6 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -78,19 +78,15 @@ cdef class Server: raise ValueError("server must be started and not shutting down") if server_queue not in self.registered_completion_queues: raise ValueError("server_queue must be a registered completion queue") - cdef OperationTag operation_tag = OperationTag(tag, None) - operation_tag.operation_call = Call() - operation_tag.request_call_details = CallDetails() - grpc_metadata_array_init(&operation_tag._c_request_metadata) - operation_tag.references.extend([self, call_queue, server_queue]) - operation_tag.is_new_request = True - cpython.Py_INCREF(operation_tag) + cdef _RequestCallTag request_call_tag = _RequestCallTag(tag) + request_call_tag.prepare() + cpython.Py_INCREF(request_call_tag) return grpc_server_request_call( - self.c_server, &operation_tag.operation_call.c_call, - &operation_tag.request_call_details.c_details, - &operation_tag._c_request_metadata, + self.c_server, &request_call_tag.call.c_call, + &request_call_tag.call_details.c_details, + &request_call_tag.c_invocation_metadata, call_queue.c_completion_queue, server_queue.c_completion_queue, - <cpython.PyObject *>operation_tag) + <cpython.PyObject *>request_call_tag) def register_completion_queue( self, CompletionQueue queue not None): @@ -131,16 +127,14 @@ cdef class Server: cdef _c_shutdown(self, CompletionQueue queue, tag): self.is_shutting_down = True - operation_tag = OperationTag(tag, None) - operation_tag.shutting_down_server = self - cpython.Py_INCREF(operation_tag) + cdef _ServerShutdownTag server_shutdown_tag = _ServerShutdownTag(tag, self) + cpython.Py_INCREF(server_shutdown_tag) with nogil: grpc_server_shutdown_and_notify( self.c_server, queue.c_completion_queue, - <cpython.PyObject *>operation_tag) + <cpython.PyObject *>server_shutdown_tag) def shutdown(self, CompletionQueue queue not None, tag): - cdef OperationTag operation_tag if queue.is_shutting_down: raise ValueError("queue must be live") elif not self.is_started: @@ -153,7 +147,8 @@ cdef class Server: self._c_shutdown(queue, tag) cdef notify_shutdown_complete(self): - # called only by a completion queue on receiving our shutdown operation tag + # called only after our server shutdown tag has emerged from a completion + # queue. self.is_shutdown = True def cancel_all_calls(self): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/tag.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/tag.pxd.pxi new file mode 100644 index 0000000000..f9a3b5e8f4 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/tag.pxd.pxi @@ -0,0 +1,58 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef class _Tag: + + cdef object event(self, grpc_event c_event) + + +cdef class _ConnectivityTag(_Tag): + + cdef readonly object _user_tag + + cdef ConnectivityEvent event(self, grpc_event c_event) + + +cdef class _RequestCallTag(_Tag): + + cdef readonly object _user_tag + cdef Call call + cdef CallDetails call_details + cdef grpc_metadata_array c_invocation_metadata + + cdef void prepare(self) + cdef RequestCallEvent event(self, grpc_event c_event) + + +cdef class _BatchOperationTag(_Tag): + + cdef object _user_tag + cdef readonly object _operations + cdef readonly object _retained_call + cdef grpc_op *c_ops + cdef size_t c_nops + + cdef void prepare(self) + cdef BatchOperationEvent event(self, grpc_event c_event) + + +cdef class _ServerShutdownTag(_Tag): + + cdef readonly object _user_tag + # This allows CompletionQueue to notify the Python Server object that the + # underlying GRPC core server has shutdown + cdef readonly Server _shutting_down_server + + cdef ServerShutdownEvent event(self, grpc_event c_event) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/tag.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/tag.pyx.pxi new file mode 100644 index 0000000000..aaca458442 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/tag.pyx.pxi @@ -0,0 +1,87 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef class _Tag: + + cdef object event(self, grpc_event c_event): + raise NotImplementedError() + + +cdef class _ConnectivityTag(_Tag): + + def __cinit__(self, user_tag): + self._user_tag = user_tag + + cdef ConnectivityEvent event(self, grpc_event c_event): + return ConnectivityEvent(c_event.type, c_event.success, self._user_tag) + + +cdef class _RequestCallTag(_Tag): + + def __cinit__(self, user_tag): + self._user_tag = user_tag + self.call = None + self.call_details = None + + cdef void prepare(self): + self.call = Call() + self.call_details = CallDetails() + grpc_metadata_array_init(&self.c_invocation_metadata) + + cdef RequestCallEvent event(self, grpc_event c_event): + cdef tuple invocation_metadata = _metadata(&self.c_invocation_metadata) + grpc_metadata_array_destroy(&self.c_invocation_metadata) + return RequestCallEvent( + c_event.type, c_event.success, self._user_tag, self.call, + self.call_details, invocation_metadata) + + +cdef class _BatchOperationTag: + + def __cinit__(self, user_tag, operations, call): + self._user_tag = user_tag + self._operations = operations + self._retained_call = call + + cdef void prepare(self): + self.c_nops = 0 if self._operations is None else len(self._operations) + if 0 < self.c_nops: + self.c_ops = <grpc_op *>gpr_malloc(sizeof(grpc_op) * self.c_nops) + for index, operation in enumerate(self._operations): + (<Operation>operation).c() + self.c_ops[index] = (<Operation>operation).c_op + + cdef BatchOperationEvent event(self, grpc_event c_event): + if 0 < self.c_nops: + for index, operation in enumerate(self._operations): + (<Operation>operation).c_op = self.c_ops[index] + (<Operation>operation).un_c() + gpr_free(self.c_ops) + return BatchOperationEvent( + c_event.type, c_event.success, self._user_tag, self._operations) + else: + return BatchOperationEvent( + c_event.type, c_event.success, self._user_tag, ()) + + +cdef class _ServerShutdownTag(_Tag): + + def __cinit__(self, user_tag, shutting_down_server): + self._user_tag = user_tag + self._shutting_down_server = shutting_down_server + + cdef ServerShutdownEvent event(self, grpc_event c_event): + self._shutting_down_server.notify_shutdown_complete() + return ServerShutdownEvent(c_event.type, c_event.success, self._user_tag)
\ No newline at end of file diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pxd b/src/python/grpcio/grpc/_cython/cygrpc.pxd index 6fc5638d5d..b32fa518fc 100644 --- a/src/python/grpcio/grpc/_cython/cygrpc.pxd +++ b/src/python/grpcio/grpc/_cython/cygrpc.pxd @@ -18,7 +18,10 @@ include "_cygrpc/call.pxd.pxi" include "_cygrpc/channel.pxd.pxi" include "_cygrpc/credentials.pxd.pxi" include "_cygrpc/completion_queue.pxd.pxi" +include "_cygrpc/event.pxd.pxi" include "_cygrpc/metadata.pxd.pxi" +include "_cygrpc/operation.pxd.pxi" include "_cygrpc/records.pxd.pxi" include "_cygrpc/security.pxd.pxi" include "_cygrpc/server.pxd.pxi" +include "_cygrpc/tag.pxd.pxi" diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx index d605229822..5106394708 100644 --- a/src/python/grpcio/grpc/_cython/cygrpc.pyx +++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx @@ -25,10 +25,13 @@ include "_cygrpc/call.pyx.pxi" include "_cygrpc/channel.pyx.pxi" include "_cygrpc/credentials.pyx.pxi" include "_cygrpc/completion_queue.pyx.pxi" +include "_cygrpc/event.pyx.pxi" include "_cygrpc/metadata.pyx.pxi" +include "_cygrpc/operation.pyx.pxi" include "_cygrpc/records.pyx.pxi" include "_cygrpc/security.pyx.pxi" include "_cygrpc/server.pyx.pxi" +include "_cygrpc/tag.pyx.pxi" # # initialize gRPC diff --git a/src/python/grpcio/grpc/_interceptor.py b/src/python/grpcio/grpc/_interceptor.py index fffb269845..56a280624f 100644 --- a/src/python/grpcio/grpc/_interceptor.py +++ b/src/python/grpcio/grpc/_interceptor.py @@ -44,9 +44,10 @@ def service_pipeline(interceptors): class _ClientCallDetails( - collections.namedtuple('_ClientCallDetails', - ('method', 'timeout', 'metadata', - 'credentials')), grpc.ClientCallDetails): + collections.namedtuple( + '_ClientCallDetails', + ('method', 'timeout', 'metadata', 'credentials')), + grpc.ClientCallDetails): pass diff --git a/src/python/grpcio/grpc/_plugin_wrapping.py b/src/python/grpcio/grpc/_plugin_wrapping.py index f7287956dc..6785e5876a 100644 --- a/src/python/grpcio/grpc/_plugin_wrapping.py +++ b/src/python/grpcio/grpc/_plugin_wrapping.py @@ -23,7 +23,9 @@ from grpc._cython import cygrpc class _AuthMetadataContext( collections.namedtuple('AuthMetadataContext', ( - 'service_url', 'method_name',)), grpc.AuthMetadataContext): + 'service_url', + 'method_name', + )), grpc.AuthMetadataContext): pass @@ -70,8 +72,9 @@ class _Plugin(object): _common.decode(service_url), _common.decode(method_name)) callback_state = _CallbackState() try: - self._metadata_plugin( - context, _AuthMetadataPluginCallback(callback_state, callback)) + self._metadata_plugin(context, + _AuthMetadataPluginCallback( + callback_state, callback)) except Exception as exception: # pylint: disable=broad-except logging.exception( 'AuthMetadataPluginCallback "%s" raised exception!', diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index 02d3af8706..1cdb2d45b6 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -50,7 +50,7 @@ _UNEXPECTED_EXIT_SERVER_GRACE = 1.0 def _serialized_request(request_event): - return request_event.batch_operations[0].received_message.bytes() + return request_event.batch_operations[0].message() def _application_code(code): @@ -78,7 +78,9 @@ def _details(state): class _HandlerCallDetails( collections.namedtuple('_HandlerCallDetails', ( - 'method', 'invocation_metadata',)), grpc.HandlerCallDetails): + 'method', + 'invocation_metadata', + )), grpc.HandlerCallDetails): pass @@ -130,13 +132,15 @@ def _abort(state, call, code, details): effective_code = _abortion_code(state, code) effective_details = details if state.details is None else state.details if state.initial_metadata_allowed: - operations = (cygrpc.operation_send_initial_metadata( - (), _EMPTY_FLAGS), cygrpc.operation_send_status_from_server( + operations = ( + cygrpc.SendInitialMetadataOperation(None, _EMPTY_FLAGS), + cygrpc.SendStatusFromServerOperation( state.trailing_metadata, effective_code, effective_details, - _EMPTY_FLAGS),) + _EMPTY_FLAGS), + ) token = _SEND_INITIAL_METADATA_AND_SEND_STATUS_FROM_SERVER_TOKEN else: - operations = (cygrpc.operation_send_status_from_server( + operations = (cygrpc.SendStatusFromServerOperation( state.trailing_metadata, effective_code, effective_details, _EMPTY_FLAGS),) token = _SEND_STATUS_FROM_SERVER_TOKEN @@ -150,8 +154,7 @@ def _receive_close_on_server(state): def receive_close_on_server(receive_close_on_server_event): with state.condition: - if receive_close_on_server_event.batch_operations[ - 0].received_cancelled: + if receive_close_on_server_event.batch_operations[0].cancelled(): state.client = _CANCELLED elif state.client is _OPEN: state.client = _CLOSED @@ -218,11 +221,10 @@ class _Context(grpc.ServicerContext): def time_remaining(self): return max( - float(self._rpc_event.request_call_details.deadline) - time.time(), - 0) + float(self._rpc_event.call_details.deadline) - time.time(), 0) def cancel(self): - self._rpc_event.operation_call.cancel() + self._rpc_event.call.cancel() def add_callback(self, callback): with self._state.condition: @@ -237,23 +239,23 @@ class _Context(grpc.ServicerContext): self._state.disable_next_compression = True def invocation_metadata(self): - return self._rpc_event.request_metadata + return self._rpc_event.invocation_metadata def peer(self): - return _common.decode(self._rpc_event.operation_call.peer()) + return _common.decode(self._rpc_event.call.peer()) def peer_identities(self): - return cygrpc.peer_identities(self._rpc_event.operation_call) + return cygrpc.peer_identities(self._rpc_event.call) def peer_identity_key(self): - id_key = cygrpc.peer_identity_key(self._rpc_event.operation_call) + id_key = cygrpc.peer_identity_key(self._rpc_event.call) return id_key if id_key is None else _common.decode(id_key) def auth_context(self): return { _common.decode(key): value for key, value in six.iteritems( - cygrpc.auth_context(self._rpc_event.operation_call)) + cygrpc.auth_context(self._rpc_event.call)) } def send_initial_metadata(self, initial_metadata): @@ -262,9 +264,9 @@ class _Context(grpc.ServicerContext): _raise_rpc_error(self._state) else: if self._state.initial_metadata_allowed: - operation = cygrpc.operation_send_initial_metadata( + operation = cygrpc.SendInitialMetadataOperation( initial_metadata, _EMPTY_FLAGS) - self._rpc_event.operation_call.start_server_batch( + self._rpc_event.call.start_server_batch( (operation,), _send_initial_metadata(self._state)) self._state.initial_metadata_allowed = False self._state.due.add(_SEND_INITIAL_METADATA_TOKEN) @@ -305,7 +307,7 @@ class _RequestIterator(object): raise StopIteration() else: self._call.start_server_batch( - (cygrpc.operation_receive_message(_EMPTY_FLAGS),), + (cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS),), _receive_message(self._state, self._call, self._request_deserializer)) self._state.due.add(_RECEIVE_MESSAGE_TOKEN) @@ -347,9 +349,9 @@ def _unary_request(rpc_event, state, request_deserializer): if state.client is _CANCELLED or state.statused: return None else: - rpc_event.operation_call.start_server_batch( - (cygrpc.operation_receive_message(_EMPTY_FLAGS),), - _receive_message(state, rpc_event.operation_call, + rpc_event.call.start_server_batch( + (cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS),), + _receive_message(state, rpc_event.call, request_deserializer)) state.due.add(_RECEIVE_MESSAGE_TOKEN) while True: @@ -357,8 +359,8 @@ def _unary_request(rpc_event, state, request_deserializer): if state.request is None: if state.client is _CLOSED: details = '"{}" requires exactly one request message.'.format( - rpc_event.request_call_details.method) - _abort(state, rpc_event.operation_call, + rpc_event.call_details.method) + _abort(state, rpc_event.call, cygrpc.StatusCode.unimplemented, _common.encode(details)) return None @@ -379,13 +381,13 @@ def _call_behavior(rpc_event, state, behavior, argument, request_deserializer): except Exception as exception: # pylint: disable=broad-except with state.condition: if exception is state.abortion: - _abort(state, rpc_event.operation_call, - cygrpc.StatusCode.unknown, b'RPC Aborted') + _abort(state, rpc_event.call, cygrpc.StatusCode.unknown, + b'RPC Aborted') elif exception not in state.rpc_errors: details = 'Exception calling application: {}'.format(exception) logging.exception(details) - _abort(state, rpc_event.operation_call, - cygrpc.StatusCode.unknown, _common.encode(details)) + _abort(state, rpc_event.call, cygrpc.StatusCode.unknown, + _common.encode(details)) return None, False @@ -397,13 +399,13 @@ def _take_response_from_response_iterator(rpc_event, state, response_iterator): except Exception as exception: # pylint: disable=broad-except with state.condition: if exception is state.abortion: - _abort(state, rpc_event.operation_call, - cygrpc.StatusCode.unknown, b'RPC Aborted') + _abort(state, rpc_event.call, cygrpc.StatusCode.unknown, + b'RPC Aborted') elif exception not in state.rpc_errors: details = 'Exception iterating responses: {}'.format(exception) logging.exception(details) - _abort(state, rpc_event.operation_call, - cygrpc.StatusCode.unknown, _common.encode(details)) + _abort(state, rpc_event.call, cygrpc.StatusCode.unknown, + _common.encode(details)) return None, False @@ -411,7 +413,7 @@ def _serialize_response(rpc_event, state, response, response_serializer): serialized_response = _common.serialize(response, response_serializer) if serialized_response is None: with state.condition: - _abort(state, rpc_event.operation_call, cygrpc.StatusCode.internal, + _abort(state, rpc_event.call, cygrpc.StatusCode.internal, b'Failed to serialize response!') return None else: @@ -424,17 +426,19 @@ def _send_response(rpc_event, state, serialized_response): return False else: if state.initial_metadata_allowed: - operations = (cygrpc.operation_send_initial_metadata( - (), _EMPTY_FLAGS), cygrpc.operation_send_message( - serialized_response, _EMPTY_FLAGS),) + operations = ( + cygrpc.SendInitialMetadataOperation(None, _EMPTY_FLAGS), + cygrpc.SendMessageOperation(serialized_response, + _EMPTY_FLAGS), + ) state.initial_metadata_allowed = False token = _SEND_INITIAL_METADATA_AND_SEND_MESSAGE_TOKEN else: - operations = (cygrpc.operation_send_message(serialized_response, - _EMPTY_FLAGS),) + operations = (cygrpc.SendMessageOperation( + serialized_response, _EMPTY_FLAGS),) token = _SEND_MESSAGE_TOKEN - rpc_event.operation_call.start_server_batch( - operations, _send_message(state, token)) + rpc_event.call.start_server_batch(operations, + _send_message(state, token)) state.due.add(token) while True: state.condition.wait() @@ -448,17 +452,17 @@ def _status(rpc_event, state, serialized_response): code = _completion_code(state) details = _details(state) operations = [ - cygrpc.operation_send_status_from_server( + cygrpc.SendStatusFromServerOperation( state.trailing_metadata, code, details, _EMPTY_FLAGS), ] if state.initial_metadata_allowed: operations.append( - cygrpc.operation_send_initial_metadata((), _EMPTY_FLAGS)) + cygrpc.SendInitialMetadataOperation(None, _EMPTY_FLAGS)) if serialized_response is not None: operations.append( - cygrpc.operation_send_message(serialized_response, - _EMPTY_FLAGS)) - rpc_event.operation_call.start_server_batch( + cygrpc.SendMessageOperation(serialized_response, + _EMPTY_FLAGS)) + rpc_event.call.start_server_batch( operations, _send_status_from_server(state, _SEND_STATUS_FROM_SERVER_TOKEN)) state.statused = True @@ -525,7 +529,7 @@ def _handle_unary_stream(rpc_event, state, method_handler, thread_pool): def _handle_stream_unary(rpc_event, state, method_handler, thread_pool): - request_iterator = _RequestIterator(state, rpc_event.operation_call, + request_iterator = _RequestIterator(state, rpc_event.call, method_handler.request_deserializer) return thread_pool.submit( _unary_response_in_pool, rpc_event, state, method_handler.stream_unary, @@ -534,7 +538,7 @@ def _handle_stream_unary(rpc_event, state, method_handler, thread_pool): def _handle_stream_stream(rpc_event, state, method_handler, thread_pool): - request_iterator = _RequestIterator(state, rpc_event.operation_call, + request_iterator = _RequestIterator(state, rpc_event.call, method_handler.request_deserializer) return thread_pool.submit( _stream_response_in_pool, rpc_event, state, @@ -552,8 +556,8 @@ def _find_method_handler(rpc_event, generic_handlers, interceptor_pipeline): return None handler_call_details = _HandlerCallDetails( - _common.decode(rpc_event.request_call_details.method), - rpc_event.request_metadata) + _common.decode(rpc_event.call_details.method), + rpc_event.invocation_metadata) if interceptor_pipeline is not None: return interceptor_pipeline.execute(query_handlers, @@ -563,21 +567,23 @@ def _find_method_handler(rpc_event, generic_handlers, interceptor_pipeline): def _reject_rpc(rpc_event, status, details): - operations = (cygrpc.operation_send_initial_metadata((), _EMPTY_FLAGS), - cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS), - cygrpc.operation_send_status_from_server((), status, details, - _EMPTY_FLAGS),) + operations = ( + cygrpc.SendInitialMetadataOperation(None, _EMPTY_FLAGS), + cygrpc.ReceiveCloseOnServerOperation(_EMPTY_FLAGS), + cygrpc.SendStatusFromServerOperation(None, status, details, + _EMPTY_FLAGS), + ) rpc_state = _RPCState() - rpc_event.operation_call.start_server_batch( - operations, lambda ignored_event: (rpc_state, (),)) + rpc_event.call.start_server_batch(operations, + lambda ignored_event: (rpc_state, (),)) return rpc_state def _handle_with_method_handler(rpc_event, method_handler, thread_pool): state = _RPCState() with state.condition: - rpc_event.operation_call.start_server_batch( - (cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),), + rpc_event.call.start_server_batch( + (cygrpc.ReceiveCloseOnServerOperation(_EMPTY_FLAGS),), _receive_close_on_server(state)) state.due.add(_RECEIVE_CLOSE_ON_SERVER_TOKEN) if method_handler.request_streaming: @@ -600,7 +606,7 @@ def _handle_call(rpc_event, generic_handlers, interceptor_pipeline, thread_pool, concurrency_exceeded): if not rpc_event.success: return None, None - if rpc_event.request_call_details.method is not None: + if rpc_event.call_details.method is not None: try: method_handler = _find_method_handler(rpc_event, generic_handlers, interceptor_pipeline) @@ -799,8 +805,8 @@ class Server(grpc.Server): return _add_insecure_port(self._state, _common.encode(address)) def add_secure_port(self, address, server_credentials): - return _add_secure_port(self._state, - _common.encode(address), server_credentials) + return _add_secure_port(self._state, _common.encode(address), + server_credentials) def start(self): _start(self._state) diff --git a/src/python/grpcio/grpc/_utilities.py b/src/python/grpcio/grpc/_utilities.py index 47cedcc867..25bd1ceae2 100644 --- a/src/python/grpcio/grpc/_utilities.py +++ b/src/python/grpcio/grpc/_utilities.py @@ -29,9 +29,15 @@ _DONE_CALLBACK_EXCEPTION_LOG_MESSAGE = ( class RpcMethodHandler( collections.namedtuple('_RpcMethodHandler', ( - 'request_streaming', 'response_streaming', 'request_deserializer', - 'response_serializer', 'unary_unary', 'unary_stream', - 'stream_unary', 'stream_stream',)), grpc.RpcMethodHandler): + 'request_streaming', + 'response_streaming', + 'request_deserializer', + 'response_serializer', + 'unary_unary', + 'unary_stream', + 'stream_unary', + 'stream_stream', + )), grpc.RpcMethodHandler): pass diff --git a/src/python/grpcio/grpc/beta/_client_adaptations.py b/src/python/grpcio/grpc/beta/_client_adaptations.py index dcaa0eeaa2..cf200a8c13 100644 --- a/src/python/grpcio/grpc/beta/_client_adaptations.py +++ b/src/python/grpcio/grpc/beta/_client_adaptations.py @@ -51,8 +51,7 @@ def _abortion(rpc_error_call): code = rpc_error_call.code() pair = _STATUS_CODE_TO_ABORTION_KIND_AND_ABORTION_ERROR_CLASS.get(code) error_kind = face.Abortion.Kind.LOCAL_FAILURE if pair is None else pair[0] - return face.Abortion(error_kind, - rpc_error_call.initial_metadata(), + return face.Abortion(error_kind, rpc_error_call.initial_metadata(), rpc_error_call.trailing_metadata(), code, rpc_error_call.details()) @@ -441,9 +440,14 @@ class _GenericStub(face.GenericStub): metadata=None, with_call=None, protocol_options=None): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _blocking_unary_unary(self._channel, group, method, timeout, with_call, protocol_options, metadata, self._metadata_transformer, request, @@ -456,9 +460,14 @@ class _GenericStub(face.GenericStub): timeout, metadata=None, protocol_options=None): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _future_unary_unary(self._channel, group, method, timeout, protocol_options, metadata, self._metadata_transformer, request, @@ -471,9 +480,14 @@ class _GenericStub(face.GenericStub): timeout, metadata=None, protocol_options=None): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _unary_stream(self._channel, group, method, timeout, protocol_options, metadata, self._metadata_transformer, request, @@ -487,9 +501,14 @@ class _GenericStub(face.GenericStub): metadata=None, with_call=None, protocol_options=None): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _blocking_stream_unary( self._channel, group, method, timeout, with_call, protocol_options, metadata, self._metadata_transformer, request_iterator, @@ -502,9 +521,14 @@ class _GenericStub(face.GenericStub): timeout, metadata=None, protocol_options=None): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _future_stream_unary( self._channel, group, method, timeout, protocol_options, metadata, self._metadata_transformer, request_iterator, request_serializer, @@ -517,9 +541,14 @@ class _GenericStub(face.GenericStub): timeout, metadata=None, protocol_options=None): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _stream_stream(self._channel, group, method, timeout, protocol_options, metadata, self._metadata_transformer, request_iterator, @@ -568,33 +597,53 @@ class _GenericStub(face.GenericStub): raise NotImplementedError() def unary_unary(self, group, method): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _UnaryUnaryMultiCallable( self._channel, group, method, self._metadata_transformer, request_serializer, response_deserializer) def unary_stream(self, group, method): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _UnaryStreamMultiCallable( self._channel, group, method, self._metadata_transformer, request_serializer, response_deserializer) def stream_unary(self, group, method): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _StreamUnaryMultiCallable( self._channel, group, method, self._metadata_transformer, request_serializer, response_deserializer) def stream_stream(self, group, method): - request_serializer = self._request_serializers.get((group, method,)) - response_deserializer = self._response_deserializers.get((group, - method,)) + request_serializer = self._request_serializers.get(( + group, + method, + )) + response_deserializer = self._response_deserializers.get(( + group, + method, + )) return _StreamStreamMultiCallable( self._channel, group, method, self._metadata_transformer, request_serializer, response_deserializer) @@ -624,8 +673,8 @@ class _DynamicStub(face.DynamicStub): elif method_cardinality is cardinality.Cardinality.STREAM_STREAM: return self._generic_stub.stream_stream(self._group, attr) else: - raise AttributeError('_DynamicStub object has no attribute "%s"!' % - attr) + raise AttributeError( + '_DynamicStub object has no attribute "%s"!' % attr) def __enter__(self): return self diff --git a/src/python/grpcio/grpc/beta/_metadata.py b/src/python/grpcio/grpc/beta/_metadata.py index e135f4dff4..b7c8535285 100644 --- a/src/python/grpcio/grpc/beta/_metadata.py +++ b/src/python/grpcio/grpc/beta/_metadata.py @@ -15,7 +15,10 @@ import collections -_Metadatum = collections.namedtuple('_Metadatum', ('key', 'value',)) +_Metadatum = collections.namedtuple('_Metadatum', ( + 'key', + 'value', +)) def _beta_metadatum(key, value): diff --git a/src/python/grpcio/grpc/beta/_server_adaptations.py b/src/python/grpcio/grpc/beta/_server_adaptations.py index 1c22dbe3bb..3c04fd7639 100644 --- a/src/python/grpcio/grpc/beta/_server_adaptations.py +++ b/src/python/grpcio/grpc/beta/_server_adaptations.py @@ -245,9 +245,15 @@ def _adapt_stream_stream_event(stream_stream_event): class _SimpleMethodHandler( collections.namedtuple('_MethodHandler', ( - 'request_streaming', 'response_streaming', 'request_deserializer', - 'response_serializer', 'unary_unary', 'unary_stream', - 'stream_unary', 'stream_stream',)), grpc.RpcMethodHandler): + 'request_streaming', + 'response_streaming', + 'request_deserializer', + 'response_serializer', + 'unary_unary', + 'unary_stream', + 'stream_unary', + 'stream_stream', + )), grpc.RpcMethodHandler): pass @@ -255,15 +261,17 @@ def _simple_method_handler(implementation, request_deserializer, response_serializer): if implementation.style is style.Service.INLINE: if implementation.cardinality is cardinality.Cardinality.UNARY_UNARY: - return _SimpleMethodHandler( - False, False, request_deserializer, response_serializer, - _adapt_unary_request_inline(implementation.unary_unary_inline), - None, None, None) + return _SimpleMethodHandler(False, False, request_deserializer, + response_serializer, + _adapt_unary_request_inline( + implementation.unary_unary_inline), + None, None, None) elif implementation.cardinality is cardinality.Cardinality.UNARY_STREAM: - return _SimpleMethodHandler( - False, True, request_deserializer, response_serializer, None, - _adapt_unary_request_inline(implementation.unary_stream_inline), - None, None) + return _SimpleMethodHandler(False, True, request_deserializer, + response_serializer, None, + _adapt_unary_request_inline( + implementation.unary_stream_inline), + None, None) elif implementation.cardinality is cardinality.Cardinality.STREAM_UNARY: return _SimpleMethodHandler(True, False, request_deserializer, response_serializer, None, None, @@ -278,26 +286,28 @@ def _simple_method_handler(implementation, request_deserializer, implementation.stream_stream_inline)) elif implementation.style is style.Service.EVENT: if implementation.cardinality is cardinality.Cardinality.UNARY_UNARY: - return _SimpleMethodHandler( - False, False, request_deserializer, response_serializer, - _adapt_unary_unary_event(implementation.unary_unary_event), - None, None, None) + return _SimpleMethodHandler(False, False, request_deserializer, + response_serializer, + _adapt_unary_unary_event( + implementation.unary_unary_event), + None, None, None) elif implementation.cardinality is cardinality.Cardinality.UNARY_STREAM: - return _SimpleMethodHandler( - False, True, request_deserializer, response_serializer, None, - _adapt_unary_stream_event(implementation.unary_stream_event), - None, None) + return _SimpleMethodHandler(False, True, request_deserializer, + response_serializer, None, + _adapt_unary_stream_event( + implementation.unary_stream_event), + None, None) elif implementation.cardinality is cardinality.Cardinality.STREAM_UNARY: - return _SimpleMethodHandler( - True, False, request_deserializer, response_serializer, None, - None, - _adapt_stream_unary_event(implementation.stream_unary_event), - None) + return _SimpleMethodHandler(True, False, request_deserializer, + response_serializer, None, None, + _adapt_stream_unary_event( + implementation.stream_unary_event), + None) elif implementation.cardinality is cardinality.Cardinality.STREAM_STREAM: - return _SimpleMethodHandler( - True, True, request_deserializer, response_serializer, None, - None, None, - _adapt_stream_stream_event(implementation.stream_stream_event)) + return _SimpleMethodHandler(True, True, request_deserializer, + response_serializer, None, None, None, + _adapt_stream_stream_event( + implementation.stream_stream_event)) def _flatten_method_pair_map(method_pair_map): @@ -325,10 +335,11 @@ class _GenericRpcHandler(grpc.GenericRpcHandler): method_implementation = self._method_implementations.get( handler_call_details.method) if method_implementation is not None: - return _simple_method_handler( - method_implementation, - self._request_deserializers.get(handler_call_details.method), - self._response_serializers.get(handler_call_details.method)) + return _simple_method_handler(method_implementation, + self._request_deserializers.get( + handler_call_details.method), + self._response_serializers.get( + handler_call_details.method)) elif self._multi_method_implementation is None: return None else: diff --git a/src/python/grpcio/grpc/beta/implementations.py b/src/python/grpcio/grpc/beta/implementations.py index 312daf033e..44dbd61c55 100644 --- a/src/python/grpcio/grpc/beta/implementations.py +++ b/src/python/grpcio/grpc/beta/implementations.py @@ -110,8 +110,8 @@ def insecure_channel(host, port): Returns: A Channel to the remote host through which RPCs may be conducted. """ - channel = grpc.insecure_channel(host - if port is None else '%s:%d' % (host, port)) + channel = grpc.insecure_channel(host if port is None else '%s:%d' % (host, + port)) return Channel(channel) diff --git a/src/python/grpcio/grpc/framework/foundation/callable_util.py b/src/python/grpcio/grpc/framework/foundation/callable_util.py index 5bdfda5301..b9b9c49f17 100644 --- a/src/python/grpcio/grpc/framework/foundation/callable_util.py +++ b/src/python/grpcio/grpc/framework/foundation/callable_util.py @@ -50,8 +50,8 @@ class _EasyOutcome( def _call_logging_exceptions(behavior, message, *args, **kwargs): try: - return _EasyOutcome(Outcome.Kind.RETURNED, - behavior(*args, **kwargs), None) + return _EasyOutcome(Outcome.Kind.RETURNED, behavior(*args, **kwargs), + None) except Exception as e: # pylint: disable=broad-except logging.exception(message) return _EasyOutcome(Outcome.Kind.RAISED, None, e) diff --git a/src/python/grpcio/grpc/framework/interfaces/base/utilities.py b/src/python/grpcio/grpc/framework/interfaces/base/utilities.py index a9163d8588..281db62b5d 100644 --- a/src/python/grpcio/grpc/framework/interfaces/base/utilities.py +++ b/src/python/grpcio/grpc/framework/interfaces/base/utilities.py @@ -19,15 +19,22 @@ from grpc.framework.interfaces.base import base class _Completion(base.Completion, - collections.namedtuple('_Completion', ('terminal_metadata', - 'code', 'message',))): + collections.namedtuple('_Completion', ( + 'terminal_metadata', + 'code', + 'message', + ))): """A trivial implementation of base.Completion.""" class _Subscription(base.Subscription, collections.namedtuple('_Subscription', ( - 'kind', 'termination_callback', 'allowance', 'operator', - 'protocol_receiver',))): + 'kind', + 'termination_callback', + 'allowance', + 'operator', + 'protocol_receiver', + ))): """A trivial implementation of base.Subscription.""" diff --git a/src/python/grpcio/grpc/framework/interfaces/face/face.py b/src/python/grpcio/grpc/framework/interfaces/face/face.py index 0b93ea0f49..5b47f11d0d 100644 --- a/src/python/grpcio/grpc/framework/interfaces/face/face.py +++ b/src/python/grpcio/grpc/framework/interfaces/face/face.py @@ -50,13 +50,20 @@ class NoSuchMethodError(Exception): self.method = method def __repr__(self): - return 'face.NoSuchMethodError(%s, %s)' % (self.group, self.method,) + return 'face.NoSuchMethodError(%s, %s)' % ( + self.group, + self.method, + ) class Abortion( - collections.namedtuple('Abortion', - ('kind', 'initial_metadata', 'terminal_metadata', - 'code', 'details',))): + collections.namedtuple('Abortion', ( + 'kind', + 'initial_metadata', + 'terminal_metadata', + 'code', + 'details', + ))): """A value describing RPC abortion. Attributes: diff --git a/src/python/grpcio_health_checking/health_commands.py b/src/python/grpcio_health_checking/health_commands.py index 19547358a1..933f965aa2 100644 --- a/src/python/grpcio_health_checking/health_commands.py +++ b/src/python/grpcio_health_checking/health_commands.py @@ -36,9 +36,9 @@ class CopyProtoModules(setuptools.Command): def run(self): if os.path.isfile(HEALTH_PROTO): - shutil.copyfile( - HEALTH_PROTO, - os.path.join(ROOT_DIR, 'grpc_health/v1/health.proto')) + shutil.copyfile(HEALTH_PROTO, + os.path.join(ROOT_DIR, + 'grpc_health/v1/health.proto')) class BuildPackageProtos(setuptools.Command): diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index c105f57509..60d309ec65 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -56,8 +56,10 @@ PACKAGE_DIRECTORIES = { '': '.', } -INSTALL_REQUIRES = ('protobuf>=3.5.0.post1', - 'grpcio>={version}'.format(version=grpc_version.VERSION),) +INSTALL_REQUIRES = ( + 'protobuf>=3.5.0.post1', + 'grpcio>={version}'.format(version=grpc_version.VERSION), +) try: import health_commands as _health_commands diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py index 26a7ba8685..0c564f10e5 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py @@ -27,7 +27,8 @@ def _not_found_error(): return reflection_pb2.ServerReflectionResponse( error_response=reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.NOT_FOUND.value[0], - error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(),)) + error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), + )) def _file_descriptor_response(descriptor): @@ -101,10 +102,11 @@ class ReflectionServicer(reflection_pb2_grpc.ServerReflectionServicer): def _list_services(self): return reflection_pb2.ServerReflectionResponse( - list_services_response=reflection_pb2.ListServiceResponse(service=[ - reflection_pb2.ServiceResponse(name=service_name) - for service_name in self._service_names - ])) + list_services_response=reflection_pb2.ListServiceResponse( + service=[ + reflection_pb2.ServiceResponse(name=service_name) + for service_name in self._service_names + ])) def ServerReflectionInfo(self, request_iterator, context): # pylint: disable=unused-argument @@ -128,7 +130,8 @@ class ReflectionServicer(reflection_pb2_grpc.ServerReflectionServicer): error_response=reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.INVALID_ARGUMENT.value[0], error_message=grpc.StatusCode.INVALID_ARGUMENT.value[1] - .encode(),)) + .encode(), + )) def enable_server_reflection(service_names, server, pool=None): diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index 760b89373a..10c4c38f19 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -57,8 +57,10 @@ PACKAGE_DIRECTORIES = { '': '.', } -INSTALL_REQUIRES = ('protobuf>=3.5.0.post1', - 'grpcio>={version}'.format(version=grpc_version.VERSION),) +INSTALL_REQUIRES = ( + 'protobuf>=3.5.0.post1', + 'grpcio>={version}'.format(version=grpc_version.VERSION), +) try: import reflection_commands as _reflection_commands diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_multi_callable.py b/src/python/grpcio_testing/grpc_testing/_channel/_multi_callable.py index 32b2f361d3..2b2f5761f5 100644 --- a/src/python/grpcio_testing/grpc_testing/_channel/_multi_callable.py +++ b/src/python/grpcio_testing/grpc_testing/_channel/_multi_callable.py @@ -27,20 +27,20 @@ class UnaryUnary(grpc.UnaryUnaryMultiCallable): def __call__(self, request, timeout=None, metadata=None, credentials=None): rpc_handler = self._channel_handler.invoke_rpc( - self._method_full_rpc_name, - _common.fuss_with_metadata(metadata), [request], True, timeout) + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [request], True, timeout) return _invocation.blocking_unary_response(rpc_handler) def with_call(self, request, timeout=None, metadata=None, credentials=None): rpc_handler = self._channel_handler.invoke_rpc( - self._method_full_rpc_name, - _common.fuss_with_metadata(metadata), [request], True, timeout) + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [request], True, timeout) return _invocation.blocking_unary_response_with_call(rpc_handler) def future(self, request, timeout=None, metadata=None, credentials=None): rpc_handler = self._channel_handler.invoke_rpc( - self._method_full_rpc_name, - _common.fuss_with_metadata(metadata), [request], True, timeout) + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [request], True, timeout) return _invocation.future_call(rpc_handler) @@ -52,8 +52,8 @@ class UnaryStream(grpc.StreamStreamMultiCallable): def __call__(self, request, timeout=None, metadata=None, credentials=None): rpc_handler = self._channel_handler.invoke_rpc( - self._method_full_rpc_name, - _common.fuss_with_metadata(metadata), [request], True, timeout) + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [request], True, timeout) return _invocation.ResponseIteratorCall(rpc_handler) @@ -69,8 +69,8 @@ class StreamUnary(grpc.StreamUnaryMultiCallable): metadata=None, credentials=None): rpc_handler = self._channel_handler.invoke_rpc( - self._method_full_rpc_name, - _common.fuss_with_metadata(metadata), [], False, timeout) + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [], False, timeout) _invocation.consume_requests(request_iterator, rpc_handler) return _invocation.blocking_unary_response(rpc_handler) @@ -80,8 +80,8 @@ class StreamUnary(grpc.StreamUnaryMultiCallable): metadata=None, credentials=None): rpc_handler = self._channel_handler.invoke_rpc( - self._method_full_rpc_name, - _common.fuss_with_metadata(metadata), [], False, timeout) + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [], False, timeout) _invocation.consume_requests(request_iterator, rpc_handler) return _invocation.blocking_unary_response_with_call(rpc_handler) @@ -91,8 +91,8 @@ class StreamUnary(grpc.StreamUnaryMultiCallable): metadata=None, credentials=None): rpc_handler = self._channel_handler.invoke_rpc( - self._method_full_rpc_name, - _common.fuss_with_metadata(metadata), [], False, timeout) + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [], False, timeout) _invocation.consume_requests(request_iterator, rpc_handler) return _invocation.future_call(rpc_handler) @@ -109,8 +109,8 @@ class StreamStream(grpc.StreamStreamMultiCallable): metadata=None, credentials=None): rpc_handler = self._channel_handler.invoke_rpc( - self._method_full_rpc_name, - _common.fuss_with_metadata(metadata), [], False, timeout) + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [], False, timeout) _invocation.consume_requests(request_iterator, rpc_handler) return _invocation.ResponseIteratorCall(rpc_handler) diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_rpc_state.py b/src/python/grpcio_testing/grpc_testing/_channel/_rpc_state.py index ee0233002d..009f675e49 100644 --- a/src/python/grpcio_testing/grpc_testing/_channel/_rpc_state.py +++ b/src/python/grpcio_testing/grpc_testing/_channel/_rpc_state.py @@ -179,8 +179,8 @@ class State(_common.ChannelRpcHandler): elif self._code is None: self._condition.wait() else: - raise ValueError( - 'Status code unexpectedly {}!'.format(self._code)) + raise ValueError('Status code unexpectedly {}!'.format( + self._code)) def is_active(self): raise NotImplementedError() diff --git a/src/python/grpcio_testing/grpc_testing/_common.py b/src/python/grpcio_testing/grpc_testing/_common.py index 05327b0ac2..cebad31b5c 100644 --- a/src/python/grpcio_testing/grpc_testing/_common.py +++ b/src/python/grpcio_testing/grpc_testing/_common.py @@ -20,9 +20,10 @@ import six def _fuss(tuplified_metadata): - return tuplified_metadata + ( - ('grpc.metadata_added_by_runtime', - 'gRPC is allowed to add metadata in transmission and does so.',),) + return tuplified_metadata + (( + 'grpc.metadata_added_by_runtime', + 'gRPC is allowed to add metadata in transmission and does so.', + ),) FUSSED_EMPTY_METADATA = _fuss(()) @@ -46,9 +47,12 @@ def rpc_names(service_descriptors): class ChannelRpcRead( - collections.namedtuple( - 'ChannelRpcRead', - ('response', 'trailing_metadata', 'code', 'details',))): + collections.namedtuple('ChannelRpcRead', ( + 'response', + 'trailing_metadata', + 'code', + 'details', + ))): pass @@ -100,8 +104,11 @@ class ChannelHandler(six.with_metaclass(abc.ABCMeta)): class ServerRpcRead( - collections.namedtuple('ServerRpcRead', - ('request', 'requests_closed', 'terminated',))): + collections.namedtuple('ServerRpcRead', ( + 'request', + 'requests_closed', + 'terminated', + ))): pass diff --git a/src/python/grpcio_testing/grpc_testing/_server/_handler.py b/src/python/grpcio_testing/grpc_testing/_server/_handler.py index 5e4730e087..d4f50f6863 100644 --- a/src/python/grpcio_testing/grpc_testing/_server/_handler.py +++ b/src/python/grpcio_testing/grpc_testing/_server/_handler.py @@ -170,8 +170,12 @@ class _Handler(Handler): if self._unary_response is None: if self._responses: self._unary_response = self._responses.pop(0) - return (self._unary_response, self._trailing_metadata, - self._code, self._details,) + return ( + self._unary_response, + self._trailing_metadata, + self._code, + self._details, + ) def stream_response_termination(self): with self._condition: diff --git a/src/python/grpcio_testing/grpc_testing/_server/_server.py b/src/python/grpcio_testing/grpc_testing/_server/_server.py index 3e358e50a9..c7effb6b55 100644 --- a/src/python/grpcio_testing/grpc_testing/_server/_server.py +++ b/src/python/grpcio_testing/grpc_testing/_server/_server.py @@ -76,7 +76,11 @@ class _Serverish(_common.Serverish): rpc, self._time, deadline) service_thread = threading.Thread( target=service_behavior, - args=(implementation, rpc, servicer_context,)) + args=( + implementation, + rpc, + servicer_context, + )) service_thread.start() def invoke_unary_unary(self, method_descriptor, handler, diff --git a/src/python/grpcio_testing/grpc_testing/_time.py b/src/python/grpcio_testing/grpc_testing/_time.py index 3b1ab4bcd8..afbdad3524 100644 --- a/src/python/grpcio_testing/grpc_testing/_time.py +++ b/src/python/grpcio_testing/grpc_testing/_time.py @@ -46,9 +46,11 @@ class _State(object): class _Delta( - collections.namedtuple('_Delta', - ('mature_behaviors', 'earliest_mature_time', - 'earliest_immature_time',))): + collections.namedtuple('_Delta', ( + 'mature_behaviors', + 'earliest_mature_time', + 'earliest_immature_time', + ))): pass diff --git a/src/python/grpcio_testing/setup.py b/src/python/grpcio_testing/setup.py index fa40424f6a..5a9d593ec1 100644 --- a/src/python/grpcio_testing/setup.py +++ b/src/python/grpcio_testing/setup.py @@ -28,8 +28,10 @@ PACKAGE_DIRECTORIES = { '': '.', } -INSTALL_REQUIRES = ('protobuf>=3.5.0.post1', - 'grpcio>={version}'.format(version=grpc_version.VERSION),) +INSTALL_REQUIRES = ( + 'protobuf>=3.5.0.post1', + 'grpcio>={version}'.format(version=grpc_version.VERSION), +) setuptools.setup( name='grpcio-testing', diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index aeb4ea9c53..250df65803 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -99,4 +99,5 @@ setuptools.setup( tests_require=TESTS_REQUIRE, test_suite=TEST_SUITE, test_loader=TEST_LOADER, - test_runner=TEST_RUNNER,) + test_runner=TEST_RUNNER, +) diff --git a/src/python/grpcio_tests/tests/_loader.py b/src/python/grpcio_tests/tests/_loader.py index 281a23c16e..31680916b4 100644 --- a/src/python/grpcio_tests/tests/_loader.py +++ b/src/python/grpcio_tests/tests/_loader.py @@ -101,5 +101,5 @@ def iterate_suite_cases(suite): elif isinstance(item, unittest.TestCase): yield item else: - raise ValueError( - 'unexpected suite item of type {}'.format(type(item))) + raise ValueError('unexpected suite item of type {}'.format( + type(item))) diff --git a/src/python/grpcio_tests/tests/_result.py b/src/python/grpcio_tests/tests/_result.py index f26fdefc97..9907c4e1f9 100644 --- a/src/python/grpcio_tests/tests/_result.py +++ b/src/python/grpcio_tests/tests/_result.py @@ -215,7 +215,8 @@ class AugmentedResult(unittest.TestResult): Args: filter (callable): A unary predicate to filter over CaseResult objects. """ - return (self.cases[case_id] for case_id in self.cases + return (self.cases[case_id] + for case_id in self.cases if filter(self.cases[case_id])) @@ -285,8 +286,8 @@ class TerminalResult(CoverageResult): def startTestRun(self): """See unittest.TestResult.startTestRun.""" super(TerminalResult, self).startTestRun() - self.out.write(_Colors.HEADER + 'Testing gRPC Python...\n' + - _Colors.END) + self.out.write( + _Colors.HEADER + 'Testing gRPC Python...\n' + _Colors.END) def stopTestRun(self): """See unittest.TestResult.stopTestRun.""" @@ -297,43 +298,43 @@ class TerminalResult(CoverageResult): def addError(self, test, error): """See unittest.TestResult.addError.""" super(TerminalResult, self).addError(test, error) - self.out.write(_Colors.FAIL + 'ERROR {}\n'.format(test.id()) + - _Colors.END) + self.out.write( + _Colors.FAIL + 'ERROR {}\n'.format(test.id()) + _Colors.END) self.out.flush() def addFailure(self, test, error): """See unittest.TestResult.addFailure.""" super(TerminalResult, self).addFailure(test, error) - self.out.write(_Colors.FAIL + 'FAILURE {}\n'.format(test.id()) + - _Colors.END) + self.out.write( + _Colors.FAIL + 'FAILURE {}\n'.format(test.id()) + _Colors.END) self.out.flush() def addSuccess(self, test): """See unittest.TestResult.addSuccess.""" super(TerminalResult, self).addSuccess(test) - self.out.write(_Colors.OK + 'SUCCESS {}\n'.format(test.id()) + - _Colors.END) + self.out.write( + _Colors.OK + 'SUCCESS {}\n'.format(test.id()) + _Colors.END) self.out.flush() def addSkip(self, test, reason): """See unittest.TestResult.addSkip.""" super(TerminalResult, self).addSkip(test, reason) - self.out.write(_Colors.INFO + 'SKIP {}\n'.format(test.id()) + - _Colors.END) + self.out.write( + _Colors.INFO + 'SKIP {}\n'.format(test.id()) + _Colors.END) self.out.flush() def addExpectedFailure(self, test, error): """See unittest.TestResult.addExpectedFailure.""" super(TerminalResult, self).addExpectedFailure(test, error) - self.out.write(_Colors.INFO + 'FAILURE_OK {}\n'.format(test.id()) + - _Colors.END) + self.out.write( + _Colors.INFO + 'FAILURE_OK {}\n'.format(test.id()) + _Colors.END) self.out.flush() def addUnexpectedSuccess(self, test): """See unittest.TestResult.addUnexpectedSuccess.""" super(TerminalResult, self).addUnexpectedSuccess(test) - self.out.write(_Colors.INFO + 'UNEXPECTED_OK {}\n'.format(test.id()) + - _Colors.END) + self.out.write( + _Colors.INFO + 'UNEXPECTED_OK {}\n'.format(test.id()) + _Colors.END) self.out.flush() diff --git a/src/python/grpcio_tests/tests/_runner.py b/src/python/grpcio_tests/tests/_runner.py index 8fb4a0e09b..8e27dc6c6d 100644 --- a/src/python/grpcio_tests/tests/_runner.py +++ b/src/python/grpcio_tests/tests/_runner.py @@ -181,8 +181,8 @@ class Runner(object): # Run the tests result.startTestRun() for augmented_case in augmented_cases: - sys.stdout.write( - 'Running {}\n'.format(augmented_case.case.id())) + sys.stdout.write('Running {}\n'.format( + augmented_case.case.id())) sys.stdout.flush() case_thread = threading.Thread( target=augmented_case.case.run, args=(result,)) @@ -196,8 +196,8 @@ class Runner(object): except: # re-raise the exception after forcing the with-block to end raise - result.set_output(augmented_case.case, - stdout_pipe.output(), stderr_pipe.output()) + result.set_output(augmented_case.case, stdout_pipe.output(), + stderr_pipe.output()) sys.stdout.write(result_out.getvalue()) sys.stdout.flush() result_out.truncate(0) diff --git a/src/python/grpcio_tests/tests/http2/negative_http2_client.py b/src/python/grpcio_tests/tests/http2/negative_http2_client.py index 8dab5b67f1..e4076827b6 100644 --- a/src/python/grpcio_tests/tests/http2/negative_http2_client.py +++ b/src/python/grpcio_tests/tests/http2/negative_http2_client.py @@ -32,14 +32,14 @@ def _validate_payload_type_and_length(response, expected_type, expected_length): def _expect_status_code(call, expected_code): if call.code() != expected_code: - raise ValueError('expected code %s, got %s' % - (expected_code, call.code())) + raise ValueError('expected code %s, got %s' % (expected_code, + call.code())) def _expect_status_details(call, expected_details): if call.details() != expected_details: - raise ValueError('expected message %s, got %s' % - (expected_details, call.details())) + raise ValueError('expected message %s, got %s' % (expected_details, + call.details())) def _validate_status_code_and_details(call, expected_code, expected_details): diff --git a/src/python/grpcio_tests/tests/interop/_intraop_test_case.py b/src/python/grpcio_tests/tests/interop/_intraop_test_case.py index ce456a679b..007db7ab41 100644 --- a/src/python/grpcio_tests/tests/interop/_intraop_test_case.py +++ b/src/python/grpcio_tests/tests/interop/_intraop_test_case.py @@ -39,8 +39,8 @@ class IntraopTestCase(object): methods.TestCase.PING_PONG.test_interoperability(self.stub, None) def testCancelAfterBegin(self): - methods.TestCase.CANCEL_AFTER_BEGIN.test_interoperability(self.stub, - None) + methods.TestCase.CANCEL_AFTER_BEGIN.test_interoperability( + self.stub, None) def testCancelAfterFirstResponse(self): methods.TestCase.CANCEL_AFTER_FIRST_RESPONSE.test_interoperability( diff --git a/src/python/grpcio_tests/tests/interop/_secure_intraop_test.py b/src/python/grpcio_tests/tests/interop/_secure_intraop_test.py index 6ce8b3715b..c89135998d 100644 --- a/src/python/grpcio_tests/tests/interop/_secure_intraop_test.py +++ b/src/python/grpcio_tests/tests/interop/_secure_intraop_test.py @@ -34,15 +34,16 @@ class SecureIntraopTest(_intraop_test_case.IntraopTestCase, unittest.TestCase): self.server) port = self.server.add_secure_port( '[::]:0', - grpc.ssl_server_credentials( - [(resources.private_key(), resources.certificate_chain())])) + grpc.ssl_server_credentials([(resources.private_key(), + resources.certificate_chain())])) self.server.start() self.stub = test_pb2_grpc.TestServiceStub( grpc.secure_channel('localhost:{}'.format(port), grpc.ssl_channel_credentials( - resources.test_root_certificates()), ( - ('grpc.ssl_target_name_override', - _SERVER_HOST_OVERRIDE,),))) + resources.test_root_certificates()), (( + 'grpc.ssl_target_name_override', + _SERVER_HOST_OVERRIDE, + ),))) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/interop/client.py b/src/python/grpcio_tests/tests/interop/client.py index 383b5f033d..3780ed9020 100644 --- a/src/python/grpcio_tests/tests/interop/client.py +++ b/src/python/grpcio_tests/tests/interop/client.py @@ -104,8 +104,10 @@ def _stub(args): channel_credentials = grpc.composite_channel_credentials( channel_credentials, call_credentials) - channel = grpc.secure_channel(target, channel_credentials, ( - ('grpc.ssl_target_name_override', args.server_host_override,),)) + channel = grpc.secure_channel(target, channel_credentials, (( + 'grpc.ssl_target_name_override', + args.server_host_override, + ),)) else: channel = grpc.insecure_channel(target) if args.test_case == "unimplemented_service": diff --git a/src/python/grpcio_tests/tests/interop/methods.py b/src/python/grpcio_tests/tests/interop/methods.py index ae9a50dd9b..b728ffd704 100644 --- a/src/python/grpcio_tests/tests/interop/methods.py +++ b/src/python/grpcio_tests/tests/interop/methods.py @@ -62,9 +62,10 @@ class TestService(test_pb2_grpc.TestServiceServicer): def UnaryCall(self, request, context): _maybe_echo_metadata(context) _maybe_echo_status_and_message(request, context) - return messages_pb2.SimpleResponse(payload=messages_pb2.Payload( - type=messages_pb2.COMPRESSABLE, - body=b'\x00' * request.response_size)) + return messages_pb2.SimpleResponse( + payload=messages_pb2.Payload( + type=messages_pb2.COMPRESSABLE, + body=b'\x00' * request.response_size)) def StreamingOutputCall(self, request, context): _maybe_echo_status_and_message(request, context) @@ -100,14 +101,14 @@ class TestService(test_pb2_grpc.TestServiceServicer): def _expect_status_code(call, expected_code): if call.code() != expected_code: - raise ValueError('expected code %s, got %s' % - (expected_code, call.code())) + raise ValueError('expected code %s, got %s' % (expected_code, + call.code())) def _expect_status_details(call, expected_details): if call.details() != expected_details: - raise ValueError('expected message %s, got %s' % - (expected_details, call.details())) + raise ValueError('expected message %s, got %s' % (expected_details, + call.details())) def _validate_status_code_and_details(call, expected_code, expected_details): @@ -152,26 +153,38 @@ def _large_unary(stub): def _client_streaming(stub): - payload_body_sizes = (27182, 8, 1828, 45904,) + payload_body_sizes = ( + 27182, + 8, + 1828, + 45904, + ) payloads = (messages_pb2.Payload(body=b'\x00' * size) for size in payload_body_sizes) requests = (messages_pb2.StreamingInputCallRequest(payload=payload) for payload in payloads) response = stub.StreamingInputCall(requests) if response.aggregated_payload_size != 74922: - raise ValueError('incorrect size %d!' % - response.aggregated_payload_size) + raise ValueError( + 'incorrect size %d!' % response.aggregated_payload_size) def _server_streaming(stub): - sizes = (31415, 9, 2653, 58979,) + sizes = ( + 31415, + 9, + 2653, + 58979, + ) request = messages_pb2.StreamingOutputCallRequest( response_type=messages_pb2.COMPRESSABLE, - response_parameters=(messages_pb2.ResponseParameters(size=sizes[0]), - messages_pb2.ResponseParameters(size=sizes[1]), - messages_pb2.ResponseParameters(size=sizes[2]), - messages_pb2.ResponseParameters(size=sizes[3]),)) + response_parameters=( + messages_pb2.ResponseParameters(size=sizes[0]), + messages_pb2.ResponseParameters(size=sizes[1]), + messages_pb2.ResponseParameters(size=sizes[2]), + messages_pb2.ResponseParameters(size=sizes[3]), + )) response_iterator = stub.StreamingOutputCall(request) for index, response in enumerate(response_iterator): _validate_payload_type_and_length(response, messages_pb2.COMPRESSABLE, @@ -218,8 +231,18 @@ class _Pipe(object): def _ping_pong(stub): - request_response_sizes = (31415, 9, 2653, 58979,) - request_payload_sizes = (27182, 8, 1828, 45904,) + request_response_sizes = ( + 31415, + 9, + 2653, + 58979, + ) + request_payload_sizes = ( + 27182, + 8, + 1828, + 45904, + ) with _Pipe() as pipe: response_iterator = stub.FullDuplexCall(pipe) @@ -247,8 +270,18 @@ def _cancel_after_begin(stub): def _cancel_after_first_response(stub): - request_response_sizes = (31415, 9, 2653, 58979,) - request_payload_sizes = (27182, 8, 1828, 45904,) + request_response_sizes = ( + 31415, + 9, + 2653, + 58979, + ) + request_payload_sizes = ( + 27182, + 8, + 1828, + 45904, + ) with _Pipe() as pipe: response_iterator = stub.FullDuplexCall(pipe) @@ -331,14 +364,14 @@ def _status_code_and_message(stub): def _unimplemented_method(test_service_stub): - response_future = ( - test_service_stub.UnimplementedCall.future(empty_pb2.Empty())) + response_future = (test_service_stub.UnimplementedCall.future( + empty_pb2.Empty())) _expect_status_code(response_future, grpc.StatusCode.UNIMPLEMENTED) def _unimplemented_service(unimplemented_service_stub): - response_future = ( - unimplemented_service_stub.UnimplementedCall.future(empty_pb2.Empty())) + response_future = (unimplemented_service_stub.UnimplementedCall.future( + empty_pb2.Empty())) _expect_status_code(response_future, grpc.StatusCode.UNIMPLEMENTED) @@ -392,11 +425,12 @@ def _oauth2_auth_token(stub, args): wanted_email = json.load(open(json_key_filename, 'rb'))['client_email'] response = _large_unary_common_behavior(stub, True, True, None) if wanted_email != response.username: - raise ValueError('expected username %s, got %s' % - (wanted_email, response.username)) + raise ValueError('expected username %s, got %s' % (wanted_email, + response.username)) if args.oauth_scope.find(response.oauth_scope) == -1: - raise ValueError('expected to find oauth scope "{}" in received "{}"'. - format(response.oauth_scope, args.oauth_scope)) + raise ValueError( + 'expected to find oauth scope "{}" in received "{}"'.format( + response.oauth_scope, args.oauth_scope)) def _jwt_token_creds(stub, args): @@ -404,8 +438,8 @@ def _jwt_token_creds(stub, args): wanted_email = json.load(open(json_key_filename, 'rb'))['client_email'] response = _large_unary_common_behavior(stub, True, False, None) if wanted_email != response.username: - raise ValueError('expected username %s, got %s' % - (wanted_email, response.username)) + raise ValueError('expected username %s, got %s' % (wanted_email, + response.username)) def _per_rpc_creds(stub, args): @@ -419,8 +453,8 @@ def _per_rpc_creds(stub, args): request=google_auth_transport_requests.Request())) response = _large_unary_common_behavior(stub, True, False, call_credentials) if wanted_email != response.username: - raise ValueError('expected username %s, got %s' % - (wanted_email, response.username)) + raise ValueError('expected username %s, got %s' % (wanted_email, + response.username)) @enum.unique @@ -479,5 +513,5 @@ class TestCase(enum.Enum): elif self is TestCase.PER_RPC_CREDS: _per_rpc_creds(stub, args) else: - raise NotImplementedError('Test case "%s" not implemented!' % - self.name) + raise NotImplementedError( + 'Test case "%s" not implemented!' % self.name) diff --git a/src/python/grpcio_tests/tests/interop/server.py b/src/python/grpcio_tests/tests/interop/server.py index dd4f5146e9..0810de2394 100644 --- a/src/python/grpcio_tests/tests/interop/server.py +++ b/src/python/grpcio_tests/tests/interop/server.py @@ -45,8 +45,8 @@ def serve(): if args.use_tls: private_key = resources.private_key() certificate_chain = resources.certificate_chain() - credentials = grpc.ssl_server_credentials(( - (private_key, certificate_chain),)) + credentials = grpc.ssl_server_credentials(((private_key, + certificate_chain),)) server.add_secure_port('[::]:{}'.format(args.port), credentials) else: server.add_insecure_port('[::]:{}'.format(args.port)) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py index 8fc539e641..6d85f43130 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py @@ -119,8 +119,11 @@ class _ServicerMethods(object): class _Service( - collections.namedtuple('_Service', ('servicer_methods', 'server', - 'stub',))): + collections.namedtuple('_Service', ( + 'servicer_methods', + 'server', + 'stub', + ))): """A live and running service. Attributes: @@ -297,8 +300,8 @@ class PythonPluginTest(unittest.TestCase): responses = service.stub.StreamingOutputCall(request) expected_responses = service.servicer_methods.StreamingOutputCall( request, 'not a real RpcContext!') - for expected_response, response in moves.zip_longest(expected_responses, - responses): + for expected_response, response in moves.zip_longest( + expected_responses, responses): self.assertEqual(expected_response, response) def testStreamingOutputCallExpired(self): @@ -388,8 +391,8 @@ class PythonPluginTest(unittest.TestCase): responses = service.stub.FullDuplexCall(_full_duplex_request_iterator()) expected_responses = service.servicer_methods.FullDuplexCall( _full_duplex_request_iterator(), 'not a real RpcContext!') - for expected_response, response in moves.zip_longest(expected_responses, - responses): + for expected_response, response in moves.zip_longest( + expected_responses, responses): self.assertEqual(expected_response, response) def testFullDuplexCallExpired(self): @@ -439,8 +442,8 @@ class PythonPluginTest(unittest.TestCase): responses = service.stub.HalfDuplexCall(half_duplex_request_iterator()) expected_responses = service.servicer_methods.HalfDuplexCall( half_duplex_request_iterator(), 'not a real RpcContext!') - for expected_response, response in moves.zip_longest(expected_responses, - responses): + for expected_response, response in moves.zip_longest( + expected_responses, responses): self.assertEqual(expected_response, response) def testHalfDuplexCallWedged(self): diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py index c732e55108..ab33775ad3 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py @@ -64,8 +64,8 @@ def _massage_proto_content(proto_content, test_name_bytes, messages_proto_relative_file_name_bytes): package_substitution = (b'package grpc_protoc_plugin.invocation_testing.' + test_name_bytes + b';') - common_namespace_substituted = proto_content.replace(_COMMON_NAMESPACE, - package_substitution) + common_namespace_substituted = proto_content.replace( + _COMMON_NAMESPACE, package_substitution) split_namespace_substituted = common_namespace_substituted.replace( _SPLIT_NAMESPACE, package_substitution) message_import_replaced = split_namespace_substituted.replace( @@ -163,8 +163,12 @@ class _GrpcBeforeProtoProtocStyle(object): return pb2_grpc_protoc_exit_code, pb2_protoc_exit_code, -_PROTOC_STYLES = (_Mid2016ProtocStyle(), _SingleProtocExecutionProtocStyle(), - _ProtoBeforeGrpcProtocStyle(), _GrpcBeforeProtoProtocStyle(),) +_PROTOC_STYLES = ( + _Mid2016ProtocStyle(), + _SingleProtocExecutionProtocStyle(), + _ProtoBeforeGrpcProtocStyle(), + _GrpcBeforeProtoProtocStyle(), +) @unittest.skipIf(platform.python_implementation() == 'PyPy', @@ -180,18 +184,22 @@ class _Test(six.with_metaclass(abc.ABCMeta, unittest.TestCase)): os.makedirs(self._python_out) proto_directories_and_names = { - (self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES, - self.MESSAGES_PROTO_FILE_NAME,), - (self.SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES, - self.SERVICES_PROTO_FILE_NAME,), + ( + self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES, + self.MESSAGES_PROTO_FILE_NAME, + ), + ( + self.SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES, + self.SERVICES_PROTO_FILE_NAME, + ), } messages_proto_relative_file_name_forward_slashes = '/'.join( - self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES + ( - self.MESSAGES_PROTO_FILE_NAME,)) - _create_directory_tree(self._proto_path, ( - relative_proto_directory_names - for relative_proto_directory_names, _ in proto_directories_and_names - )) + self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES + + (self.MESSAGES_PROTO_FILE_NAME,)) + _create_directory_tree(self._proto_path, + (relative_proto_directory_names + for relative_proto_directory_names, _ in + proto_directories_and_names)) self._absolute_proto_file_names = set() for relative_directory_names, file_name in proto_directories_and_names: absolute_proto_file_name = path.join( @@ -200,8 +208,7 @@ class _Test(six.with_metaclass(abc.ABCMeta, unittest.TestCase)): 'tests.protoc_plugin.protos.invocation_testing', path.join(*relative_directory_names + (file_name,))) massaged_proto_content = _massage_proto_content( - raw_proto_content, - self.NAME.encode(), + raw_proto_content, self.NAME.encode(), messages_proto_relative_file_name_forward_slashes.encode()) with open(absolute_proto_file_name, 'wb') as proto_file: proto_file.write(massaged_proto_content) @@ -275,7 +282,9 @@ def _create_test_case_class(split_proto, protoc_style): if split_proto: attributes['MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES'] = ( - 'split_messages', 'sub',) + 'split_messages', + 'sub', + ) attributes['MESSAGES_PROTO_FILE_NAME'] = 'messages.proto' attributes['SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES'] = ( 'split_services',) @@ -301,7 +310,10 @@ def _create_test_case_class(split_proto, protoc_style): def _create_test_case_classes(): - for split_proto in (False, True,): + for split_proto in ( + False, + True, + ): for protoc_style in _PROTOC_STYLES: yield _create_test_case_class(split_proto, protoc_style) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py index 424b153ff8..ad0ecf0079 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py @@ -36,10 +36,28 @@ _RELATIVE_PROTO_PATH = 'relative_proto_path' _RELATIVE_PYTHON_OUT = 'relative_python_out' _PROTO_FILES_PATH_COMPONENTS = ( - ('beta_grpc_plugin_test', 'payload', 'test_payload.proto',), - ('beta_grpc_plugin_test', 'requests', 'r', 'test_requests.proto',), - ('beta_grpc_plugin_test', 'responses', 'test_responses.proto',), - ('beta_grpc_plugin_test', 'service', 'test_service.proto',),) + ( + 'beta_grpc_plugin_test', + 'payload', + 'test_payload.proto', + ), + ( + 'beta_grpc_plugin_test', + 'requests', + 'r', + 'test_requests.proto', + ), + ( + 'beta_grpc_plugin_test', + 'responses', + 'test_responses.proto', + ), + ( + 'beta_grpc_plugin_test', + 'service', + 'test_service.proto', + ), +) _PAYLOAD_PB2 = 'beta_grpc_plugin_test.payload.test_payload_pb2' _REQUESTS_PB2 = 'beta_grpc_plugin_test.requests.r.test_requests_pb2' diff --git a/src/python/grpcio_tests/tests/qps/benchmark_client.py b/src/python/grpcio_tests/tests/qps/benchmark_client.py index 17fa61ea36..e6392a8b8c 100644 --- a/src/python/grpcio_tests/tests/qps/benchmark_client.py +++ b/src/python/grpcio_tests/tests/qps/benchmark_client.py @@ -155,7 +155,8 @@ class _SyncStream(object): _TIMEOUT) for _ in response_stream: self._handle_response( - self, time.time() - self._send_time_queue.get_nowait()) + self, + time.time() - self._send_time_queue.get_nowait()) def stop(self): self._is_streaming = False diff --git a/src/python/grpcio_tests/tests/qps/worker_server.py b/src/python/grpcio_tests/tests/qps/worker_server.py index fef4fb0459..41e2403c8f 100644 --- a/src/python/grpcio_tests/tests/qps/worker_server.py +++ b/src/python/grpcio_tests/tests/qps/worker_server.py @@ -72,8 +72,8 @@ class WorkerServer(services_pb2_grpc.WorkerServiceServicer): server = test_common.test_server(max_workers=server_threads) if config.server_type == control_pb2.ASYNC_SERVER: servicer = benchmark_server.BenchmarkServer() - services_pb2_grpc.add_BenchmarkServiceServicer_to_server(servicer, - server) + services_pb2_grpc.add_BenchmarkServiceServicer_to_server( + servicer, server) elif config.server_type == control_pb2.ASYNC_GENERIC_SERVER: resp_size = config.payload_config.bytebuf_params.resp_size servicer = benchmark_server.GenericBenchmarkServer(resp_size) @@ -87,12 +87,12 @@ class WorkerServer(services_pb2_grpc.WorkerServiceServicer): 'grpc.testing.BenchmarkService', method_implementations) server.add_generic_rpc_handlers((handler,)) else: - raise Exception( - 'Unsupported server type {}'.format(config.server_type)) + raise Exception('Unsupported server type {}'.format( + config.server_type)) if config.HasField('security_params'): # Use SSL - server_creds = grpc.ssl_server_credentials(( - (resources.private_key(), resources.certificate_chain()),)) + server_creds = grpc.ssl_server_credentials( + ((resources.private_key(), resources.certificate_chain()),)) port = server.add_secure_port('[::]:{}'.format(config.port), server_creds) else: @@ -156,8 +156,8 @@ class WorkerServer(services_pb2_grpc.WorkerServiceServicer): else: raise Exception('Async streaming client not supported') else: - raise Exception( - 'Unsupported client type {}'.format(config.client_type)) + raise Exception('Unsupported client type {}'.format( + config.client_type)) # In multi-channel tests, we split the load across all channels load_factor = float(config.client_channels) diff --git a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py index 86037e258a..7ffdba6a67 100644 --- a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py +++ b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py @@ -33,7 +33,13 @@ _EMPTY_PROTO_SYMBOL_NAME = 'grpc.testing.Empty' _SERVICE_NAMES = ('Angstrom', 'Bohr', 'Curie', 'Dyson', 'Einstein', 'Feynman', 'Galilei') _EMPTY_EXTENSIONS_SYMBOL_NAME = 'grpc.testing.proto2.EmptyWithExtensions' -_EMPTY_EXTENSIONS_NUMBERS = (124, 125, 126, 127, 128,) +_EMPTY_EXTENSIONS_NUMBERS = ( + 124, + 125, + 126, + 127, + 128, +) def _file_descriptor_to_proto(descriptor): @@ -54,10 +60,12 @@ class ReflectionServicerTest(unittest.TestCase): self._stub = reflection_pb2_grpc.ServerReflectionStub(channel) def testFileByName(self): - requests = (reflection_pb2.ServerReflectionRequest( - file_by_filename=_EMPTY_PROTO_FILE_NAME), - reflection_pb2.ServerReflectionRequest( - file_by_filename='i-donut-exist'),) + requests = ( + reflection_pb2.ServerReflectionRequest( + file_by_filename=_EMPTY_PROTO_FILE_NAME), + reflection_pb2.ServerReflectionRequest( + file_by_filename='i-donut-exist'), + ) responses = tuple(self._stub.ServerReflectionInfo(iter(requests))) expected_responses = ( reflection_pb2.ServerReflectionResponse( @@ -70,14 +78,18 @@ class ReflectionServicerTest(unittest.TestCase): error_response=reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.NOT_FOUND.value[0], error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), - )),) + )), + ) self.assertSequenceEqual(expected_responses, responses) def testFileBySymbol(self): - requests = (reflection_pb2.ServerReflectionRequest( - file_containing_symbol=_EMPTY_PROTO_SYMBOL_NAME - ), reflection_pb2.ServerReflectionRequest( - file_containing_symbol='i.donut.exist.co.uk.org.net.me.name.foo'),) + requests = ( + reflection_pb2.ServerReflectionRequest( + file_containing_symbol=_EMPTY_PROTO_SYMBOL_NAME), + reflection_pb2.ServerReflectionRequest( + file_containing_symbol='i.donut.exist.co.uk.org.net.me.name.foo' + ), + ) responses = tuple(self._stub.ServerReflectionInfo(iter(requests))) expected_responses = ( reflection_pb2.ServerReflectionResponse( @@ -90,18 +102,23 @@ class ReflectionServicerTest(unittest.TestCase): error_response=reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.NOT_FOUND.value[0], error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), - )),) + )), + ) self.assertSequenceEqual(expected_responses, responses) def testFileContainingExtension(self): - requests = (reflection_pb2.ServerReflectionRequest( - file_containing_extension=reflection_pb2.ExtensionRequest( - containing_type=_EMPTY_EXTENSIONS_SYMBOL_NAME, - extension_number=125,), - ), reflection_pb2.ServerReflectionRequest( - file_containing_extension=reflection_pb2.ExtensionRequest( - containing_type='i.donut.exist.co.uk.org.net.me.name.foo', - extension_number=55,),),) + requests = ( + reflection_pb2.ServerReflectionRequest( + file_containing_extension=reflection_pb2.ExtensionRequest( + containing_type=_EMPTY_EXTENSIONS_SYMBOL_NAME, + extension_number=125, + ),), + reflection_pb2.ServerReflectionRequest( + file_containing_extension=reflection_pb2.ExtensionRequest( + containing_type='i.donut.exist.co.uk.org.net.me.name.foo', + extension_number=55, + ),), + ) responses = tuple(self._stub.ServerReflectionInfo(iter(requests))) expected_responses = ( reflection_pb2.ServerReflectionResponse( @@ -114,14 +131,18 @@ class ReflectionServicerTest(unittest.TestCase): error_response=reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.NOT_FOUND.value[0], error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), - )),) + )), + ) self.assertSequenceEqual(expected_responses, responses) def testExtensionNumbersOfType(self): - requests = (reflection_pb2.ServerReflectionRequest( - all_extension_numbers_of_type=_EMPTY_EXTENSIONS_SYMBOL_NAME - ), reflection_pb2.ServerReflectionRequest( - all_extension_numbers_of_type='i.donut.exist.co.uk.net.name.foo'),) + requests = ( + reflection_pb2.ServerReflectionRequest( + all_extension_numbers_of_type=_EMPTY_EXTENSIONS_SYMBOL_NAME), + reflection_pb2.ServerReflectionRequest( + all_extension_numbers_of_type='i.donut.exist.co.uk.net.name.foo' + ), + ) responses = tuple(self._stub.ServerReflectionInfo(iter(requests))) expected_responses = ( reflection_pb2.ServerReflectionResponse( @@ -135,12 +156,12 @@ class ReflectionServicerTest(unittest.TestCase): error_response=reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.NOT_FOUND.value[0], error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), - )),) + )), + ) self.assertSequenceEqual(expected_responses, responses) def testListServices(self): - requests = (reflection_pb2.ServerReflectionRequest( - list_services='',),) + requests = (reflection_pb2.ServerReflectionRequest(list_services='',),) responses = tuple(self._stub.ServerReflectionInfo(iter(requests))) expected_responses = (reflection_pb2.ServerReflectionResponse( valid_host='', diff --git a/src/python/grpcio_tests/tests/stress/client.py b/src/python/grpcio_tests/tests/stress/client.py index 40caa3926a..41f2e1b6c2 100644 --- a/src/python/grpcio_tests/tests/stress/client.py +++ b/src/python/grpcio_tests/tests/stress/client.py @@ -102,8 +102,10 @@ def _get_channel(target, args): root_certificates = None # will load default roots. channel_credentials = grpc.ssl_channel_credentials( root_certificates=root_certificates) - options = (('grpc.ssl_target_name_override', - args.server_host_override,),) + options = (( + 'grpc.ssl_target_name_override', + args.server_host_override, + ),) channel = grpc.secure_channel( target, channel_credentials, options=options) else: diff --git a/src/python/grpcio_tests/tests/testing/_client_application.py b/src/python/grpcio_tests/tests/testing/_client_application.py index aff32fb4dc..7d0d74c8c4 100644 --- a/src/python/grpcio_tests/tests/testing/_client_application.py +++ b/src/python/grpcio_tests/tests/testing/_client_application.py @@ -235,8 +235,8 @@ def run(scenario, channel): elif scenario is Scenario.INFINITE_REQUEST_STREAM: return _run_infinite_request_stream(stub) except grpc.RpcError as rpc_error: - return Outcome(Outcome.Kind.RPC_ERROR, - rpc_error.code(), rpc_error.details()) + return Outcome(Outcome.Kind.RPC_ERROR, rpc_error.code(), + rpc_error.details()) _IMPLEMENTATIONS = { @@ -256,5 +256,5 @@ def run(scenario, channel): try: return _IMPLEMENTATIONS[scenario](stub) except grpc.RpcError as rpc_error: - return Outcome(Outcome.Kind.RPC_ERROR, - rpc_error.code(), rpc_error.details()) + return Outcome(Outcome.Kind.RPC_ERROR, rpc_error.code(), + rpc_error.details()) diff --git a/src/python/grpcio_tests/tests/testing/_client_test.py b/src/python/grpcio_tests/tests/testing/_client_test.py index 172f386d7b..5b051c3939 100644 --- a/src/python/grpcio_tests/tests/testing/_client_test.py +++ b/src/python/grpcio_tests/tests/testing/_client_test.py @@ -193,8 +193,10 @@ class ClientTest(unittest.TestCase): rpc.take_request() rpc.take_request() rpc.requests_closed() - rpc.send_initial_metadata(( - ('my_metadata_key', 'My Metadata Value!',),)) + rpc.send_initial_metadata((( + 'my_metadata_key', + 'My Metadata Value!', + ),)) for rpc in rpcs[:-1]: rpc.terminate(_application_common.STREAM_UNARY_RESPONSE, (), grpc.StatusCode.OK, '') diff --git a/src/python/grpcio_tests/tests/testing/_server_application.py b/src/python/grpcio_tests/tests/testing/_server_application.py index 06f09c8cb4..02769ca68d 100644 --- a/src/python/grpcio_tests/tests/testing/_server_application.py +++ b/src/python/grpcio_tests/tests/testing/_server_application.py @@ -41,8 +41,10 @@ class FirstServiceServicer(services_pb2_grpc.FirstServiceServicer): yield services_pb2.Strange() def StreUn(self, request_iterator, context): - context.send_initial_metadata(( - ('server_application_metadata_key', 'Hi there!',),)) + context.send_initial_metadata((( + 'server_application_metadata_key', + 'Hi there!', + ),)) for request in request_iterator: if request != _application_common.STREAM_UNARY_REQUEST: context.set_code(grpc.StatusCode.INVALID_ARGUMENT) diff --git a/src/python/grpcio_tests/tests/testing/_server_test.py b/src/python/grpcio_tests/tests/testing/_server_test.py index 7897bcce01..4f4abd7708 100644 --- a/src/python/grpcio_tests/tests/testing/_server_test.py +++ b/src/python/grpcio_tests/tests/testing/_server_test.py @@ -110,14 +110,19 @@ class FirstServiceServicerTest(unittest.TestCase): second_termination = rpc.termination() third_termination = rpc.termination() - for later_initial_metadata in (second_initial_metadata, - third_initial_metadata,): + for later_initial_metadata in ( + second_initial_metadata, + third_initial_metadata, + ): self.assertEqual(first_initial_metadata, later_initial_metadata) response = first_termination[0] terminal_metadata = first_termination[1] code = first_termination[2] details = first_termination[3] - for later_termination in (second_termination, third_termination,): + for later_termination in ( + second_termination, + third_termination, + ): self.assertEqual(response, later_termination[0]) self.assertEqual(terminal_metadata, later_termination[1]) self.assertIs(code, later_termination[2]) diff --git a/src/python/grpcio_tests/tests/testing/_time_test.py b/src/python/grpcio_tests/tests/testing/_time_test.py index 797394ae20..9dfe36fb83 100644 --- a/src/python/grpcio_tests/tests/testing/_time_test.py +++ b/src/python/grpcio_tests/tests/testing/_time_test.py @@ -105,8 +105,8 @@ class TimeTest(object): test_event.set, _QUANTUM * (2 + random.random())) for _ in range(_MANY): background_noise_futures.append( - self._time.call_in(threading.Event().set, _QUANTUM * 1000 * - random.random())) + self._time.call_in(threading.Event().set, + _QUANTUM * 1000 * random.random())) self._time.sleep_for(_QUANTUM) cancelled = set() for test_event, test_future in possibly_cancelled_futures.items(): diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 3bf5308749..e033c1063f 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -34,6 +34,7 @@ "unit._cython._no_messages_server_completion_queue_per_call_test.Test", "unit._cython._no_messages_single_server_completion_queue_test.Test", "unit._cython._read_some_but_not_all_responses_test.ReadSomeButNotAllResponsesTest", + "unit._cython._server_test.Test", "unit._cython.cygrpc_test.InsecureServerInsecureClient", "unit._cython.cygrpc_test.SecureServerSecureClient", "unit._cython.cygrpc_test.TypeSmokeTest", diff --git a/src/python/grpcio_tests/tests/unit/_api_test.py b/src/python/grpcio_tests/tests/unit/_api_test.py index d6f4447532..f6245be77d 100644 --- a/src/python/grpcio_tests/tests/unit/_api_test.py +++ b/src/python/grpcio_tests/tests/unit/_api_test.py @@ -26,28 +26,57 @@ class AllTest(unittest.TestCase): def testAll(self): expected_grpc_code_elements = ( - 'FutureTimeoutError', 'FutureCancelledError', 'Future', - 'ChannelConnectivity', 'StatusCode', 'RpcError', 'RpcContext', - 'Call', 'ChannelCredentials', 'CallCredentials', - 'AuthMetadataContext', 'AuthMetadataPluginCallback', - 'AuthMetadataPlugin', 'ServerCertificateConfiguration', - 'ServerCredentials', 'UnaryUnaryMultiCallable', - 'UnaryStreamMultiCallable', 'StreamUnaryMultiCallable', - 'StreamStreamMultiCallable', 'UnaryUnaryClientInterceptor', - 'UnaryStreamClientInterceptor', 'StreamUnaryClientInterceptor', - 'StreamStreamClientInterceptor', 'Channel', 'ServicerContext', - 'RpcMethodHandler', 'HandlerCallDetails', 'GenericRpcHandler', - 'ServiceRpcHandler', 'Server', 'ServerInterceptor', - 'unary_unary_rpc_method_handler', 'unary_stream_rpc_method_handler', - 'stream_unary_rpc_method_handler', 'ClientCallDetails', + 'FutureTimeoutError', + 'FutureCancelledError', + 'Future', + 'ChannelConnectivity', + 'StatusCode', + 'RpcError', + 'RpcContext', + 'Call', + 'ChannelCredentials', + 'CallCredentials', + 'AuthMetadataContext', + 'AuthMetadataPluginCallback', + 'AuthMetadataPlugin', + 'ServerCertificateConfiguration', + 'ServerCredentials', + 'UnaryUnaryMultiCallable', + 'UnaryStreamMultiCallable', + 'StreamUnaryMultiCallable', + 'StreamStreamMultiCallable', + 'UnaryUnaryClientInterceptor', + 'UnaryStreamClientInterceptor', + 'StreamUnaryClientInterceptor', + 'StreamStreamClientInterceptor', + 'Channel', + 'ServicerContext', + 'RpcMethodHandler', + 'HandlerCallDetails', + 'GenericRpcHandler', + 'ServiceRpcHandler', + 'Server', + 'ServerInterceptor', + 'unary_unary_rpc_method_handler', + 'unary_stream_rpc_method_handler', + 'stream_unary_rpc_method_handler', + 'ClientCallDetails', 'stream_stream_rpc_method_handler', - 'method_handlers_generic_handler', 'ssl_channel_credentials', - 'metadata_call_credentials', 'access_token_call_credentials', - 'composite_call_credentials', 'composite_channel_credentials', - 'ssl_server_credentials', 'ssl_server_certificate_configuration', - 'dynamic_ssl_server_credentials', 'channel_ready_future', - 'insecure_channel', 'secure_channel', 'intercept_channel', - 'server',) + 'method_handlers_generic_handler', + 'ssl_channel_credentials', + 'metadata_call_credentials', + 'access_token_call_credentials', + 'composite_call_credentials', + 'composite_channel_credentials', + 'ssl_server_credentials', + 'ssl_server_certificate_configuration', + 'dynamic_ssl_server_credentials', + 'channel_ready_future', + 'insecure_channel', + 'secure_channel', + 'intercept_channel', + 'server', + ) six.assertCountEqual(self, expected_grpc_code_elements, _from_grpc_import_star.GRPC_ELEMENTS) @@ -56,12 +85,13 @@ class AllTest(unittest.TestCase): class ChannelConnectivityTest(unittest.TestCase): def testChannelConnectivity(self): - self.assertSequenceEqual( - (grpc.ChannelConnectivity.IDLE, grpc.ChannelConnectivity.CONNECTING, - grpc.ChannelConnectivity.READY, - grpc.ChannelConnectivity.TRANSIENT_FAILURE, - grpc.ChannelConnectivity.SHUTDOWN,), - tuple(grpc.ChannelConnectivity)) + self.assertSequenceEqual(( + grpc.ChannelConnectivity.IDLE, + grpc.ChannelConnectivity.CONNECTING, + grpc.ChannelConnectivity.READY, + grpc.ChannelConnectivity.TRANSIENT_FAILURE, + grpc.ChannelConnectivity.SHUTDOWN, + ), tuple(grpc.ChannelConnectivity)) class ChannelTest(unittest.TestCase): diff --git a/src/python/grpcio_tests/tests/unit/_auth_context_test.py b/src/python/grpcio_tests/tests/unit/_auth_context_test.py index ebc04a71e0..468869a03e 100644 --- a/src/python/grpcio_tests/tests/unit/_auth_context_test.py +++ b/src/python/grpcio_tests/tests/unit/_auth_context_test.py @@ -29,8 +29,12 @@ _RESPONSE = b'\x00\x00\x00' _UNARY_UNARY = '/test/UnaryUnary' _SERVER_HOST_OVERRIDE = 'foo.test.google.fr' -_CLIENT_IDS = (b'*.test.google.fr', b'waterzooi.test.google.be', - b'*.test.youtube.com', b'192.168.1.3',) +_CLIENT_IDS = ( + b'*.test.google.fr', + b'waterzooi.test.google.be', + b'*.test.youtube.com', + b'192.168.1.3', +) _ID = 'id' _ID_KEY = 'id_key' _AUTH_CTX = 'auth_ctx' @@ -39,7 +43,10 @@ _PRIVATE_KEY = resources.private_key() _CERTIFICATE_CHAIN = resources.certificate_chain() _TEST_ROOT_CERTIFICATES = resources.test_root_certificates() _SERVER_CERTS = ((_PRIVATE_KEY, _CERTIFICATE_CHAIN),) -_PROPERTY_OPTIONS = (('grpc.ssl_target_name_override', _SERVER_HOST_OVERRIDE,),) +_PROPERTY_OPTIONS = (( + 'grpc.ssl_target_name_override', + _SERVER_HOST_OVERRIDE, +),) def handle_unary_unary(request, servicer_context): diff --git a/src/python/grpcio_tests/tests/unit/_channel_args_test.py b/src/python/grpcio_tests/tests/unit/_channel_args_test.py index 0a6b512866..1a2d2c0117 100644 --- a/src/python/grpcio_tests/tests/unit/_channel_args_test.py +++ b/src/python/grpcio_tests/tests/unit/_channel_args_test.py @@ -24,8 +24,13 @@ class TestPointerWrapper(object): return 123456 -TEST_CHANNEL_ARGS = (('arg1', b'bytes_val'), ('arg2', 'str_val'), ('arg3', 1), - (b'arg4', 'str_val'), ('arg6', TestPointerWrapper()),) +TEST_CHANNEL_ARGS = ( + ('arg1', b'bytes_val'), + ('arg2', 'str_val'), + ('arg3', 1), + (b'arg4', 'str_val'), + ('arg6', TestPointerWrapper()), +) class ChannelArgsTest(unittest.TestCase): diff --git a/src/python/grpcio_tests/tests/unit/_compression_test.py b/src/python/grpcio_tests/tests/unit/_compression_test.py index 93e599d8f8..7550cd39ba 100644 --- a/src/python/grpcio_tests/tests/unit/_compression_test.py +++ b/src/python/grpcio_tests/tests/unit/_compression_test.py @@ -26,16 +26,16 @@ _STREAM_STREAM = '/test/StreamStream' def handle_unary(request, servicer_context): - servicer_context.send_initial_metadata( - [('grpc-internal-encoding-request', 'gzip')]) + servicer_context.send_initial_metadata([('grpc-internal-encoding-request', + 'gzip')]) return request def handle_stream(request_iterator, servicer_context): # TODO(issue:#6891) We should be able to remove this loop, # and replace with return; yield - servicer_context.send_initial_metadata( - [('grpc-internal-encoding-request', 'gzip')]) + servicer_context.send_initial_metadata([('grpc-internal-encoding-request', + 'gzip')]) for request in request_iterator: yield request diff --git a/src/python/grpcio_tests/tests/unit/_credentials_test.py b/src/python/grpcio_tests/tests/unit/_credentials_test.py index 097898b987..f487fe66a2 100644 --- a/src/python/grpcio_tests/tests/unit/_credentials_test.py +++ b/src/python/grpcio_tests/tests/unit/_credentials_test.py @@ -26,8 +26,8 @@ class CredentialsTest(unittest.TestCase): third = grpc.access_token_call_credentials('ghi') first_and_second = grpc.composite_call_credentials(first, second) - first_second_and_third = grpc.composite_call_credentials(first, second, - third) + first_second_and_third = grpc.composite_call_credentials( + first, second, third) self.assertIsInstance(first_and_second, grpc.CallCredentials) self.assertIsInstance(first_second_and_third, grpc.CallCredentials) diff --git a/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py b/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py index 7df8e2fde6..b81d6fbc61 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py @@ -53,7 +53,7 @@ class _Handler(object): self._state = state self._lock = threading.Lock() self._completion_queue = completion_queue - self._call = rpc_event.operation_call + self._call = rpc_event.call def __call__(self): with self._state.condition: @@ -65,10 +65,10 @@ class _Handler(object): with self._lock: self._call.start_server_batch( - (cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),), + (cygrpc.ReceiveCloseOnServerOperation(_EMPTY_FLAGS),), _RECEIVE_CLOSE_ON_SERVER_TAG) self._call.start_server_batch( - (cygrpc.operation_receive_message(_EMPTY_FLAGS),), + (cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS),), _RECEIVE_MESSAGE_TAG) first_event = self._completion_queue.poll() if _is_cancellation_event(first_event): @@ -76,12 +76,13 @@ class _Handler(object): else: with self._lock: operations = ( - cygrpc.operation_send_initial_metadata(_EMPTY_METADATA, - _EMPTY_FLAGS), - cygrpc.operation_send_message(b'\x79\x57', _EMPTY_FLAGS), - cygrpc.operation_send_status_from_server( + cygrpc.SendInitialMetadataOperation(_EMPTY_METADATA, + _EMPTY_FLAGS), + cygrpc.SendMessageOperation(b'\x79\x57', _EMPTY_FLAGS), + cygrpc.SendStatusFromServerOperation( _EMPTY_METADATA, cygrpc.StatusCode.ok, b'test details!', - _EMPTY_FLAGS),) + _EMPTY_FLAGS), + ) self._call.start_server_batch(operations, _SERVER_COMPLETE_CALL_TAG) self._completion_queue.poll() @@ -151,8 +152,12 @@ class CancelManyCallsTest(unittest.TestCase): state = _State() - server_thread_args = (state, server, server_completion_queue, - server_thread_pool,) + server_thread_args = ( + state, + server, + server_completion_queue, + server_thread_pool, + ) server_thread = threading.Thread(target=_serve, args=server_thread_args) server_thread.start() @@ -170,13 +175,14 @@ class CancelManyCallsTest(unittest.TestCase): None, _EMPTY_FLAGS, client_completion_queue, b'/twinkies', None, _INFINITE_FUTURE) operations = ( - cygrpc.operation_send_initial_metadata(_EMPTY_METADATA, - _EMPTY_FLAGS), - cygrpc.operation_send_message(b'\x45\x56', _EMPTY_FLAGS), - cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), - cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS), - cygrpc.operation_receive_message(_EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) + cygrpc.SendInitialMetadataOperation(_EMPTY_METADATA, + _EMPTY_FLAGS), + cygrpc.SendMessageOperation(b'\x45\x56', _EMPTY_FLAGS), + cygrpc.SendCloseFromClientOperation(_EMPTY_FLAGS), + cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS), + cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS), + ) tag = 'client_complete_call_{0:04d}_tag'.format(index) client_call.start_client_batch(operations, tag) client_due.add(tag) @@ -193,8 +199,8 @@ class CancelManyCallsTest(unittest.TestCase): state.condition.notify_all() break - client_driver.events(test_constants.RPC_CONCURRENCY * - _SUCCESS_CALL_FRACTION) + client_driver.events( + test_constants.RPC_CONCURRENCY * _SUCCESS_CALL_FRACTION) with client_condition: for client_call in client_calls: client_call.cancel() diff --git a/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py b/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py index 1d57ea7ec1..4eeb34b92e 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py @@ -56,7 +56,10 @@ class ChannelTest(unittest.TestCase): def test_single_channel_lonely_connectivity(self): channel, completion_queue = _channel_and_completion_queue() - _in_parallel(_connectivity_loop, (channel, completion_queue,)) + _in_parallel(_connectivity_loop, ( + channel, + completion_queue, + )) completion_queue.shutdown() def test_multiple_channels_lonely_connectivity(self): diff --git a/src/python/grpcio_tests/tests/unit/_cython/_common.py b/src/python/grpcio_tests/tests/unit/_cython/_common.py index c5acd36bf2..ffd226fa95 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_common.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_common.py @@ -23,14 +23,20 @@ RPC_COUNT = 4000 INFINITE_FUTURE = cygrpc.Timespec(float('+inf')) EMPTY_FLAGS = 0 -INVOCATION_METADATA = (('client-md-key', 'client-md-key'), - ('client-md-key-bin', b'\x00\x01' * 3000),) +INVOCATION_METADATA = ( + ('client-md-key', 'client-md-key'), + ('client-md-key-bin', b'\x00\x01' * 3000), +) -INITIAL_METADATA = (('server-initial-md-key', 'server-initial-md-value'), - ('server-initial-md-key-bin', b'\x00\x02' * 3000),) +INITIAL_METADATA = ( + ('server-initial-md-key', 'server-initial-md-value'), + ('server-initial-md-key-bin', b'\x00\x02' * 3000), +) -TRAILING_METADATA = (('server-trailing-md-key', 'server-trailing-md-value'), - ('server-trailing-md-key-bin', b'\x00\x03' * 3000),) +TRAILING_METADATA = ( + ('server-trailing-md-key', 'server-trailing-md-value'), + ('server-trailing-md-key-bin', b'\x00\x03' * 3000), +) class QueueDriver(object): @@ -76,7 +82,10 @@ def execute_many_times(behavior): class OperationResult( collections.namedtuple('OperationResult', ( - 'start_batch_result', 'completion_type', 'success',))): + 'start_batch_result', + 'completion_type', + 'success', + ))): pass diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py index d08003af44..4ef4ad33e5 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py @@ -49,18 +49,19 @@ class Test(_common.RpcTest, unittest.TestCase): with self.client_condition: client_receive_initial_metadata_start_batch_result = ( client_call.start_client_batch([ - cygrpc.operation_receive_initial_metadata( - _common.EMPTY_FLAGS), + cygrpc.ReceiveInitialMetadataOperation(_common.EMPTY_FLAGS), ], client_receive_initial_metadata_tag)) + self.assertEqual(cygrpc.CallError.ok, + client_receive_initial_metadata_start_batch_result) client_complete_rpc_start_batch_result = client_call.start_client_batch( [ - cygrpc.operation_send_initial_metadata( + cygrpc.SendInitialMetadataOperation( _common.INVOCATION_METADATA, _common.EMPTY_FLAGS), - cygrpc.operation_send_close_from_client( - _common.EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client( - _common.EMPTY_FLAGS), + cygrpc.SendCloseFromClientOperation(_common.EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_common.EMPTY_FLAGS), ], client_complete_rpc_tag) + self.assertEqual(cygrpc.CallError.ok, + client_complete_rpc_start_batch_result) self.client_driver.add_due({ client_receive_initial_metadata_tag, client_complete_rpc_tag, @@ -71,8 +72,8 @@ class Test(_common.RpcTest, unittest.TestCase): with server_call_condition: server_send_initial_metadata_start_batch_result = ( - server_request_call_event.operation_call.start_server_batch([ - cygrpc.operation_send_initial_metadata( + server_request_call_event.call.start_server_batch([ + cygrpc.SendInitialMetadataOperation( _common.INITIAL_METADATA, _common.EMPTY_FLAGS), ], server_send_initial_metadata_tag)) server_call_driver.add_due({ @@ -83,10 +84,9 @@ class Test(_common.RpcTest, unittest.TestCase): with server_call_condition: server_complete_rpc_start_batch_result = ( - server_request_call_event.operation_call.start_server_batch([ - cygrpc.operation_receive_close_on_server( - _common.EMPTY_FLAGS), - cygrpc.operation_send_status_from_server( + server_request_call_event.call.start_server_batch([ + cygrpc.ReceiveCloseOnServerOperation(_common.EMPTY_FLAGS), + cygrpc.SendStatusFromServerOperation( _common.TRAILING_METADATA, cygrpc.StatusCode.ok, b'test details', _common.EMPTY_FLAGS), ], server_complete_rpc_tag)) @@ -101,27 +101,29 @@ class Test(_common.RpcTest, unittest.TestCase): client_complete_rpc_event = self.client_driver.event_with_tag( client_complete_rpc_tag) - return (_common.OperationResult(server_request_call_start_batch_result, - server_request_call_event.type, - server_request_call_event.success), - _common.OperationResult( - client_receive_initial_metadata_start_batch_result, - client_receive_initial_metadata_event.type, - client_receive_initial_metadata_event.success), - _common.OperationResult(client_complete_rpc_start_batch_result, - client_complete_rpc_event.type, - client_complete_rpc_event.success), - _common.OperationResult( - server_send_initial_metadata_start_batch_result, - server_send_initial_metadata_event.type, - server_send_initial_metadata_event.success), - _common.OperationResult(server_complete_rpc_start_batch_result, - server_complete_rpc_event.type, - server_complete_rpc_event.success),) + return ( + _common.OperationResult(server_request_call_start_batch_result, + server_request_call_event.completion_type, + server_request_call_event.success), + _common.OperationResult( + client_receive_initial_metadata_start_batch_result, + client_receive_initial_metadata_event.completion_type, + client_receive_initial_metadata_event.success), + _common.OperationResult(client_complete_rpc_start_batch_result, + client_complete_rpc_event.completion_type, + client_complete_rpc_event.success), + _common.OperationResult( + server_send_initial_metadata_start_batch_result, + server_send_initial_metadata_event.completion_type, + server_send_initial_metadata_event.success), + _common.OperationResult(server_complete_rpc_start_batch_result, + server_complete_rpc_event.completion_type, + server_complete_rpc_event.success), + ) def test_rpcs(self): - expecteds = [(_common.SUCCESSFUL_OPERATION_RESULT,) * - 5] * _common.RPC_COUNT + expecteds = [( + _common.SUCCESSFUL_OPERATION_RESULT,) * 5] * _common.RPC_COUNT actuallys = _common.execute_many_times(self._do_rpcs) self.assertSequenceEqual(expecteds, actuallys) diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py index d0166a2b29..85395c9680 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py @@ -44,17 +44,14 @@ class Test(_common.RpcTest, unittest.TestCase): with self.client_condition: client_receive_initial_metadata_start_batch_result = ( client_call.start_client_batch([ - cygrpc.operation_receive_initial_metadata( - _common.EMPTY_FLAGS), + cygrpc.ReceiveInitialMetadataOperation(_common.EMPTY_FLAGS), ], client_receive_initial_metadata_tag)) client_complete_rpc_start_batch_result = client_call.start_client_batch( [ - cygrpc.operation_send_initial_metadata( + cygrpc.SendInitialMetadataOperation( _common.INVOCATION_METADATA, _common.EMPTY_FLAGS), - cygrpc.operation_send_close_from_client( - _common.EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client( - _common.EMPTY_FLAGS), + cygrpc.SendCloseFromClientOperation(_common.EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_common.EMPTY_FLAGS), ], client_complete_rpc_tag) self.client_driver.add_due({ client_receive_initial_metadata_tag, @@ -66,8 +63,8 @@ class Test(_common.RpcTest, unittest.TestCase): with self.server_condition: server_send_initial_metadata_start_batch_result = ( - server_request_call_event.operation_call.start_server_batch([ - cygrpc.operation_send_initial_metadata( + server_request_call_event.call.start_server_batch([ + cygrpc.SendInitialMetadataOperation( _common.INITIAL_METADATA, _common.EMPTY_FLAGS), ], server_send_initial_metadata_tag)) self.server_driver.add_due({ @@ -78,12 +75,11 @@ class Test(_common.RpcTest, unittest.TestCase): with self.server_condition: server_complete_rpc_start_batch_result = ( - server_request_call_event.operation_call.start_server_batch([ - cygrpc.operation_receive_close_on_server( - _common.EMPTY_FLAGS), - cygrpc.operation_send_status_from_server( + server_request_call_event.call.start_server_batch([ + cygrpc.ReceiveCloseOnServerOperation(_common.EMPTY_FLAGS), + cygrpc.SendStatusFromServerOperation( _common.TRAILING_METADATA, cygrpc.StatusCode.ok, - b'test details', _common.EMPTY_FLAGS), + 'test details', _common.EMPTY_FLAGS), ], server_complete_rpc_tag)) self.server_driver.add_due({ server_complete_rpc_tag, @@ -96,27 +92,29 @@ class Test(_common.RpcTest, unittest.TestCase): client_complete_rpc_event = self.client_driver.event_with_tag( client_complete_rpc_tag) - return (_common.OperationResult(server_request_call_start_batch_result, - server_request_call_event.type, - server_request_call_event.success), - _common.OperationResult( - client_receive_initial_metadata_start_batch_result, - client_receive_initial_metadata_event.type, - client_receive_initial_metadata_event.success), - _common.OperationResult(client_complete_rpc_start_batch_result, - client_complete_rpc_event.type, - client_complete_rpc_event.success), - _common.OperationResult( - server_send_initial_metadata_start_batch_result, - server_send_initial_metadata_event.type, - server_send_initial_metadata_event.success), - _common.OperationResult(server_complete_rpc_start_batch_result, - server_complete_rpc_event.type, - server_complete_rpc_event.success),) + return ( + _common.OperationResult(server_request_call_start_batch_result, + server_request_call_event.completion_type, + server_request_call_event.success), + _common.OperationResult( + client_receive_initial_metadata_start_batch_result, + client_receive_initial_metadata_event.completion_type, + client_receive_initial_metadata_event.success), + _common.OperationResult(client_complete_rpc_start_batch_result, + client_complete_rpc_event.completion_type, + client_complete_rpc_event.success), + _common.OperationResult( + server_send_initial_metadata_start_batch_result, + server_send_initial_metadata_event.completion_type, + server_send_initial_metadata_event.success), + _common.OperationResult(server_complete_rpc_start_batch_result, + server_complete_rpc_event.completion_type, + server_complete_rpc_event.success), + ) def test_rpcs(self): - expecteds = [(_common.SUCCESSFUL_OPERATION_RESULT,) * - 5] * _common.RPC_COUNT + expecteds = [( + _common.SUCCESSFUL_OPERATION_RESULT,) * 5] * _common.RPC_COUNT actuallys = _common.execute_many_times(self._do_rpcs) self.assertSequenceEqual(expecteds, actuallys) diff --git a/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py b/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py index c7d19058da..82ef25b2a7 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py @@ -137,9 +137,12 @@ class ReadSomeButNotAllResponsesTest(unittest.TestCase): server_send_first_message_tag = 'server_send_first_message_tag' server_send_second_message_tag = 'server_send_second_message_tag' server_complete_rpc_tag = 'server_complete_rpc_tag' - server_call_due = set( - (server_send_initial_metadata_tag, server_send_first_message_tag, - server_send_second_message_tag, server_complete_rpc_tag,)) + server_call_due = set(( + server_send_initial_metadata_tag, + server_send_first_message_tag, + server_send_second_message_tag, + server_complete_rpc_tag, + )) server_call_completion_queue = cygrpc.CompletionQueue() server_call_driver = _QueueDriver(server_call_condition, server_call_completion_queue, @@ -159,15 +162,15 @@ class ReadSomeButNotAllResponsesTest(unittest.TestCase): with client_condition: client_receive_initial_metadata_start_batch_result = ( client_call.start_client_batch([ - cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS), + cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS), ], client_receive_initial_metadata_tag)) client_due.add(client_receive_initial_metadata_tag) client_complete_rpc_start_batch_result = ( client_call.start_client_batch([ - cygrpc.operation_send_initial_metadata(_EMPTY_METADATA, - _EMPTY_FLAGS), - cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS), + cygrpc.SendInitialMetadataOperation(_EMPTY_METADATA, + _EMPTY_FLAGS), + cygrpc.SendCloseFromClientOperation(_EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS), ], client_complete_rpc_tag)) client_due.add(client_complete_rpc_tag) @@ -175,13 +178,13 @@ class ReadSomeButNotAllResponsesTest(unittest.TestCase): with server_call_condition: server_send_initial_metadata_start_batch_result = ( - server_rpc_event.operation_call.start_server_batch([ - cygrpc.operation_send_initial_metadata(_EMPTY_METADATA, - _EMPTY_FLAGS), + server_rpc_event.call.start_server_batch([ + cygrpc.SendInitialMetadataOperation(_EMPTY_METADATA, + _EMPTY_FLAGS), ], server_send_initial_metadata_tag)) server_send_first_message_start_batch_result = ( - server_rpc_event.operation_call.start_server_batch([ - cygrpc.operation_send_message(b'\x07', _EMPTY_FLAGS), + server_rpc_event.call.start_server_batch([ + cygrpc.SendMessageOperation(b'\x07', _EMPTY_FLAGS), ], server_send_first_message_tag)) server_send_initial_metadata_event = server_call_driver.event_with_tag( server_send_initial_metadata_tag) @@ -189,13 +192,13 @@ class ReadSomeButNotAllResponsesTest(unittest.TestCase): server_send_first_message_tag) with server_call_condition: server_send_second_message_start_batch_result = ( - server_rpc_event.operation_call.start_server_batch([ - cygrpc.operation_send_message(b'\x07', _EMPTY_FLAGS), + server_rpc_event.call.start_server_batch([ + cygrpc.SendMessageOperation(b'\x07', _EMPTY_FLAGS), ], server_send_second_message_tag)) server_complete_rpc_start_batch_result = ( - server_rpc_event.operation_call.start_server_batch([ - cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS), - cygrpc.operation_send_status_from_server( + server_rpc_event.call.start_server_batch([ + cygrpc.ReceiveCloseOnServerOperation(_EMPTY_FLAGS), + cygrpc.SendStatusFromServerOperation( (), cygrpc.StatusCode.ok, b'test details', _EMPTY_FLAGS), ], server_complete_rpc_tag)) @@ -209,7 +212,7 @@ class ReadSomeButNotAllResponsesTest(unittest.TestCase): client_receive_first_message_tag = 'client_receive_first_message_tag' client_receive_first_message_start_batch_result = ( client_call.start_client_batch([ - cygrpc.operation_receive_message(_EMPTY_FLAGS), + cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), ], client_receive_first_message_tag)) client_due.add(client_receive_first_message_tag) client_receive_first_message_event = client_driver.event_with_tag( @@ -232,9 +235,8 @@ class ReadSomeButNotAllResponsesTest(unittest.TestCase): self.assertEqual(cygrpc.CallError.ok, client_call_cancel_result) self.assertIs(server_rpc_tag, server_rpc_event.tag) self.assertEqual(cygrpc.CompletionType.operation_complete, - server_rpc_event.type) - self.assertIsInstance(server_rpc_event.operation_call, cygrpc.Call) - self.assertEqual(0, len(server_rpc_event.batch_operations)) + server_rpc_event.completion_type) + self.assertIsInstance(server_rpc_event.call, cygrpc.Call) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/unit/_cython/_server_test.py b/src/python/grpcio_tests/tests/unit/_cython/_server_test.py new file mode 100644 index 0000000000..12bf40be6b --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_cython/_server_test.py @@ -0,0 +1,49 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test servers at the level of the Cython API.""" + +import threading +import time +import unittest + +from grpc._cython import cygrpc + + +class Test(unittest.TestCase): + + def test_lonely_server(self): + server_call_completion_queue = cygrpc.CompletionQueue() + server_shutdown_completion_queue = cygrpc.CompletionQueue() + server = cygrpc.Server(cygrpc.ChannelArgs([])) + server.register_completion_queue(server_call_completion_queue) + server.register_completion_queue(server_shutdown_completion_queue) + port = server.add_http2_port(b'[::]:0') + server.start() + + server_request_call_tag = 'server_request_call_tag' + server_request_call_start_batch_result = server.request_call( + server_call_completion_queue, server_call_completion_queue, + server_request_call_tag) + + time.sleep(4) + + server_shutdown_tag = 'server_shutdown_tag' + server_shutdown_result = server.shutdown( + server_shutdown_completion_queue, server_shutdown_tag) + server_request_call_event = server_call_completion_queue.poll() + server_shutdown_event = server_shutdown_completion_queue.poll() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py index 33a35ae235..5f9b74ba98 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py @@ -29,17 +29,14 @@ _EMPTY_FLAGS = 0 def _metadata_plugin(context, callback): - callback(((_CALL_CREDENTIALS_METADATA_KEY, - _CALL_CREDENTIALS_METADATA_VALUE,),), cygrpc.StatusCode.ok, b'') + callback((( + _CALL_CREDENTIALS_METADATA_KEY, + _CALL_CREDENTIALS_METADATA_VALUE, + ),), cygrpc.StatusCode.ok, b'') class TypeSmokeTest(unittest.TestCase): - def testOperationFlags(self): - operation = cygrpc.operation_send_message(b'asdf', - cygrpc.WriteFlag.no_compress) - self.assertEqual(cygrpc.WriteFlag.no_compress, operation.flags) - def testTimespec(self): now = time.time() now_timespec_a = cygrpc.Timespec(now) @@ -92,7 +89,8 @@ class TypeSmokeTest(unittest.TestCase): shutdown_tag = object() server.shutdown(completion_queue, shutdown_tag) event = completion_queue.poll() - self.assertEqual(cygrpc.CompletionType.operation_complete, event.type) + self.assertEqual(cygrpc.CompletionType.operation_complete, + event.completion_type) self.assertIs(shutdown_tag, event.tag) del server del completion_queue @@ -117,13 +115,12 @@ class ServerClientMixin(object): cygrpc.ChannelArg(cygrpc.ChannelArgKey.ssl_target_name_override, host_override) ]) - self.client_channel = cygrpc.Channel( - 'localhost:{}'.format(self.port).encode(), - client_channel_arguments, client_credentials) + self.client_channel = cygrpc.Channel('localhost:{}'.format( + self.port).encode(), client_channel_arguments, + client_credentials) else: - self.client_channel = cygrpc.Channel( - 'localhost:{}'.format(self.port).encode(), - cygrpc.ChannelArgs([])) + self.client_channel = cygrpc.Channel('localhost:{}'.format( + self.port).encode(), cygrpc.ChannelArgs([])) if host_override: self.host_argument = None # default host self.expected_host = host_override @@ -152,12 +149,12 @@ class ServerClientMixin(object): self.assertEqual(cygrpc.CallError.ok, call_result) event = queue.poll(deadline) self.assertEqual(cygrpc.CompletionType.operation_complete, - event.type) + event.completion_type) self.assertTrue(event.success) self.assertIs(tag, event.tag) except Exception as error: - raise Exception( - "Error in '{}': {}".format(description, error.message)) + raise Exception("Error in '{}': {}".format( + description, error.message)) return event return test_utilities.SimpleFuture(performer) @@ -174,7 +171,7 @@ class ServerClientMixin(object): SERVER_TRAILING_METADATA_KEY = 'california_is_in_a_drought' SERVER_TRAILING_METADATA_VALUE = 'zomg it is' SERVER_STATUS_CODE = cygrpc.StatusCode.ok - SERVER_STATUS_DETAILS = b'our work is never over' + SERVER_STATUS_DETAILS = 'our work is never over' REQUEST = b'in death a member of project mayhem has a name' RESPONSE = b'his name is robert paulson' METHOD = b'twinkies' @@ -193,16 +190,23 @@ class ServerClientMixin(object): None, 0, self.client_completion_queue, METHOD, self.host_argument, cygrpc_deadline) client_initial_metadata = ( - (CLIENT_METADATA_ASCII_KEY, CLIENT_METADATA_ASCII_VALUE,), - (CLIENT_METADATA_BIN_KEY, CLIENT_METADATA_BIN_VALUE,),) + ( + CLIENT_METADATA_ASCII_KEY, + CLIENT_METADATA_ASCII_VALUE, + ), + ( + CLIENT_METADATA_BIN_KEY, + CLIENT_METADATA_BIN_VALUE, + ), + ) client_start_batch_result = client_call.start_client_batch([ - cygrpc.operation_send_initial_metadata(client_initial_metadata, - _EMPTY_FLAGS), - cygrpc.operation_send_message(REQUEST, _EMPTY_FLAGS), - cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), - cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS), - cygrpc.operation_receive_message(_EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS) + cygrpc.SendInitialMetadataOperation(client_initial_metadata, + _EMPTY_FLAGS), + cygrpc.SendMessageOperation(REQUEST, _EMPTY_FLAGS), + cygrpc.SendCloseFromClientOperation(_EMPTY_FLAGS), + cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS), + cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS), ], client_call_tag) self.assertEqual(cygrpc.CallError.ok, client_start_batch_result) client_event_future = test_utilities.CompletionQueuePollFuture( @@ -210,33 +214,35 @@ class ServerClientMixin(object): request_event = self.server_completion_queue.poll(cygrpc_deadline) self.assertEqual(cygrpc.CompletionType.operation_complete, - request_event.type) - self.assertIsInstance(request_event.operation_call, cygrpc.Call) + request_event.completion_type) + self.assertIsInstance(request_event.call, cygrpc.Call) self.assertIs(server_request_tag, request_event.tag) - self.assertEqual(0, len(request_event.batch_operations)) self.assertTrue( test_common.metadata_transmitted(client_initial_metadata, - request_event.request_metadata)) - self.assertEqual(METHOD, request_event.request_call_details.method) - self.assertEqual(self.expected_host, - request_event.request_call_details.host) + request_event.invocation_metadata)) + self.assertEqual(METHOD, request_event.call_details.method) + self.assertEqual(self.expected_host, request_event.call_details.host) self.assertLess( - abs(DEADLINE - float(request_event.request_call_details.deadline)), + abs(DEADLINE - float(request_event.call_details.deadline)), DEADLINE_TOLERANCE) server_call_tag = object() - server_call = request_event.operation_call - server_initial_metadata = ( - (SERVER_INITIAL_METADATA_KEY, SERVER_INITIAL_METADATA_VALUE,),) - server_trailing_metadata = ( - (SERVER_TRAILING_METADATA_KEY, SERVER_TRAILING_METADATA_VALUE,),) + server_call = request_event.call + server_initial_metadata = (( + SERVER_INITIAL_METADATA_KEY, + SERVER_INITIAL_METADATA_VALUE, + ),) + server_trailing_metadata = (( + SERVER_TRAILING_METADATA_KEY, + SERVER_TRAILING_METADATA_VALUE, + ),) server_start_batch_result = server_call.start_server_batch([ - cygrpc.operation_send_initial_metadata( - server_initial_metadata, - _EMPTY_FLAGS), cygrpc.operation_receive_message(_EMPTY_FLAGS), - cygrpc.operation_send_message(RESPONSE, _EMPTY_FLAGS), - cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS), - cygrpc.operation_send_status_from_server( + cygrpc.SendInitialMetadataOperation(server_initial_metadata, + _EMPTY_FLAGS), + cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), + cygrpc.SendMessageOperation(RESPONSE, _EMPTY_FLAGS), + cygrpc.ReceiveCloseOnServerOperation(_EMPTY_FLAGS), + cygrpc.SendStatusFromServerOperation( server_trailing_metadata, SERVER_STATUS_CODE, SERVER_STATUS_DETAILS, _EMPTY_FLAGS) ], server_call_tag) @@ -249,25 +255,24 @@ class ServerClientMixin(object): found_client_op_types = set() for client_result in client_event.batch_operations: # we expect each op type to be unique - self.assertNotIn(client_result.type, found_client_op_types) - found_client_op_types.add(client_result.type) - if client_result.type == cygrpc.OperationType.receive_initial_metadata: + self.assertNotIn(client_result.type(), found_client_op_types) + found_client_op_types.add(client_result.type()) + if client_result.type( + ) == cygrpc.OperationType.receive_initial_metadata: self.assertTrue( test_common.metadata_transmitted( server_initial_metadata, - client_result.received_metadata)) - elif client_result.type == cygrpc.OperationType.receive_message: - self.assertEqual(RESPONSE, - client_result.received_message.bytes()) - elif client_result.type == cygrpc.OperationType.receive_status_on_client: + client_result.initial_metadata())) + elif client_result.type() == cygrpc.OperationType.receive_message: + self.assertEqual(RESPONSE, client_result.message()) + elif client_result.type( + ) == cygrpc.OperationType.receive_status_on_client: self.assertTrue( test_common.metadata_transmitted( server_trailing_metadata, - client_result.received_metadata)) - self.assertEqual(SERVER_STATUS_DETAILS, - client_result.received_status_details) - self.assertEqual(SERVER_STATUS_CODE, - client_result.received_status_code) + client_result.trailing_metadata())) + self.assertEqual(SERVER_STATUS_DETAILS, client_result.details()) + self.assertEqual(SERVER_STATUS_CODE, client_result.code()) self.assertEqual( set([ cygrpc.OperationType.send_initial_metadata, @@ -281,13 +286,13 @@ class ServerClientMixin(object): self.assertEqual(5, len(server_event.batch_operations)) found_server_op_types = set() for server_result in server_event.batch_operations: - self.assertNotIn(client_result.type, found_server_op_types) - found_server_op_types.add(server_result.type) - if server_result.type == cygrpc.OperationType.receive_message: - self.assertEqual(REQUEST, - server_result.received_message.bytes()) - elif server_result.type == cygrpc.OperationType.receive_close_on_server: - self.assertFalse(server_result.received_cancelled) + self.assertNotIn(client_result.type(), found_server_op_types) + found_server_op_types.add(server_result.type()) + if server_result.type() == cygrpc.OperationType.receive_message: + self.assertEqual(REQUEST, server_result.message()) + elif server_result.type( + ) == cygrpc.OperationType.receive_close_on_server: + self.assertFalse(server_result.cancelled()) self.assertEqual( set([ cygrpc.OperationType.send_initial_metadata, @@ -323,13 +328,12 @@ class ServerClientMixin(object): cygrpc_deadline, description) client_event_future = perform_client_operations([ - cygrpc.operation_send_initial_metadata(empty_metadata, - _EMPTY_FLAGS), - cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS), + cygrpc.SendInitialMetadataOperation(empty_metadata, _EMPTY_FLAGS), + cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS), ], "Client prologue") request_event = self.server_completion_queue.poll(cygrpc_deadline) - server_call = request_event.operation_call + server_call = request_event.call def perform_server_operations(operations, description): return self._perform_operations(operations, server_call, @@ -337,8 +341,7 @@ class ServerClientMixin(object): cygrpc_deadline, description) server_event_future = perform_server_operations([ - cygrpc.operation_send_initial_metadata(empty_metadata, - _EMPTY_FLAGS), + cygrpc.SendInitialMetadataOperation(empty_metadata, _EMPTY_FLAGS), ], "Server prologue") client_event_future.result() # force completion @@ -347,12 +350,12 @@ class ServerClientMixin(object): # Messaging for _ in range(10): client_event_future = perform_client_operations([ - cygrpc.operation_send_message(b'', _EMPTY_FLAGS), - cygrpc.operation_receive_message(_EMPTY_FLAGS), + cygrpc.SendMessageOperation(b'', _EMPTY_FLAGS), + cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), ], "Client message") server_event_future = perform_server_operations([ - cygrpc.operation_send_message(b'', _EMPTY_FLAGS), - cygrpc.operation_receive_message(_EMPTY_FLAGS), + cygrpc.SendMessageOperation(b'', _EMPTY_FLAGS), + cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), ], "Server receive") client_event_future.result() # force completion @@ -360,13 +363,13 @@ class ServerClientMixin(object): # Epilogue client_event_future = perform_client_operations([ - cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), - cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS) + cygrpc.SendCloseFromClientOperation(_EMPTY_FLAGS), + cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS) ], "Client epilogue") server_event_future = perform_server_operations([ - cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS), - cygrpc.operation_send_status_from_server( + cygrpc.ReceiveCloseOnServerOperation(_EMPTY_FLAGS), + cygrpc.SendStatusFromServerOperation( empty_metadata, cygrpc.StatusCode.ok, b'', _EMPTY_FLAGS) ], "Server epilogue") @@ -386,10 +389,11 @@ class InsecureServerInsecureClient(unittest.TestCase, ServerClientMixin): class SecureServerSecureClient(unittest.TestCase, ServerClientMixin): def setUp(self): - server_credentials = cygrpc.server_credentials_ssl(None, [ - cygrpc.SslPemKeyCertPair(resources.private_key(), - resources.certificate_chain()) - ], False) + server_credentials = cygrpc.server_credentials_ssl( + None, [ + cygrpc.SslPemKeyCertPair(resources.private_key(), + resources.certificate_chain()) + ], False) client_credentials = cygrpc.SSLChannelCredentials( resources.test_root_certificates(), None, None) self.setUpMixin(server_credentials, client_credentials, diff --git a/src/python/grpcio_tests/tests/unit/_empty_message_test.py b/src/python/grpcio_tests/tests/unit/_empty_message_test.py index 14695bc13f..c55ef61c13 100644 --- a/src/python/grpcio_tests/tests/unit/_empty_message_test.py +++ b/src/python/grpcio_tests/tests/unit/_empty_message_test.py @@ -106,13 +106,13 @@ class EmptyMessageTest(unittest.TestCase): list(response_iterator)) def testStreamUnary(self): - response = self._channel.stream_unary(_STREAM_UNARY)( - iter([_REQUEST] * test_constants.STREAM_LENGTH)) + response = self._channel.stream_unary(_STREAM_UNARY)(iter( + [_REQUEST] * test_constants.STREAM_LENGTH)) self.assertEqual(_RESPONSE, response) def testStreamStream(self): - response_iterator = self._channel.stream_stream(_STREAM_STREAM)( - iter([_REQUEST] * test_constants.STREAM_LENGTH)) + response_iterator = self._channel.stream_stream(_STREAM_STREAM)(iter( + [_REQUEST] * test_constants.STREAM_LENGTH)) self.assertSequenceEqual([_RESPONSE] * test_constants.STREAM_LENGTH, list(response_iterator)) diff --git a/src/python/grpcio_tests/tests/unit/_interceptor_test.py b/src/python/grpcio_tests/tests/unit/_interceptor_test.py index 2aee298df2..3d547b71cd 100644 --- a/src/python/grpcio_tests/tests/unit/_interceptor_test.py +++ b/src/python/grpcio_tests/tests/unit/_interceptor_test.py @@ -65,7 +65,10 @@ class _Handler(object): def handle_unary_unary(self, request, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) return request def handle_unary_stream(self, request, servicer_context): @@ -74,7 +77,10 @@ class _Handler(object): yield request self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) def handle_stream_unary(self, request_iterator, servicer_context): if servicer_context is not None: @@ -86,13 +92,19 @@ class _Handler(object): response_elements.append(request) self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) return b''.join(response_elements) def handle_stream_stream(self, request_iterator, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) for request in request_iterator: self._control.control() yield request @@ -162,9 +174,10 @@ def _stream_stream_multi_callable(channel): class _ClientCallDetails( - collections.namedtuple('_ClientCallDetails', - ('method', 'timeout', 'metadata', - 'credentials')), grpc.ClientCallDetails): + collections.namedtuple( + '_ClientCallDetails', + ('method', 'timeout', 'metadata', 'credentials')), + grpc.ClientCallDetails): pass @@ -262,7 +275,10 @@ def _append_request_header_interceptor(header, value): metadata = [] if client_call_details.metadata: metadata = list(client_call_details.metadata) - metadata.append((header, value,)) + metadata.append(( + header, + value, + )) client_call_details = _ClientCallDetails( client_call_details.method, client_call_details.timeout, metadata, client_call_details.credentials) @@ -306,9 +322,11 @@ class InterceptorTest(unittest.TestCase): self._server = grpc.server( self._server_pool, options=(('grpc.so_reuseport', 0),), - interceptors=(_LoggingInterceptor('s1', self._record), - conditional_interceptor, - _LoggingInterceptor('s2', self._record),)) + interceptors=( + _LoggingInterceptor('s1', self._record), + conditional_interceptor, + _LoggingInterceptor('s2', self._record), + )) port = self._server.add_insecure_port('[::]:0') self._server.add_generic_rpc_handlers((_GenericHandler(self._handler),)) self._server.start() @@ -333,8 +351,8 @@ class InterceptorTest(unittest.TestCase): interceptor = _wrap_request_iterator_stream_interceptor(triple) channel = grpc.intercept_channel(self._channel, interceptor) - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) multi_callable = _stream_stream_multi_callable(channel) response_iterator = multi_callable( @@ -365,8 +383,8 @@ class InterceptorTest(unittest.TestCase): multi_callable = _unary_unary_multi_callable(defective_channel) call_future = multi_callable.future( request, - metadata=( - ('test', 'InterceptedUnaryRequestBlockingUnaryResponse'),)) + metadata=(('test', + 'InterceptedUnaryRequestBlockingUnaryResponse'),)) self.assertIsNotNone(call_future.exception()) self.assertEqual(call_future.code(), grpc.StatusCode.INTERNAL) @@ -374,12 +392,14 @@ class InterceptorTest(unittest.TestCase): def testInterceptedHeaderManipulationWithServerSideVerification(self): request = b'\x07\x08' - channel = grpc.intercept_channel( - self._channel, _append_request_header_interceptor('secret', '42')) - channel = grpc.intercept_channel( - channel, - _LoggingInterceptor('c1', self._record), - _LoggingInterceptor('c2', self._record)) + channel = grpc.intercept_channel(self._channel, + _append_request_header_interceptor( + 'secret', '42')) + channel = grpc.intercept_channel(channel, + _LoggingInterceptor( + 'c1', self._record), + _LoggingInterceptor( + 'c2', self._record)) self._record[:] = [] @@ -401,16 +421,17 @@ class InterceptorTest(unittest.TestCase): self._record[:] = [] - channel = grpc.intercept_channel( - self._channel, - _LoggingInterceptor('c1', self._record), - _LoggingInterceptor('c2', self._record)) + channel = grpc.intercept_channel(self._channel, + _LoggingInterceptor( + 'c1', self._record), + _LoggingInterceptor( + 'c2', self._record)) multi_callable = _unary_unary_multi_callable(channel) multi_callable( request, - metadata=( - ('test', 'InterceptedUnaryRequestBlockingUnaryResponse'),)) + metadata=(('test', + 'InterceptedUnaryRequestBlockingUnaryResponse'),)) self.assertSequenceEqual(self._record, [ 'c1:intercept_unary_unary', 'c2:intercept_unary_unary', @@ -420,10 +441,11 @@ class InterceptorTest(unittest.TestCase): def testInterceptedUnaryRequestBlockingUnaryResponseWithCall(self): request = b'\x07\x08' - channel = grpc.intercept_channel( - self._channel, - _LoggingInterceptor('c1', self._record), - _LoggingInterceptor('c2', self._record)) + channel = grpc.intercept_channel(self._channel, + _LoggingInterceptor( + 'c1', self._record), + _LoggingInterceptor( + 'c2', self._record)) self._record[:] = [] @@ -443,10 +465,11 @@ class InterceptorTest(unittest.TestCase): request = b'\x07\x08' self._record[:] = [] - channel = grpc.intercept_channel( - self._channel, - _LoggingInterceptor('c1', self._record), - _LoggingInterceptor('c2', self._record)) + channel = grpc.intercept_channel(self._channel, + _LoggingInterceptor( + 'c1', self._record), + _LoggingInterceptor( + 'c2', self._record)) multi_callable = _unary_unary_multi_callable(channel) response_future = multi_callable.future( @@ -463,10 +486,11 @@ class InterceptorTest(unittest.TestCase): request = b'\x37\x58' self._record[:] = [] - channel = grpc.intercept_channel( - self._channel, - _LoggingInterceptor('c1', self._record), - _LoggingInterceptor('c2', self._record)) + channel = grpc.intercept_channel(self._channel, + _LoggingInterceptor( + 'c1', self._record), + _LoggingInterceptor( + 'c2', self._record)) multi_callable = _unary_stream_multi_callable(channel) response_iterator = multi_callable( @@ -480,21 +504,22 @@ class InterceptorTest(unittest.TestCase): ]) def testInterceptedStreamRequestBlockingUnaryResponse(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) self._record[:] = [] - channel = grpc.intercept_channel( - self._channel, - _LoggingInterceptor('c1', self._record), - _LoggingInterceptor('c2', self._record)) + channel = grpc.intercept_channel(self._channel, + _LoggingInterceptor( + 'c1', self._record), + _LoggingInterceptor( + 'c2', self._record)) multi_callable = _stream_unary_multi_callable(channel) multi_callable( request_iterator, - metadata=( - ('test', 'InterceptedStreamRequestBlockingUnaryResponse'),)) + metadata=(('test', + 'InterceptedStreamRequestBlockingUnaryResponse'),)) self.assertSequenceEqual(self._record, [ 'c1:intercept_stream_unary', 'c2:intercept_stream_unary', @@ -502,15 +527,16 @@ class InterceptorTest(unittest.TestCase): ]) def testInterceptedStreamRequestBlockingUnaryResponseWithCall(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) self._record[:] = [] - channel = grpc.intercept_channel( - self._channel, - _LoggingInterceptor('c1', self._record), - _LoggingInterceptor('c2', self._record)) + channel = grpc.intercept_channel(self._channel, + _LoggingInterceptor( + 'c1', self._record), + _LoggingInterceptor( + 'c2', self._record)) multi_callable = _stream_unary_multi_callable(channel) multi_callable.with_call( @@ -525,15 +551,16 @@ class InterceptorTest(unittest.TestCase): ]) def testInterceptedStreamRequestFutureUnaryResponse(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) self._record[:] = [] - channel = grpc.intercept_channel( - self._channel, - _LoggingInterceptor('c1', self._record), - _LoggingInterceptor('c2', self._record)) + channel = grpc.intercept_channel(self._channel, + _LoggingInterceptor( + 'c1', self._record), + _LoggingInterceptor( + 'c2', self._record)) multi_callable = _stream_unary_multi_callable(channel) response_future = multi_callable.future( @@ -547,15 +574,16 @@ class InterceptorTest(unittest.TestCase): ]) def testInterceptedStreamRequestStreamResponse(self): - requests = tuple(b'\x77\x58' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x77\x58' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) self._record[:] = [] - channel = grpc.intercept_channel( - self._channel, - _LoggingInterceptor('c1', self._record), - _LoggingInterceptor('c2', self._record)) + channel = grpc.intercept_channel(self._channel, + _LoggingInterceptor( + 'c1', self._record), + _LoggingInterceptor( + 'c2', self._record)) multi_callable = _stream_stream_multi_callable(channel) response_iterator = multi_callable( diff --git a/src/python/grpcio_tests/tests/unit/_invalid_metadata_test.py b/src/python/grpcio_tests/tests/unit/_invalid_metadata_test.py index 8f4c075e2c..4edf0fc4ad 100644 --- a/src/python/grpcio_tests/tests/unit/_invalid_metadata_test.py +++ b/src/python/grpcio_tests/tests/unit/_invalid_metadata_test.py @@ -106,8 +106,8 @@ class InvalidMetadataTest(unittest.TestCase): self.assertEqual(response_iterator.code(), grpc.StatusCode.INTERNAL) def testStreamRequestBlockingUnaryResponse(self): - request_iterator = (b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + request_iterator = ( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) metadata = (('InVaLiD', 'StreamRequestBlockingUnaryResponse'),) expected_error_details = "metadata was invalid: %s" % metadata with self.assertRaises(ValueError) as exception_context: @@ -115,8 +115,8 @@ class InvalidMetadataTest(unittest.TestCase): self.assertIn(expected_error_details, str(exception_context.exception)) def testStreamRequestBlockingUnaryResponseWithCall(self): - request_iterator = (b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + request_iterator = ( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) metadata = (('InVaLiD', 'StreamRequestBlockingUnaryResponseWithCall'),) expected_error_details = "metadata was invalid: %s" % metadata multi_callable = _stream_unary_multi_callable(self._channel) @@ -125,8 +125,8 @@ class InvalidMetadataTest(unittest.TestCase): self.assertIn(expected_error_details, str(exception_context.exception)) def testStreamRequestFutureUnaryResponse(self): - request_iterator = (b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + request_iterator = ( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) metadata = (('InVaLiD', 'StreamRequestFutureUnaryResponse'),) expected_error_details = "metadata was invalid: %s" % metadata response_future = self._stream_unary.future( @@ -141,8 +141,8 @@ class InvalidMetadataTest(unittest.TestCase): self.assertEqual(response_future.code(), grpc.StatusCode.INTERNAL) def testStreamRequestStreamResponse(self): - request_iterator = (b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + request_iterator = ( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) metadata = (('InVaLiD', 'StreamRequestStreamResponse'),) expected_error_details = "metadata was invalid: %s" % metadata response_iterator = self._stream_stream( diff --git a/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py b/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py index b46d176d04..e40cca8b24 100644 --- a/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py +++ b/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py @@ -62,7 +62,10 @@ class _Handler(object): def handle_unary_unary(self, request, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) return request def handle_unary_stream(self, request, servicer_context): @@ -71,7 +74,10 @@ class _Handler(object): yield request self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) def handle_stream_unary(self, request_iterator, servicer_context): if servicer_context is not None: @@ -83,13 +89,19 @@ class _Handler(object): response_elements.append(request) self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) return b''.join(response_elements) def handle_stream_stream(self, request_iterator, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) for request in request_iterator: self._control.control() yield request @@ -208,8 +220,8 @@ class InvocationDefectsTest(unittest.TestCase): with self.assertRaises(grpc.RpcError): response = multi_callable( requests, - metadata=( - ('test', 'IterableStreamRequestBlockingUnaryResponse'),)) + metadata=(('test', + 'IterableStreamRequestBlockingUnaryResponse'),)) def testIterableStreamRequestFutureUnaryResponse(self): requests = [b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)] diff --git a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py index ec67f99fbc..bb6ac70497 100644 --- a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py +++ b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py @@ -36,16 +36,16 @@ _UNARY_STREAM = 'UnaryStream' _STREAM_UNARY = 'StreamUnary' _STREAM_STREAM = 'StreamStream' -_CLIENT_METADATA = (('client-md-key', 'client-md-key'), - ('client-md-key-bin', b'\x00\x01')) +_CLIENT_METADATA = (('client-md-key', 'client-md-key'), ('client-md-key-bin', + b'\x00\x01')) -_SERVER_INITIAL_METADATA = ( - ('server-initial-md-key', 'server-initial-md-value'), - ('server-initial-md-key-bin', b'\x00\x02')) +_SERVER_INITIAL_METADATA = (('server-initial-md-key', + 'server-initial-md-value'), + ('server-initial-md-key-bin', b'\x00\x02')) -_SERVER_TRAILING_METADATA = ( - ('server-trailing-md-key', 'server-trailing-md-value'), - ('server-trailing-md-key-bin', b'\x00\x03')) +_SERVER_TRAILING_METADATA = (('server-trailing-md-key', + 'server-trailing-md-value'), + ('server-trailing-md-key-bin', b'\x00\x03')) _NON_OK_CODE = grpc.StatusCode.NOT_FOUND _DETAILS = 'Test details!' @@ -193,17 +193,33 @@ class MetadataCodeDetailsTest(unittest.TestCase): channel = grpc.insecure_channel('localhost:{}'.format(port)) self._unary_unary = channel.unary_unary( - '/'.join(('', _SERVICE, _UNARY_UNARY,)), + '/'.join(( + '', + _SERVICE, + _UNARY_UNARY, + )), request_serializer=_REQUEST_SERIALIZER, - response_deserializer=_RESPONSE_DESERIALIZER,) - self._unary_stream = channel.unary_stream( - '/'.join(('', _SERVICE, _UNARY_STREAM,)),) - self._stream_unary = channel.stream_unary( - '/'.join(('', _SERVICE, _STREAM_UNARY,)),) + response_deserializer=_RESPONSE_DESERIALIZER, + ) + self._unary_stream = channel.unary_stream('/'.join(( + '', + _SERVICE, + _UNARY_STREAM, + )),) + self._stream_unary = channel.stream_unary('/'.join(( + '', + _SERVICE, + _STREAM_UNARY, + )),) self._stream_stream = channel.stream_stream( - '/'.join(('', _SERVICE, _STREAM_STREAM,)), + '/'.join(( + '', + _SERVICE, + _STREAM_STREAM, + )), request_serializer=_REQUEST_SERIALIZER, - response_deserializer=_RESPONSE_DESERIALIZER,) + response_deserializer=_RESPONSE_DESERIALIZER, + ) def testSuccessfulUnaryUnary(self): self._servicer.set_details(_DETAILS) diff --git a/src/python/grpcio_tests/tests/unit/_metadata_test.py b/src/python/grpcio_tests/tests/unit/_metadata_test.py index f2dac7bdc5..a918066ea4 100644 --- a/src/python/grpcio_tests/tests/unit/_metadata_test.py +++ b/src/python/grpcio_tests/tests/unit/_metadata_test.py @@ -33,18 +33,50 @@ _UNARY_STREAM = '/test/UnaryStream' _STREAM_UNARY = '/test/StreamUnary' _STREAM_STREAM = '/test/StreamStream' -_INVOCATION_METADATA = ((b'invocation-md-key', u'invocation-md-value',), - (u'invocation-md-key-bin', b'\x00\x01',),) -_EXPECTED_INVOCATION_METADATA = (('invocation-md-key', 'invocation-md-value',), - ('invocation-md-key-bin', b'\x00\x01',),) +_INVOCATION_METADATA = ( + ( + b'invocation-md-key', + u'invocation-md-value', + ), + ( + u'invocation-md-key-bin', + b'\x00\x01', + ), +) +_EXPECTED_INVOCATION_METADATA = ( + ( + 'invocation-md-key', + 'invocation-md-value', + ), + ( + 'invocation-md-key-bin', + b'\x00\x01', + ), +) _INITIAL_METADATA = ((b'initial-md-key', u'initial-md-value'), (u'initial-md-key-bin', b'\x00\x02')) -_EXPECTED_INITIAL_METADATA = (('initial-md-key', 'initial-md-value',), - ('initial-md-key-bin', b'\x00\x02',),) - -_TRAILING_METADATA = (('server-trailing-md-key', 'server-trailing-md-value',), - ('server-trailing-md-key-bin', b'\x00\x03',),) +_EXPECTED_INITIAL_METADATA = ( + ( + 'initial-md-key', + 'initial-md-value', + ), + ( + 'initial-md-key-bin', + b'\x00\x02', + ), +) + +_TRAILING_METADATA = ( + ( + 'server-trailing-md-key', + 'server-trailing-md-value', + ), + ( + 'server-trailing-md-key-bin', + b'\x00\x03', + ), +) _EXPECTED_TRAILING_METADATA = _TRAILING_METADATA @@ -146,8 +178,8 @@ class MetadataTest(unittest.TestCase): def setUp(self): self._server = test_common.test_server() - self._server.add_generic_rpc_handlers( - (_GenericHandler(weakref.proxy(self)),)) + self._server.add_generic_rpc_handlers((_GenericHandler( + weakref.proxy(self)),)) port = self._server.add_insecure_port('[::]:0') self._server.start() self._channel = grpc.insecure_channel( diff --git a/src/python/grpcio_tests/tests/unit/_rpc_test.py b/src/python/grpcio_tests/tests/unit/_rpc_test.py index 1515a87d93..54f01d9f8d 100644 --- a/src/python/grpcio_tests/tests/unit/_rpc_test.py +++ b/src/python/grpcio_tests/tests/unit/_rpc_test.py @@ -64,7 +64,10 @@ class _Handler(object): def handle_unary_unary(self, request, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) # TODO(https://github.com/grpc/grpc/issues/8483): test the values # returned by these methods rather than only "smoke" testing that # the return after having been called. @@ -78,7 +81,10 @@ class _Handler(object): yield request self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) def handle_stream_unary(self, request_iterator, servicer_context): if servicer_context is not None: @@ -90,13 +96,19 @@ class _Handler(object): response_elements.append(request) self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) return b''.join(response_elements) def handle_stream_stream(self, request_iterator, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) + servicer_context.set_trailing_metadata((( + 'testkey', + 'testvalue', + ),)) for request in request_iterator: self._control.control() yield request @@ -244,8 +256,8 @@ class RPCTest(unittest.TestCase): self.assertSequenceEqual(expected_responses, responses) def testSuccessfulStreamRequestBlockingUnaryResponse(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) expected_response = self._handler.handle_stream_unary( iter(requests), None) request_iterator = iter(requests) @@ -253,14 +265,14 @@ class RPCTest(unittest.TestCase): multi_callable = _stream_unary_multi_callable(self._channel) response = multi_callable( request_iterator, - metadata=( - ('test', 'SuccessfulStreamRequestBlockingUnaryResponse'),)) + metadata=(('test', + 'SuccessfulStreamRequestBlockingUnaryResponse'),)) self.assertEqual(expected_response, response) def testSuccessfulStreamRequestBlockingUnaryResponseWithCall(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) expected_response = self._handler.handle_stream_unary( iter(requests), None) request_iterator = iter(requests) @@ -276,8 +288,8 @@ class RPCTest(unittest.TestCase): self.assertIs(grpc.StatusCode.OK, call.code()) def testSuccessfulStreamRequestFutureUnaryResponse(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) expected_response = self._handler.handle_stream_unary( iter(requests), None) request_iterator = iter(requests) @@ -293,8 +305,8 @@ class RPCTest(unittest.TestCase): self.assertIsNone(response_future.traceback()) def testSuccessfulStreamRequestStreamResponse(self): - requests = tuple(b'\x77\x58' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x77\x58' for _ in range(test_constants.STREAM_LENGTH)) expected_responses = tuple( self._handler.handle_stream_stream(iter(requests), None)) request_iterator = iter(requests) @@ -326,8 +338,8 @@ class RPCTest(unittest.TestCase): def testConcurrentBlockingInvocations(self): pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) expected_response = self._handler.handle_stream_unary( iter(requests), None) expected_responses = [expected_response @@ -342,15 +354,15 @@ class RPCTest(unittest.TestCase): request_iterator, metadata=(('test', 'ConcurrentBlockingInvocations'),)) response_futures[index] = response_future - responses = tuple(response_future.result() - for response_future in response_futures) + responses = tuple( + response_future.result() for response_future in response_futures) pool.shutdown(wait=True) self.assertSequenceEqual(expected_responses, responses) def testConcurrentFutureInvocations(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) expected_response = self._handler.handle_stream_unary( iter(requests), None) expected_responses = [expected_response @@ -364,8 +376,8 @@ class RPCTest(unittest.TestCase): request_iterator, metadata=(('test', 'ConcurrentFutureInvocations'),)) response_futures[index] = response_future - responses = tuple(response_future.result() - for response_future in response_futures) + responses = tuple( + response_future.result() for response_future in response_futures) self.assertSequenceEqual(expected_responses, responses) @@ -424,14 +436,14 @@ class RPCTest(unittest.TestCase): multi_callable = _unary_stream_multi_callable(self._channel) response_iterator = multi_callable( request, - metadata=( - ('test', 'ConsumingSomeButNotAllStreamResponsesUnaryRequest'),)) + metadata=(('test', + 'ConsumingSomeButNotAllStreamResponsesUnaryRequest'),)) for _ in range(test_constants.STREAM_LENGTH // 2): next(response_iterator) def testConsumingSomeButNotAllStreamResponsesStreamRequest(self): - requests = tuple(b'\x67\x88' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x67\x88' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_stream_multi_callable(self._channel) @@ -443,15 +455,15 @@ class RPCTest(unittest.TestCase): next(response_iterator) def testConsumingTooManyStreamResponsesStreamRequest(self): - requests = tuple(b'\x67\x88' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x67\x88' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_stream_multi_callable(self._channel) response_iterator = multi_callable( request_iterator, - metadata=( - ('test', 'ConsumingTooManyStreamResponsesStreamRequest'),)) + metadata=(('test', + 'ConsumingTooManyStreamResponsesStreamRequest'),)) for _ in range(test_constants.STREAM_LENGTH): next(response_iterator) for _ in range(test_constants.STREAM_LENGTH): @@ -503,8 +515,8 @@ class RPCTest(unittest.TestCase): self.assertIsNotNone(response_iterator.trailing_metadata()) def testCancelledStreamRequestUnaryResponse(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_unary_multi_callable(self._channel) @@ -528,8 +540,8 @@ class RPCTest(unittest.TestCase): self.assertIsNotNone(response_future.trailing_metadata()) def testCancelledStreamRequestStreamResponse(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_stream_multi_callable(self._channel) @@ -555,8 +567,8 @@ class RPCTest(unittest.TestCase): multi_callable.with_call( request, timeout=test_constants.SHORT_TIMEOUT, - metadata=( - ('test', 'ExpiredUnaryRequestBlockingUnaryResponse'),)) + metadata=(('test', + 'ExpiredUnaryRequestBlockingUnaryResponse'),)) self.assertIsInstance(exception_context.exception, grpc.Call) self.assertIsNotNone(exception_context.exception.initial_metadata()) @@ -610,8 +622,8 @@ class RPCTest(unittest.TestCase): response_iterator.code()) def testExpiredStreamRequestBlockingUnaryResponse(self): - requests = tuple(b'\x07\x08' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_unary_multi_callable(self._channel) @@ -620,8 +632,8 @@ class RPCTest(unittest.TestCase): multi_callable( request_iterator, timeout=test_constants.SHORT_TIMEOUT, - metadata=( - ('test', 'ExpiredStreamRequestBlockingUnaryResponse'),)) + metadata=(('test', + 'ExpiredStreamRequestBlockingUnaryResponse'),)) self.assertIsInstance(exception_context.exception, grpc.RpcError) self.assertIsInstance(exception_context.exception, grpc.Call) @@ -632,8 +644,8 @@ class RPCTest(unittest.TestCase): self.assertIsNotNone(exception_context.exception.trailing_metadata()) def testExpiredStreamRequestFutureUnaryResponse(self): - requests = tuple(b'\x07\x18' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x18' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) callback = _Callback() @@ -644,8 +656,8 @@ class RPCTest(unittest.TestCase): timeout=test_constants.SHORT_TIMEOUT, metadata=(('test', 'ExpiredStreamRequestFutureUnaryResponse'),)) with self.assertRaises(grpc.FutureTimeoutError): - response_future.result(timeout=test_constants.SHORT_TIMEOUT / - 2.0) + response_future.result( + timeout=test_constants.SHORT_TIMEOUT / 2.0) response_future.add_done_callback(callback) value_passed_to_callback = callback.value() @@ -663,8 +675,8 @@ class RPCTest(unittest.TestCase): self.assertIsNotNone(response_future.trailing_metadata()) def testExpiredStreamRequestStreamResponse(self): - requests = tuple(b'\x67\x18' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x67\x18' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_stream_multi_callable(self._channel) @@ -689,8 +701,8 @@ class RPCTest(unittest.TestCase): with self.assertRaises(grpc.RpcError) as exception_context: multi_callable.with_call( request, - metadata=( - ('test', 'FailedUnaryRequestBlockingUnaryResponse'),)) + metadata=(('test', + 'FailedUnaryRequestBlockingUnaryResponse'),)) self.assertIs(grpc.StatusCode.UNKNOWN, exception_context.exception.code()) @@ -734,8 +746,8 @@ class RPCTest(unittest.TestCase): exception_context.exception.code()) def testFailedStreamRequestBlockingUnaryResponse(self): - requests = tuple(b'\x47\x58' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x47\x58' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_unary_multi_callable(self._channel) @@ -743,15 +755,15 @@ class RPCTest(unittest.TestCase): with self.assertRaises(grpc.RpcError) as exception_context: multi_callable( request_iterator, - metadata=( - ('test', 'FailedStreamRequestBlockingUnaryResponse'),)) + metadata=(('test', + 'FailedStreamRequestBlockingUnaryResponse'),)) self.assertIs(grpc.StatusCode.UNKNOWN, exception_context.exception.code()) def testFailedStreamRequestFutureUnaryResponse(self): - requests = tuple(b'\x07\x18' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x18' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) callback = _Callback() @@ -773,8 +785,8 @@ class RPCTest(unittest.TestCase): self.assertIs(response_future, value_passed_to_callback) def testFailedStreamRequestStreamResponse(self): - requests = tuple(b'\x67\x88' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x67\x88' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_stream_multi_callable(self._channel) @@ -805,8 +817,8 @@ class RPCTest(unittest.TestCase): request, metadata=(('test', 'IgnoredUnaryRequestStreamResponse'),)) def testIgnoredStreamRequestFutureUnaryResponse(self): - requests = tuple(b'\x07\x18' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x07\x18' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_unary_multi_callable(self._channel) @@ -815,8 +827,8 @@ class RPCTest(unittest.TestCase): metadata=(('test', 'IgnoredStreamRequestFutureUnaryResponse'),)) def testIgnoredStreamRequestStreamResponse(self): - requests = tuple(b'\x67\x88' - for _ in range(test_constants.STREAM_LENGTH)) + requests = tuple( + b'\x67\x88' for _ in range(test_constants.STREAM_LENGTH)) request_iterator = iter(requests) multi_callable = _stream_stream_multi_callable(self._channel) diff --git a/src/python/grpcio_tests/tests/unit/_server_ssl_cert_config_test.py b/src/python/grpcio_tests/tests/unit/_server_ssl_cert_config_test.py index 2c513da5d0..0d78034b7b 100644 --- a/src/python/grpcio_tests/tests/unit/_server_ssl_cert_config_test.py +++ b/src/python/grpcio_tests/tests/unit/_server_ssl_cert_config_test.py @@ -74,7 +74,8 @@ def _create_client_stub( expect_success, root_certificates=None, private_key=None, - certificate_chain=None,): + certificate_chain=None, +): channel = grpc.secure_channel('localhost:{}'.format(port), grpc.ssl_channel_credentials( root_certificates=root_certificates, diff --git a/src/python/grpcio_tests/tests/unit/_thread_cleanup_test.py b/src/python/grpcio_tests/tests/unit/_thread_cleanup_test.py index fe3e71d686..18f5af058a 100644 --- a/src/python/grpcio_tests/tests/unit/_thread_cleanup_test.py +++ b/src/python/grpcio_tests/tests/unit/_thread_cleanup_test.py @@ -52,7 +52,9 @@ class CleanupThreadTest(unittest.TestCase): target=target, name='test-name', args=('arg1', 'arg2'), - kwargs={'arg3': 'arg3'}) + kwargs={ + 'arg3': 'arg3' + }) cleanup_thread.start() cleanup_thread.join() self.assertEqual(cleanup_thread.name, 'test-name') diff --git a/src/python/grpcio_tests/tests/unit/beta/_beta_features_test.py b/src/python/grpcio_tests/tests/unit/beta/_beta_features_test.py index eb8dc80a6e..61c03f64ba 100644 --- a/src/python/grpcio_tests/tests/unit/beta/_beta_features_test.py +++ b/src/python/grpcio_tests/tests/unit/beta/_beta_features_test.py @@ -163,7 +163,10 @@ class BetaFeaturesTest(unittest.TestCase): self._server = implementations.server( method_implementations, options=server_options) server_credentials = implementations.ssl_server_credentials([ - (resources.private_key(), resources.certificate_chain(),), + ( + resources.private_key(), + resources.certificate_chain(), + ), ]) port = self._server.add_secure_port('[::]:0', server_credentials) self._server.start() @@ -289,7 +292,10 @@ class ContextManagementAndLifecycleTest(unittest.TestCase): self._server_options = implementations.server_options( thread_pool_size=test_constants.POOL_SIZE) self._server_credentials = implementations.ssl_server_credentials([ - (resources.private_key(), resources.certificate_chain(),), + ( + resources.private_key(), + resources.certificate_chain(), + ), ]) self._channel_credentials = implementations.ssl_channel_credentials( resources.test_root_certificates()) diff --git a/src/python/grpcio_tests/tests/unit/beta/_face_interface_test.py b/src/python/grpcio_tests/tests/unit/beta/_face_interface_test.py index e4b81e7e78..c99738e085 100644 --- a/src/python/grpcio_tests/tests/unit/beta/_face_interface_test.py +++ b/src/python/grpcio_tests/tests/unit/beta/_face_interface_test.py @@ -32,8 +32,11 @@ _SERVER_HOST_OVERRIDE = 'foo.test.google.fr' class _SerializationBehaviors( collections.namedtuple('_SerializationBehaviors', ( - 'request_serializers', 'request_deserializers', - 'response_serializers', 'response_deserializers',))): + 'request_serializers', + 'request_deserializers', + 'response_serializers', + 'response_deserializers', + ))): pass @@ -73,7 +76,10 @@ class _Implementation(test_interfaces.Implementation): server = implementations.server( method_implementations, options=server_options) server_credentials = implementations.ssl_server_credentials([ - (resources.private_key(), resources.certificate_chain(),), + ( + resources.private_key(), + resources.certificate_chain(), + ), ]) port = server.add_secure_port('[::]:0', server_credentials) server.start() @@ -116,9 +122,10 @@ class _Implementation(test_interfaces.Implementation): def load_tests(loader, tests, pattern): - return unittest.TestSuite(tests=tuple( - loader.loadTestsFromTestCase(test_case_class) - for test_case_class in test_cases.test_cases(_Implementation()))) + return unittest.TestSuite( + tests=tuple( + loader.loadTestsFromTestCase(test_case_class) + for test_case_class in test_cases.test_cases(_Implementation()))) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/unit/beta/_implementations_test.py b/src/python/grpcio_tests/tests/unit/beta/_implementations_test.py index 75a615eeff..5a53766d29 100644 --- a/src/python/grpcio_tests/tests/unit/beta/_implementations_test.py +++ b/src/python/grpcio_tests/tests/unit/beta/_implementations_test.py @@ -41,8 +41,8 @@ class CallCredentialsTest(unittest.TestCase): def test_google_call_credentials(self): creds = oauth2client_client.GoogleCredentials( 'token', 'client_id', 'secret', 'refresh_token', - datetime.datetime(2008, 6, 24), 'https://refresh.uri.com/', - 'user_agent') + datetime.datetime(2008, 6, + 24), 'https://refresh.uri.com/', 'user_agent') call_creds = implementations.google_call_credentials(creds) self.assertIsInstance(call_creds, implementations.CallCredentials) diff --git a/src/python/grpcio_tests/tests/unit/beta/test_utilities.py b/src/python/grpcio_tests/tests/unit/beta/test_utilities.py index 65da0f2020..c8d920d35e 100644 --- a/src/python/grpcio_tests/tests/unit/beta/test_utilities.py +++ b/src/python/grpcio_tests/tests/unit/beta/test_utilities.py @@ -33,6 +33,8 @@ def not_really_secure_channel(host, port, channel_credentials, conducted. """ target = '%s:%d' % (host, port) - channel = grpc.secure_channel(target, channel_credentials, ( - ('grpc.ssl_target_name_override', server_host_override,),)) + channel = grpc.secure_channel(target, channel_credentials, (( + 'grpc.ssl_target_name_override', + server_host_override, + ),)) return implementations.Channel(channel) diff --git a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py index 45fd321ed6..5d8679aa62 100644 --- a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py +++ b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_blocking_invocation_inline_service.py @@ -70,8 +70,8 @@ class TestCase( self.implementation.destantiate(self._memo) def testSuccessfulUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -81,8 +81,8 @@ class TestCase( test_messages.verify(request, response, self) def testSuccessfulUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_stream_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -93,8 +93,8 @@ class TestCase( test_messages.verify(request, responses, self) def testSuccessfulStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() @@ -104,8 +104,8 @@ class TestCase( test_messages.verify(requests, response, self) def testSuccessfulStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_stream_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() @@ -116,8 +116,8 @@ class TestCase( test_messages.verify(requests, responses, self) def testSequentialInvocations(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: first_request = test_messages.request() second_request = test_messages.request() @@ -134,8 +134,8 @@ class TestCase( def testParallelInvocations(self): pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = [] response_futures = [] @@ -158,8 +158,8 @@ class TestCase( def testWaitingForSomeButNotAllParallelInvocations(self): pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = [] response_futures_to_indices = {} @@ -197,8 +197,8 @@ class TestCase( raise NotImplementedError() def testExpiredUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -208,8 +208,8 @@ class TestCase( request, _3069_test_constant.REALLY_SHORT_TIMEOUT) def testExpiredUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_stream_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -220,33 +220,33 @@ class TestCase( list(response_iterator) def testExpiredStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() with self._control.pause(), self.assertRaises( face.ExpirationError): - self._invoker.blocking(group, method)( - iter(requests), - _3069_test_constant.REALLY_SHORT_TIMEOUT) + self._invoker.blocking( + group, method)(iter(requests), + _3069_test_constant.REALLY_SHORT_TIMEOUT) def testExpiredStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_stream_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() with self._control.pause(), self.assertRaises( face.ExpirationError): - response_iterator = self._invoker.blocking(group, method)( - iter(requests), - _3069_test_constant.REALLY_SHORT_TIMEOUT) + response_iterator = self._invoker.blocking( + group, method)(iter(requests), + _3069_test_constant.REALLY_SHORT_TIMEOUT) list(response_iterator) def testFailedUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -255,8 +255,8 @@ class TestCase( request, test_constants.LONG_TIMEOUT) def testFailedUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_stream_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -266,8 +266,8 @@ class TestCase( list(response_iterator) def testFailedStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() @@ -276,8 +276,8 @@ class TestCase( iter(requests), test_constants.LONG_TIMEOUT) def testFailedStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_stream_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() diff --git a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_digest.py b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_digest.py index 0e399c4bc4..b1c33da43a 100644 --- a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_digest.py +++ b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_digest.py @@ -34,11 +34,15 @@ _IDENTITY = lambda x: x class TestServiceDigest( collections.namedtuple('TestServiceDigest', ( - 'methods', 'inline_method_implementations', - 'event_method_implementations', 'multi_method_implementation', - 'unary_unary_messages_sequences', 'unary_stream_messages_sequences', + 'methods', + 'inline_method_implementations', + 'event_method_implementations', + 'multi_method_implementation', + 'unary_unary_messages_sequences', + 'unary_stream_messages_sequences', 'stream_unary_messages_sequences', - 'stream_stream_messages_sequences',))): + 'stream_stream_messages_sequences', + ))): """A transformation of a service.TestService. Attributes: @@ -421,8 +425,8 @@ def digest(service, control, pool): events.update(stream_unary.events) events.update(stream_stream.events) - return TestServiceDigest( - methods, inlines, events, - _MultiMethodImplementation(adaptations, control, pool), - unary_unary.messages, unary_stream.messages, stream_unary.messages, - stream_stream.messages) + return TestServiceDigest(methods, inlines, events, + _MultiMethodImplementation(adaptations, control, + pool), + unary_unary.messages, unary_stream.messages, + stream_unary.messages, stream_stream.messages) diff --git a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py index bc65bf4c80..3d9b2816aa 100644 --- a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py +++ b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_future_invocation_asynchronous_event_service.py @@ -134,8 +134,8 @@ class TestCase( self._digest_pool.shutdown(wait=True) def testSuccessfulUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() callback = _Callback() @@ -151,8 +151,8 @@ class TestCase( self.assertIsNone(response_future.traceback()) def testSuccessfulUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_stream_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -163,8 +163,8 @@ class TestCase( test_messages.verify(request, responses, self) def testSuccessfulStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() request_iterator = _PauseableIterator(iter(requests)) @@ -185,8 +185,8 @@ class TestCase( self.assertIsNone(response_future.traceback()) def testSuccessfulStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_stream_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() request_iterator = _PauseableIterator(iter(requests)) @@ -201,8 +201,8 @@ class TestCase( test_messages.verify(requests, responses, self) def testSequentialInvocations(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: first_request = test_messages.request() second_request = test_messages.request() @@ -220,8 +220,8 @@ class TestCase( test_messages.verify(second_request, second_response, self) def testParallelInvocations(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: first_request = test_messages.request() second_request = test_messages.request() @@ -236,8 +236,8 @@ class TestCase( test_messages.verify(first_request, first_response, self) test_messages.verify(second_request, second_response, self) - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = [] response_futures = [] @@ -258,8 +258,8 @@ class TestCase( def testWaitingForSomeButNotAllParallelInvocations(self): pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = [] response_futures_to_indices = {} @@ -282,8 +282,8 @@ class TestCase( pool.shutdown(wait=True) def testCancelledUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() callback = _Callback() @@ -305,8 +305,8 @@ class TestCase( response_future.traceback() def testCancelledUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_stream_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -319,8 +319,8 @@ class TestCase( next(response_iterator) def testCancelledStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() callback = _Callback() @@ -342,8 +342,8 @@ class TestCase( response_future.traceback() def testCancelledStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_stream_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() @@ -356,8 +356,8 @@ class TestCase( next(response_iterator) def testExpiredUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() callback = _Callback() @@ -376,8 +376,8 @@ class TestCase( self.assertIsNotNone(response_future.traceback()) def testExpiredUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_stream_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -388,16 +388,16 @@ class TestCase( list(response_iterator) def testExpiredStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() callback = _Callback() with self._control.pause(): - response_future = self._invoker.future(group, method)( - iter(requests), - _3069_test_constant.REALLY_SHORT_TIMEOUT) + response_future = self._invoker.future( + group, method)(iter(requests), + _3069_test_constant.REALLY_SHORT_TIMEOUT) response_future.add_done_callback(callback) self.assertIs(callback.future(), response_future) self.assertIsInstance(response_future.exception(), @@ -409,21 +409,21 @@ class TestCase( self.assertIsNotNone(response_future.traceback()) def testExpiredStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_stream_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() with self._control.pause(): - response_iterator = self._invoker.future(group, method)( - iter(requests), - _3069_test_constant.REALLY_SHORT_TIMEOUT) + response_iterator = self._invoker.future( + group, method)(iter(requests), + _3069_test_constant.REALLY_SHORT_TIMEOUT) with self.assertRaises(face.ExpirationError): list(response_iterator) def testFailedUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_unary_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() callback = _Callback() @@ -448,8 +448,8 @@ class TestCase( self.assertIsNotNone(abortion_callback.future()) def testFailedUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.unary_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.unary_stream_messages_sequences)): for test_messages in test_messages_sequence: request = test_messages.request() @@ -464,17 +464,17 @@ class TestCase( list(response_iterator) def testFailedStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_unary_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_unary_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() callback = _Callback() abortion_callback = _Callback() with self._control.fail(): - response_future = self._invoker.future(group, method)( - iter(requests), - _3069_test_constant.REALLY_SHORT_TIMEOUT) + response_future = self._invoker.future( + group, method)(iter(requests), + _3069_test_constant.REALLY_SHORT_TIMEOUT) response_future.add_done_callback(callback) response_future.add_abortion_callback(abortion_callback) @@ -491,8 +491,8 @@ class TestCase( self.assertIsNotNone(abortion_callback.future()) def testFailedStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - six.iteritems(self._digest.stream_stream_messages_sequences)): + for (group, method), test_messages_sequence in (six.iteritems( + self._digest.stream_stream_messages_sequences)): for test_messages in test_messages_sequence: requests = test_messages.requests() @@ -502,7 +502,7 @@ class TestCase( # expiration of the RPC. with self._control.fail(), self.assertRaises( face.ExpirationError): - response_iterator = self._invoker.future(group, method)( - iter(requests), - _3069_test_constant.REALLY_SHORT_TIMEOUT) + response_iterator = self._invoker.future( + group, method)(iter(requests), + _3069_test_constant.REALLY_SHORT_TIMEOUT) list(response_iterator) diff --git a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_invocation.py b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_invocation.py index fd55f4e09f..efc93d56b0 100644 --- a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_invocation.py +++ b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_invocation.py @@ -191,5 +191,8 @@ def invoker_constructors(): Returns: A sequence of InvokerConstructors. """ - return (_GenericInvokerConstructor(), _MultiCallableInvokerConstructor(), - _DynamicInvokerConstructor(),) + return ( + _GenericInvokerConstructor(), + _MultiCallableInvokerConstructor(), + _DynamicInvokerConstructor(), + ) diff --git a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_stock_service.py b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_stock_service.py index 69c7ac2d73..a84e02a79a 100644 --- a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_stock_service.py +++ b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_stock_service.py @@ -33,8 +33,8 @@ def _get_last_trade_price(stock_request, stock_reply_callback, control, active): if active(): stock_reply_callback( stock_pb2.StockReply( - symbol=stock_request.symbol, price=_price( - stock_request.symbol))) + symbol=stock_request.symbol, + price=_price(stock_request.symbol))) else: raise abandonment.Abandoned() diff --git a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/test_cases.py b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/test_cases.py index d1c5b8f76b..cff4b7cdea 100644 --- a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/test_cases.py +++ b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/test_cases.py @@ -24,7 +24,8 @@ from tests.unit.framework.interfaces.face import test_interfaces # pylint: disa _TEST_CASE_SUPERCLASSES = ( _blocking_invocation_inline_service.TestCase, - _future_invocation_asynchronous_event_service.TestCase,) + _future_invocation_asynchronous_event_service.TestCase, +) def test_cases(implementation): @@ -42,8 +43,9 @@ def test_cases(implementation): for invoker_constructor in _invocation.invoker_constructors(): for super_class in _TEST_CASE_SUPERCLASSES: test_case_classes.append( - type(invoker_constructor.name() + super_class.NAME, ( - super_class,), { + type( + invoker_constructor.name() + super_class.NAME, + (super_class,), { 'implementation': implementation, 'invoker_constructor': invoker_constructor, '__module__': implementation.__module__, diff --git a/src/python/grpcio_tests/tests/unit/resources.py b/src/python/grpcio_tests/tests/unit/resources.py index 11ef9e8565..51a8979f58 100644 --- a/src/python/grpcio_tests/tests/unit/resources.py +++ b/src/python/grpcio_tests/tests/unit/resources.py @@ -58,7 +58,8 @@ def cert_hier_1_client_1_key(): def cert_hier_1_client_1_cert(): return pkg_resources.resource_string( __name__, - 'credentials/certificate_hierarchy_1/intermediate/certs/client.cert.pem') + 'credentials/certificate_hierarchy_1/intermediate/certs/client.cert.pem' + ) def cert_hier_1_server_1_key(): @@ -97,7 +98,8 @@ def cert_hier_2_client_1_key(): def cert_hier_2_client_1_cert(): return pkg_resources.resource_string( __name__, - 'credentials/certificate_hierarchy_2/intermediate/certs/client.cert.pem') + 'credentials/certificate_hierarchy_2/intermediate/certs/client.cert.pem' + ) def cert_hier_2_server_1_key(): diff --git a/src/python/grpcio_tests/tests/unit/test_common.py b/src/python/grpcio_tests/tests/unit/test_common.py index 6334a32b0e..61717ae135 100644 --- a/src/python/grpcio_tests/tests/unit/test_common.py +++ b/src/python/grpcio_tests/tests/unit/test_common.py @@ -19,9 +19,21 @@ from concurrent import futures import grpc import six -INVOCATION_INITIAL_METADATA = (('0', 'abc'), ('1', 'def'), ('2', 'ghi'),) -SERVICE_INITIAL_METADATA = (('3', 'jkl'), ('4', 'mno'), ('5', 'pqr'),) -SERVICE_TERMINAL_METADATA = (('6', 'stu'), ('7', 'vwx'), ('8', 'yza'),) +INVOCATION_INITIAL_METADATA = ( + ('0', 'abc'), + ('1', 'def'), + ('2', 'ghi'), +) +SERVICE_INITIAL_METADATA = ( + ('3', 'jkl'), + ('4', 'mno'), + ('5', 'pqr'), +) +SERVICE_TERMINAL_METADATA = ( + ('6', 'stu'), + ('7', 'vwx'), + ('8', 'yza'), +) DETAILS = 'test details' @@ -80,8 +92,10 @@ def test_secure_channel(target, channel_credentials, server_host_override): An implementations.Channel to the remote host through which RPCs may be conducted. """ - channel = grpc.secure_channel(target, channel_credentials, ( - ('grpc.ssl_target_name_override', server_host_override,),)) + channel = grpc.secure_channel(target, channel_credentials, (( + 'grpc.ssl_target_name_override', + server_host_override, + ),)) return channel diff --git a/test/core/end2end/end2end_nosec_tests.cc b/test/core/end2end/end2end_nosec_tests.cc index 3236feea56..6318550ad8 100644 --- a/test/core/end2end/end2end_nosec_tests.cc +++ b/test/core/end2end/end2end_nosec_tests.cc @@ -68,6 +68,8 @@ extern void filter_causes_close(grpc_end2end_test_config config); extern void filter_causes_close_pre_init(void); extern void filter_latency(grpc_end2end_test_config config); extern void filter_latency_pre_init(void); +extern void filter_status_code(grpc_end2end_test_config config); +extern void filter_status_code_pre_init(void); extern void graceful_server_shutdown(grpc_end2end_test_config config); extern void graceful_server_shutdown_pre_init(void); extern void high_initial_seqno(grpc_end2end_test_config config); @@ -170,6 +172,7 @@ void grpc_end2end_tests_pre_init(void) { filter_call_init_fails_pre_init(); filter_causes_close_pre_init(); filter_latency_pre_init(); + filter_status_code_pre_init(); graceful_server_shutdown_pre_init(); high_initial_seqno_pre_init(); hpack_size_pre_init(); @@ -237,6 +240,7 @@ void grpc_end2end_tests(int argc, char **argv, filter_call_init_fails(config); filter_causes_close(config); filter_latency(config); + filter_status_code(config); graceful_server_shutdown(config); high_initial_seqno(config); hpack_size(config); @@ -356,6 +360,10 @@ void grpc_end2end_tests(int argc, char **argv, filter_latency(config); continue; } + if (0 == strcmp("filter_status_code", argv[i])) { + filter_status_code(config); + continue; + } if (0 == strcmp("graceful_server_shutdown", argv[i])) { graceful_server_shutdown(config); continue; diff --git a/test/core/end2end/end2end_tests.cc b/test/core/end2end/end2end_tests.cc index ca9443b642..9d8dfd6723 100644 --- a/test/core/end2end/end2end_tests.cc +++ b/test/core/end2end/end2end_tests.cc @@ -70,6 +70,8 @@ extern void filter_causes_close(grpc_end2end_test_config config); extern void filter_causes_close_pre_init(void); extern void filter_latency(grpc_end2end_test_config config); extern void filter_latency_pre_init(void); +extern void filter_status_code(grpc_end2end_test_config config); +extern void filter_status_code_pre_init(void); extern void graceful_server_shutdown(grpc_end2end_test_config config); extern void graceful_server_shutdown_pre_init(void); extern void high_initial_seqno(grpc_end2end_test_config config); @@ -173,6 +175,7 @@ void grpc_end2end_tests_pre_init(void) { filter_call_init_fails_pre_init(); filter_causes_close_pre_init(); filter_latency_pre_init(); + filter_status_code_pre_init(); graceful_server_shutdown_pre_init(); high_initial_seqno_pre_init(); hpack_size_pre_init(); @@ -241,6 +244,7 @@ void grpc_end2end_tests(int argc, char **argv, filter_call_init_fails(config); filter_causes_close(config); filter_latency(config); + filter_status_code(config); graceful_server_shutdown(config); high_initial_seqno(config); hpack_size(config); @@ -364,6 +368,10 @@ void grpc_end2end_tests(int argc, char **argv, filter_latency(config); continue; } + if (0 == strcmp("filter_status_code", argv[i])) { + filter_status_code(config); + continue; + } if (0 == strcmp("graceful_server_shutdown", argv[i])) { graceful_server_shutdown(config); continue; diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index 7c8e7f420a..e7cf97b2d0 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -101,6 +101,7 @@ END2END_TESTS = { 'filter_causes_close': default_test_options._replace(cpu_cost=LOWCPU), 'filter_call_init_fails': default_test_options, 'filter_latency': default_test_options._replace(cpu_cost=LOWCPU), + 'filter_status_code': default_test_options._replace(cpu_cost=LOWCPU), 'graceful_server_shutdown': default_test_options._replace(cpu_cost=LOWCPU,exclude_inproc=True), 'hpack_size': default_test_options._replace(proxyable=False, traceable=False, diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index b9a42bdb88..1d759e1ecb 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -146,6 +146,7 @@ END2END_TESTS = { 'trailing_metadata': test_options(), 'authority_not_supported': test_options(), 'filter_latency': test_options(), + 'filter_status_code': test_options(), 'workaround_cronet_compression': test_options(), 'write_buffering': test_options(needs_write_buffering=True), 'write_buffering_at_end': test_options(needs_write_buffering=True), diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc new file mode 100644 index 0000000000..261ddd93ec --- /dev/null +++ b/test/core/end2end/tests/filter_status_code.cc @@ -0,0 +1,353 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "test/core/end2end/end2end_tests.h" + +#include <limits.h> +#include <stdbool.h> +#include <stdio.h> +#include <string.h> + +#include <grpc/byte_buffer.h> +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/time.h> +#include <grpc/support/useful.h> + +#include "src/core/lib/channel/channel_stack_builder.h" +#include "src/core/lib/surface/channel_init.h" +#include "test/core/end2end/cq_verifier.h" + +static bool g_enable_filter = false; +static gpr_mu g_mu; +static bool g_client_code_recv; +static bool g_server_code_recv; +static gpr_cv g_client_code_cv; +static gpr_cv g_server_code_cv; +static grpc_status_code g_client_status_code; +static grpc_status_code g_server_status_code; + +static void* tag(intptr_t t) { return (void*)t; } + +static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, + const char* test_name, + grpc_channel_args* client_args, + grpc_channel_args* server_args) { + grpc_end2end_test_fixture f; + gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name); + f = config.create_fixture(client_args, server_args); + config.init_server(&f, server_args); + config.init_client(&f, client_args); + return f; +} + +static gpr_timespec n_seconds_from_now(int n) { + return grpc_timeout_seconds_to_deadline(n); +} + +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} + +static void drain_cq(grpc_completion_queue* cq) { + grpc_event ev; + do { + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), nullptr); + } while (ev.type != GRPC_QUEUE_SHUTDOWN); +} + +static void shutdown_server(grpc_end2end_test_fixture* f) { + if (!f->server) return; + grpc_server_shutdown_and_notify(f->server, f->shutdown_cq, tag(1000)); + GPR_ASSERT(grpc_completion_queue_pluck(f->shutdown_cq, tag(1000), + grpc_timeout_seconds_to_deadline(5), + nullptr) + .type == GRPC_OP_COMPLETE); + grpc_server_destroy(f->server); + f->server = nullptr; +} + +static void shutdown_client(grpc_end2end_test_fixture* f) { + if (!f->client) return; + grpc_channel_destroy(f->client); + f->client = nullptr; +} + +static void end_test(grpc_end2end_test_fixture* f) { + shutdown_server(f); + shutdown_client(f); + + grpc_completion_queue_shutdown(f->cq); + drain_cq(f->cq); + grpc_completion_queue_destroy(f->cq); + grpc_completion_queue_destroy(f->shutdown_cq); +} + +// Simple request via a server filter that saves the reported status code. +static void test_request(grpc_end2end_test_config config) { + grpc_call* c; + grpc_call* s; + grpc_end2end_test_fixture f = + begin_test(config, "filter_status_code", nullptr, nullptr); + cq_verifier* cqv = cq_verifier_create(f.cq); + grpc_op ops[6]; + grpc_op* op; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + grpc_slice details; + int was_cancelled = 2; + + gpr_mu_lock(&g_mu); + g_client_status_code = GRPC_STATUS_OK; + g_server_status_code = GRPC_STATUS_OK; + gpr_mu_unlock(&g_mu); + + gpr_timespec deadline = five_seconds_from_now(); + c = grpc_channel_create_call( + f.client, nullptr, GRPC_PROPAGATE_DEFAULTS, f.cq, + grpc_slice_from_static_string("/foo"), + get_host_override_slice("foo.test.google.fr", config), deadline, nullptr); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->data.send_initial_metadata.metadata = nullptr; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->flags = 0; + op->reserved = nullptr; + op++; + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), nullptr); + GPR_ASSERT(GRPC_CALL_OK == error); + + error = + grpc_server_request_call(f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(101), 1); + cq_verify(cqv); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; + op->data.send_status_from_server.trailing_metadata_count = 0; + op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED; + grpc_slice status_string = grpc_slice_from_static_string("xyz"); + op->data.send_status_from_server.status_details = &status_string; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; + op->data.recv_close_on_server.cancelled = &was_cancelled; + op->flags = 0; + op->reserved = nullptr; + op++; + error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), nullptr); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(102), 1); + CQ_EXPECT_COMPLETION(cqv, tag(1), 1); + cq_verify(cqv); + + GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); + GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); + + grpc_slice_unref(details); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + + grpc_call_unref(s); + grpc_call_unref(c); + + cq_verifier_destroy(cqv); + + end_test(&f); + config.tear_down_data(&f); + + // Perform checks after test tear-down + // Guards against the case that there's outstanding channel-related work on a + // call prior to verification + // TODO(https://github.com/grpc/grpc/issues/13915) enable this for windows +#ifndef GPR_WINDOWS + gpr_mu_lock(&g_mu); + if (!g_client_code_recv) { + GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu, + grpc_timeout_seconds_to_deadline(3))); + } + if (!g_server_code_recv) { + GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu, + grpc_timeout_seconds_to_deadline(3))); + } + GPR_ASSERT(g_client_status_code == GRPC_STATUS_UNIMPLEMENTED); + GPR_ASSERT(g_server_status_code == GRPC_STATUS_UNIMPLEMENTED); + gpr_mu_unlock(&g_mu); +#endif // GPR_WINDOWS +} + +/******************************************************************************* + * Test status_code filter + */ + +static grpc_error* init_call_elem(grpc_call_element* elem, + const grpc_call_element_args* args) { + return GRPC_ERROR_NONE; +} + +static void client_destroy_call_elem(grpc_call_element* elem, + const grpc_call_final_info* final_info, + grpc_closure* ignored) { + gpr_mu_lock(&g_mu); + g_client_status_code = final_info->final_status; + g_client_code_recv = true; + gpr_cv_signal(&g_client_code_cv); + gpr_mu_unlock(&g_mu); +} + +static void server_destroy_call_elem(grpc_call_element* elem, + const grpc_call_final_info* final_info, + grpc_closure* ignored) { + gpr_mu_lock(&g_mu); + g_server_status_code = final_info->final_status; + g_server_code_recv = true; + gpr_cv_signal(&g_server_code_cv); + gpr_mu_unlock(&g_mu); +} + +static grpc_error* init_channel_elem(grpc_channel_element* elem, + grpc_channel_element_args* args) { + return GRPC_ERROR_NONE; +} + +static void destroy_channel_elem(grpc_channel_element* elem) {} + +static const grpc_channel_filter test_client_filter = { + grpc_call_next_op, + grpc_channel_next_op, + 0, + init_call_elem, + grpc_call_stack_ignore_set_pollset_or_pollset_set, + client_destroy_call_elem, + 0, + init_channel_elem, + destroy_channel_elem, + grpc_channel_next_get_info, + "client_filter_status_code"}; + +static const grpc_channel_filter test_server_filter = { + grpc_call_next_op, + grpc_channel_next_op, + 0, + init_call_elem, + grpc_call_stack_ignore_set_pollset_or_pollset_set, + server_destroy_call_elem, + 0, + init_channel_elem, + destroy_channel_elem, + grpc_channel_next_get_info, + "server_filter_status_code"}; + +/******************************************************************************* + * Registration + */ + +static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { + grpc_channel_filter* filter = (grpc_channel_filter*)arg; + if (g_enable_filter) { + // Want to add the filter as close to the end as possible, to make + // sure that all of the filters work well together. However, we + // can't add it at the very end, because the + // connected_channel/client_channel filter must be the last one. + // So we add it right before the last one. + grpc_channel_stack_builder_iterator* it = + grpc_channel_stack_builder_create_iterator_at_last(builder); + GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); + const bool retval = grpc_channel_stack_builder_add_filter_before( + it, filter, nullptr, nullptr); + grpc_channel_stack_builder_iterator_destroy(it); + return retval; + } else { + return true; + } +} + +static void init_plugin(void) { + gpr_mu_init(&g_mu); + gpr_cv_init(&g_client_code_cv); + gpr_cv_init(&g_server_code_cv); + g_client_code_recv = false; + g_server_code_recv = false; + + grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, + maybe_add_filter, + (void*)&test_client_filter); + grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, + maybe_add_filter, + (void*)&test_client_filter); + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, + maybe_add_filter, + (void*)&test_server_filter); +} + +static void destroy_plugin(void) { + gpr_cv_destroy(&g_client_code_cv); + gpr_cv_destroy(&g_server_code_cv); + gpr_mu_destroy(&g_mu); +} + +void filter_status_code(grpc_end2end_test_config config) { + g_enable_filter = true; + test_request(config); + g_enable_filter = false; +} + +void filter_status_code_pre_init(void) { + grpc_register_plugin(init_plugin, destroy_plugin); +} diff --git a/test/core/transport/chttp2/settings_timeout_test.cc b/test/core/transport/chttp2/settings_timeout_test.cc index 08473c72b6..d7d6ee7508 100644 --- a/test/core/transport/chttp2/settings_timeout_test.cc +++ b/test/core/transport/chttp2/settings_timeout_test.cc @@ -21,6 +21,7 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> +#include <functional> #include <memory> #include <thread> diff --git a/third_party/rake-compiler-dock/Dockerfile b/third_party/rake-compiler-dock/Dockerfile index b4a5158535..06c721c39b 100644 --- a/third_party/rake-compiler-dock/Dockerfile +++ b/third_party/rake-compiler-dock/Dockerfile @@ -1,182 +1,4 @@ -FROM ubuntu:17.04 - -RUN apt-get -y update && \ - apt-get install -y curl git-core xz-utils build-essential wget unzip sudo gpg dirmngr - -# Add "rvm" as system group, to avoid conflicts with host GIDs typically starting with 1000 -RUN groupadd -r rvm && useradd -r -g rvm -G sudo -p "" --create-home rvm && \ - echo "source /etc/profile.d/rvm.sh" >> /etc/rubybashrc - -USER root -RUN apt-get -y update && \ - apt-get install -y gcc-mingw-w64-x86-64 gcc-mingw-w64-i686 g++-mingw-w64-x86-64 g++-mingw-w64-i686 \ - gcc-multilib moreutils -USER rvm - -# install rvm, RVM 1.26.0+ has signed releases, source rvm for usage outside of package scripts -RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 && \ - (curl -L http://get.rvm.io | sudo bash -s stable) && \ - bash -c " \ - source /etc/rubybashrc && \ - rvmsudo rvm cleanup all " - -# Regenerate the following using build.sh if the build folder changes. -RUN echo \\ -H4sIANslcFgAA+08a3fbxo75uvwVqOxTSbFIidTDVhKnTW0n8d28ju2ku6fO5R2RQ4kVRXJnSMve\ -OPe3LzAk9fIrvk3UuylxUnk0DwADYDADzKiD1A/c5oNvCi2E7W7rgdlpdbe3Tfxumj2zpepzeGC2\ -W+1Oy7TaZvcBNpvYHbrflq0MUpkwAfCAxVHgXIQ39uPhcB3srBsGSv8ydSMu5DeiQQrudW7Wv9Xq\ -rejf6nZ7D6D1jfhZgr+4/ve5x9IgkZADD8/sMecxbO1CZZQksR2L6PwCqCjzsjerPdrbt1++PT6x\ -j97/8t/2u1fPTp6/PXq9Uv3h4Oj48O0bVXv4+tmLA1DVe3tFS0X7s6Xw14Vs/Ys0ZN9q9d/t/zsd\ -c3X9W1a3XP/rgI0fmgM/bA6YHGnaUERpzFwX9CFUNl8c7lfoz9Hb9+8q2mTs+gL0GJrJJG6OognX\ -UsnFcm89xdJ7VXoB4mzSoI2FBlWwZjAfCvqEOh4fHFU07eXb1we7mzViAZ48eQIV7owi+GfWXNeC\ -EHQJzVSKZhA5LGgKNua6E01iP+AC0dD4StNYqta0jHJakAFdlzzRFfFTjTzdL8+OX9oHbz7sNnni\ -4AoYXBAHwsmbdR2H/vzdu6Z8//eH3tRwvhGNO/Z/s2etnv+sTrtdrv91QPOhBg/hZORLwC19KNgE\ -Rix0Ay7h+PDF4ZsTwG/gRWLKhCvBTyCJsCpKRrj2cITDpTQIxWECiMMPEx663KVeAw64rwCT8O5w\ -H0zqRR33I2eMY0nsiSxQILmpn4yg4matOLCyNBLehvDKD9PzBiBpcLNjC6DhhizIWRbEZzYEsD4S\ -iJWFF3mnjM0T5JtjNw57iQj0PWBjVsyU+FfDXIgRWU6DhjU1bcMPnSB1OTzJ0Bmjp4t1F7KZXMRc\ -Xq2eMj+hWg1lA7Hvwi60HmvaWeS7Wsa3jRipsUYfhD2d1LVP6IV8r4YD6sofjf0goG+NnNv6Y+1z\ -hnPC/FANZWLoNMAZoT0/xPLZbx/rhEUmInUUYuYkfhRCyKd2VnxMRIhowpJUIme6iawBNB/CMU8g\ -jZWwMwQpygy1KmPu+N6FakBEkCEySEawgNmQzC60sgtL8ySiWOKTOLlAnwy1H5eHTZgc1x9fweYF\ -bChz4cHCbGqZPBqwgKYBNZLvw3qrrnpnYke1j2sKcy7YT0qypKDaj5kIVCuA4DjdEH49+K/Dk+OT\ -Zyfvj2sL7Z95IHk2diY4fs6ds7imxG5+RGbyUo4Q7UlEolahbpVlIhkKhRc1uu71n/n/mCXOiMtv\ -lAf48vjfsnpdk/x/r9Uu4/91wLL+6RSkm8aOsa3H7e3OV7KH++d/OttWp9T/OuB2/YdRbKi2P0Tj\ -rvyP2VqN/3rdtlWe/9YBru95GOsM8WDHmk4Uev6QNvrBvIxnDJefQ7drdreZZxi9XrvFBwxMWtTd\ -7z0++t5hZf0vxtB6y+gb3a+wB9zf//eoqvT/a4C79U9fWaJTeKYPUnWiN8hrfDmN2/2/2eq2V/L/\ -ltUt83/rgecimkDH9HbanU6n7zmdttUeWF6/3+U722aXse5Ox3L7nttvtzvwGuOtYx6DuQ2t1iP1\ -DyxUoUZoHsErJiT8JwsxkH4SYPnnoeD+/+r4EUo3Ep7h8qfaPkv4IzhmSQPaLfgbCxGD2YPWziP6\ -tw1baBEt7Tgd/M6d5BH89u7Zyd7Lj7CHMeSQq7DTQyuFCcdAFylN8lRDZqb+IOBZJqEwVU3DkB9i\ -wc/8CCO1lbFTDAkx3B9cwCiKxn44hChEfEhFqwj+P6mPe2FVDLLNsFoBhwWBoal0SZYxQXoTJOsg\ -Cw5LJS/oqkyICLH7BeSIJBSIGtgrATfiMqwmGvGA7AuMB7GzHwJTXLp0F+OLKJzwMDG0w4S4CrHJ\ -DyTNOYiYCxhF45cRo+CfAwtkBDETCUReHrlj4M2EC4E/EExcNLAw5trvEvVIeR3hRk4+HeRP3QMh\ -fV/KFLmlrtm9z6NmEw8Io3RgoJSXncSKy6D0TTMb39zJRL8icSQVRlPKUqA6VZ6IJD8TO8zFPhmL\ -QS7yhjYd+c6IBicXse8osS5OlJ8nJFrwkIuGmpqv8lFTSvfMjMaFOGCJF4mJdsYCmqOSmpOkCmHI\ -uctdkgfKc8ouchwkaBzKPFRood6mxOA9boCMMukThVzGyFcyUpg1ym4wB1XrkmXlE3zBJ8SlEaCo\ -GkVliDZ0xrMMV47I5xLtROWqtJnVL5gEsUbZjjOONv4u4AyNL4wS3phzpJw7uChNJ4mQMxQWgyGO\ -CMlw1HxlsXwQgeBuA097gPziTLF+qHJALi5YNyNPjEsDjiM0XkRVEbj6JjHE4yHowqsYmq7rGvGv\ -zKKJWuGh9FG3TI4NMYBLaHdhawn060ADU9EqzKQBZgctU3JB2R1Z26o3wDLB5QHPKvS6pi0dZW/m\ -YXBzW37UbbVbzOv3DMPrm52+2aejbq/TocndhlnD2dyK/eefQTd7/cYObGV/sOJv7IzBrCfIUZSi\ -ygbKY3kcfQLaY3E3Cxtq6dCVy6LNS0f4cTLv5UxcePIE2EDaeRdD8EAZmE22aXvorGvUjLpTNXUN\ -NH1OpHBTJK2pHwQ5O1IZnQdTrpaNIyIpIVv5aN5zBNglMbC/WsjST9Bi8oToT7XKxqeC6OfmAp1K\ -XXk3bWvOBRnc/TjYujcHOY2COswhipVhGXEqR7WHipKdcWvnTTXlS+r1+SgeuihI0nK71220+7Cl\ -/rbuqecNyjz7Ic9nxlSWk2xIqvx2NuvCkf1QjMrG2LkntlX/Gvawi54NoOjePuNiQd0btML8sBB2\ -Y0n35EYn44lHZfQGr7GPWpRDHnKh2CpEvoLmS0bmbKvqJZ0UnSuw+xR+u1Zdjbmx3Q43mltDu3vw\ -VQz5lCoflwQYxVk2XE2F1pb6ptSkk8iBri3VnPWb5rzImpp2UWFT34/kbS+TywVpFzSLfrfSJXXM\ -x5IOcOfkM2WRUZHvxOWCy0uNRCMhM5XFqKscF6q4hduZDJ/TdhfFPKwlRsgmuENVp9W66uZdLqrS\ -M6YCV+vCGSAnUz0NT8PKfIHPu1IHWwxutvSrQxQ7gjO3tuokPF/IZHVBLxTnK1QpgMwhE/x1yi6o\ -XhFdYUVKdPRlWWwzeTjxFS+mGMQNXolxyeMWM4YnUDGNfgV+/DEvPNmFxWc4i/K+RjFiK7lWNchO\ -RHd7Cd2jKOnNtXG7freuQ3GnDq6OMmQ6+KHW/HttZhz1wu5q1fpmE6menppkD6enVnUJw73kszjn\ -K7Sz2gFyPob66cNafkcoZ/SRdjaFav1uTPkN0B9FpM6Tp0Y9Fhh01KDI3CkzPK1J4dAe45+f1mfI\ -4zSRmZSWVEyKneIZaLU2Wzc54dXGRKQhmjyvYZEHwRK3fBHXwhdVnLvQe7mkG53ozCWtLqR7+5+7\ -nMo1fqgQztLUF4vqXNDpd+n019npNcz7HgtIUBWHhXjGz2I/mWJEpMSA51JUW9tq4hEpCs4qFPaE\ -efSSrxYMARnFL1G4hPCYZ1Hjo+sCveR36YxSDOhWYjx+pg+i8yLM687xzSNmxUd1LqirsfTcm6JJ\ -BwyPNG9VxL8oXMEn0RmdaUKZwKOjwZ4aumhgKz0WXyHe0e2a5X5Nr/2D472jw3cn9H4RvUh2yHJ/\ -qq024kLSQbOMrtEyaAVFU2mY2v+D5Pjd+T/KpkRpoqOVynsm/nK46/5n27JW8n9m22qV+b91wFLQ\ -rIIG9R5wfqBR7+owrL2xLQ+ae8xxdkwMmnuMWayzEjTfPDoLmm9uV0FzW7lN+rNNXrPw/fSyz6bn\ -f5+byo5lc+PT68M3L35Vr48/47eVl8bXRBeEI3MCFV1XWHbzYb+8P3y1/7kIFaq6zkM2CLguR5Qp\ -qc7rXV+qBh/9BgsCSoJV8xAFW4v1k4yvq3SC3EvmlbTMdquE+iM9XEEX/TwSDodfEXnkjC21NR7R\ -hmgaOw386BfvlFQmx0/u6XOuvf/tG+2v+RTkX7j/t6xeef+zDrhR/1/n6l/Bv3D/j19K/78OKO//\ -/9pwy/q3JQYu6qriD/qBu95/W2ZnZf1vd8v33+uBL17/O/2d7XbLMQzH4z2r2y7Wf3a+m/fOznPz\ -7+r81un16ei2VRToDIfnGObYXho6dOfoTSJXvXGOh+pBbkd90rtderULMmbT8AzkhaS7P7DtouSM\ -RIQB8ZBr8B9ZFCd5InEY5SLoFyuSc/XLFc8ZEQknkiOQfjiChOFHEA0tEFEauqeLCFJEgH+G+Mdl\ -GBViZM4DDE7tgIkhtz2XbvbwP448pWFWOF1NKU/GiT/hQB/DCQwnVLAFBrt4jrOHPFGt+d/Ic9kF\ -oAkGENPn6WqGO6ZkGRQrEuQoTTDIDNU7aDz7JcwZgxvYfkL5dW7HI1csRtILKL5kJOVGHtF8mLRR\ -SWdM7P5jE8vqlzkVrHHOlObszVyFFbgE6pAIW47+oSnNhjb2G3GcrFJyZfPV4ZuDN2/pN0WzUZXN\ -GYXv/qc2/5Zwjf+3jLbR+rPP/2Z5/l8L3Kj/9e3/rU6vfXX/L/M/a4Ev3v/5gLfdHdz/WbvdG2zf\ -b//f6Xfaav9XhR7t/7Md5tadojDBytKuM6vV1FU/lwlUzjev6wC7cH7B5ePs7ZQi6LAEnj4lBl3u\ -SYN+c2o/2zt4+1zTN/KL95fPPhzYxwdv9p8fvjoAU4O8A2jg+d/XNnXt+u/8Wf7fsjC4zPI/5fvf\ -tcCN+v9T/L9lYQfl/ztl/mctUPr/0v+7TZkIP7angsUxF1+dxh3+3zTN7ZXzX3sb3UC5/tcAGz+o\ -/7MG3bxRMoM2AE3bgGMyiOyeLQrVm6YAvChw1cN2mVAiQT0m/uALejw9iM4huxbLeyGKJFKv64Gp\ -9AouQF/CIB3OXxpMp1PjbDbeiMSwmfjOmCfNHdwRNG32ZiDhk5hYqc6r6Gua+IGsapoyXlzolY1P\ -6qXHgOHyZxNe22zVPxs4s4qmqYfL2OfZ0YsPhuD00ODTpXcJ3u4/m3/Xm/DZmLBYvQhhYkhP1+hZ\ -8y6c5JQN+qV7VRGqZo2GE0SSa9njkvfEieHE9Av8hmqlFynY+BuWG1T7UaNHIFr+dLRgJEsqLTMC\ -W/kra2To02XSiC7pFRiig8+alobYIikNhjKpgWKoAQ8LtPT2hp+jKzetbUXQiVyOxDZ/Mqg6+6V5\ -LgyDM2eUPZUhHiPhq3nPpyMmqnJ1isXssiHZtBRNovV9+cYSSiihhBJKKKGEEkoooYQSSiihhBJK\ -KKGEEkoooYQSSiihhBJKKKGEEkoooYQSSiihhH9H+D9i0BbqAHgAAA==\ -| base64 -d | tar xzC /tmp - -# Import patch files for ruby and gems -RUN cp -r /tmp/build/patches /home/rvm/patches/ -ENV BASH_ENV /etc/rubybashrc - -# install rubies and fix permissions on -RUN bash -c " \ - export CFLAGS='-s -O3 -fno-fast-math -fPIC' && \ - echo 'about to install patches for ruby 2.4.0 from:' && \ - ls -r ~/patches && \ - for v in 2.4.0 ; do \ - rvm install \$v --patch \$(echo ~/patches/ruby-\$v/* | tr ' ' ','); \ - done && \ - rvm cleanup all && \ - find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw " - -# Install rake-compiler and typical gems in all Rubies -# do not generate documentation for gems -RUN echo "gem: --no-ri --no-rdoc" >> ~/.gemrc && \ - bash -c " \ - rvm all do gem install bundler rake-compiler hoe mini_portile rubygems-tasks && \ - rvm 2.4.0 do gem install mini_portile2 && \ - find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw " - -RUN bash -c "gem env" -RUN bash -c "gem list rake-compiler" - -# Install rake-compiler's cross rubies in global dir instead of /root -RUN sudo mkdir -p /usr/local/rake-compiler && \ - sudo chown rvm.rvm /usr/local/rake-compiler && \ - ln -s /usr/local/rake-compiler ~/.rake-compiler - -# Patch rake-compiler to avoid build of ruby extensions -RUN cd /usr/local/rvm/gems/ruby-2.4.0/gems/rake-compiler-0.9.5 && git apply /home/rvm/patches/rake-compiler-0.9.5/*.diff ; \ - true - -RUN bash -c "rvm use 2.4.0 --default && \ - export MAKE=\"make -j`nproc`\" CFLAGS='-s -O1 -fno-omit-frame-pointer -fno-fast-math' && \ - rake-compiler cross-ruby VERSION=2.4.0 HOST=i686-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.4.0 HOST=x86_64-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.4.0 HOST=x86_64-linux-gnu && \ - rake-compiler cross-ruby VERSION=2.3.0 HOST=i686-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.3.0 HOST=x86_64-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.3.0 HOST=x86_64-linux-gnu && \ - rake-compiler cross-ruby VERSION=2.2.2 HOST=i686-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.2.2 HOST=x86_64-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.2.2 HOST=x86_64-linux-gnu && \ - rake-compiler cross-ruby VERSION=2.1.5 HOST=i686-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.1.5 HOST=x86_64-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.1.5 HOST=x86_64-linux-gnu && \ - rake-compiler cross-ruby VERSION=2.0.0-p645 HOST=i686-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.0.0-p645 HOST=x86_64-w64-mingw32 && \ - rake-compiler cross-ruby VERSION=2.0.0-p645 HOST=x86_64-linux-gnu && \ - rm -rf ~/.rake-compiler/tmp/builds ~/.rake-compiler/sources && \ - find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw " - -RUN bash -c "rvm use 2.4.0 --default && \ - export MAKE=\"make -j`nproc`\" CFLAGS='-m32 -s -O1 -fno-omit-frame-pointer -fno-fast-math' LDFLAGS='-m32' && \ - rake-compiler cross-ruby VERSION=2.4.0 HOST=i686-linux-gnu && \ - rake-compiler cross-ruby VERSION=2.3.0 HOST=i686-linux-gnu && \ - rake-compiler cross-ruby VERSION=2.2.2 HOST=i686-linux-gnu && \ - rake-compiler cross-ruby VERSION=2.1.5 HOST=i686-linux-gnu && \ - rake-compiler cross-ruby VERSION=2.0.0-p645 HOST=i686-linux-gnu && \ - rm -rf ~/.rake-compiler/tmp/builds ~/.rake-compiler/sources && \ - find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw " - -RUN bash -c " \ - rvm alias create 2.4 2.4.0 " - -USER root - -# Fix paths in rake-compiler/config.yml and add rvm and mingw-tools to the global bashrc -RUN sed -i -- "s:/root/.rake-compiler:/usr/local/rake-compiler:g" /usr/local/rake-compiler/config.yml && \ - echo "source /etc/profile.d/rvm.sh" >> /etc/bash.bashrc && \ - echo "export PATH=\$PATH:/opt/mingw/mingw32/bin" >> /etc/bash.bashrc && \ - echo "export PATH=\$PATH:/opt/mingw/mingw64/bin" >> /etc/bash.bashrc - -# Install wrappers for strip commands as a workaround for "Protocol error" in boot2docker. -RUN cp /tmp/build/strip_wrapper /root/ -RUN sudo chmod +rx /root/strip_wrapper -RUN mv /usr/bin/i686-w64-mingw32-strip /usr/bin/i686-w64-mingw32-strip.bin && \ - mv /usr/bin/x86_64-w64-mingw32-strip /usr/bin/x86_64-w64-mingw32-strip.bin && \ - ln /root/strip_wrapper /usr/bin/i686-w64-mingw32-strip && \ - ln /root/strip_wrapper /usr/bin/x86_64-w64-mingw32-strip +FROM larskanis/rake-compiler-dock:0.6.2 RUN find / -name rbconfig.rb | while read f ; do sed -i 's/0x0501/0x0600/' $f ; done RUN find / -name win32.h | while read f ; do sed -i 's/gettimeofday/rb_gettimeofday/' $f ; done @@ -184,26 +6,6 @@ RUN sed -i 's/defined.__MINGW64__.$/1/' /usr/local/rake-compiler/ruby/i686-w64-m RUN find / -name libwinpthread.dll.a | xargs rm RUN find / -name libwinpthread-1.dll | xargs rm RUN find / -name *msvcrt-ruby*.dll.a | while read f ; do n=`echo $f | sed s/.dll//` ; mv $f $n ; done -RUN find /usr/local/rake-compiler/ruby -name libruby.so | xargs rm -RUN find /usr/local/rake-compiler/ruby -name libruby-static.a | while read f ; do ar t $f | xargs ar d $f ; done -RUN find /usr/local/rake-compiler/ruby -name libruby-static.a | while read f ; do mv $f `echo $f | sed s/-static//` ; done - -# Install SIGINT forwarder -RUN cp /tmp/build/sigfw.c /root/ -RUN gcc $HOME/sigfw.c -o /usr/local/bin/sigfw - -# Install user mapper -RUN cp /tmp/build/runas /usr/local/bin/ - -# Install sudoers configuration -RUN cp /tmp/build/sudoers /etc/sudoers.d/rake-compiler-dock - -# Fixup Ruby 2.4 'static' compilation issue. -RUN echo '!<arch>' > /usr/local/rake-compiler/ruby/x86_64-linux-gnu/ruby-2.4.0/lib/libruby.a -RUN echo '!<arch>' > /usr/local/rake-compiler/ruby/i686-linux-gnu/ruby-2.4.0/lib/libruby.a - -ENV RUBY_CC_VERSION 2.4.0:2.3.0:2.2.2:2.1.5:2.0.0 - RUN apt-get install -y g++-multilib CMD bash diff --git a/third_party/rake-compiler-dock/build.sh b/third_party/rake-compiler-dock/build.sh deleted file mode 100755 index ca01fa6d4d..0000000000 --- a/third_party/rake-compiler-dock/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -# Run this to produce the snipplet of data to insert in the Dockerfile. - -echo 'RUN echo \\' -tar cz build | base64 | sed 's/$/\\/' -echo '| base64 -d | tar xzC /tmp' diff --git a/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/compat-with-bundler.diff b/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/compat-with-bundler.diff deleted file mode 100644 index ea22bd928e..0000000000 --- a/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/compat-with-bundler.diff +++ /dev/null @@ -1,105 +0,0 @@ -From 41f834449fc4323b2f995e8715aa5842d9fd9334 Mon Sep 17 00:00:00 2001 -From: Lars Kanis <lars@greiz-reinsdorf.de> -Date: Sat, 30 Jan 2016 08:08:07 +0100 -Subject: [PATCH] Change the fake mechanism to be compatible with bundler. - -The previous fake mechanism worked by hooking onto the -"require 'rbconfig'" call. -This is problematic because bundler internally requires rbconfig, but doesn't -work corretly in a faked environment. -It then fails to load gems that are also part of the standard library, like -json and rdoc. -This results in issues like https://github.com/rake-compiler/rake-compiler-dock/issues/8 - -The fake mechanism is now changed to hook onto the "require 'mkrb'" call, -which is typically part of the extconf file, and it is where the faked platform -values are actually needed. -That way it is loaded after bundler/setup, so that the library paths are -set according to the Gemfile.lock, to the native Linux libraries, before -the fake environment is active. - -Please note, that the build directory of a given gem needs to be cleared, -in order to get updated fake files. So do a "rm tmp pkg -rf". ---- - lib/rake/extensiontask.rb | 35 ++++++++++++++--------------------- - 1 file changed, 14 insertions(+), 21 deletions(-) - -diff --git a/lib/rake/extensiontask.rb b/lib/rake/extensiontask.rb -index 030af96..f914919 100644 ---- a/lib/rake/extensiontask.rb -+++ b/lib/rake/extensiontask.rb -@@ -169,8 +169,8 @@ Java extension should be preferred. - # now add the extconf script - cmd << abs_extconf.relative_path_from(abs_tmp_path) - -- # rbconfig.rb will be present if we are cross compiling -- if t.prerequisites.include?("#{tmp_path}/rbconfig.rb") then -+ # fake.rb will be present if we are cross compiling -+ if t.prerequisites.include?("#{tmp_path}/fake.rb") then - options.push(*cross_config_options(platf)) - end - -@@ -365,39 +365,30 @@ Java extension should be preferred. - # define compilation tasks for cross platform! - define_compile_tasks(for_platform, ruby_ver) - -- # chain fake.rb, rbconfig.rb and mkmf.rb to Makefile generation -+ # chain fake.rb and mkmf.rb to Makefile generation - file "#{tmp_path}/Makefile" => ["#{tmp_path}/fake.rb", -- "#{tmp_path}/rbconfig.rb", - "#{tmp_path}/mkmf.rb"] - -- # copy the file from the cross-ruby location -- file "#{tmp_path}/rbconfig.rb" => [rbconfig_file] do |t| -+ # copy the rbconfig from the cross-ruby location and -+ # genearte fake.rb for different ruby versions -+ file "#{tmp_path}/fake.rb" => [rbconfig_file] do |t| - File.open(t.name, 'w') do |f| -- f.write "require 'fake.rb'\n\n" -+ f.write fake_rb(for_platform, ruby_ver) - f.write File.read(t.prerequisites.first) - end - end - - # copy mkmf from cross-ruby location - file "#{tmp_path}/mkmf.rb" => [mkmf_file] do |t| -- cp t.prerequisites.first, t.name -- if ruby_ver < "1.9" && "1.9" <= RUBY_VERSION -- File.open(t.name, 'r+t') do |f| -- content = f.read -+ File.open(t.name, 'w') do |f| -+ content = File.read(t.prerequisites.first) -+ content.sub!(/^(require ')rbconfig(')$/, '\\1fake\\2') -+ if ruby_ver < "1.9" && "1.9" <= RUBY_VERSION - content.sub!(/^( break )\*(defaults)$/, '\\1\\2.first') - content.sub!(/^( return )\*(defaults)$/, '\\1\\2.first') - content.sub!(/^( mfile\.)print( configuration\(srcprefix\))$/, '\\1puts\\2') -- f.rewind -- f.write content -- f.truncate(f.tell) - end -- end -- end -- -- # genearte fake.rb for different ruby versions -- file "#{tmp_path}/fake.rb" do |t| -- File.open(t.name, 'w') do |f| -- f.write fake_rb(for_platform, ruby_ver) -+ f.write content - end - end - -@@ -495,8 +486,10 @@ Java extension should be preferred. - # "cannot load such file -- win32/resolv" when it is required later on. - # See also: https://github.com/tjschuck/rake-compiler-dev-box/issues/5 - require 'resolv' -+ require 'rbconfig' - - class Object -+ remove_const :RbConfig - remove_const :RUBY_PLATFORM - remove_const :RUBY_VERSION - remove_const :RUBY_DESCRIPTION if defined?(RUBY_DESCRIPTION) --- -2.5.0.windows.1 - diff --git a/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/without-exts.diff b/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/without-exts.diff deleted file mode 100644 index 07739d33ec..0000000000 --- a/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/without-exts.diff +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake -index 6acc816..6aa2a49 100644 ---- a/tasks/bin/cross-ruby.rake -+++ b/tasks/bin/cross-ruby.rake -@@ -135,8 +135,7 @@ file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile" => ["#{USER - "--build=#{RUBY_BUILD}", - '--enable-shared', - '--disable-install-doc', -- '--without-tk', -- '--without-tcl' -+ '--with-ext=' - ] - - # Force Winsock2 for Ruby 1.8, 1.9 defaults to it diff --git a/third_party/rake-compiler-dock/build/patches/ruby-1.8.7-p374/nop.patch b/third_party/rake-compiler-dock/build/patches/ruby-1.8.7-p374/nop.patch deleted file mode 100644 index fac8525da6..0000000000 --- a/third_party/rake-compiler-dock/build/patches/ruby-1.8.7-p374/nop.patch +++ /dev/null @@ -1,2 +0,0 @@ -diff --git a/configure b/configure -index 55157af..6630eba 100755 diff --git a/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/no_sendfile.patch b/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/no_sendfile.patch deleted file mode 100644 index d8f339e814..0000000000 --- a/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/no_sendfile.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/configure b/configure -index 898730c..cfe6253 100755 ---- a/configure -+++ b/configure -@@ -14695,7 +14695,7 @@ for ac_func in fmod killpg wait4 waitpid fork spawnv syscall __syscall chroot ge - setsid telldir seekdir fchmod cosh sinh tanh log2 round\ - setuid setgid daemon select_large_fdset setenv unsetenv\ - mktime timegm gmtime_r clock_gettime gettimeofday poll ppoll\ -- pread sendfile shutdown sigaltstack dl_iterate_phdr -+ pread shutdown sigaltstack dl_iterate_phdr - do : - as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` - ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" diff --git a/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/nop.patch b/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/nop.patch deleted file mode 100644 index fac8525da6..0000000000 --- a/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/nop.patch +++ /dev/null @@ -1,2 +0,0 @@ -diff --git a/configure b/configure -index 55157af..6630eba 100755 diff --git a/third_party/rake-compiler-dock/build/patches/ruby-2.3.0/no_sendfile.patch b/third_party/rake-compiler-dock/build/patches/ruby-2.3.0/no_sendfile.patch deleted file mode 100644 index 915fc7b790..0000000000 --- a/third_party/rake-compiler-dock/build/patches/ruby-2.3.0/no_sendfile.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/configure b/configure -index ebe3d8c..a336b73 100755 ---- a/configure -+++ b/configure -@@ -18943,7 +18943,6 @@ do : - ac_fn_c_check_func "$LINENO" "sendfile" "ac_cv_func_sendfile" - if test "x$ac_cv_func_sendfile" = xyes; then : - cat >>confdefs.h <<_ACEOF --#define HAVE_SENDFILE 1 - _ACEOF - - fi diff --git a/third_party/rake-compiler-dock/build/patches/ruby-2.4.0/no_sendfile.patch b/third_party/rake-compiler-dock/build/patches/ruby-2.4.0/no_sendfile.patch deleted file mode 100644 index 915fc7b790..0000000000 --- a/third_party/rake-compiler-dock/build/patches/ruby-2.4.0/no_sendfile.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/configure b/configure -index ebe3d8c..a336b73 100755 ---- a/configure -+++ b/configure -@@ -18943,7 +18943,6 @@ do : - ac_fn_c_check_func "$LINENO" "sendfile" "ac_cv_func_sendfile" - if test "x$ac_cv_func_sendfile" = xyes; then : - cat >>confdefs.h <<_ACEOF --#define HAVE_SENDFILE 1 - _ACEOF - - fi diff --git a/third_party/rake-compiler-dock/build/runas b/third_party/rake-compiler-dock/build/runas deleted file mode 100755 index b29ce31fcc..0000000000 --- a/third_party/rake-compiler-dock/build/runas +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -groupadd -g "$GID" "$GROUP" -mkdir -p /tmp/home -useradd -g "$GID" -u "$UID" -G rvm,sudo -p "" -b /tmp/home -m "$USER" - -HOME=$(bash <<< "echo ~$USER") -ln -s /usr/local/rake-compiler "$HOME"/.rake-compiler - -sudo -u "$USER" --set-home \ - BASH_ENV=/etc/rubybashrc \ - -- "$@" diff --git a/third_party/rake-compiler-dock/build/sigfw.c b/third_party/rake-compiler-dock/build/sigfw.c deleted file mode 100644 index 291d76cec8..0000000000 --- a/third_party/rake-compiler-dock/build/sigfw.c +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This program handles SIGINT and forwards it to another process. - * It is intended to be run as PID 1. - * - * Docker starts processes with "docker run" as PID 1. - * On Linux, the default signal handler for PID 1 ignores any signals. - * Therefore Ctrl-C aka SIGINT is ignored per default. - */ - -#include <signal.h> -#include <sys/types.h> -#include <sys/wait.h> - -int pid = 0; - -void -handle_sigint (int signum) -{ - if(pid) - kill(pid, SIGINT); -} - -int main(int argc, char *argv[]){ - struct sigaction new_action; - int status = -1; - - /* Set up the structure to specify the new action. */ - new_action.sa_handler = handle_sigint; - sigemptyset (&new_action.sa_mask); - new_action.sa_flags = 0; - - sigaction (SIGINT, &new_action, (void*)0); - - pid = fork(); - if(pid){ - wait(&status); - return WEXITSTATUS(status); - }else{ - status = execvp(argv[1], &argv[1]); - perror("exec"); - return status; - } -} diff --git a/third_party/rake-compiler-dock/build/strip_wrapper b/third_party/rake-compiler-dock/build/strip_wrapper deleted file mode 100755 index 7f8a1346a1..0000000000 --- a/third_party/rake-compiler-dock/build/strip_wrapper +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env ruby - -# Strip file on local folder instead of a Virtualbox shared folder -# to work around this bug: https://www.virtualbox.org/ticket/8463 - -require 'tempfile' -require 'fileutils' - -strip = "#{File.basename($0)}.bin" - -files = ARGV.reject{|f| f=~/^-/ }.map do |arg| - tmp = Tempfile.new 'strip' - tmp.close - FileUtils.cp arg, tmp.path - [tmp, arg] -end - -options = ARGV.select{|f| f=~/^-/ } + files.map{|t,o| t.path } - -unless system( strip, *options ) - exit 127 -end -code = $?.exitstatus - -files.each do |tmp, orig| - FileUtils.rm orig - FileUtils.cp tmp.path, orig -end - -exit code diff --git a/third_party/rake-compiler-dock/build/sudoers b/third_party/rake-compiler-dock/build/sudoers deleted file mode 100644 index f9f9b97c95..0000000000 --- a/third_party/rake-compiler-dock/build/sudoers +++ /dev/null @@ -1 +0,0 @@ -Defaults env_keep += "http_proxy https_proxy ftp_proxy RCD_HOST_RUBY_PLATFORM RCD_HOST_RUBY_VERSION RCD_IMAGE RUBY_CC_VERSION" diff --git a/tools/bazel.rc b/tools/bazel.rc index c554f03971..8af2fc981d 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -1,4 +1,5 @@ build --client_env=CC=clang +build --copt -DGRPC_BAZEL_BUILD build:asan --strip=never build:asan --copt -fsanitize-coverage=edge diff --git a/tools/buildgen/bunch.py b/tools/buildgen/bunch.py index 0ce9a6b233..f3bfc81102 100755 --- a/tools/buildgen/bunch.py +++ b/tools/buildgen/bunch.py @@ -48,5 +48,6 @@ def merge_json(dst, add): elif isinstance(dst, list) and isinstance(add, list): dst.extend(add) else: - raise Exception('Tried to merge incompatible objects %s %s\n\n%r\n\n%r' - % (type(dst).__name__, type(add).__name__, dst, add)) + raise Exception( + 'Tried to merge incompatible objects %s %s\n\n%r\n\n%r' % + (type(dst).__name__, type(add).__name__, dst, add)) diff --git a/tools/buildgen/mako_renderer.py b/tools/buildgen/mako_renderer.py index 0569fa14d2..acd72bd2e3 100755 --- a/tools/buildgen/mako_renderer.py +++ b/tools/buildgen/mako_renderer.py @@ -99,10 +99,10 @@ def main(argv): elif opt == '-P': assert not got_preprocessed_input assert json_dict == {} - sys.path.insert( - 0, - os.path.abspath( - os.path.join(os.path.dirname(sys.argv[0]), 'plugins'))) + sys.path.insert(0, + os.path.abspath( + os.path.join( + os.path.dirname(sys.argv[0]), 'plugins'))) with open(arg, 'r') as dict_file: dictionary = pickle.load(dict_file) got_preprocessed_input = True diff --git a/tools/buildgen/plugins/expand_filegroups.py b/tools/buildgen/plugins/expand_filegroups.py index 886a59cdbb..99d9463b3f 100755 --- a/tools/buildgen/plugins/expand_filegroups.py +++ b/tools/buildgen/plugins/expand_filegroups.py @@ -104,8 +104,7 @@ def mako_plugin(dictionary): # build reverse dependency map things = {} - for thing in dictionary['libs'] + dictionary['targets'] + dictionary[ - 'filegroups']: + for thing in dictionary['libs'] + dictionary['targets'] + dictionary['filegroups']: things[thing['name']] = thing thing['used_by'] = [] thing_deps = lambda t: t.get('uses', []) + t.get('filegroups', []) + t.get('deps', []) @@ -148,7 +147,7 @@ def mako_plugin(dictionary): lib[lst] = vals lib['plugins'] = plugins if lib.get('generate_plugin_registry', False): - lib['src'].append('src/core/plugin_registry/%s_plugin_registry.cc' % - lib['name']) + lib['src'].append( + 'src/core/plugin_registry/%s_plugin_registry.cc' % lib['name']) for lst in FILEGROUP_LISTS: lib[lst] = uniquify(lib.get(lst, [])) diff --git a/tools/buildgen/plugins/generate_vsprojects.py b/tools/buildgen/plugins/generate_vsprojects.py index f6251d4677..f7ef49288f 100755 --- a/tools/buildgen/plugins/generate_vsprojects.py +++ b/tools/buildgen/plugins/generate_vsprojects.py @@ -56,11 +56,12 @@ def mako_plugin(dictionary): target['vs_props'] = [] target['vs_proj_dir'] = target.get('vs_proj_dir', default_test_dir) if target.get('vs_project_guid', - None) is None and 'windows' in target.get('platforms', - ['windows']): + None) is None and 'windows' in target.get( + 'platforms', ['windows']): name = target['name'] guid = re.sub('(........)(....)(....)(....)(.*)', - r'{\1-\2-\3-\4-\5}', hashlib.md5(name).hexdigest()) + r'{\1-\2-\3-\4-\5}', + hashlib.md5(name).hexdigest()) target['vs_project_guid'] = guid.upper() # Exclude projects without a visual project guid, such as the tests. projects = [ @@ -69,9 +70,9 @@ def mako_plugin(dictionary): projects = [ project for project in projects - if project['language'] != 'c++' or project['build'] == 'all' or project[ - 'build'] == 'protoc' or (project['language'] == 'c++' and (project[ - 'build'] == 'test' or project['build'] == 'private')) + if project['language'] != 'c++' or project['build'] == 'all' or + project['build'] == 'protoc' or (project['language'] == 'c++' and ( + project['build'] == 'test' or project['build'] == 'private')) ] project_dict = dict([(p['name'], p) for p in projects]) diff --git a/tools/buildgen/plugins/transitive_dependencies.py b/tools/buildgen/plugins/transitive_dependencies.py index 5373bca2d1..258e10bff5 100644 --- a/tools/buildgen/plugins/transitive_dependencies.py +++ b/tools/buildgen/plugins/transitive_dependencies.py @@ -54,5 +54,5 @@ def mako_plugin(dictionary): target['transitive_deps'] = transitive_deps(target, libs) python_dependencies = dictionary.get('python_dependencies') - python_dependencies['transitive_deps'] = ( - transitive_deps(python_dependencies, libs)) + python_dependencies['transitive_deps'] = (transitive_deps( + python_dependencies, libs)) diff --git a/tools/codegen/core/gen_settings_ids.py b/tools/codegen/core/gen_settings_ids.py index bc4380605a..5c3f066b15 100755 --- a/tools/codegen/core/gen_settings_ids.py +++ b/tools/codegen/core/gen_settings_ids.py @@ -174,10 +174,13 @@ for decorated_setting in sorted(decorated_settings): print >> C, "{NULL, 0, 0, 0, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_HTTP2_PROTOCOL_ERROR}," i += 1 print >> C, "{\"%s\", %du, %du, %du, GRPC_CHTTP2_%s, GRPC_HTTP2_%s}," % ( - decorated_setting.name, decorated_setting.setting.default, - decorated_setting.setting.min, decorated_setting.setting.max, + decorated_setting.name, + decorated_setting.setting.default, + decorated_setting.setting.min, + decorated_setting.setting.max, decorated_setting.setting.on_error.behavior, - decorated_setting.setting.on_error.code,) + decorated_setting.setting.on_error.code, + ) i += 1 print >> C, "};" diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index 9d99bbf096..3a171bbfc2 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -387,8 +387,8 @@ for i, elem in enumerate(all_strs): print >> H, '#define %s (grpc_static_slice_table[%d])' % ( mangle(elem).upper(), i) print >> H -print >> C, 'static uint8_t g_bytes[] = {%s};' % ( - ','.join('%d' % ord(c) for c in ''.join(all_strs))) +print >> C, 'static uint8_t g_bytes[] = {%s};' % (','.join( + '%d' % ord(c) for c in ''.join(all_strs))) print >> C print >> C, 'static void static_ref(void *unused) {}' print >> C, 'static void static_unref(void *unused) {}' @@ -444,8 +444,8 @@ for i, elem in enumerate(all_elems): print >> H print >> C, ('uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] ' '= {') -print >> C, ' %s' % ','.join('%d' % static_userdata.get(elem, 0) - for elem in all_elems) +print >> C, ' %s' % ','.join( + '%d' % static_userdata.get(elem, 0) for elem in all_elems) print >> C, '};' print >> C @@ -520,8 +520,8 @@ for i, k in enumerate(elem_keys): idxs[h] = i print >> C, 'static const uint16_t elem_keys[] = {%s};' % ','.join( '%d' % k for k in keys) -print >> C, 'static const uint8_t elem_idxs[] = {%s};' % ','.join('%d' % i - for i in idxs) +print >> C, 'static const uint8_t elem_idxs[] = {%s};' % ','.join( + '%d' % i for i in idxs) print >> C print >> H, 'grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b);' @@ -579,8 +579,8 @@ print >> H, 'extern const uint8_t grpc_static_accept_stream_encoding_metadata[%d 1 << len(STREAM_COMPRESSION_ALGORITHMS)) print >> C, 'const uint8_t grpc_static_accept_stream_encoding_metadata[%d] = {' % ( 1 << len(STREAM_COMPRESSION_ALGORITHMS)) -print >> C, '0,%s' % ','.join('%d' % md_idx(elem) - for elem in stream_compression_elems) +print >> C, '0,%s' % ','.join( + '%d' % md_idx(elem) for elem in stream_compression_elems) print >> C, '};' print >> H, '#define GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS(algs) (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[grpc_static_accept_stream_encoding_metadata[(algs)]], GRPC_MDELEM_STORAGE_STATIC))' diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 93f2035c45..5c9d9e5ea5 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -28,8 +28,8 @@ REQUIRED_FIELDS = ['name', 'doc'] def make_type(name, fields): - return (collections.namedtuple( - name, ' '.join(list(set(REQUIRED_FIELDS + fields)))), []) + return (collections.namedtuple(name, ' '.join( + list(set(REQUIRED_FIELDS + fields)))), []) def c_str(s, encoding='ascii'): @@ -44,7 +44,10 @@ def c_str(s, encoding='ascii'): return '"' + result + '"' -types = (make_type('Counter', []), make_type('Histogram', ['max', 'buckets']),) +types = ( + make_type('Counter', []), + make_type('Histogram', ['max', 'buckets']), +) inst_map = dict((t[0].__name__, t[1]) for t in types) @@ -349,8 +352,8 @@ with open('src/core/lib/debug/stats_data.cc', 'w') as C: print >> C, "const int grpc_stats_histo_start[%d] = {%s};" % ( len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start)) print >> C, "const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % ( - len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x - for x in histo_bucket_boundaries)) + len(inst_map['Histogram']), ','.join( + 'grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >> C, "void (*const grpc_stats_inc_histogram[%d])(int x) = {%s};" % ( len(inst_map['Histogram']), ','.join( 'grpc_stats_inc_%s' % histogram.name.lower() diff --git a/tools/debug/core/error_ref_leak.py b/tools/debug/core/error_ref_leak.py index 7806338683..7e206c26b2 100644 --- a/tools/debug/core/error_ref_leak.py +++ b/tools/debug/core/error_ref_leak.py @@ -39,7 +39,7 @@ for line in data: elif line[0] == "realloc": errs.remove(line[1]) errs.append(line[3]) - # explicitly look for the last dereference + # explicitly look for the last dereference elif line[1] == "1" and line[3] == "0": assert (err in errs) errs.remove(err) diff --git a/tools/distrib/build_ruby_environment_macos.sh b/tools/distrib/build_ruby_environment_macos.sh index fe0c5a4d70..af36740255 100644 --- a/tools/distrib/build_ruby_environment_macos.sh +++ b/tools/distrib/build_ruby_environment_macos.sh @@ -47,7 +47,7 @@ EOF MAKE="make -j8" -for v in 2.4.0 2.3.0 2.2.2 2.1.5 2.0.0-p645 ; do +for v in 2.5.0 2.4.0 2.3.0 2.2.2 2.1.6 2.0.0-p645 ; do ccache -c rake -f $CROSS_RUBY cross-ruby VERSION=$v HOST=x86_64-darwin11 done diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py index f6e73624a7..0eb2cbe1a2 100755 --- a/tools/distrib/check_copyright.py +++ b/tools/distrib/check_copyright.py @@ -48,6 +48,7 @@ LICENSE_PREFIX = { '.cc': r'\s*(?://|\*)\s*', '.h': r'\s*(?://|\*)\s*', '.m': r'\s*\*\s*', + '.mm': r'\s*\*\s*', '.php': r'\s*\*\s*', '.js': r'\s*\*\s*', '.py': r'#\s*', @@ -84,13 +85,15 @@ _EXEMPT = frozenset(( # census.proto copied from github 'tools/grpcz/census.proto', # status.proto copied from googleapis - 'src/proto/grpc/status/status.proto',)) + 'src/proto/grpc/status/status.proto', +)) RE_YEAR = r'Copyright (?P<first_year>[0-9]+\-)?(?P<last_year>[0-9]+) gRPC authors.' -RE_LICENSE = dict((k, r'\n'.join( - LICENSE_PREFIX[k] + (RE_YEAR - if re.search(RE_YEAR, line) else re.escape(line)) - for line in LICENSE_NOTICE)) for k, v in LICENSE_PREFIX.iteritems()) +RE_LICENSE = dict( + (k, r'\n'.join(LICENSE_PREFIX[k] + + (RE_YEAR if re.search(RE_YEAR, line) else re.escape(line)) + for line in LICENSE_NOTICE)) + for k, v in LICENSE_PREFIX.iteritems()) if args.precommit: FILE_LIST_COMMAND = 'git status -z | grep -Poz \'(?<=^[MARC][MARCD ] )[^\s]+\'' diff --git a/tools/distrib/check_include_guards.py b/tools/distrib/check_include_guards.py index 24e076afda..6fc606f2ef 100755 --- a/tools/distrib/check_include_guards.py +++ b/tools/distrib/check_include_guards.py @@ -95,14 +95,14 @@ class GuardValidator(object): # Does the guard end with a '_H'? running_guard = match.group(1) if not running_guard.endswith('_H'): - fcontents = self.fail(fpath, match.re, match.string, - match.group(1), valid_guard, fix) + fcontents = self.fail(fpath, match.re, match.string, match.group(1), + valid_guard, fix) if fix: save(fpath, fcontents) # Is it the expected one based on the file path? if running_guard != valid_guard: - fcontents = self.fail(fpath, match.re, match.string, - match.group(1), valid_guard, fix) + fcontents = self.fail(fpath, match.re, match.string, match.group(1), + valid_guard, fix) if fix: save(fpath, fcontents) # Is there a #define? Is it the same as the #ifndef one? @@ -114,8 +114,8 @@ class GuardValidator(object): # Is the #define guard the same as the #ifndef guard? if match.group(1) != running_guard: - fcontents = self.fail(fpath, match.re, match.string, - match.group(1), valid_guard, fix) + fcontents = self.fail(fpath, match.re, match.string, match.group(1), + valid_guard, fix) if fix: save(fpath, fcontents) # Is there a properly commented #endif? @@ -138,8 +138,8 @@ class GuardValidator(object): self.fail(fpath, endif_re, flines[-1], '', '', False) elif match.group(1) != running_guard: # Is the #endif guard the same as the #ifndef and #define guards? - fcontents = self.fail(fpath, endif_re, fcontents, - match.group(1), valid_guard, fix) + fcontents = self.fail(fpath, endif_re, fcontents, match.group(1), + valid_guard, fix) if fix: save(fpath, fcontents) return not self.failed # Did the check succeed? (ie, not failed) diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/command.py b/tools/distrib/python/grpcio_tools/grpc_tools/command.py index c0f9d93b0a..7ede05f140 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/command.py +++ b/tools/distrib/python/grpcio_tools/grpc_tools/command.py @@ -30,8 +30,8 @@ def build_package_protos(package_root): proto_files.append( os.path.abspath(os.path.join(root, filename))) - well_known_protos_include = pkg_resources.resource_filename('grpc_tools', - '_proto') + well_known_protos_include = pkg_resources.resource_filename( + 'grpc_tools', '_proto') for proto_file in proto_files: command = [ diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 342a220d5e..c6bcee497f 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -196,7 +196,8 @@ def extension_modules(): language='c++', define_macros=list(DEFINE_MACROS), extra_compile_args=list(EXTRA_COMPILE_ARGS), - extra_link_args=list(EXTRA_LINK_ARGS),) + extra_link_args=list(EXTRA_LINK_ARGS), + ) extensions = [plugin_ext] if BUILD_WITH_CYTHON: from Cython import Build @@ -220,4 +221,5 @@ setuptools.setup( 'protobuf>=3.5.0.post1', 'grpcio>={version}'.format(version=grpc_version.VERSION), ], - package_data=package_data(),) + package_data=package_data(), +) diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index 216492a2f3..4847233217 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -107,12 +107,14 @@ def get_deps(): `out_file`.""" cc_files_output = bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY) cc_files = [ - name[len(PROTOBUF_CC_PREFIX):] for name in cc_files_output + name[len(PROTOBUF_CC_PREFIX):] + for name in cc_files_output if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX) ] proto_files_output = bazel_query(BAZEL_DEPS_COMMON_PROTOS_QUERY) proto_files = [ - name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output + name[len(PROTOBUF_PROTO_PREFIX):] + for name in proto_files_output if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX) ] commit_hash = protobuf_submodule_commit_hash() diff --git a/tools/distrib/run_clang_tidy.py b/tools/distrib/run_clang_tidy.py index 3ac712ea9e..72d7956b68 100755 --- a/tools/distrib/run_clang_tidy.py +++ b/tools/distrib/run_clang_tidy.py @@ -54,7 +54,8 @@ argp.set_defaults(fix=False) args = argp.parse_args() cmdline = [ - clang_tidy, '--checks=-*,%s' % ','.join(GRPC_CHECKS), + clang_tidy, + '--checks=-*,%s' % ','.join(GRPC_CHECKS), '--warnings-as-errors=%s' % ','.join(GRPC_CHECKS) ] + ['--extra-arg-before=%s' % arg for arg in extra_args] @@ -65,6 +66,7 @@ jobs = [] for filename in args.files: jobs.append(jobset.JobSpec( cmdline + [filename], - shortname=filename,)) #verbose_success=True)) + shortname=filename, + )) #verbose_success=True)) jobset.run(jobs, maxjobs=args.jobs) diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh index fb14f36c66..d188a0242c 100755 --- a/tools/distrib/yapf_code.sh +++ b/tools/distrib/yapf_code.sh @@ -34,7 +34,7 @@ virtualenv $VIRTUALENV PYTHON=$(realpath "${VIRTUALENV}/bin/python") $PYTHON -m pip install --upgrade pip==9.0.1 $PYTHON -m pip install --upgrade futures -$PYTHON -m pip install yapf==0.16.0 +$PYTHON -m pip install yapf==0.20.0 yapf() { local exclusion exclusion_args=() diff --git a/tools/failures/detect_new_failures.py b/tools/failures/detect_new_failures.py index 87fd1d9f02..4063978fce 100644 --- a/tools/failures/detect_new_failures.py +++ b/tools/failures/detect_new_failures.py @@ -49,8 +49,8 @@ def gh(url, data=None): if 200 <= response.getcode() < 300: return json.loads(response.read()) else: - raise ValueError('Error ({}) accessing {}'.format(response.getcode(), - response.geturl())) + raise ValueError('Error ({}) accessing {}'.format( + response.getcode(), response.geturl())) def search_gh_issues(search_term, status='open'): @@ -92,11 +92,11 @@ def create_issues(new_flakes, always_create): else: preexisting_issues = search_gh_issues(test_name) if preexisting_issues['total_count'] > 0: - print('\nFound {} issues for "{}":'.format(preexisting_issues[ - 'total_count'], test_name)) + print('\nFound {} issues for "{}":'.format( + preexisting_issues['total_count'], test_name)) for issue in preexisting_issues['items']: - print('\t"{}" ; URL: {}'.format(issue['title'], issue[ - 'html_url'])) + print('\t"{}" ; URL: {}'.format(issue['title'], + issue['html_url'])) else: print( '\nNo preexisting issues found for "{}"'.format(test_name)) @@ -148,10 +148,11 @@ def get_new_failures(dates): rows = page.get('rows') if rows: return { - row['f'][0]['v']: Row(poll_strategy=row['f'][1]['v'], - job_name=row['f'][2]['v'], - build_id=row['f'][3]['v'], - timestamp=row['f'][4]['v']) + row['f'][0]['v']: Row( + poll_strategy=row['f'][1]['v'], + job_name=row['f'][2]['v'], + build_id=row['f'][3]['v'], + timestamp=row['f'][4]['v']) for row in rows } else: @@ -294,13 +295,14 @@ def main(): elif args.format == 'csv': if args.count_only: print('from_date,to_date,count') - print('{},{},{}'.format(dates['reporting']['begin'].isoformat( - ), dates['reporting']['end'].isoformat(), len(new_flakes))) + print('{},{},{}'.format(dates['reporting']['begin'].isoformat(), + dates['reporting']['end'].isoformat(), + len(new_flakes))) else: print_table(new_flakes, 'csv') else: - raise ValueError( - 'Invalid argument for --format: {}'.format(args.format)) + raise ValueError('Invalid argument for --format: {}'.format( + args.format)) if __name__ == '__main__': diff --git a/tools/github/pr_latency.py b/tools/github/pr_latency.py index 0131e60bbc..34870a5326 100644 --- a/tools/github/pr_latency.py +++ b/tools/github/pr_latency.py @@ -72,9 +72,13 @@ def output(pr, errors)) elif mode == 'csv': print(','.join([ - str(pr), str(base_time), str(test_time), str( - int((test_time - base_time).total_seconds())), str(successes), - str(failures), str(errors) + str(pr), + str(base_time), + str(test_time), + str(int((test_time - base_time).total_seconds())), + str(successes), + str(failures), + str(errors) ])) @@ -117,8 +121,7 @@ def get_status_data(statuses_url, system): if system == 'kokoro': string_in_target_url = 'kokoro' elif system == 'jenkins': string_in_target_url = 'grpc-testing' for status in statuses['statuses']: - if not status['target_url'] or string_in_target_url not in status[ - 'target_url']: + if not status['target_url'] or string_in_target_url not in status['target_url']: continue # Ignore jenkins if status['state'] == 'pending': return None elif status['state'] == 'success': successes += 1 @@ -130,8 +133,8 @@ def get_status_data(statuses_url, system): latest_datetime = max(latest_datetime, parse_timestamp(status['updated_at'])) # First status is the most recent one. - if any([successes, failures, errors]) and sum( - [successes, failures, errors]) > 15: + if any([successes, failures, errors + ]) and sum([successes, failures, errors]) > 15: return { 'latest_datetime': latest_datetime, 'successes': successes, diff --git a/tools/interop_matrix/create_matrix_images.py b/tools/interop_matrix/create_matrix_images.py index ad7bb59331..ef9f6a5990 100755 --- a/tools/interop_matrix/create_matrix_images.py +++ b/tools/interop_matrix/create_matrix_images.py @@ -173,8 +173,8 @@ def build_all_images_for_lang(lang): releases = client_matrix.get_release_tags(lang) else: # Build a particular release. - if args.release not in ['master'] + client_matrix.get_release_tags( - lang): + if args.release not in ['master' + ] + client_matrix.get_release_tags(lang): jobset.message( 'SKIPPED', '%s for %s is not defined' % (args.release, lang), diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py index ff42bd77b0..3391ef51cc 100755 --- a/tools/interop_matrix/run_interop_matrix_tests.py +++ b/tools/interop_matrix/run_interop_matrix_tests.py @@ -105,8 +105,8 @@ def find_all_images_for_lang(lang): releases = ['master'] + client_matrix.get_release_tags(lang) else: # Look for a particular release. - if args.release not in ['master'] + client_matrix.get_release_tags( - lang): + if args.release not in ['master' + ] + client_matrix.get_release_tags(lang): jobset.message( 'SKIPPED', '%s for %s is not defined' % (args.release, lang), @@ -136,7 +136,8 @@ def find_all_images_for_lang(lang): 'Skipped images (no-tag/unknown-tag): %d' % skipped, do_newline=True) # Filter tags based on the releases. - images[runtime] = [(tag, '%s:%s' % (image_path, tag)) for tag in tags + images[runtime] = [(tag, '%s:%s' % (image_path, tag)) + for tag in tags if tag in releases] return images diff --git a/tools/line_count/collect-history.py b/tools/line_count/collect-history.py index d2d5c95705..c8e33c9ed2 100755 --- a/tools/line_count/collect-history.py +++ b/tools/line_count/collect-history.py @@ -31,7 +31,8 @@ end_date = datetime.date(2017, 3, 29) for dt in daterange(start_date, end_date): dmy = dt.strftime('%Y-%m-%d') sha1 = subprocess.check_output( - ['git', 'rev-list', '-n', '1', '--before=%s' % dmy, 'master']).strip() + ['git', 'rev-list', '-n', '1', + '--before=%s' % dmy, 'master']).strip() subprocess.check_call(['git', 'checkout', sha1]) subprocess.check_call(['git', 'submodule', 'update']) subprocess.check_call(['git', 'clean', '-f', '-x', '-d']) diff --git a/tools/line_count/summarize-history.py b/tools/line_count/summarize-history.py index 80b0ed7a7e..4a085999a5 100755 --- a/tools/line_count/summarize-history.py +++ b/tools/line_count/summarize-history.py @@ -32,6 +32,7 @@ for dt in daterange(start_date, end_date): dmy = dt.strftime('%Y-%m-%d') print dmy subprocess.check_call([ - 'tools/line_count/yaml2csv.py', '-i', '../count/%s.yaml' % dmy, '-d', - dmy, '-o', '../count/%s.csv' % dmy + 'tools/line_count/yaml2csv.py', '-i', + '../count/%s.yaml' % dmy, '-d', dmy, '-o', + '../count/%s.csv' % dmy ]) diff --git a/tools/mkowners/mkowners.py b/tools/mkowners/mkowners.py index d8b3d3c332..2ea7265f88 100755 --- a/tools/mkowners/mkowners.py +++ b/tools/mkowners/mkowners.py @@ -47,7 +47,8 @@ args = argp.parse_args() # owners_files = [ - os.path.join(root, 'OWNERS') for root, dirs, files in os.walk(git_root) + os.path.join(root, 'OWNERS') + for root, dirs, files in os.walk(git_root) if 'OWNERS' in files ] diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 746b643b43..e084e28dcf 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -28,9 +28,9 @@ import subprocess columns = [] for row in json.loads( - subprocess.check_output([ - 'bq', '--format=json', 'show', 'microbenchmarks.microbenchmarks' - ]))['schema']['fields']: + subprocess.check_output( + ['bq', '--format=json', 'show', + 'microbenchmarks.microbenchmarks']))['schema']['fields']: columns.append((row['name'], row['type'].lower())) SANITIZE = { diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_build.py b/tools/profiling/microbenchmarks/bm_diff/bm_build.py index a4cd61707d..4197ba3632 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_build.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_build.py @@ -43,7 +43,8 @@ def _args(): '-n', '--name', type=str, - help='Unique name of this build. To be used as a handle to pass to the other bm* scripts' + help= + 'Unique name of this build. To be used as a handle to pass to the other bm* scripts' ) argp.add_argument('--counters', dest='counters', action='store_true') argp.add_argument('--no-counters', dest='counters', action='store_false') @@ -71,7 +72,8 @@ def build(name, benchmarks, jobs, counters): subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) os.rename( 'bins', - 'bm_diff_%s' % name,) + 'bm_diff_%s' % name, + ) if __name__ == '__main__': diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py index cff29dbe08..5719e42620 100644 --- a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py @@ -22,10 +22,12 @@ _AVAILABLE_BENCHMARK_TESTS = [ 'bm_metadata', 'bm_fullstack_trickle' ] -_INTERESTING = ( - 'cpu_time', 'real_time', 'call_initial_size-median', 'locks_per_iteration', - 'allocs_per_iteration', 'writes_per_iteration', 'atm_cas_per_iteration', - 'atm_add_per_iteration', 'nows_per_iteration', - 'cli_transport_stalls_per_iteration', 'cli_stream_stalls_per_iteration', - 'svr_transport_stalls_per_iteration', 'svr_stream_stalls_per_iteration', - 'http2_pings_sent_per_iteration') +_INTERESTING = ('cpu_time', 'real_time', 'call_initial_size-median', + 'locks_per_iteration', 'allocs_per_iteration', + 'writes_per_iteration', 'atm_cas_per_iteration', + 'atm_add_per_iteration', 'nows_per_iteration', + 'cli_transport_stalls_per_iteration', + 'cli_stream_stalls_per_iteration', + 'svr_transport_stalls_per_iteration', + 'svr_stream_stalls_per_iteration', + 'http2_pings_sent_per_iteration') diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py index b8a3b22861..f975a8b402 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py @@ -65,7 +65,8 @@ def _args(): '--loops', type=int, default=20, - help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py' + help= + 'Number of times to loops the benchmarks. Must match what was passed to bm_run.py' ) argp.add_argument( '-r', diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py index 137c22bf8e..96c63ba060 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_main.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py @@ -78,7 +78,8 @@ def _args(): '--loops', type=int, default=10, - help='Number of times to loops the benchmarks. More loops cuts down on noise' + help= + 'Number of times to loops the benchmarks. More loops cuts down on noise' ) argp.add_argument( '-j', diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_run.py b/tools/profiling/microbenchmarks/bm_diff/bm_run.py index 08894bbe4d..dfb9b178fa 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_run.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_run.py @@ -51,7 +51,8 @@ def _args(): '-n', '--name', type=str, - help='Unique name of the build to run. Needs to match the handle passed to bm_build.py' + help= + 'Unique name of the build to run. Needs to match the handle passed to bm_build.py' ) argp.add_argument( '-r', @@ -64,7 +65,8 @@ def _args(): '--loops', type=int, default=20, - help='Number of times to loops the benchmarks. More loops cuts down on noise' + help= + 'Number of times to loops the benchmarks. More loops cuts down on noise' ) argp.add_argument('--counters', dest='counters', action='store_true') argp.add_argument('--no-counters', dest='counters', action='store_false') @@ -82,13 +84,14 @@ def _collect_bm_data(bm, cfg, name, regex, idx, loops): 'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_list_tests', '--benchmark_filter=%s' % regex ]).splitlines(): - stripped_line = line.strip().replace("/", "_").replace( - "<", "_").replace(">", "_").replace(", ", "_") + stripped_line = line.strip().replace("/", + "_").replace("<", "_").replace( + ">", "_").replace(", ", "_") cmd = [ 'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_filter=^%s$' % line, - '--benchmark_out=%s.%s.%s.%s.%d.json' % - (bm, stripped_line, cfg, name, idx), + '--benchmark_out=%s.%s.%s.%s.%d.json' % (bm, stripped_line, cfg, + name, idx), '--benchmark_out_format=json', ] jobs_list.append( diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index 1dd9f65dbf..497d7ca813 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -183,7 +183,8 @@ def expand_json(js, js2=None): context = js['context'] if 'label' in bm: labels_list = [ - s.split(':') for s in bm['label'].strip().split(' ') + s.split(':') + for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#' ] for el in labels_list: diff --git a/tools/profiling/qps/qps_diff.py b/tools/profiling/qps/qps_diff.py index 55a81f034d..393f862b4d 100755 --- a/tools/profiling/qps/qps_diff.py +++ b/tools/profiling/qps/qps_diff.py @@ -48,7 +48,8 @@ def _args(): '--loops', type=int, default=4, - help='Number of loops for each benchmark. More loops cuts down on noise') + help='Number of loops for each benchmark. More loops cuts down on noise' + ) argp.add_argument( '-j', '--jobs', @@ -128,8 +129,8 @@ def diff(scenarios, loops, old, new): rows = [] for sn in scenarios: mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn])) - print('%s: %s=%r %s=%r mdn_diff=%r' % - (sn, new, new_data[sn], old, old_data[sn], mdn_diff)) + print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old, + old_data[sn], mdn_diff)) s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5) if abs(s) > 3 and mdn_diff > 0.5: rows.append([sn, '%+d%%' % s]) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index dc0803b3c7..efc4ca0d8a 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -271,8 +271,8 @@ class PHPArtifact: def build_jobspec(self): if self.platform == 'linux': return create_docker_jobspec( - self.name, - 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), + self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( + self.arch), 'tools/run_tests/artifacts/build_artifact_php.sh') else: return create_jobspec( @@ -337,36 +337,38 @@ def targets(): for Cls in (CSharpExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64') ] + [ - PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact( - 'linux', 'x86', 'cp27-cp27mu'), PythonArtifact( - 'linux', 'x86', 'cp34-cp34m'), PythonArtifact( - 'linux', 'x86', 'cp35-cp35m'), PythonArtifact( - 'linux', 'x86', 'cp36-cp36m'), PythonArtifact( - 'linux_extra', 'armv7', '2.7'), PythonArtifact( - 'linux_extra', 'armv7', '3.4'), PythonArtifact( - 'linux_extra', 'armv7', '3.5'), - PythonArtifact('linux_extra', 'armv7', '3.6'), PythonArtifact( - 'linux_extra', 'armv6', '2.7'), PythonArtifact( - 'linux_extra', 'armv6', '3.4'), PythonArtifact( - 'linux_extra', 'armv6', '3.5'), PythonArtifact( - 'linux_extra', 'armv6', '3.6'), PythonArtifact( - 'linux', 'x64', 'cp27-cp27m'), PythonArtifact( - 'linux', 'x64', 'cp27-cp27mu'), PythonArtifact( - 'linux', 'x64', 'cp34-cp34m'), - PythonArtifact('linux', 'x64', 'cp35-cp35m'), PythonArtifact( - 'linux', 'x64', 'cp36-cp36m'), PythonArtifact( - 'macos', 'x64', 'python2.7'), PythonArtifact( - 'macos', 'x64', 'python3.4'), PythonArtifact('macos', 'x64', - 'python3.5'), - PythonArtifact('macos', 'x64', 'python3.6'), PythonArtifact( - 'windows', 'x86', 'Python27_32bits'), PythonArtifact( - 'windows', 'x86', 'Python34_32bits'), PythonArtifact( - 'windows', 'x86', 'Python35_32bits'), PythonArtifact( - 'windows', 'x86', 'Python36_32bits'), PythonArtifact( - 'windows', 'x64', 'Python27'), - PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact( - 'windows', 'x64', 'Python35'), PythonArtifact( - 'windows', 'x64', 'Python36'), RubyArtifact( - 'linux', 'x64'), RubyArtifact('macos', 'x64'), PHPArtifact( - 'linux', 'x64'), PHPArtifact('macos', 'x64') + PythonArtifact('linux', 'x86', 'cp27-cp27m'), + PythonArtifact('linux', 'x86', 'cp27-cp27mu'), + PythonArtifact('linux', 'x86', 'cp34-cp34m'), + PythonArtifact('linux', 'x86', 'cp35-cp35m'), + PythonArtifact('linux', 'x86', 'cp36-cp36m'), + PythonArtifact('linux_extra', 'armv7', '2.7'), + PythonArtifact('linux_extra', 'armv7', '3.4'), + PythonArtifact('linux_extra', 'armv7', '3.5'), + PythonArtifact('linux_extra', 'armv7', '3.6'), + PythonArtifact('linux_extra', 'armv6', '2.7'), + PythonArtifact('linux_extra', 'armv6', '3.4'), + PythonArtifact('linux_extra', 'armv6', '3.5'), + PythonArtifact('linux_extra', 'armv6', '3.6'), + PythonArtifact('linux', 'x64', 'cp27-cp27m'), + PythonArtifact('linux', 'x64', 'cp27-cp27mu'), + PythonArtifact('linux', 'x64', 'cp34-cp34m'), + PythonArtifact('linux', 'x64', 'cp35-cp35m'), + PythonArtifact('linux', 'x64', 'cp36-cp36m'), + PythonArtifact('macos', 'x64', 'python2.7'), + PythonArtifact('macos', 'x64', 'python3.4'), + PythonArtifact('macos', 'x64', 'python3.5'), + PythonArtifact('macos', 'x64', 'python3.6'), + PythonArtifact('windows', 'x86', 'Python27_32bits'), + PythonArtifact('windows', 'x86', 'Python34_32bits'), + PythonArtifact('windows', 'x86', 'Python35_32bits'), + PythonArtifact('windows', 'x86', 'Python36_32bits'), + PythonArtifact('windows', 'x64', 'Python27'), + PythonArtifact('windows', 'x64', 'Python34'), + PythonArtifact('windows', 'x64', 'Python35'), + PythonArtifact('windows', 'x64', 'Python36'), + RubyArtifact('linux', 'x64'), + RubyArtifact('macos', 'x64'), + PHPArtifact('linux', 'x64'), + PHPArtifact('macos', 'x64') ]) diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 94a2d53770..b2cc16acd4 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -106,8 +106,8 @@ class CSharpDistribTest(object): if self.platform == 'linux': return create_docker_jobspec( self.name, - 'tools/dockerfile/distribtest/csharp_%s_%s' % ( - self.docker_suffix, self.arch), + 'tools/dockerfile/distribtest/csharp_%s_%s' % + (self.docker_suffix, self.arch), 'test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix, copy_rel_path='test/distrib') @@ -260,8 +260,8 @@ class CppDistribTest(object): def build_jobspec(self): if self.platform == 'linux': return create_docker_jobspec( - self.name, 'tools/dockerfile/distribtest/cpp_%s_%s' % ( - self.docker_suffix, self.arch), + self.name, 'tools/dockerfile/distribtest/cpp_%s_%s' % + (self.docker_suffix, self.arch), 'test/distrib/cpp/run_distrib_test_%s.sh' % self.testcase) elif self.platform == 'windows': return create_jobspec( diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index 52908454f2..abf1b5ebed 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -152,6 +152,9 @@ class PHPPackage: def targets(): """Gets list of supported targets""" return [ - CSharpPackage(), CSharpPackage(linux=True), RubyPackage(), - PythonPackage(), PHPPackage() + CSharpPackage(), + CSharpPackage(linux=True), + RubyPackage(), + PythonPackage(), + PHPPackage() ] diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 067bf43834..4e2a1b2863 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7633,6 +7633,7 @@ "test/core/end2end/tests/filter_call_init_fails.cc", "test/core/end2end/tests/filter_causes_close.cc", "test/core/end2end/tests/filter_latency.cc", + "test/core/end2end/tests/filter_status_code.cc", "test/core/end2end/tests/graceful_server_shutdown.cc", "test/core/end2end/tests/high_initial_seqno.cc", "test/core/end2end/tests/hpack_size.cc", @@ -7714,6 +7715,7 @@ "test/core/end2end/tests/filter_call_init_fails.cc", "test/core/end2end/tests/filter_causes_close.cc", "test/core/end2end/tests/filter_latency.cc", + "test/core/end2end/tests/filter_status_code.cc", "test/core/end2end/tests/graceful_server_shutdown.cc", "test/core/end2end/tests/high_initial_seqno.cc", "test/core/end2end/tests/hpack_size.cc", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 82157d095b..cbfce5b11e 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -6823,6 +6823,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_census_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -8161,6 +8184,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -9456,6 +9502,28 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_fakesec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -10662,6 +10730,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_fd_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -11929,6 +12020,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -13185,6 +13299,25 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_test", + "platforms": [ + "linux" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -14369,6 +14502,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -15661,6 +15817,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+workarounds_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -17017,6 +17196,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_http_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -18415,6 +18618,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_load_reporting_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -19771,6 +19997,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_oauth2_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -21091,6 +21341,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -22195,6 +22469,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -23419,6 +23717,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -24603,6 +24925,32 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "msan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_1byte_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -25949,6 +26297,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -27233,6 +27604,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_ssl_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -28390,6 +28785,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_uds_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -29586,6 +30004,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "inproc_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "high_initial_seqno" ], "ci_platforms": [ @@ -30623,6 +31064,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_census_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -31938,6 +32402,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -33136,6 +33623,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_fd_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -34380,6 +34890,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -35617,6 +36150,25 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_nosec_test", + "platforms": [ + "linux" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -36778,6 +37330,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -38047,6 +38622,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+workarounds_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -39379,6 +39977,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_http_proxy_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -40754,6 +41376,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_load_reporting_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -42014,6 +42659,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_proxy_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -43094,6 +43763,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -44294,6 +44987,30 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -45452,6 +46169,32 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "msan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_1byte_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -46750,6 +47493,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_uds_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "graceful_server_shutdown" ], "ci_platforms": [ @@ -47923,6 +48689,29 @@ }, { "args": [ + "filter_status_code" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "inproc_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "high_initial_seqno" ], "ci_platforms": [ diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 37f6e7aae0..790202c43e 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -18,8 +18,7 @@ import massage_qps_stats_helpers def massage_qps_stats(scenario_result): - for stats in scenario_result["serverStats"] + scenario_result[ - "clientStats"]: + for stats in scenario_result["serverStats"] + scenario_result["clientStats"]: if "coreStats" not in stats: return core_stats = stats["coreStats"] del stats["coreStats"] @@ -294,8 +293,8 @@ def massage_qps_stats(scenario_result): core_stats, "cq_ev_queue_transient_pop_failures") h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size") stats["core_call_initial_size"] = ",".join("%f" % x for x in h.buckets) - stats["core_call_initial_size_bkts"] = ",".join("%f" % x - for x in h.boundaries) + stats["core_call_initial_size_bkts"] = ",".join( + "%f" % x for x in h.boundaries) stats[ "core_call_initial_size_50p"] = massage_qps_stats_helpers.percentile( h.buckets, 50, h.boundaries) @@ -307,8 +306,8 @@ def massage_qps_stats(scenario_result): h.buckets, 99, h.boundaries) h = massage_qps_stats_helpers.histogram(core_stats, "poll_events_returned") - stats["core_poll_events_returned"] = ",".join("%f" % x - for x in h.buckets) + stats["core_poll_events_returned"] = ",".join( + "%f" % x for x in h.buckets) stats["core_poll_events_returned_bkts"] = ",".join( "%f" % x for x in h.boundaries) stats[ @@ -322,8 +321,8 @@ def massage_qps_stats(scenario_result): h.buckets, 99, h.boundaries) h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size") stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets) - stats["core_tcp_write_size_bkts"] = ",".join("%f" % x - for x in h.boundaries) + stats["core_tcp_write_size_bkts"] = ",".join( + "%f" % x for x in h.boundaries) stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile( h.buckets, 50, h.boundaries) stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile( @@ -333,8 +332,8 @@ def massage_qps_stats(scenario_result): h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size") stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets) - stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x - for x in h.boundaries) + stats["core_tcp_write_iov_size_bkts"] = ",".join( + "%f" % x for x in h.boundaries) stats[ "core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile( h.buckets, 50, h.boundaries) @@ -346,8 +345,8 @@ def massage_qps_stats(scenario_result): h.buckets, 99, h.boundaries) h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size") stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets) - stats["core_tcp_read_size_bkts"] = ",".join("%f" % x - for x in h.boundaries) + stats["core_tcp_read_size_bkts"] = ",".join( + "%f" % x for x in h.boundaries) stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile( h.buckets, 50, h.boundaries) stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile( @@ -356,8 +355,8 @@ def massage_qps_stats(scenario_result): h.buckets, 99, h.boundaries) h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer") stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets) - stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x - for x in h.boundaries) + stats["core_tcp_read_offer_bkts"] = ",".join( + "%f" % x for x in h.boundaries) stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile( h.buckets, 50, h.boundaries) stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile( @@ -366,8 +365,8 @@ def massage_qps_stats(scenario_result): h.buckets, 99, h.boundaries) h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size") - stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x - for x in h.buckets) + stats["core_tcp_read_offer_iov_size"] = ",".join( + "%f" % x for x in h.buckets) stats["core_tcp_read_offer_iov_size_bkts"] = ",".join( "%f" % x for x in h.boundaries) stats[ @@ -381,8 +380,8 @@ def massage_qps_stats(scenario_result): h.buckets, 99, h.boundaries) h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size") - stats["core_http2_send_message_size"] = ",".join("%f" % x - for x in h.buckets) + stats["core_http2_send_message_size"] = ",".join( + "%f" % x for x in h.buckets) stats["core_http2_send_message_size_bkts"] = ",".join( "%f" % x for x in h.boundaries) stats[ @@ -457,8 +456,8 @@ def massage_qps_stats(scenario_result): h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked") stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets) - stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x - for x in h.boundaries) + stats["core_server_cqs_checked_bkts"] = ",".join( + "%f" % x for x in h.boundaries) stats[ "core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile( h.buckets, 50, h.boundaries) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 7af33f92f0..f05753154e 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -178,8 +178,8 @@ def _ping_pong_scenario(name, # clamp buffer usage to something reasonable (16 gig for now) MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024 if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE: - outstanding_calls = max(1, MAX_MEMORY_USE / max(req_size, - resp_size)) + outstanding_calls = max(1, + MAX_MEMORY_USE / max(req_size, resp_size)) wide = channels if channels is not None else WIDE deep = int(math.ceil(1.0 * outstanding_calls / wide)) @@ -503,8 +503,8 @@ class CXXLanguage: ]: for synchronicity in ['sync', 'async']: yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_ping_pong_%s' % - (synchronicity, rpc_type, secstr), + 'cpp_protobuf_%s_%s_ping_pong_%s' % (synchronicity, + rpc_type, secstr), rpc_type=rpc_type.upper(), client_type='%s_CLIENT' % synchronicity.upper(), server_type='%s_SERVER' % synchronicity.upper(), @@ -580,10 +580,10 @@ class CXXLanguage: minimal_stack=not secure, categories=[SWEEP]) - for channels in geometric_progression(1, 20000, - math.sqrt(10)): - for outstanding in geometric_progression(1, 200000, - math.sqrt(10)): + for channels in geometric_progression( + 1, 20000, math.sqrt(10)): + for outstanding in geometric_progression( + 1, 200000, math.sqrt(10)): if synchronicity == 'sync' and outstanding > 1200: continue if outstanding < channels: continue diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index d2941c0811..2d22dc13a0 100755 --- a/tools/run_tests/python_utils/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -50,8 +50,8 @@ def docker_mapped_port(cid, port, timeout_seconds=15): return int(output.split(':', 2)[1]) except subprocess.CalledProcessError as e: pass - raise Exception('Failed to get exposed port %s for container %s.' % - (port, cid)) + raise Exception('Failed to get exposed port %s for container %s.' % (port, + cid)) def wait_for_healthy(cid, shortname, timeout_seconds): diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 8e0dc708dd..4c09b3414e 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -124,10 +124,10 @@ def _get_changed_files(base_branch): """ # Get file changes between branch and merge-base of specified branch # Not combined to be Windows friendly - base_commit = check_output( - ["git", "merge-base", base_branch, "HEAD"]).rstrip() - return check_output( - ["git", "diff", base_commit, "--name-only", "HEAD"]).splitlines() + base_commit = check_output(["git", "merge-base", base_branch, + "HEAD"]).rstrip() + return check_output(["git", "diff", base_commit, "--name-only", + "HEAD"]).splitlines() def _can_skip_tests(file_names, triggers): diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 454d09bf0d..6a3391337e 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -133,12 +133,13 @@ def message(tag, msg, explanatory_text=None, do_newline=False): logging.info(explanatory_text) logging.info('%s: %s', tag, msg) else: - sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % ( - _BEGINNING_OF_LINE, _CLEAR_LINE, '\n%s' % explanatory_text - if explanatory_text is not None else '', - _COLORS[_TAG_COLOR[tag]][1], _COLORS[_TAG_COLOR[tag]][0], - tag, msg, '\n' - if do_newline or explanatory_text is not None else '')) + sys.stdout.write( + '%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % + (_BEGINNING_OF_LINE, _CLEAR_LINE, '\n%s' % explanatory_text + if explanatory_text is not None else '', + _COLORS[_TAG_COLOR[tag]][1], _COLORS[_TAG_COLOR[tag]][0], + tag, msg, '\n' + if do_newline or explanatory_text is not None else '')) sys.stdout.flush() return except IOError, e: @@ -210,8 +211,8 @@ class JobSpec(object): def __str__(self): return '%s: %s %s' % (self.shortname, ' '.join( - '%s=%s' % kv - for kv in self.environ.items()), ' '.join(self.cmdline)) + '%s=%s' % kv for kv in self.environ.items()), + ' '.join(self.cmdline)) class JobResult(object): @@ -284,8 +285,9 @@ class Job(object): self._process = try_start() break except OSError: - message('WARNING', 'Failed to start %s, retrying in %f seconds' - % (self._spec.shortname, delay)) + message('WARNING', + 'Failed to start %s, retrying in %f seconds' % + (self._spec.shortname, delay)) time.sleep(delay) delay *= 2 else: @@ -343,8 +345,8 @@ class Job(object): if real > 0.5: cores = (user + sys) / real self.result.cpu_measured = float('%.01f' % cores) - self.result.cpu_estimated = float('%.01f' % - self._spec.cpu_cost) + self.result.cpu_estimated = float( + '%.01f' % self._spec.cpu_cost) measurement = '; cpu_cost=%.01f; estimated=%.01f' % ( self.result.cpu_measured, self.result.cpu_estimated) if not self._quiet_success: @@ -378,8 +380,8 @@ class Job(object): else: message( 'TIMEOUT', - '%s [pid=%d, time=%.1fsec]' % - (self._spec.shortname, self._process.pid, elapsed), + '%s [pid=%d, time=%.1fsec]' % (self._spec.shortname, + self._process.pid, elapsed), stdout(), do_newline=True) self.kill() diff --git a/tools/run_tests/python_utils/start_port_server.py b/tools/run_tests/python_utils/start_port_server.py index 5572cdcfe7..37995acbdf 100644 --- a/tools/run_tests/python_utils/start_port_server.py +++ b/tools/run_tests/python_utils/start_port_server.py @@ -43,16 +43,16 @@ def start_port_server(): if running: current_version = int( subprocess.check_output([ - sys.executable, os.path.abspath( - 'tools/run_tests/python_utils/port_server.py'), + sys.executable, + os.path.abspath('tools/run_tests/python_utils/port_server.py'), 'dump_version' ])) logging.info('my port server is version %d', current_version) running = (version >= current_version) if not running: logging.info('port_server version mismatch: killing the old one') - urllib.urlopen('http://localhost:%d/quitquitquit' % - _PORT_SERVER_PORT).read() + urllib.urlopen( + 'http://localhost:%d/quitquitquit' % _PORT_SERVER_PORT).read() time.sleep(1) if not running: fd, logfile = tempfile.mkstemp() @@ -61,7 +61,8 @@ def start_port_server(): args = [ sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), - '-p', '%d' % _PORT_SERVER_PORT, '-l', logfile + '-p', + '%d' % _PORT_SERVER_PORT, '-l', logfile ] env = dict(os.environ) env['BUILD_ID'] = 'pleaseDontKillMeJenkins' @@ -91,8 +92,8 @@ def start_port_server(): # try one final time: maybe another build managed to start one time.sleep(1) try: - urllib.urlopen('http://localhost:%d/get' % - _PORT_SERVER_PORT).read() + urllib.urlopen( + 'http://localhost:%d/get' % _PORT_SERVER_PORT).read() logging.info( 'last ditch attempt to contact port server succeeded') break diff --git a/tools/run_tests/run_build_statistics.py b/tools/run_tests/run_build_statistics.py index 4af00a47a6..405533279e 100755 --- a/tools/run_tests/run_build_statistics.py +++ b/tools/run_tests/run_build_statistics.py @@ -61,7 +61,7 @@ _BUILDS = { } _URL_BASE = 'https://grpc-testing.appspot.com/job' -# This is a dynamic list where known and active issues should be added. +# This is a dynamic list where known and active issues should be added. # Fixed ones should be removed. # Also try not to add multiple messages from the same failure. _KNOWN_ERRORS = [ @@ -106,8 +106,8 @@ def _scrape_for_known_errors(html): 'description': known_error, 'count': this_error_count }) - print('====> %d failures due to %s' % - (this_error_count, known_error)) + print('====> %d failures due to %s' % (this_error_count, + known_error)) return error_list @@ -116,8 +116,9 @@ def _no_report_files_found(html): def _get_last_processed_buildnumber(build_name): - query = 'SELECT max(build_number) FROM [%s:%s.%s];' % ( - _PROJECT_ID, _DATASET_ID, build_name) + query = 'SELECT max(build_number) FROM [%s:%s.%s];' % (_PROJECT_ID, + _DATASET_ID, + build_name) query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query) page = bq.jobs().getQueryResults( pageToken=None, **query_job['jobReference']).execute(num_retries=3) @@ -167,8 +168,8 @@ def _process_build(json_url, console_url): html = urllib.urlopen(console_url).read() build_result['pass_count'] = 0 build_result['failure_count'] = 1 - # In this case, the string doesn't exist in the result html but the fact - # that we fail to parse the result html indicates Jenkins failure and hence + # In this case, the string doesn't exist in the result html but the fact + # that we fail to parse the result html indicates Jenkins failure and hence # no report files were generated. build_result['no_report_files_found'] = True error_list = _scrape_for_known_errors(html) @@ -223,7 +224,7 @@ for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds: if build.get_status() == 'ABORTED': continue # If any build is still running, stop processing this job. Next time, we - # start from where it was left so that all builds are processed + # start from where it was left so that all builds are processed # sequentially. if build.is_running(): print('====> Build %d is still running.' % build_number) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 99f4298813..44a6ec26e8 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -685,7 +685,8 @@ def cloud_to_prod_jobspec(language, cmdargs = [ '--server_host=%s' % server_host_detail[0], '--server_host_override=%s' % server_host_detail[1], - '--server_port=443', '--use_tls=true', '--test_case=%s' % test_case + '--server_port=443', '--use_tls=true', + '--test_case=%s' % test_case ] environ = dict(language.cloud_to_prod_env(), **language.global_env()) if auth: @@ -696,18 +697,19 @@ def cloud_to_prod_jobspec(language, cwd = language.client_cwd if docker_image: - container_name = dockerjob.random_name('interop_client_%s' % - language.safename) + container_name = dockerjob.random_name( + 'interop_client_%s' % language.safename) cmdline = docker_run_cmdline( cmdline, image=docker_image, cwd=cwd, environ=environ, - docker_args=['--net=host', '--name=%s' % container_name]) + docker_args=['--net=host', + '--name=%s' % container_name]) if manual_cmd_log is not None: if manual_cmd_log == []: - manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % - docker_image) + manual_cmd_log.append( + 'echo "Testing ${docker_image:=%s}"' % docker_image) manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) cwd = None environ = None @@ -775,18 +777,19 @@ def cloud_to_cloud_jobspec(language, environ = language.global_env() if docker_image and language.safename != 'objc': # we can't run client in docker for objc. - container_name = dockerjob.random_name('interop_client_%s' % - language.safename) + container_name = dockerjob.random_name( + 'interop_client_%s' % language.safename) cmdline = docker_run_cmdline( cmdline, image=docker_image, environ=environ, cwd=cwd, - docker_args=['--net=host', '--name=%s' % container_name]) + docker_args=['--net=host', + '--name=%s' % container_name]) if manual_cmd_log is not None: if manual_cmd_log == []: - manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % - docker_image) + manual_cmd_log.append( + 'echo "Testing ${docker_image:=%s}"' % docker_image) manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) cwd = None @@ -807,12 +810,12 @@ def cloud_to_cloud_jobspec(language, def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): """Create jobspec for running a server""" - container_name = dockerjob.random_name('interop_server_%s' % - language.safename) + container_name = dockerjob.random_name( + 'interop_server_%s' % language.safename) cmdline = bash_cmdline( language.server_cmd([ - '--port=%s' % _DEFAULT_SERVER_PORT, '--use_tls=%s' % ( - 'false' if insecure else 'true') + '--port=%s' % _DEFAULT_SERVER_PORT, + '--use_tls=%s' % ('false' if insecure else 'true') ])) environ = language.global_env() docker_args = ['--name=%s' % container_name] @@ -821,9 +824,9 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): # with the server port. These ports are used for http2 interop test # (one test case per port). docker_args += list( - itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i)) - for i in range( - len(_HTTP2_SERVER_TEST_CASES)))) + itertools.chain.from_iterable( + ('-p', str(_DEFAULT_SERVER_PORT + i)) + for i in range(len(_HTTP2_SERVER_TEST_CASES)))) # Enable docker's healthcheck mechanism. # This runs a Python script inside the container every second. The script # pings the http2 server to verify it is ready. The 'health-retries' flag @@ -834,8 +837,8 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): # command line. docker_args += [ '--health-cmd=python test/http2_test/http2_server_health_check.py ' - '--server_host=%s --server_port=%d' % - ('localhost', _DEFAULT_SERVER_PORT), + '--server_host=%s --server_port=%d' % ('localhost', + _DEFAULT_SERVER_PORT), '--health-interval=1s', '--health-retries=5', '--health-timeout=10s', @@ -852,8 +855,8 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): docker_args=docker_args) if manual_cmd_log is not None: if manual_cmd_log == []: - manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % - docker_image) + manual_cmd_log.append( + 'echo "Testing ${docker_image:=%s}"' % docker_image) manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) server_job = jobset.JobSpec( cmdline=docker_cmdline, @@ -974,7 +977,8 @@ argp.add_argument( '--override_server', action='append', type=lambda kv: kv.split('='), - help='Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', + help= + 'Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', default=[]) argp.add_argument( '-t', '--travis', default=False, action='store_const', const=True) @@ -993,7 +997,8 @@ argp.add_argument( default=False, action='store_const', const=True, - help='Allow flaky tests to show as passing (re-runs failed tests up to five times)' + help= + 'Allow flaky tests to show as passing (re-runs failed tests up to five times)' ) argp.add_argument( '--manual_run', @@ -1014,7 +1019,8 @@ argp.add_argument( default=False, action='store_const', const=True, - help='Enable HTTP/2 server edge case testing. (Includes positive and negative tests' + help= + 'Enable HTTP/2 server edge case testing. (Includes positive and negative tests' ) argp.add_argument( '--insecure', @@ -1039,8 +1045,8 @@ args = argp.parse_args() servers = set( s - for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x] - for x in args.server)) + for s in itertools.chain.from_iterable( + _SERVERS if x == 'all' else [x] for x in args.server)) if args.use_docker: if not args.travis: @@ -1067,10 +1073,9 @@ if not args.use_docker and servers: # we want to include everything but objc in 'all' # because objc won't run on non-mac platforms all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc']) -languages = set( - _LANGUAGES[l] - for l in itertools.chain.from_iterable(all_but_objc if x == 'all' else [x] - for x in args.language)) +languages = set(_LANGUAGES[l] + for l in itertools.chain.from_iterable( + all_but_objc if x == 'all' else [x] for x in args.language)) languages_http2_clients_for_http2_server_interop = set() if args.http2_server_interop: diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 561217ceb1..4e4d05cdcd 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -92,8 +92,9 @@ def collect_latency(bm_name, args): benchmarks.append( jobset.JobSpec( [ - 'bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' % - line, '--benchmark_min_time=0.05' + 'bins/basicprof/%s' % bm_name, + '--benchmark_filter=^%s$' % line, + '--benchmark_min_time=0.05' ], environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}, shortname='profile-%s' % fnize(line))) @@ -102,8 +103,9 @@ def collect_latency(bm_name, args): [ sys.executable, 'tools/profiling/latency_profile/profile_analyzer.py', - '--source', '%s.trace' % fnize(line), '--fmt', 'simple', - '--out', 'reports/%s.txt' % fnize(line) + '--source', + '%s.trace' % fnize(line), '--fmt', 'simple', '--out', + 'reports/%s.txt' % fnize(line) ], timeout_seconds=20 * 60, shortname='analyze-%s' % fnize(line))) @@ -116,7 +118,8 @@ def collect_latency(bm_name, args): # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) jobset.run( - benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2)) + benchmarks, maxjobs=max(1, + multiprocessing.cpu_count() / 2)) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) benchmarks = [] @@ -145,8 +148,9 @@ def collect_perf(bm_name, args): benchmarks.append( jobset.JobSpec( [ - 'perf', 'record', '-o', '%s-perf.data' % fnize( - line), '-g', '-F', '997', 'bins/mutrace/%s' % bm_name, + 'perf', 'record', '-o', + '%s-perf.data' % fnize(line), '-g', '-F', '997', + 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10' ], shortname='perf-%s' % fnize(line))) @@ -183,12 +187,14 @@ def collect_perf(bm_name, args): def run_summary(bm_name, cfg, base_json_name): subprocess.check_call([ - 'make', bm_name, 'CONFIG=%s' % cfg, '-j', + 'make', bm_name, + 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count() ]) cmd = [ - 'bins/%s/%s' % (cfg, bm_name), '--benchmark_out=%s.%s.json' % - (base_json_name, cfg), '--benchmark_out_format=json' + 'bins/%s/%s' % (cfg, bm_name), + '--benchmark_out=%s.%s.json' % (base_json_name, cfg), + '--benchmark_out_format=json' ] if args.summary_time is not None: cmd += ['--benchmark_min_time=%d' % args.summary_time] @@ -205,10 +211,12 @@ def collect_summary(bm_name, args): f.write( subprocess.check_output([ 'tools/profiling/microbenchmarks/bm2bq.py', - '%s.counters.json' % bm_name, '%s.opt.json' % bm_name + '%s.counters.json' % bm_name, + '%s.opt.json' % bm_name ])) subprocess.check_call([ - 'bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name + 'bq', 'load', 'microbenchmarks.microbenchmarks', + '%s.csv' % bm_name ]) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 03b684b318..9a9f74e9e5 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -100,7 +100,8 @@ def create_qpsworker_job(language, jobspec = jobset.JobSpec( cmdline=cmdline, shortname=shortname, - timeout_seconds=worker_timeout, # workers get restarted after each scenario + timeout_seconds= + worker_timeout, # workers get restarted after each scenario verbose_success=True) return QpsWorkerJob(jobspec, language, host_and_port, perf_file_base_name) @@ -298,7 +299,8 @@ def create_qpsworkers(languages, worker_hosts, perf_cmd=None): port=worker[1] + language.worker_port_offset(), remote_host=worker[0], perf_cmd=perf_cmd) - for language in languages for worker_idx, worker in enumerate(workers) + for language in languages + for worker_idx, worker in enumerate(workers) ] @@ -367,10 +369,10 @@ def create_scenarios(languages, workers = workers_by_lang[str(language)][:] # 'SERVER_LANGUAGE' is an indicator for this script to pick # a server in different language. - custom_server_lang = scenario_json.get('SERVER_LANGUAGE', - None) - custom_client_lang = scenario_json.get('CLIENT_LANGUAGE', - None) + custom_server_lang = scenario_json.get( + 'SERVER_LANGUAGE', None) + custom_client_lang = scenario_json.get( + 'CLIENT_LANGUAGE', None) scenario_json = scenario_config.remove_nonproto_fields( scenario_json) if custom_server_lang and custom_client_lang: @@ -480,8 +482,8 @@ def main(): argp.add_argument( '--remote_driver_host', default=None, - help='Run QPS driver on given host. By default, QPS driver is run locally.' - ) + help= + 'Run QPS driver on given host. By default, QPS driver is run locally.') argp.add_argument( '--remote_worker_host', nargs='+', @@ -560,7 +562,8 @@ def main(): '--flame_graph_reports', default='perf_reports', type=str, - help='Name of directory to output flame graph profiles to, if any are created.' + help= + 'Name of directory to output flame graph profiles to, if any are created.' ) argp.add_argument( '-u', @@ -662,15 +665,16 @@ def main(): six.iteritems(resultset))) finally: # Consider qps workers that need to be killed as failures - qps_workers_killed += finish_qps_workers(scenario.workers, - qpsworker_jobs) + qps_workers_killed += finish_qps_workers( + scenario.workers, qpsworker_jobs) if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs: workers_and_base_names = {} for worker in scenario.workers: if not worker.perf_file_base_name: raise Exception( - 'using perf buf perf report filename is unspecified') + 'using perf buf perf report filename is unspecified' + ) workers_and_base_names[ worker.host_and_port] = worker.perf_file_base_name perf_report_failures += run_collect_perf_profile_jobs( diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 3aa9eb87c7..c8e917f117 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -182,15 +182,15 @@ def get_c_tests(travis, test_lang): js = json.load(f) return [ tgt for tgt in js - if tgt['language'] == test_lang and platform_string() in tgt[ - platforms_str] and not (travis and tgt['flaky']) + if tgt['language'] == test_lang and platform_string() in + tgt[platforms_str] and not (travis and tgt['flaky']) ] def _check_compiler(compiler, supported_compilers): if compiler not in supported_compilers: - raise Exception('Compiler %s not supported (on this platform).' % - compiler) + raise Exception( + 'Compiler %s not supported (on this platform).' % compiler) def _check_arch(arch, supported_archs): @@ -263,9 +263,9 @@ class CLanguage(object): self.config = config self.args = args if self.platform == 'windows': - _check_compiler(self.args.compiler, [ - 'default', 'cmake', 'cmake_vs2015', 'cmake_vs2017' - ]) + _check_compiler( + self.args.compiler, + ['default', 'cmake', 'cmake_vs2015', 'cmake_vs2017']) _check_arch(self.args.arch, ['default', 'x64', 'x86']) self._cmake_generator_option = 'Visual Studio 15 2017' if self.args.compiler == 'cmake_vs2017' else 'Visual Studio 14 2015' self._cmake_arch_option = 'x64' if self.args.arch == 'x64' else 'Win32' @@ -305,9 +305,9 @@ class CLanguage(object): # cmake doesn't build boringssl tests continue auto_timeout_scaling = target.get('auto_timeout_scaling', True) - polling_strategies = ( - _POLLING_STRATEGIES.get(self.platform, ['all']) - if target.get('uses_polling', True) else ['none']) + polling_strategies = (_POLLING_STRATEGIES.get( + self.platform, ['all']) if target.get('uses_polling', True) else + ['none']) if self.args.iomgr_platform == 'uv': polling_strategies = ['all'] for polling_strategy in polling_strategies: @@ -377,7 +377,8 @@ class CLanguage(object): for line in tests.split('\n'): test = line.strip() if not test: continue - cmdline = [binary, '--benchmark_filter=%s$' % test + cmdline = [binary, + '--benchmark_filter=%s$' % test ] + target['args'] out.append( self.config.job_spec( @@ -408,7 +409,8 @@ class CLanguage(object): assert base is not None assert line[1] == ' ' test = base + line.strip() - cmdline = [binary, '--gtest_filter=%s' % test + cmdline = [binary, + '--gtest_filter=%s' % test ] + target['args'] out.append( self.config.job_spec( @@ -445,8 +447,8 @@ class CLanguage(object): # don't build tools on windows just yet return ['buildtests_%s' % self.make_target] return [ - 'buildtests_%s' % self.make_target, 'tools_%s' % self.make_target, - 'check_epollexclusive' + 'buildtests_%s' % self.make_target, + 'tools_%s' % self.make_target, 'check_epollexclusive' ] def make_options(self): @@ -480,14 +482,18 @@ class CLanguage(object): def _clang_make_options(self, version_suffix=''): return [ - 'CC=clang%s' % version_suffix, 'CXX=clang++%s' % version_suffix, - 'LD=clang%s' % version_suffix, 'LDXX=clang++%s' % version_suffix + 'CC=clang%s' % version_suffix, + 'CXX=clang++%s' % version_suffix, + 'LD=clang%s' % version_suffix, + 'LDXX=clang++%s' % version_suffix ] def _gcc_make_options(self, version_suffix): return [ - 'CC=gcc%s' % version_suffix, 'CXX=g++%s' % version_suffix, - 'LD=gcc%s' % version_suffix, 'LDXX=g++%s' % version_suffix + 'CC=gcc%s' % version_suffix, + 'CXX=g++%s' % version_suffix, + 'LD=gcc%s' % version_suffix, + 'LDXX=g++%s' % version_suffix ] def _compiler_options(self, use_docker, compiler): @@ -700,8 +706,8 @@ class PythonLanguage(object): environ=dict( list(environment.items()) + [( 'GRPC_PYTHON_TESTRUNNER_FILTER', str(suite_name))]), - shortname='%s.test.%s' % (config.name, suite_name),) - for suite_name in tests_json for config in self.pythons + shortname='%s.test.%s' % (config.name, suite_name), + ) for suite_name in tests_json for config in self.pythons ] def pre_build_steps(self): @@ -801,7 +807,10 @@ class PythonLanguage(object): if os.name == 'nt': return (python35_config,) else: - return (python27_config, python34_config,) + return ( + python27_config, + python34_config, + ) elif args.compiler == 'python2.7': return (python27_config,) elif args.compiler == 'python3.4': @@ -817,8 +826,12 @@ class PythonLanguage(object): elif args.compiler == 'python_alpine': return (python27_config,) elif args.compiler == 'all_the_cpythons': - return (python27_config, python34_config, python35_config, - python36_config,) + return ( + python27_config, + python34_config, + python35_config, + python36_config, + ) else: raise Exception('Compiler %s not supported.' % args.compiler) @@ -921,13 +934,15 @@ class CSharpLanguage(object): specs = [] for assembly in six.iterkeys(tests_by_assembly): - assembly_file = 'src/csharp/%s/%s/%s%s' % ( - assembly, assembly_subdir, assembly, assembly_extension) + assembly_file = 'src/csharp/%s/%s/%s%s' % (assembly, + assembly_subdir, + assembly, + assembly_extension) if self.config.build_config != 'gcov' or self.platform != 'windows': # normally, run each test as a separate process for test in tests_by_assembly[assembly]: - cmdline = runtime_cmd + [assembly_file, '--test=%s' % test - ] + nunit_args + cmdline = runtime_cmd + [assembly_file, + '--test=%s' % test] + nunit_args specs.append( self.config.job_spec( cmdline, @@ -1147,8 +1162,8 @@ class Sanity(object): # different configurations we can run under with open('tools/run_tests/generated/configs.json') as f: - _CONFIGS = dict((cfg['config'], Config(**cfg)) - for cfg in ast.literal_eval(f.read())) + _CONFIGS = dict( + (cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read())) _LANGUAGES = { 'c++': CLanguage('cxx', 'c++'), @@ -1298,13 +1313,15 @@ argp.add_argument( default=False, action='store_const', const=True, - help='Allow flaky tests to show as passing (re-runs failed tests up to five times)' + help= + 'Allow flaky tests to show as passing (re-runs failed tests up to five times)' ) argp.add_argument( '--arch', choices=['default', 'x86', 'x64'], default='default', - help='Selects architecture to target. For some platforms "default" is the only supported choice.' + help= + 'Selects architecture to target. For some platforms "default" is the only supported choice.' ) argp.add_argument( '--compiler', @@ -1316,7 +1333,8 @@ argp.add_argument( 'cmake_vs2015', 'cmake_vs2017' ], default='default', - help='Selects compiler to use. Allowed values depend on the platform and language.' + help= + 'Selects compiler to use. Allowed values depend on the platform and language.' ) argp.add_argument( '--iomgr_platform', @@ -1339,7 +1357,8 @@ argp.add_argument( '--update_submodules', default=[], nargs='*', - help='Update some submodules before building. If any are updated, also run generate_projects. ' + help= + 'Update some submodules before building. If any are updated, also run generate_projects. ' + 'Submodules are specified as SUBMODULE_NAME:BRANCH; if BRANCH is omitted, master is assumed.' ) @@ -1360,7 +1379,8 @@ argp.add_argument( default=False, action='store_const', const=True, - help='Don\'t print anything when a test passes. Passing tests also will not be reported in XML report. ' + help= + 'Don\'t print anything when a test passes. Passing tests also will not be reported in XML report. ' + 'Useful when running many iterations of each test (argument -n).') argp.add_argument( '--force_default_poller', @@ -1399,8 +1419,8 @@ if not args.disable_auto_set_flakes: if test.flaky: flaky_tests.add(test.name) if test.cpu > 0: shortname_to_cpu[test.name] = test.cpu except: - print("Unexpected error getting flaky tests: %s" % - traceback.format_exc()) + print( + "Unexpected error getting flaky tests: %s" % traceback.format_exc()) if args.force_default_poller: _POLLING_STRATEGIES = {} @@ -1473,7 +1493,8 @@ if any(language.make_options() for language in languages): language_make_options = list( set([ make_option - for lang in languages for make_option in lang.make_options() + for lang in languages + for make_option in lang.make_options() ])) if args.use_docker: @@ -1530,8 +1551,8 @@ def make_jobspec(cfg, targets, makefile='Makefile'): return [ jobset.JobSpec( [ - 'cmake', '--build', '.', '--target', '%s' % target, - '--config', _MSBUILD_CONFIG[cfg] + 'cmake', '--build', '.', '--target', + '%s' % target, '--config', _MSBUILD_CONFIG[cfg] ], cwd=os.path.dirname(makefile), timeout_seconds=None) for target in targets @@ -1541,8 +1562,8 @@ def make_jobspec(cfg, targets, makefile='Makefile'): # With cmake, we've passed all the build configuration in the pre-build step already return [ jobset.JobSpec( - [os.getenv('MAKE', 'make'), '-j', '%d' % args.jobs] + - targets, + [os.getenv('MAKE', 'make'), '-j', + '%d' % args.jobs] + targets, cwd='cmake/build', timeout_seconds=None) ] @@ -1550,10 +1571,11 @@ def make_jobspec(cfg, targets, makefile='Makefile'): return [ jobset.JobSpec( [ - os.getenv('MAKE', 'make'), '-f', makefile, '-j', '%d' % - args.jobs, + os.getenv('MAKE', 'make'), '-f', makefile, '-j', + '%d' % args.jobs, 'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' % - args.slowdown, 'CONFIG=%s' % cfg, 'Q=' + args.slowdown, + 'CONFIG=%s' % cfg, 'Q=' ] + language_make_options + ([] if not args.travis else ['JENKINS_BUILD=1']) + targets, timeout_seconds=None) @@ -1565,8 +1587,8 @@ def make_jobspec(cfg, targets, makefile='Makefile'): make_targets = {} for l in languages: makefile = l.makefile_name() - make_targets[makefile] = make_targets.get( - makefile, set()).union(set(l.make_targets())) + make_targets[makefile] = make_targets.get(makefile, set()).union( + set(l.make_targets())) def build_step_environ(cfg): @@ -1581,7 +1603,8 @@ build_steps = list( set( jobset.JobSpec( cmdline, environ=build_step_environ(build_config), flake_retries=2) - for l in languages for cmdline in l.pre_build_steps())) + for l in languages + for cmdline in l.pre_build_steps())) if make_targets: make_commands = itertools.chain.from_iterable( make_jobspec(build_config, list(targets), makefile) @@ -1593,12 +1616,14 @@ build_steps.extend( cmdline, environ=build_step_environ(build_config), timeout_seconds=None) - for l in languages for cmdline in l.build_steps())) + for l in languages + for cmdline in l.build_steps())) post_tests_steps = list( set( jobset.JobSpec(cmdline, environ=build_step_environ(build_config)) - for l in languages for cmdline in l.post_tests_steps())) + for l in languages + for cmdline in l.post_tests_steps())) runs_per_test = args.runs_per_test forever = args.forever @@ -1612,8 +1637,8 @@ def _shut_down_legacy_server(legacy_server_port): except: pass else: - urllib.request.urlopen('http://localhost:%d/quitquitquit' % - legacy_server_port).read() + urllib.request.urlopen( + 'http://localhost:%d/quitquitquit' % legacy_server_port).read() def _calculate_num_runs_failures(list_of_results): @@ -1679,8 +1704,8 @@ def _build_and_run(check_cancelled, return [] if not args.travis and not _has_epollexclusive() and platform_string( - ) in _POLLING_STRATEGIES and 'epollex' in _POLLING_STRATEGIES[ - platform_string()]: + ) in _POLLING_STRATEGIES and 'epollex' in _POLLING_STRATEGIES[platform_string( + )]: print('\n\nOmitting EPOLLEXCLUSIVE tests\n\n') _POLLING_STRATEGIES[platform_string()].remove('epollex') @@ -1694,12 +1719,11 @@ def _build_and_run(check_cancelled, num_test_failures = 0 try: infinite_runs = runs_per_test == 0 - one_run = set(spec - for language in languages - for spec in language.test_specs() - if (re.search(args.regex, spec.shortname) and ( - args.regex_exclude == '' or not re.search( - args.regex_exclude, spec.shortname)))) + one_run = set( + spec for language in languages for spec in language.test_specs() + if (re.search(args.regex, spec.shortname) and + (args.regex_exclude == '' or + not re.search(args.regex_exclude, spec.shortname)))) # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if args.travis and args.max_time <= 0: @@ -1722,8 +1746,9 @@ def _build_and_run(check_cancelled, if infinite_runs: assert len(massaged_one_run ) > 0, 'Must have at least one test for a -n inf run' - runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs - else itertools.repeat(massaged_one_run, runs_per_test)) + runs_sequence = (itertools.repeat(massaged_one_run) + if infinite_runs else itertools.repeat( + massaged_one_run, runs_per_test)) all_runs = itertools.chain.from_iterable(runs_sequence) if args.quiet_success: @@ -1750,8 +1775,8 @@ def _build_and_run(check_cancelled, else: jobset.message( 'FLAKE', - '%s [%d/%d runs flaked]' % - (k, num_failures, num_runs), + '%s [%d/%d runs flaked]' % (k, num_failures, + num_runs), do_newline=True) finally: for antagonist in antagonists: diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 49be8f1d7e..ac90bef5ac 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -65,8 +65,10 @@ def _docker_jobspec(name, test_job = jobset.JobSpec( cmdline=[ 'python', 'tools/run_tests/run_tests.py', '--use_docker', '-t', - '-j', str(inner_jobs), '-x', _report_filename(name), - '--report_suite_name', '%s' % name + '-j', + str(inner_jobs), '-x', + _report_filename(name), '--report_suite_name', + '%s' % name ] + runtests_args, environ=runtests_envs, shortname='run_tests_%s' % name, @@ -90,8 +92,10 @@ def _workspace_jobspec(name, test_job = jobset.JobSpec( cmdline=[ 'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh', - '-t', '-j', str(inner_jobs), '-x', '../%s' % _report_filename(name), - '--report_suite_name', '%s' % name + '-t', '-j', + str(inner_jobs), '-x', + '../%s' % _report_filename(name), '--report_suite_name', + '%s' % name ] + runtests_args, environ=env, shortname='run_tests_%s' % name, @@ -492,8 +496,8 @@ if __name__ == "__main__": jobs = [] for job in all_jobs: - if not args.filter or all(filter in job.labels - for filter in args.filter): + if not args.filter or all( + filter in job.labels for filter in args.filter): if not any(exclude_label in job.labels for exclude_label in args.exclude): jobs.append(job) diff --git a/tools/run_tests/sanity/check_test_filtering.py b/tools/run_tests/sanity/check_test_filtering.py index c2a6399ae8..ebbb1a93b3 100755 --- a/tools/run_tests/sanity/check_test_filtering.py +++ b/tools/run_tests/sanity/check_test_filtering.py @@ -80,7 +80,8 @@ class TestFilteringTest(unittest.TestCase): if (label in job.labels): jobs_matching_labels += 1 self.assertEquals( - len(filtered_jobs), len(all_jobs) - jobs_matching_labels) + len(filtered_jobs), + len(all_jobs) - jobs_matching_labels) def test_individual_language_filters(self): # Changing unlisted file should trigger all languages |