diff options
31 files changed, 360 insertions, 149 deletions
@@ -177,7 +177,9 @@ EXPORTS grpc_slice_buffer_move_into grpc_slice_buffer_trim_end grpc_slice_buffer_move_first + grpc_slice_buffer_move_first_into_buffer grpc_slice_buffer_take_first + grpc_slice_buffer_undo_take_first gpr_malloc gpr_free gpr_realloc diff --git a/include/grpc/impl/codegen/slice.h b/include/grpc/impl/codegen/slice.h index 3c9c7b0285..0b09a0bfd8 100644 --- a/include/grpc/impl/codegen/slice.h +++ b/include/grpc/impl/codegen/slice.h @@ -106,11 +106,16 @@ struct grpc_slice { /* Represents an expandable array of slices, to be interpreted as a single item. */ typedef struct { - /* slices in the array */ + /* This is for internal use only. External users (i.e any code outside grpc + * core) MUST NOT use this field */ + grpc_slice *base_slices; + + /* slices in the array (Points to the first valid grpc_slice in the array) */ grpc_slice *slices; /* the number of slices in the array */ size_t count; - /* the number of slices allocated in the array */ + /* the number of slices allocated in the array. External users (i.e any code + * outside grpc core) MUST NOT use this field */ size_t capacity; /* the combined length of all slices in the array */ size_t length; diff --git a/include/grpc/slice_buffer.h b/include/grpc/slice_buffer.h index f1de653af4..2ed896645b 100644 --- a/include/grpc/slice_buffer.h +++ b/include/grpc/slice_buffer.h @@ -77,8 +77,15 @@ GPRAPI void grpc_slice_buffer_trim_end(grpc_slice_buffer *src, size_t n, /* move the first n bytes of src into dst */ GPRAPI void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, grpc_slice_buffer *dst); +/* move the first n bytes of src into dst (copying them) */ +GPRAPI void grpc_slice_buffer_move_first_into_buffer(grpc_exec_ctx *exec_ctx, + grpc_slice_buffer *src, + size_t n, void *dst); /* take the first slice in the slice buffer */ GPRAPI grpc_slice grpc_slice_buffer_take_first(grpc_slice_buffer *src); +/* undo the above with (a possibly different) \a slice */ +GPRAPI void grpc_slice_buffer_undo_take_first(grpc_slice_buffer *src, + grpc_slice slice); #ifdef __cplusplus } diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 39b5c0032e..0a565565bd 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -796,7 +796,7 @@ static polling_island *polling_island_merge(polling_island *p, gpr_atm_rel_store(&p->merged_to, (gpr_atm)q); PI_ADD_REF(q, "pi_merge"); /* To account for the new incoming ref from p */ - workqueue_move_items_to_parent(q); + workqueue_move_items_to_parent(p); } /* else if p == q, nothing needs to be done */ diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 5bc5621443..9477ac3688 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -413,9 +413,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, const char *reason) { fd->on_done_closure = on_done; fd->released = release_fd != NULL; - if (!fd->released) { - shutdown(fd->fd, SHUT_RDWR); - } else { + if (fd->released) { *release_fd = fd->fd; } gpr_mu_lock(&fd->mu); diff --git a/src/core/lib/slice/slice_buffer.c b/src/core/lib/slice/slice_buffer.c index 08eaf4963a..9176dc8a42 100644 --- a/src/core/lib/slice/slice_buffer.c +++ b/src/core/lib/slice/slice_buffer.c @@ -46,15 +46,27 @@ #define GROW(x) (3 * (x) / 2) static void maybe_embiggen(grpc_slice_buffer *sb) { - if (sb->count == sb->capacity) { + if (sb->base_slices != sb->slices) { + memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice)); + sb->slices = sb->base_slices; + } + + /* How far away from sb->base_slices is sb->slices pointer */ + size_t slice_offset = (size_t)(sb->slices - sb->base_slices); + size_t slice_count = sb->count + slice_offset; + + if (slice_count == sb->capacity) { sb->capacity = GROW(sb->capacity); - GPR_ASSERT(sb->capacity > sb->count); - if (sb->slices == sb->inlined) { - sb->slices = gpr_malloc(sb->capacity * sizeof(grpc_slice)); - memcpy(sb->slices, sb->inlined, sb->count * sizeof(grpc_slice)); + GPR_ASSERT(sb->capacity > slice_count); + if (sb->base_slices == sb->inlined) { + sb->base_slices = gpr_malloc(sb->capacity * sizeof(grpc_slice)); + memcpy(sb->base_slices, sb->inlined, slice_count * sizeof(grpc_slice)); } else { - sb->slices = gpr_realloc(sb->slices, sb->capacity * sizeof(grpc_slice)); + sb->base_slices = + gpr_realloc(sb->base_slices, sb->capacity * sizeof(grpc_slice)); } + + sb->slices = sb->base_slices + slice_offset; } } @@ -62,14 +74,14 @@ void grpc_slice_buffer_init(grpc_slice_buffer *sb) { sb->count = 0; sb->length = 0; sb->capacity = GRPC_SLICE_BUFFER_INLINE_ELEMENTS; - sb->slices = sb->inlined; + sb->base_slices = sb->slices = sb->inlined; } void grpc_slice_buffer_destroy_internal(grpc_exec_ctx *exec_ctx, grpc_slice_buffer *sb) { grpc_slice_buffer_reset_and_unref_internal(exec_ctx, sb); - if (sb->slices != sb->inlined) { - gpr_free(sb->slices); + if (sb->base_slices != sb->inlined) { + gpr_free(sb->base_slices); } } @@ -166,7 +178,6 @@ void grpc_slice_buffer_pop(grpc_slice_buffer *sb) { void grpc_slice_buffer_reset_and_unref_internal(grpc_exec_ctx *exec_ctx, grpc_slice_buffer *sb) { size_t i; - for (i = 0; i < sb->count; i++) { grpc_slice_unref_internal(exec_ctx, sb->slices[i]); } @@ -182,32 +193,45 @@ void grpc_slice_buffer_reset_and_unref(grpc_slice_buffer *sb) { } void grpc_slice_buffer_swap(grpc_slice_buffer *a, grpc_slice_buffer *b) { - GPR_SWAP(size_t, a->count, b->count); - GPR_SWAP(size_t, a->capacity, b->capacity); - GPR_SWAP(size_t, a->length, b->length); + size_t a_offset = (size_t)(a->slices - a->base_slices); + size_t b_offset = (size_t)(b->slices - b->base_slices); - if (a->slices == a->inlined) { - if (b->slices == b->inlined) { + size_t a_count = a->count + a_offset; + size_t b_count = b->count + b_offset; + + if (a->base_slices == a->inlined) { + if (b->base_slices == b->inlined) { /* swap contents of inlined buffer */ grpc_slice temp[GRPC_SLICE_BUFFER_INLINE_ELEMENTS]; - memcpy(temp, a->slices, b->count * sizeof(grpc_slice)); - memcpy(a->slices, b->slices, a->count * sizeof(grpc_slice)); - memcpy(b->slices, temp, b->count * sizeof(grpc_slice)); + memcpy(temp, a->base_slices, a_count * sizeof(grpc_slice)); + memcpy(a->base_slices, b->base_slices, b_count * sizeof(grpc_slice)); + memcpy(b->base_slices, temp, a_count * sizeof(grpc_slice)); } else { /* a is inlined, b is not - copy a inlined into b, fix pointers */ - a->slices = b->slices; - b->slices = b->inlined; - memcpy(b->slices, a->inlined, b->count * sizeof(grpc_slice)); + a->base_slices = b->base_slices; + b->base_slices = b->inlined; + memcpy(b->base_slices, a->inlined, a_count * sizeof(grpc_slice)); } - } else if (b->slices == b->inlined) { + } else if (b->base_slices == b->inlined) { /* b is inlined, a is not - copy b inlined int a, fix pointers */ - b->slices = a->slices; - a->slices = a->inlined; - memcpy(a->slices, b->inlined, a->count * sizeof(grpc_slice)); + b->base_slices = a->base_slices; + a->base_slices = a->inlined; + memcpy(a->base_slices, b->inlined, b_count * sizeof(grpc_slice)); } else { /* no inlining: easy swap */ - GPR_SWAP(grpc_slice *, a->slices, b->slices); + GPR_SWAP(grpc_slice *, a->base_slices, b->base_slices); } + + /* Update the slices pointers (cannot do a GPR_SWAP on slices fields here). + * Also note that since the base_slices pointers are already swapped we need + * use 'b_offset' for 'a->base_slices' and vice versa */ + a->slices = a->base_slices + b_offset; + b->slices = b->base_slices + a_offset; + + /* base_slices and slices fields are correctly set. Swap all other fields */ + GPR_SWAP(size_t, a->count, b->count); + GPR_SWAP(size_t, a->capacity, b->capacity); + GPR_SWAP(size_t, a->length, b->length); } void grpc_slice_buffer_move_into(grpc_slice_buffer *src, @@ -229,7 +253,6 @@ void grpc_slice_buffer_move_into(grpc_slice_buffer *src, void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, grpc_slice_buffer *dst) { - size_t src_idx; size_t output_len = dst->length + n; size_t new_input_len = src->length - n; GPR_ASSERT(src->length >= n); @@ -237,34 +260,55 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, grpc_slice_buffer_move_into(src, dst); return; } - src_idx = 0; - while (src_idx < src->capacity) { - grpc_slice slice = src->slices[src_idx]; + + while (src->count > 0) { + grpc_slice slice = grpc_slice_buffer_take_first(src); size_t slice_len = GRPC_SLICE_LENGTH(slice); if (n > slice_len) { grpc_slice_buffer_add(dst, slice); n -= slice_len; - src_idx++; } else if (n == slice_len) { grpc_slice_buffer_add(dst, slice); - src_idx++; break; } else { /* n < slice_len */ - src->slices[src_idx] = grpc_slice_split_tail(&slice, n); + grpc_slice_buffer_undo_take_first(src, grpc_slice_split_tail(&slice, n)); GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); - GPR_ASSERT(GRPC_SLICE_LENGTH(src->slices[src_idx]) == slice_len - n); grpc_slice_buffer_add(dst, slice); break; } } GPR_ASSERT(dst->length == output_len); - memmove(src->slices, src->slices + src_idx, - sizeof(grpc_slice) * (src->count - src_idx)); - src->count -= src_idx; - src->length = new_input_len; + GPR_ASSERT(src->length == new_input_len); GPR_ASSERT(src->count > 0); } +void grpc_slice_buffer_move_first_into_buffer(grpc_exec_ctx *exec_ctx, + grpc_slice_buffer *src, size_t n, + void *dst) { + char *dstp = dst; + GPR_ASSERT(src->length >= n); + + while (n > 0) { + grpc_slice slice = grpc_slice_buffer_take_first(src); + size_t slice_len = GRPC_SLICE_LENGTH(slice); + if (slice_len > n) { + memcpy(dstp, GRPC_SLICE_START_PTR(slice), n); + grpc_slice_buffer_undo_take_first( + src, grpc_slice_sub_no_ref(slice, n, slice_len)); + n = 0; + } else if (slice_len == n) { + memcpy(dstp, GRPC_SLICE_START_PTR(slice), n); + grpc_slice_unref_internal(exec_ctx, slice); + n = 0; + } else { + memcpy(dstp, GRPC_SLICE_START_PTR(slice), slice_len); + dstp += slice_len; + n -= slice_len; + grpc_slice_unref_internal(exec_ctx, slice); + } + } +} + void grpc_slice_buffer_trim_end(grpc_slice_buffer *sb, size_t n, grpc_slice_buffer *garbage) { GPR_ASSERT(n <= sb->length); @@ -293,8 +337,17 @@ grpc_slice grpc_slice_buffer_take_first(grpc_slice_buffer *sb) { grpc_slice slice; GPR_ASSERT(sb->count > 0); slice = sb->slices[0]; - memmove(&sb->slices[0], &sb->slices[1], (sb->count - 1) * sizeof(grpc_slice)); + sb->slices++; sb->count--; sb->length -= GRPC_SLICE_LENGTH(slice); + return slice; } + +void grpc_slice_buffer_undo_take_first(grpc_slice_buffer *sb, + grpc_slice slice) { + sb->slices--; + sb->slices[0] = slice; + sb->count++; + sb->length += GRPC_SLICE_LENGTH(slice); +} diff --git a/src/node/ext/call_credentials.cc b/src/node/ext/call_credentials.cc index 4d172d4ddf..afcc363131 100644 --- a/src/node/ext/call_credentials.cc +++ b/src/node/ext/call_credentials.cc @@ -35,7 +35,7 @@ #include <nan.h> #include <uv.h> -#include <list> +#include <queue> #include "grpc/grpc.h" #include "grpc/grpc_security.h" @@ -170,7 +170,7 @@ NAN_METHOD(CallCredentials::CreateFromPlugin) { grpc_metadata_credentials_plugin plugin; plugin_state *state = new plugin_state; state->callback = new Nan::Callback(info[0].As<Function>()); - state->pending_callbacks = new std::list<plugin_callback_data*>(); + state->pending_callbacks = new std::queue<plugin_callback_data*>(); uv_mutex_init(&state->plugin_mutex); uv_async_init(uv_default_loop(), &state->plugin_async, @@ -231,13 +231,13 @@ NAN_METHOD(PluginCallback) { NAUV_WORK_CB(SendPluginCallback) { Nan::HandleScope scope; plugin_state *state = reinterpret_cast<plugin_state*>(async->data); - std::list<plugin_callback_data*> callbacks; + std::queue<plugin_callback_data*> callbacks; uv_mutex_lock(&state->plugin_mutex); - callbacks.splice(callbacks.begin(), *state->pending_callbacks); + state->pending_callbacks->swap(callbacks); uv_mutex_unlock(&state->plugin_mutex); while (!callbacks.empty()) { plugin_callback_data *data = callbacks.front(); - callbacks.pop_front(); + callbacks.pop(); Local<Object> callback_data = Nan::New<Object>(); Nan::Set(callback_data, Nan::New("cb").ToLocalChecked(), Nan::New<v8::External>(reinterpret_cast<void*>(data->cb))); @@ -266,7 +266,7 @@ void plugin_get_metadata(void *state, grpc_auth_metadata_context context, data->user_data = user_data; uv_mutex_lock(&p_state->plugin_mutex); - p_state->pending_callbacks->push_back(data); + p_state->pending_callbacks->push(data); uv_mutex_unlock(&p_state->plugin_mutex); uv_async_send(&p_state->plugin_async); diff --git a/src/node/ext/call_credentials.h b/src/node/ext/call_credentials.h index 04c852bea1..21a4b8923e 100644 --- a/src/node/ext/call_credentials.h +++ b/src/node/ext/call_credentials.h @@ -34,7 +34,7 @@ #ifndef GRPC_NODE_CALL_CREDENTIALS_H_ #define GRPC_NODE_CALL_CREDENTIALS_H_ -#include <list> +#include <queue> #include <node.h> #include <nan.h> @@ -84,7 +84,7 @@ typedef struct plugin_callback_data { typedef struct plugin_state { Nan::Callback *callback; - std::list<plugin_callback_data*> *pending_callbacks; + std::queue<plugin_callback_data*> *pending_callbacks; uv_mutex_t plugin_mutex; // async.data == this uv_async_t plugin_async; diff --git a/src/node/ext/node_grpc.cc b/src/node/ext/node_grpc.cc index 682af0e5ad..95e273f8ac 100644 --- a/src/node/ext/node_grpc.cc +++ b/src/node/ext/node_grpc.cc @@ -31,7 +31,7 @@ * */ -#include <list> +#include <queue> #include <node.h> #include <nan.h> @@ -77,7 +77,7 @@ typedef struct log_args { typedef struct logger_state { Nan::Callback *callback; - std::list<log_args *> *pending_args; + std::queue<log_args *> *pending_args; uv_mutex_t mutex; uv_async_t async; // Indicates that a logger has been set @@ -338,14 +338,14 @@ NAN_METHOD(SetDefaultRootsPem) { NAUV_WORK_CB(LogMessagesCallback) { Nan::HandleScope scope; - std::list<log_args *> args; + std::queue<log_args *> args; uv_mutex_lock(&grpc_logger_state.mutex); - args.splice(args.begin(), *grpc_logger_state.pending_args); + grpc_logger_state.pending_args->swap(args); uv_mutex_unlock(&grpc_logger_state.mutex); /* Call the callback with each log message */ while (!args.empty()) { log_args *arg = args.front(); - args.pop_front(); + args.pop(); Local<Value> file = Nan::New(arg->core_args.file).ToLocalChecked(); Local<Value> line = Nan::New<Uint32, uint32_t>(arg->core_args.line); Local<Value> severity = Nan::New( @@ -372,7 +372,7 @@ void node_log_func(gpr_log_func_args *args) { args_copy->timestamp = gpr_now(GPR_CLOCK_REALTIME); uv_mutex_lock(&grpc_logger_state.mutex); - grpc_logger_state.pending_args->push_back(args_copy); + grpc_logger_state.pending_args->push(args_copy); uv_mutex_unlock(&grpc_logger_state.mutex); uv_async_send(&grpc_logger_state.async); @@ -380,7 +380,7 @@ void node_log_func(gpr_log_func_args *args) { void init_logger() { memset(&grpc_logger_state, 0, sizeof(logger_state)); - grpc_logger_state.pending_args = new std::list<log_args *>(); + grpc_logger_state.pending_args = new std::queue<log_args *>(); uv_mutex_init(&grpc_logger_state.mutex); uv_async_init(uv_default_loop(), &grpc_logger_state.async, diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index e09f922591..d813df5f44 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -271,12 +271,12 @@ class BuildExt(build_ext.build_ext): compiler = self.compiler.compiler_type if compiler in BuildExt.C_OPTIONS: for extension in self.extensions: - extension.extra_compile_args += list(BuildExt.C_OPTIONS[ - compiler]) + extension.extra_compile_args += list( + BuildExt.C_OPTIONS[compiler]) if compiler in BuildExt.LINK_OPTIONS: for extension in self.extensions: - extension.extra_link_args += list(BuildExt.LINK_OPTIONS[ - compiler]) + extension.extra_link_args += list( + BuildExt.LINK_OPTIONS[compiler]) if not check_and_update_cythonization(self.extensions): self.extensions = try_cythonize(self.extensions) try: @@ -284,8 +284,8 @@ class BuildExt(build_ext.build_ext): except Exception as error: formatted_exception = traceback.format_exc() support.diagnose_build_ext_error(self, error, formatted_exception) - raise CommandError("Failed `build_ext` step:\n{}".format( - formatted_exception)) + raise CommandError( + "Failed `build_ext` step:\n{}".format(formatted_exception)) class Gather(setuptools.Command): diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 77412236cc..5a8a3d487a 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -842,8 +842,8 @@ def _poll_connectivity(state, channel, initial_try_to_connect): connectivity = channel.check_connectivity_state(try_to_connect) with state.lock: state.connectivity = ( - _common.CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[ - connectivity]) + _common. + CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[connectivity]) callbacks = tuple(callback for callback, unused_but_known_to_be_none_connectivity in state.callbacks_and_connectivities) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index 04872b9c09..4d988192df 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -29,6 +29,8 @@ cimport cpython +import traceback + cdef class ChannelCredentials: @@ -138,15 +140,22 @@ cdef class AuthMetadataContext: cdef void plugin_get_metadata( void *state, grpc_auth_metadata_context context, grpc_credentials_plugin_metadata_cb cb, void *user_data) with gil: + called_flag = [False] def python_callback( Metadata metadata, grpc_status_code status, bytes error_details): cb(user_data, metadata.c_metadata_array.metadata, metadata.c_metadata_array.count, status, error_details) + called_flag[0] = True cdef CredentialsMetadataPlugin self = <CredentialsMetadataPlugin>state cdef AuthMetadataContext cy_context = AuthMetadataContext() cy_context.context = context - self.plugin_callback(cy_context, python_callback) + try: + self.plugin_callback(cy_context, python_callback) + except Exception as error: + if not called_flag[0]: + cb(user_data, Metadata([]).c_metadata_array.metadata, + 0, StatusCode.unknown, traceback.format_exc().encode()) cdef void plugin_destroy_c_plugin_state(void *state) with gil: cpython.Py_DECREF(<CredentialsMetadataPlugin>state) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi index d052b3f8bc..a9163c75bf 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi @@ -194,6 +194,25 @@ cdef class Timespec: def infinite_past(): return Timespec(float("-inf")) + def __richcmp__(Timespec self not None, Timespec other not None, int op): + cdef gpr_timespec self_c_time = self.c_time + cdef gpr_timespec other_c_time = other.c_time + cdef int result = gpr_time_cmp(self_c_time, other_c_time) + if op == 0: # < + return result < 0 + elif op == 2: # == + return result == 0 + elif op == 4: # > + return result > 0 + elif op == 1: # <= + return result <= 0 + elif op == 3: # != + return result != 0 + elif op == 5: # >= + return result >= 0 + else: + raise ValueError('__richcmp__ `op` contract violated') + cdef class CallDetails: diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index 845b7f598c..af0ffe3475 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -121,8 +121,8 @@ class BuildProtoModules(setuptools.Command): '--grpc_python_out={}'.format(PROTO_STEM), ] + [path] if protoc.main(command) != 0: - sys.stderr.write('warning: Command:\n{}\nFailed'.format( - command)) + sys.stderr.write( + 'warning: Command:\n{}\nFailed'.format(command)) # Generated proto directories dont include __init__.py, but # these are needed for python package resolution diff --git a/src/python/grpcio_tests/tests/_loader.py b/src/python/grpcio_tests/tests/_loader.py index 42cf9ab4ca..165bc53fb7 100644 --- a/src/python/grpcio_tests/tests/_loader.py +++ b/src/python/grpcio_tests/tests/_loader.py @@ -116,5 +116,5 @@ def iterate_suite_cases(suite): elif isinstance(item, unittest.TestCase): yield item else: - raise ValueError('unexpected suite item of type {}'.format( - type(item))) + raise ValueError( + 'unexpected suite item of type {}'.format(type(item))) diff --git a/src/python/grpcio_tests/tests/_runner.py b/src/python/grpcio_tests/tests/_runner.py index 59964b271c..1138a2279d 100644 --- a/src/python/grpcio_tests/tests/_runner.py +++ b/src/python/grpcio_tests/tests/_runner.py @@ -196,8 +196,8 @@ class Runner(object): # Run the tests result.startTestRun() for augmented_case in augmented_cases: - sys.stdout.write('Running {}\n'.format(augmented_case.case.id( - ))) + sys.stdout.write( + 'Running {}\n'.format(augmented_case.case.id())) sys.stdout.flush() case_thread = threading.Thread( target=augmented_case.case.run, args=(result,)) diff --git a/src/python/grpcio_tests/tests/interop/methods.py b/src/python/grpcio_tests/tests/interop/methods.py index e1f8722168..bdb258591e 100644 --- a/src/python/grpcio_tests/tests/interop/methods.py +++ b/src/python/grpcio_tests/tests/interop/methods.py @@ -428,8 +428,8 @@ def _compute_engine_creds(stub, args): def _oauth2_auth_token(stub, args): - json_key_filename = os.environ[ - oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS] + json_key_filename = os.environ[oauth2client_client. + GOOGLE_APPLICATION_CREDENTIALS] wanted_email = json.load(open(json_key_filename, 'rb'))['client_email'] response = _large_unary_common_behavior(stub, True, True, None) if wanted_email != response.username: @@ -441,8 +441,8 @@ def _oauth2_auth_token(stub, args): def _jwt_token_creds(stub, args): - json_key_filename = os.environ[ - oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS] + json_key_filename = os.environ[oauth2client_client. + GOOGLE_APPLICATION_CREDENTIALS] wanted_email = json.load(open(json_key_filename, 'rb'))['client_email'] response = _large_unary_common_behavior(stub, True, False, None) if wanted_email != response.username: @@ -451,11 +451,10 @@ def _jwt_token_creds(stub, args): def _per_rpc_creds(stub, args): - json_key_filename = os.environ[ - oauth2client_client.GOOGLE_APPLICATION_CREDENTIALS] + json_key_filename = os.environ[oauth2client_client. + GOOGLE_APPLICATION_CREDENTIALS] wanted_email = json.load(open(json_key_filename, 'rb'))['client_email'] - credentials = oauth2client_client.GoogleCredentials.get_application_default( - ) + credentials = oauth2client_client.GoogleCredentials.get_application_default() scoped_credentials = credentials.create_scoped([args.oauth_scope]) # TODO(https://github.com/grpc/grpc/issues/6799): Eliminate this last # remaining use of the Beta API. diff --git a/src/python/grpcio_tests/tests/qps/worker_server.py b/src/python/grpcio_tests/tests/qps/worker_server.py index 1deb7ed698..ca1a777611 100644 --- a/src/python/grpcio_tests/tests/qps/worker_server.py +++ b/src/python/grpcio_tests/tests/qps/worker_server.py @@ -102,8 +102,8 @@ class WorkerServer(services_pb2.WorkerServiceServicer): 'grpc.testing.BenchmarkService', method_implementations) server.add_generic_rpc_handlers((handler,)) else: - raise Exception('Unsupported server type {}'.format( - config.server_type)) + raise Exception( + 'Unsupported server type {}'.format(config.server_type)) if config.HasField('security_params'): # Use SSL server_creds = grpc.ssl_server_credentials(( @@ -171,8 +171,8 @@ class WorkerServer(services_pb2.WorkerServiceServicer): else: raise Exception('Async streaming client not supported') else: - raise Exception('Unsupported client type {}'.format( - config.client_type)) + raise Exception( + 'Unsupported client type {}'.format(config.client_type)) # In multi-channel tests, we split the load across all channels load_factor = float(config.client_channels) diff --git a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py index 7aec316b95..b4efe87730 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py @@ -95,8 +95,18 @@ class TypeSmokeTest(unittest.TestCase): def testTimespec(self): now = time.time() - timespec = cygrpc.Timespec(now) - self.assertAlmostEqual(now, float(timespec), places=8) + now_timespec_a = cygrpc.Timespec(now) + now_timespec_b = cygrpc.Timespec(now) + self.assertAlmostEqual(now, float(now_timespec_a), places=8) + self.assertEqual(now_timespec_a, now_timespec_b) + self.assertLess(cygrpc.Timespec(now - 1), cygrpc.Timespec(now)) + self.assertGreater(cygrpc.Timespec(now + 1), cygrpc.Timespec(now)) + self.assertGreaterEqual(cygrpc.Timespec(now + 1), cygrpc.Timespec(now)) + self.assertGreaterEqual(cygrpc.Timespec(now), cygrpc.Timespec(now)) + self.assertLessEqual(cygrpc.Timespec(now - 1), cygrpc.Timespec(now)) + self.assertLessEqual(cygrpc.Timespec(now), cygrpc.Timespec(now)) + self.assertNotEqual(cygrpc.Timespec(now - 1), cygrpc.Timespec(now)) + self.assertNotEqual(cygrpc.Timespec(now + 1), cygrpc.Timespec(now)) def testCompletionQueueUpDown(self): completion_queue = cygrpc.CompletionQueue() @@ -204,8 +214,8 @@ class ServerClientMixin(object): self.assertTrue(event.success) self.assertIs(tag, event.tag) except Exception as error: - raise Exception("Error in '{}': {}".format(description, - error.message)) + raise Exception( + "Error in '{}': {}".format(description, error.message)) return event return test_utilities.SimpleFuture(performer) diff --git a/src/python/grpcio_tests/tests/unit/_empty_message_test.py b/src/python/grpcio_tests/tests/unit/_empty_message_test.py index 4588688ea6..155173807f 100644 --- a/src/python/grpcio_tests/tests/unit/_empty_message_test.py +++ b/src/python/grpcio_tests/tests/unit/_empty_message_test.py @@ -122,13 +122,13 @@ class EmptyMessageTest(unittest.TestCase): list(response_iterator)) def testStreamUnary(self): - response = self._channel.stream_unary(_STREAM_UNARY)(iter( - [_REQUEST] * test_constants.STREAM_LENGTH)) + response = self._channel.stream_unary(_STREAM_UNARY)( + iter([_REQUEST] * test_constants.STREAM_LENGTH)) self.assertEqual(_RESPONSE, response) def testStreamStream(self): - response_iterator = self._channel.stream_stream(_STREAM_STREAM)(iter( - [_REQUEST] * test_constants.STREAM_LENGTH)) + response_iterator = self._channel.stream_stream(_STREAM_STREAM)( + iter([_REQUEST] * test_constants.STREAM_LENGTH)) self.assertSequenceEqual([_RESPONSE] * test_constants.STREAM_LENGTH, list(response_iterator)) diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 230682e72d..d8d76cc1c5 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -215,7 +215,9 @@ grpc_slice_buffer_swap_type grpc_slice_buffer_swap_import; grpc_slice_buffer_move_into_type grpc_slice_buffer_move_into_import; grpc_slice_buffer_trim_end_type grpc_slice_buffer_trim_end_import; grpc_slice_buffer_move_first_type grpc_slice_buffer_move_first_import; +grpc_slice_buffer_move_first_into_buffer_type grpc_slice_buffer_move_first_into_buffer_import; grpc_slice_buffer_take_first_type grpc_slice_buffer_take_first_import; +grpc_slice_buffer_undo_take_first_type grpc_slice_buffer_undo_take_first_import; gpr_malloc_type gpr_malloc_import; gpr_free_type gpr_free_import; gpr_realloc_type gpr_realloc_import; @@ -504,7 +506,9 @@ void grpc_rb_load_imports(HMODULE library) { grpc_slice_buffer_move_into_import = (grpc_slice_buffer_move_into_type) GetProcAddress(library, "grpc_slice_buffer_move_into"); grpc_slice_buffer_trim_end_import = (grpc_slice_buffer_trim_end_type) GetProcAddress(library, "grpc_slice_buffer_trim_end"); grpc_slice_buffer_move_first_import = (grpc_slice_buffer_move_first_type) GetProcAddress(library, "grpc_slice_buffer_move_first"); + grpc_slice_buffer_move_first_into_buffer_import = (grpc_slice_buffer_move_first_into_buffer_type) GetProcAddress(library, "grpc_slice_buffer_move_first_into_buffer"); grpc_slice_buffer_take_first_import = (grpc_slice_buffer_take_first_type) GetProcAddress(library, "grpc_slice_buffer_take_first"); + grpc_slice_buffer_undo_take_first_import = (grpc_slice_buffer_undo_take_first_type) GetProcAddress(library, "grpc_slice_buffer_undo_take_first"); gpr_malloc_import = (gpr_malloc_type) GetProcAddress(library, "gpr_malloc"); gpr_free_import = (gpr_free_type) GetProcAddress(library, "gpr_free"); gpr_realloc_import = (gpr_realloc_type) GetProcAddress(library, "gpr_realloc"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 4c4f655b86..4eab9639ea 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -596,9 +596,15 @@ extern grpc_slice_buffer_trim_end_type grpc_slice_buffer_trim_end_import; typedef void(*grpc_slice_buffer_move_first_type)(grpc_slice_buffer *src, size_t n, grpc_slice_buffer *dst); extern grpc_slice_buffer_move_first_type grpc_slice_buffer_move_first_import; #define grpc_slice_buffer_move_first grpc_slice_buffer_move_first_import +typedef void(*grpc_slice_buffer_move_first_into_buffer_type)(grpc_exec_ctx *exec_ctx, grpc_slice_buffer *src, size_t n, void *dst); +extern grpc_slice_buffer_move_first_into_buffer_type grpc_slice_buffer_move_first_into_buffer_import; +#define grpc_slice_buffer_move_first_into_buffer grpc_slice_buffer_move_first_into_buffer_import typedef grpc_slice(*grpc_slice_buffer_take_first_type)(grpc_slice_buffer *src); extern grpc_slice_buffer_take_first_type grpc_slice_buffer_take_first_import; #define grpc_slice_buffer_take_first grpc_slice_buffer_take_first_import +typedef void(*grpc_slice_buffer_undo_take_first_type)(grpc_slice_buffer *src, grpc_slice slice); +extern grpc_slice_buffer_undo_take_first_type grpc_slice_buffer_undo_take_first_import; +#define grpc_slice_buffer_undo_take_first grpc_slice_buffer_undo_take_first_import typedef void *(*gpr_malloc_type)(size_t size); extern gpr_malloc_type gpr_malloc_import; #define gpr_malloc gpr_malloc_import diff --git a/src/ruby/tools/bin/grpc_tools_ruby_protoc b/src/ruby/tools/bin/grpc_tools_ruby_protoc index dab06e7958..7e619e74a9 100755 --- a/src/ruby/tools/bin/grpc_tools_ruby_protoc +++ b/src/ruby/tools/bin/grpc_tools_ruby_protoc @@ -30,7 +30,7 @@ require 'rbconfig' -require_relative '../os_check' +require_relative '../platform_check' ext = RbConfig::CONFIG['EXEEXT'] @@ -39,7 +39,7 @@ protoc_name = 'protoc' + ext plugin_name = 'grpc_ruby_plugin' + ext protoc_dir = File.join(File.dirname(__FILE__), - RbConfig::CONFIG['host_cpu'] + '-' + OS.os_name) + PLATFORM.architecture + '-' + PLATFORM.os_name) protoc_path = File.join(protoc_dir, protoc_name) diff --git a/src/ruby/tools/bin/grpc_tools_ruby_protoc_plugin b/src/ruby/tools/bin/grpc_tools_ruby_protoc_plugin index 4b296dedc7..e6af2fe365 100755 --- a/src/ruby/tools/bin/grpc_tools_ruby_protoc_plugin +++ b/src/ruby/tools/bin/grpc_tools_ruby_protoc_plugin @@ -30,12 +30,12 @@ require 'rbconfig' -require_relative '../os_check' +require_relative '../platform_check' plugin_name = 'grpc_ruby_plugin' + RbConfig::CONFIG['EXEEXT'] plugin_path = File.join(File.dirname(__FILE__), - RbConfig::CONFIG['host_cpu'] + '-' + OS.os_name, + PLATFORM.architecture + '-' + PLATFORM.os_name, plugin_name) exec([ plugin_path, plugin_path ], *ARGV) diff --git a/src/ruby/tools/grpc-tools.gemspec b/src/ruby/tools/grpc-tools.gemspec index 68e2a7a113..bc142ae3cb 100644 --- a/src/ruby/tools/grpc-tools.gemspec +++ b/src/ruby/tools/grpc-tools.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.description = 'protoc and the Ruby gRPC protoc plugin' s.license = 'BSD-3-Clause' - s.files = %w( version.rb os_check.rb README.md ) + s.files = %w( version.rb platform_check.rb README.md ) s.files += Dir.glob('bin/**/*') s.bindir = 'bin' diff --git a/src/ruby/tools/os_check.rb b/src/ruby/tools/platform_check.rb index 2677306457..1f4d5a68b7 100644 --- a/src/ruby/tools/os_check.rb +++ b/src/ruby/tools/platform_check.rb @@ -27,19 +27,28 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# This is based on http://stackoverflow.com/a/171011/159388 by Aaron Hinni - require 'rbconfig' -module OS - def OS.os_name +# This is based on http://stackoverflow.com/a/171011/159388 by Aaron Hinni + +module PLATFORM + def PLATFORM.os_name case RbConfig::CONFIG['host_os'] - when /cygwin|mswin|mingw|bccwin|wince|emx/ - 'windows' - when /darwin/ - 'macos' - else - 'linux' + when /cygwin|mswin|mingw|bccwin|wince|emx/ + 'windows' + when /darwin/ + 'macos' + else + 'linux' + end + end + + def PLATFORM.architecture + case RbConfig::CONFIG['host_cpu'] + when /x86_64/ + 'x86_64' + else + 'x86' end end end diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template index 38a5ca725d..6204c3e2cb 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template @@ -33,6 +33,8 @@ <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> + RUN pip install twisted h2 + # Define the default command. CMD ["bash"] diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 0d06f64795..a10be7f81b 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -45,6 +45,7 @@ #include <grpc/support/log.h> #include "src/core/lib/iomgr/iomgr.h" +#include "src/core/lib/iomgr/workqueue.h" #include "test/core/util/test_config.h" typedef struct test_pollset { @@ -60,6 +61,22 @@ typedef struct test_fd { /* num_fds should be an even number */ static void test_fd_init(test_fd *tfds, int *fds, int num_fds) { int i; + int r; + + /* Create some dummy file descriptors. Currently using pipe file descriptors + * for this test but we could use any other type of file descriptors. Also, + * since pipe() used in this test creates two fds in each call, num_fds should + * be an even number */ + GPR_ASSERT((num_fds % 2) == 0); + for (i = 0; i < num_fds; i = i + 2) { + r = pipe(fds + i); + if (r != 0) { + gpr_log(GPR_ERROR, "Error in creating pipe. %d (%s)", errno, + strerror(errno)); + return; + } + } + for (i = 0; i < num_fds; i++) { tfds[i].inner_fd = fds[i]; tfds[i].fd = grpc_fd_create(fds[i], "test_fd"); @@ -111,8 +128,80 @@ static void test_pollset_cleanup(grpc_exec_ctx *exec_ctx, } } -#define NUM_FDS 8 -#define NUM_POLLSETS 4 +static void increment(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + ++*(int *)arg; +} + +/* + * Validate that merging two workqueues preserves the closures in each queue. + * This is a regression test for a bug in + * polling_island_merge()[ev_epoll_linux.c], where the parent relationship was + * inverted. + */ +static void test_pollset_queue_merge_items() { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + const int num_fds = 2; + const int num_pollsets = 2; + const int num_closures = 4; + test_fd tfds[num_fds]; + int fds[num_fds]; + test_pollset pollsets[num_pollsets]; + grpc_closure closures[num_closures]; + int i; + int result = 0; + + test_fd_init(tfds, fds, num_fds); + test_pollset_init(pollsets, num_pollsets); + + /* Two distinct polling islands, each with their own FD and pollset. */ + for (i = 0; i < num_fds; i++) { + grpc_pollset_add_fd(&exec_ctx, pollsets[i].pollset, tfds[i].fd); + grpc_exec_ctx_flush(&exec_ctx); + } + + /* Enqeue the closures, 3 to polling island 0 and 1 to polling island 1. */ + grpc_closure_init( + closures, increment, &result, + grpc_workqueue_scheduler(grpc_fd_get_polling_island(tfds[0].fd))); + grpc_closure_init( + closures + 1, increment, &result, + grpc_workqueue_scheduler(grpc_fd_get_polling_island(tfds[0].fd))); + grpc_closure_init( + closures + 2, increment, &result, + grpc_workqueue_scheduler(grpc_fd_get_polling_island(tfds[0].fd))); + grpc_closure_init( + closures + 3, increment, &result, + grpc_workqueue_scheduler(grpc_fd_get_polling_island(tfds[1].fd))); + for (i = 0; i < num_closures; ++i) { + grpc_closure_sched(&exec_ctx, closures + i, GRPC_ERROR_NONE); + } + + /* Merge the two polling islands. */ + grpc_pollset_add_fd(&exec_ctx, pollsets[0].pollset, tfds[1].fd); + grpc_exec_ctx_flush(&exec_ctx); + + /* + * Execute the closures, verify we see each one execute when executing work on + * the merged polling island. + */ + grpc_pollset_worker *worker = NULL; + for (i = 0; i < num_closures; ++i) { + const gpr_timespec deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(2, GPR_TIMESPAN)); + gpr_mu_lock(pollsets[1].mu); + GRPC_LOG_IF_ERROR( + "grpc_pollset_work", + grpc_pollset_work(&exec_ctx, pollsets[1].pollset, &worker, + gpr_now(GPR_CLOCK_MONOTONIC), deadline)); + gpr_mu_unlock(pollsets[1].mu); + } + GPR_ASSERT(result == num_closures); + + test_fd_cleanup(&exec_ctx, tfds, num_fds); + test_pollset_cleanup(&exec_ctx, pollsets, num_pollsets); + grpc_exec_ctx_finish(&exec_ctx); +} + /* * Cases to test: * case 1) Polling islands of both fd and pollset are NULL @@ -125,28 +214,16 @@ static void test_pollset_cleanup(grpc_exec_ctx *exec_ctx, * */ static void test_add_fd_to_pollset() { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - test_fd tfds[NUM_FDS]; - int fds[NUM_FDS]; - test_pollset pollsets[NUM_POLLSETS]; + const int num_fds = 8; + const int num_pollsets = 4; + test_fd tfds[num_fds]; + int fds[num_fds]; + test_pollset pollsets[num_pollsets]; void *expected_pi = NULL; int i; - int r; - /* Create some dummy file descriptors. Currently using pipe file descriptors - * for this test but we could use any other type of file descriptors. Also, - * since pipe() used in this test creates two fds in each call, NUM_FDS should - * be an even number */ - for (i = 0; i < NUM_FDS; i = i + 2) { - r = pipe(fds + i); - if (r != 0) { - gpr_log(GPR_ERROR, "Error in creating pipe. %d (%s)", errno, - strerror(errno)); - return; - } - } - - test_fd_init(tfds, fds, NUM_FDS); - test_pollset_init(pollsets, NUM_POLLSETS); + test_fd_init(tfds, fds, num_fds); + test_pollset_init(pollsets, num_pollsets); /*Step 1. * Create three polling islands (This will exercise test case 1 and 2) with @@ -207,19 +284,19 @@ static void test_add_fd_to_pollset() { /* Compare Fd:0's polling island with that of all other Fds */ expected_pi = grpc_fd_get_polling_island(tfds[0].fd); - for (i = 1; i < NUM_FDS; i++) { + for (i = 1; i < num_fds; i++) { GPR_ASSERT(grpc_are_polling_islands_equal( expected_pi, grpc_fd_get_polling_island(tfds[i].fd))); } /* Compare Fd:0's polling island with that of all other pollsets */ - for (i = 0; i < NUM_POLLSETS; i++) { + for (i = 0; i < num_pollsets; i++) { GPR_ASSERT(grpc_are_polling_islands_equal( expected_pi, grpc_pollset_get_polling_island(pollsets[i].pollset))); } - test_fd_cleanup(&exec_ctx, tfds, NUM_FDS); - test_pollset_cleanup(&exec_ctx, pollsets, NUM_POLLSETS); + test_fd_cleanup(&exec_ctx, tfds, num_fds); + test_pollset_cleanup(&exec_ctx, pollsets, num_pollsets); grpc_exec_ctx_finish(&exec_ctx); } @@ -231,6 +308,7 @@ int main(int argc, char **argv) { poll_strategy = grpc_get_poll_strategy_name(); if (poll_strategy != NULL && strcmp(poll_strategy, "epoll") == 0) { test_add_fd_to_pollset(); + test_pollset_queue_merge_items(); } else { gpr_log(GPR_INFO, "Skipping the test. The test is only relevant for 'epoll' " diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index 38ffcd6e0e..fddaa2ba3e 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -28,11 +28,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + import argparse import os import os.path import shutil import subprocess +import sys import tempfile parser = argparse.ArgumentParser() @@ -99,6 +102,7 @@ if args.submit: python_doc_dir = os.path.join(repo_dir, 'python') doc_branch = args.doc_branch + print('Cloning your repository...') subprocess.check_call([ 'git', 'clone', 'https://{}@github.com/{}/grpc'.format( github_user, github_repository_owner) @@ -110,13 +114,20 @@ if args.submit: subprocess.check_call([ 'git', 'checkout', 'upstream/gh-pages', '-b', doc_branch ], cwd=repo_dir) + print('Updating documentation...') shutil.rmtree(python_doc_dir, ignore_errors=True) shutil.copytree(DOC_PATH, python_doc_dir) - subprocess.check_call(['git', 'add', '--all'], cwd=repo_dir) - subprocess.check_call([ - 'git', 'commit', '-m', 'Auto-update Python documentation' - ], cwd=repo_dir) - subprocess.check_call([ - 'git', 'push', '--set-upstream', 'origin', doc_branch - ], cwd=repo_dir) + print('Attempting to push documentation...') + try: + subprocess.check_call(['git', 'add', '--all'], cwd=repo_dir) + subprocess.check_call([ + 'git', 'commit', '-m', 'Auto-update Python documentation' + ], cwd=repo_dir) + subprocess.check_call([ + 'git', 'push', '--set-upstream', 'origin', doc_branch + ], cwd=repo_dir) + except subprocess.CalledProcessError: + print('Failed to push documentation. Examine this directory and push ' + 'manually: {}'.format(repo_parent_dir)) + sys.exit(1) shutil.rmtree(repo_parent_dir) diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 05e963d1e6..3a5e15d21b 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -47,5 +47,7 @@ RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install twisted h2 + # Define the default command. CMD ["bash"] diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 0e88e96765..5647d9c2fc 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -163,22 +163,18 @@ pip_install_dir() { PWD=`pwd` cd $1 ($VENV_PYTHON setup.py build_ext -c $TOOLCHAIN || true) - # install the dependencies - $VENV_PYTHON -m pip install --upgrade . - # ensure that we've reinstalled the test packages - $VENV_PYTHON -m pip install --upgrade --force-reinstall --no-deps . + $VENV_PYTHON -m pip install --no-deps . cd $PWD } $VENV_PYTHON -m pip install --upgrade pip $VENV_PYTHON -m pip install setuptools $VENV_PYTHON -m pip install cython +$VENV_PYTHON -m pip install six enum34 protobuf futures pip_install_dir $ROOT + $VENV_PYTHON $ROOT/tools/distrib/python/make_grpcio_tools.py pip_install_dir $ROOT/tools/distrib/python/grpcio_tools -# TODO(atash) figure out namespace packages and grpcio-tools and auditwheel -# etc... -pip_install_dir $ROOT # Build/install health checking $VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess @@ -191,6 +187,7 @@ $VENV_PYTHON $ROOT/src/python/grpcio_reflection/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_reflection # Build/install tests +$VENV_PYTHON -m pip install coverage oauth2client $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_tests |