diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/README.md | 6 | ||||
| -rw-r--r-- | tools/dockerfile/grpc_java/Dockerfile | 5 | ||||
| -rw-r--r-- | tools/dockerfile/grpc_java_base/Dockerfile | 28 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.c++.internal | 4 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.core | 2 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.core.internal | 4 | ||||
| -rwxr-xr-x | tools/gce_setup/grpc_docker.sh | 34 | ||||
| -rwxr-xr-x | tools/gce_setup/shared_startup_funcs.sh | 3 | ||||
| -rw-r--r-- | tools/jenkins/grpc_jenkins_slave/Dockerfile | 140 | ||||
| -rwxr-xr-x | tools/jenkins/run_jenkins.sh | 96 | ||||
| -rwxr-xr-x | tools/run_tests/build_node.sh | 2 | ||||
| -rwxr-xr-x | tools/run_tests/build_python.sh | 4 | ||||
| -rwxr-xr-x | tools/run_tests/jobset.py | 56 | ||||
| -rwxr-xr-x | tools/run_tests/prepare_travis.sh | 6 | ||||
| -rwxr-xr-x | tools/run_tests/run_tests.py | 51 | ||||
| -rw-r--r-- | tools/run_tests/tests.json | 174 |
16 files changed, 521 insertions, 94 deletions
diff --git a/tools/README.md b/tools/README.md index 1dbbad886f..3daf73228c 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,5 +1,11 @@ buildgen: contains the template renderer for our build system. + distpackages: contains script to generate debian packages. + dockerfile: contains all of the docker files to test gRPC. + gce_setup: contains boilerplate for running the docker files under GCE. + +jenkins: support for running tests on Jenkins + run_tests: contains python scripts to properly run the tests in parallel. diff --git a/tools/dockerfile/grpc_java/Dockerfile b/tools/dockerfile/grpc_java/Dockerfile index fa67cb2627..15fce276ba 100644 --- a/tools/dockerfile/grpc_java/Dockerfile +++ b/tools/dockerfile/grpc_java/Dockerfile @@ -34,7 +34,10 @@ RUN git clone --recursive --depth 1 https://github.com/grpc/grpc-java.git /var/l RUN cd /var/local/git/grpc-java/lib/netty && \ mvn -pl codec-http2 -am -DskipTests install clean RUN cd /var/local/git/grpc-java && \ - ./gradlew build installDist + ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true + +# Add a service_account directory containing the auth creds file +ADD service_account service_account # Specify the default command such that the interop server runs on its known testing port CMD ["/var/local/git/grpc-java/run-test-server.sh", "--use_tls=true", "--port=8030"] diff --git a/tools/dockerfile/grpc_java_base/Dockerfile b/tools/dockerfile/grpc_java_base/Dockerfile index 57b1b90fcd..55fbe94eaf 100644 --- a/tools/dockerfile/grpc_java_base/Dockerfile +++ b/tools/dockerfile/grpc_java_base/Dockerfile @@ -27,18 +27,21 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Base Dockerfile for the gRPC Java dev image -FROM grpc/base +FROM debian:latest -# Install JDK 8 +# Install JDK 8 and Git # # TODO(temiola): simplify this if/when a simpler process is available. # -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections -RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list -RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 -RUN apt-get update && apt-get -y install oracle-java8-installer && \ +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \ + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list && \ + echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && \ + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 && \ + apt-get update && \ + apt-get -y install \ + git \ + oracle-java8-installer \ + && \ apt-get clean && rm -r /var/cache/oracle-jdk8-installer/ # Install maven @@ -50,15 +53,6 @@ ENV M2_HOME /var/local/apache-maven-3.2.1 ENV PATH $PATH:$JAVA_HOME/bin:$M2_HOME/bin ENV LD_LIBRARY_PATH /usr/local/lib -# Get the protobuf source from GitHub and install it -RUN wget -O - https://github.com/google/protobuf/archive/v3.0.0-alpha-2.tar.gz | \ - tar xz && \ - cd protobuf-3.0.0-alpha-2 && \ - ./autogen.sh && \ - ./configure --prefix=/usr && \ - make -j12 && make check && make install && \ - rm -r "$(pwd)" - # Trigger download of as many Maven and Gradle artifacts as possible. We don't build grpc-java # because we don't want to install netty RUN git clone --recursive --depth 1 https://github.com/grpc/grpc-java.git && \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 3071693dd1..6d323274c9 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -60,7 +60,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = doc/ref/c++ +OUTPUT_DIRECTORY = doc/ref/c++.internal # If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and @@ -760,7 +760,7 @@ WARN_LOGFILE = # spaces. # Note: If this tag is empty the current directory is searched. -INPUT = include/grpc++/async_generic_service.h include/grpc++/async_unary_call.h include/grpc++/byte_buffer.h include/grpc++/channel_arguments.h include/grpc++/channel_interface.h include/grpc++/client_context.h include/grpc++/completion_queue.h include/grpc++/config.h include/grpc++/create_channel.h include/grpc++/credentials.h include/grpc++/generic_stub.h include/grpc++/impl/call.h include/grpc++/impl/client_unary_call.h include/grpc++/impl/grpc_library.h include/grpc++/impl/internal_stub.h include/grpc++/impl/rpc_method.h include/grpc++/impl/rpc_service_method.h include/grpc++/impl/service_type.h include/grpc++/impl/sync.h include/grpc++/impl/sync_cxx11.h include/grpc++/impl/sync_no_cxx11.h include/grpc++/impl/thd.h include/grpc++/impl/thd_cxx11.h include/grpc++/impl/thd_no_cxx11.h include/grpc++/server.h include/grpc++/server_builder.h include/grpc++/server_context.h include/grpc++/server_credentials.h include/grpc++/slice.h include/grpc++/status.h include/grpc++/status_code_enum.h include/grpc++/stream.h include/grpc++/thread_pool_interface.h include/grpc++/time.h src/cpp/client/secure_credentials.h src/cpp/server/secure_server_credentials.h src/cpp/client/channel.h src/cpp/proto/proto_utils.h src/cpp/server/thread_pool.h src/cpp/client/secure_credentials.cc src/cpp/server/secure_server_credentials.cc src/cpp/client/channel.cc src/cpp/client/channel_arguments.cc src/cpp/client/client_context.cc src/cpp/client/client_unary_call.cc src/cpp/client/create_channel.cc src/cpp/client/credentials.cc src/cpp/client/generic_stub.cc src/cpp/client/insecure_credentials.cc src/cpp/client/internal_stub.cc src/cpp/common/call.cc src/cpp/common/completion_queue.cc src/cpp/common/rpc_method.cc src/cpp/proto/proto_utils.cc src/cpp/server/async_generic_service.cc src/cpp/server/insecure_server_credentials.cc src/cpp/server/server.cc src/cpp/server/server_builder.cc src/cpp/server/server_context.cc src/cpp/server/server_credentials.cc src/cpp/server/thread_pool.cc src/cpp/util/byte_buffer.cc src/cpp/util/slice.cc src/cpp/util/status.cc src/cpp/util/time.cc +INPUT = include/grpc++/async_generic_service.h include/grpc++/async_unary_call.h include/grpc++/byte_buffer.h include/grpc++/channel_arguments.h include/grpc++/channel_interface.h include/grpc++/client_context.h include/grpc++/completion_queue.h include/grpc++/config.h include/grpc++/create_channel.h include/grpc++/credentials.h include/grpc++/generic_stub.h include/grpc++/impl/call.h include/grpc++/impl/client_unary_call.h include/grpc++/impl/grpc_library.h include/grpc++/impl/internal_stub.h include/grpc++/impl/rpc_method.h include/grpc++/impl/rpc_service_method.h include/grpc++/impl/service_type.h include/grpc++/impl/sync.h include/grpc++/impl/sync_cxx11.h include/grpc++/impl/sync_no_cxx11.h include/grpc++/impl/thd.h include/grpc++/impl/thd_cxx11.h include/grpc++/impl/thd_no_cxx11.h include/grpc++/server.h include/grpc++/server_builder.h include/grpc++/server_context.h include/grpc++/server_credentials.h include/grpc++/slice.h include/grpc++/status.h include/grpc++/status_code_enum.h include/grpc++/stream.h include/grpc++/thread_pool_interface.h include/grpc++/time.h src/cpp/client/secure_credentials.h src/cpp/server/secure_server_credentials.h src/cpp/client/channel.h src/cpp/proto/proto_utils.h src/cpp/server/thread_pool.h src/cpp/client/secure_credentials.cc src/cpp/server/secure_server_credentials.cc src/cpp/client/channel.cc src/cpp/client/channel_arguments.cc src/cpp/client/client_context.cc src/cpp/client/client_unary_call.cc src/cpp/client/create_channel.cc src/cpp/client/credentials.cc src/cpp/client/generic_stub.cc src/cpp/client/insecure_credentials.cc src/cpp/client/internal_stub.cc src/cpp/common/call.cc src/cpp/common/completion_queue.cc src/cpp/common/rpc_method.cc src/cpp/proto/proto_utils.cc src/cpp/server/async_generic_service.cc src/cpp/server/create_default_thread_pool.cc src/cpp/server/insecure_server_credentials.cc src/cpp/server/server.cc src/cpp/server/server_builder.cc src/cpp/server/server_context.cc src/cpp/server/server_credentials.cc src/cpp/server/thread_pool.cc src/cpp/util/byte_buffer.cc src/cpp/util/slice.cc src/cpp/util/status.cc src/cpp/util/time.cc # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 0fe97c6d2d..7cc96b2e06 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -760,7 +760,7 @@ WARN_LOGFILE = # spaces. # Note: If this tag is empty the current directory is searched. -INPUT = include/grpc/grpc_security.h include/grpc/byte_buffer.h include/grpc/byte_buffer_reader.h include/grpc/grpc.h include/grpc/status.h include/grpc/census.h include/grpc/support/alloc.h include/grpc/support/atm.h include/grpc/support/atm_gcc_atomic.h include/grpc/support/atm_gcc_sync.h include/grpc/support/atm_win32.h include/grpc/support/cancellable_platform.h include/grpc/support/cmdline.h include/grpc/support/cpu.h include/grpc/support/histogram.h include/grpc/support/host_port.h include/grpc/support/log.h include/grpc/support/log_win32.h include/grpc/support/port_platform.h include/grpc/support/slice.h include/grpc/support/slice_buffer.h include/grpc/support/subprocess.h include/grpc/support/sync.h include/grpc/support/sync_generic.h include/grpc/support/sync_posix.h include/grpc/support/sync_win32.h include/grpc/support/thd.h include/grpc/support/time.h include/grpc/support/tls.h include/grpc/support/tls_gcc.h include/grpc/support/tls_msvc.h include/grpc/support/tls_pthread.h include/grpc/support/useful.h +INPUT = include/grpc/grpc_security.h include/grpc/byte_buffer.h include/grpc/byte_buffer_reader.h include/grpc/compression.h include/grpc/grpc.h include/grpc/status.h include/grpc/census.h include/grpc/support/alloc.h include/grpc/support/atm.h include/grpc/support/atm_gcc_atomic.h include/grpc/support/atm_gcc_sync.h include/grpc/support/atm_win32.h include/grpc/support/cancellable_platform.h include/grpc/support/cmdline.h include/grpc/support/cpu.h include/grpc/support/histogram.h include/grpc/support/host_port.h include/grpc/support/log.h include/grpc/support/log_win32.h include/grpc/support/port_platform.h include/grpc/support/slice.h include/grpc/support/slice_buffer.h include/grpc/support/string_util.h include/grpc/support/subprocess.h include/grpc/support/sync.h include/grpc/support/sync_generic.h include/grpc/support/sync_posix.h include/grpc/support/sync_win32.h include/grpc/support/thd.h include/grpc/support/time.h include/grpc/support/tls.h include/grpc/support/tls_gcc.h include/grpc/support/tls_msvc.h include/grpc/support/tls_pthread.h include/grpc/support/useful.h # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 34295a5c2a..c71676568e 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -60,7 +60,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = doc/ref/core +OUTPUT_DIRECTORY = doc/ref/core.internal # If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and @@ -760,7 +760,7 @@ WARN_LOGFILE = # spaces. # Note: If this tag is empty the current directory is searched. -INPUT = include/grpc/grpc_security.h include/grpc/byte_buffer.h include/grpc/byte_buffer_reader.h include/grpc/grpc.h include/grpc/status.h include/grpc/census.h src/core/httpcli/format_request.h src/core/httpcli/httpcli.h src/core/httpcli/httpcli_security_connector.h src/core/httpcli/parser.h src/core/security/auth_filters.h src/core/security/base64.h src/core/security/credentials.h src/core/security/json_token.h src/core/security/secure_endpoint.h src/core/security/secure_transport_setup.h src/core/security/security_connector.h src/core/security/security_context.h src/core/tsi/fake_transport_security.h src/core/tsi/ssl_transport_security.h src/core/tsi/transport_security.h src/core/tsi/transport_security_interface.h src/core/census/grpc_context.h src/core/channel/channel_args.h src/core/channel/channel_stack.h src/core/channel/child_channel.h src/core/channel/client_channel.h src/core/channel/client_setup.h src/core/channel/connected_channel.h src/core/channel/http_client_filter.h src/core/channel/http_server_filter.h src/core/channel/noop_filter.h src/core/compression/algorithm.h src/core/compression/message_compress.h src/core/debug/trace.h src/core/iomgr/alarm.h src/core/iomgr/alarm_heap.h src/core/iomgr/alarm_internal.h src/core/iomgr/endpoint.h src/core/iomgr/endpoint_pair.h src/core/iomgr/fd_posix.h src/core/iomgr/iocp_windows.h src/core/iomgr/iomgr.h src/core/iomgr/iomgr_internal.h src/core/iomgr/iomgr_posix.h src/core/iomgr/pollset.h src/core/iomgr/pollset_kick_posix.h src/core/iomgr/pollset_posix.h src/core/iomgr/pollset_set_posix.h src/core/iomgr/pollset_set_windows.h src/core/iomgr/pollset_windows.h src/core/iomgr/resolve_address.h src/core/iomgr/sockaddr.h src/core/iomgr/sockaddr_posix.h src/core/iomgr/sockaddr_utils.h src/core/iomgr/sockaddr_win32.h src/core/iomgr/socket_utils_posix.h src/core/iomgr/socket_windows.h src/core/iomgr/tcp_client.h src/core/iomgr/tcp_posix.h src/core/iomgr/tcp_server.h src/core/iomgr/tcp_windows.h src/core/iomgr/time_averaged_stats.h src/core/iomgr/wakeup_fd_pipe.h src/core/iomgr/wakeup_fd_posix.h src/core/json/json.h src/core/json/json_common.h src/core/json/json_reader.h src/core/json/json_writer.h src/core/profiling/timers.h src/core/profiling/timers_preciseclock.h src/core/surface/byte_buffer_queue.h src/core/surface/call.h src/core/surface/channel.h src/core/surface/client.h src/core/surface/completion_queue.h src/core/surface/event_string.h src/core/surface/init.h src/core/surface/server.h src/core/surface/surface_trace.h src/core/transport/chttp2/alpn.h src/core/transport/chttp2/bin_encoder.h src/core/transport/chttp2/frame.h src/core/transport/chttp2/frame_data.h src/core/transport/chttp2/frame_goaway.h src/core/transport/chttp2/frame_ping.h src/core/transport/chttp2/frame_rst_stream.h src/core/transport/chttp2/frame_settings.h src/core/transport/chttp2/frame_window_update.h src/core/transport/chttp2/hpack_parser.h src/core/transport/chttp2/hpack_table.h src/core/transport/chttp2/http2_errors.h src/core/transport/chttp2/huffsyms.h src/core/transport/chttp2/status_conversion.h src/core/transport/chttp2/stream_encoder.h src/core/transport/chttp2/stream_map.h src/core/transport/chttp2/timeout_encoding.h src/core/transport/chttp2/varint.h src/core/transport/chttp2_transport.h src/core/transport/metadata.h src/core/transport/stream_op.h src/core/transport/transport.h src/core/transport/transport_impl.h src/core/census/context.h src/core/httpcli/format_request.c src/core/httpcli/httpcli.c src/core/httpcli/httpcli_security_connector.c src/core/httpcli/parser.c src/core/security/base64.c src/core/security/client_auth_filter.c src/core/security/credentials.c src/core/security/credentials_metadata.c src/core/security/credentials_posix.c src/core/security/credentials_win32.c src/core/security/google_default_credentials.c src/core/security/json_token.c src/core/security/secure_endpoint.c src/core/security/secure_transport_setup.c src/core/security/security_connector.c src/core/security/security_context.c src/core/security/server_auth_filter.c src/core/security/server_secure_chttp2.c src/core/surface/init_secure.c src/core/surface/secure_channel_create.c src/core/tsi/fake_transport_security.c src/core/tsi/ssl_transport_security.c src/core/tsi/transport_security.c src/core/census/grpc_context.c src/core/channel/channel_args.c src/core/channel/channel_stack.c src/core/channel/child_channel.c src/core/channel/client_channel.c src/core/channel/client_setup.c src/core/channel/connected_channel.c src/core/channel/http_client_filter.c src/core/channel/http_server_filter.c src/core/channel/noop_filter.c src/core/compression/algorithm.c src/core/compression/message_compress.c src/core/debug/trace.c src/core/iomgr/alarm.c src/core/iomgr/alarm_heap.c src/core/iomgr/endpoint.c src/core/iomgr/endpoint_pair_posix.c src/core/iomgr/endpoint_pair_windows.c src/core/iomgr/fd_posix.c src/core/iomgr/iocp_windows.c src/core/iomgr/iomgr.c src/core/iomgr/iomgr_posix.c src/core/iomgr/iomgr_windows.c src/core/iomgr/pollset_kick_posix.c src/core/iomgr/pollset_multipoller_with_epoll.c src/core/iomgr/pollset_multipoller_with_poll_posix.c src/core/iomgr/pollset_posix.c src/core/iomgr/pollset_set_posix.c src/core/iomgr/pollset_set_windows.c src/core/iomgr/pollset_windows.c src/core/iomgr/resolve_address_posix.c src/core/iomgr/resolve_address_windows.c src/core/iomgr/sockaddr_utils.c src/core/iomgr/socket_utils_common_posix.c src/core/iomgr/socket_utils_linux.c src/core/iomgr/socket_utils_posix.c src/core/iomgr/socket_windows.c src/core/iomgr/tcp_client_posix.c src/core/iomgr/tcp_client_windows.c src/core/iomgr/tcp_posix.c src/core/iomgr/tcp_server_posix.c src/core/iomgr/tcp_server_windows.c src/core/iomgr/tcp_windows.c src/core/iomgr/time_averaged_stats.c src/core/iomgr/wakeup_fd_eventfd.c src/core/iomgr/wakeup_fd_nospecial.c src/core/iomgr/wakeup_fd_pipe.c src/core/iomgr/wakeup_fd_posix.c src/core/json/json.c src/core/json/json_reader.c src/core/json/json_string.c src/core/json/json_writer.c src/core/profiling/basic_timers.c src/core/profiling/stap_timers.c src/core/surface/byte_buffer.c src/core/surface/byte_buffer_queue.c src/core/surface/byte_buffer_reader.c src/core/surface/call.c src/core/surface/call_details.c src/core/surface/call_log_batch.c src/core/surface/channel.c src/core/surface/channel_create.c src/core/surface/client.c src/core/surface/completion_queue.c src/core/surface/event_string.c src/core/surface/init.c src/core/surface/lame_client.c src/core/surface/metadata_array.c src/core/surface/server.c src/core/surface/server_chttp2.c src/core/surface/server_create.c src/core/surface/surface_trace.c src/core/transport/chttp2/alpn.c src/core/transport/chttp2/bin_encoder.c src/core/transport/chttp2/frame_data.c src/core/transport/chttp2/frame_goaway.c src/core/transport/chttp2/frame_ping.c src/core/transport/chttp2/frame_rst_stream.c src/core/transport/chttp2/frame_settings.c src/core/transport/chttp2/frame_window_update.c src/core/transport/chttp2/hpack_parser.c src/core/transport/chttp2/hpack_table.c src/core/transport/chttp2/huffsyms.c src/core/transport/chttp2/status_conversion.c src/core/transport/chttp2/stream_encoder.c src/core/transport/chttp2/stream_map.c src/core/transport/chttp2/timeout_encoding.c src/core/transport/chttp2/varint.c src/core/transport/chttp2_transport.c src/core/transport/metadata.c src/core/transport/stream_op.c src/core/transport/transport.c src/core/transport/transport_op_string.c src/core/census/context.c src/core/census/initialize.c include/grpc/support/alloc.h include/grpc/support/atm.h include/grpc/support/atm_gcc_atomic.h include/grpc/support/atm_gcc_sync.h include/grpc/support/atm_win32.h include/grpc/support/cancellable_platform.h include/grpc/support/cmdline.h include/grpc/support/cpu.h include/grpc/support/histogram.h include/grpc/support/host_port.h include/grpc/support/log.h include/grpc/support/log_win32.h include/grpc/support/port_platform.h include/grpc/support/slice.h include/grpc/support/slice_buffer.h include/grpc/support/subprocess.h include/grpc/support/sync.h include/grpc/support/sync_generic.h include/grpc/support/sync_posix.h include/grpc/support/sync_win32.h include/grpc/support/thd.h include/grpc/support/time.h include/grpc/support/tls.h include/grpc/support/tls_gcc.h include/grpc/support/tls_msvc.h include/grpc/support/tls_pthread.h include/grpc/support/useful.h src/core/support/env.h src/core/support/file.h src/core/support/murmur_hash.h src/core/support/string.h src/core/support/string_win32.h src/core/support/thd_internal.h src/core/support/alloc.c src/core/support/cancellable.c src/core/support/cmdline.c src/core/support/cpu_iphone.c src/core/support/cpu_linux.c src/core/support/cpu_posix.c src/core/support/cpu_windows.c src/core/support/env_linux.c src/core/support/env_posix.c src/core/support/env_win32.c src/core/support/file.c src/core/support/file_posix.c src/core/support/file_win32.c src/core/support/histogram.c src/core/support/host_port.c src/core/support/log.c src/core/support/log_android.c src/core/support/log_linux.c src/core/support/log_posix.c src/core/support/log_win32.c src/core/support/murmur_hash.c src/core/support/slice.c src/core/support/slice_buffer.c src/core/support/string.c src/core/support/string_posix.c src/core/support/string_win32.c src/core/support/subprocess_posix.c src/core/support/sync.c src/core/support/sync_posix.c src/core/support/sync_win32.c src/core/support/thd.c src/core/support/thd_posix.c src/core/support/thd_win32.c src/core/support/time.c src/core/support/time_posix.c src/core/support/time_win32.c src/core/support/tls_pthread.c +INPUT = include/grpc/grpc_security.h include/grpc/byte_buffer.h include/grpc/byte_buffer_reader.h include/grpc/compression.h include/grpc/grpc.h include/grpc/status.h include/grpc/census.h src/core/httpcli/format_request.h src/core/httpcli/httpcli.h src/core/httpcli/httpcli_security_connector.h src/core/httpcli/parser.h src/core/security/auth_filters.h src/core/security/base64.h src/core/security/credentials.h src/core/security/json_token.h src/core/security/secure_endpoint.h src/core/security/secure_transport_setup.h src/core/security/security_connector.h src/core/security/security_context.h src/core/tsi/fake_transport_security.h src/core/tsi/ssl_transport_security.h src/core/tsi/transport_security.h src/core/tsi/transport_security_interface.h src/core/census/grpc_context.h src/core/channel/census_filter.h src/core/channel/channel_args.h src/core/channel/channel_stack.h src/core/channel/child_channel.h src/core/channel/client_channel.h src/core/channel/client_setup.h src/core/channel/connected_channel.h src/core/channel/context.h src/core/channel/http_client_filter.h src/core/channel/http_server_filter.h src/core/channel/noop_filter.h src/core/compression/message_compress.h src/core/debug/trace.h src/core/iomgr/alarm.h src/core/iomgr/alarm_heap.h src/core/iomgr/alarm_internal.h src/core/iomgr/endpoint.h src/core/iomgr/endpoint_pair.h src/core/iomgr/fd_posix.h src/core/iomgr/iocp_windows.h src/core/iomgr/iomgr.h src/core/iomgr/iomgr_internal.h src/core/iomgr/iomgr_posix.h src/core/iomgr/pollset.h src/core/iomgr/pollset_kick_posix.h src/core/iomgr/pollset_posix.h src/core/iomgr/pollset_set_posix.h src/core/iomgr/pollset_set_windows.h src/core/iomgr/pollset_windows.h src/core/iomgr/resolve_address.h src/core/iomgr/sockaddr.h src/core/iomgr/sockaddr_posix.h src/core/iomgr/sockaddr_utils.h src/core/iomgr/sockaddr_win32.h src/core/iomgr/socket_utils_posix.h src/core/iomgr/socket_windows.h src/core/iomgr/tcp_client.h src/core/iomgr/tcp_posix.h src/core/iomgr/tcp_server.h src/core/iomgr/tcp_windows.h src/core/iomgr/time_averaged_stats.h src/core/iomgr/wakeup_fd_pipe.h src/core/iomgr/wakeup_fd_posix.h src/core/json/json.h src/core/json/json_common.h src/core/json/json_reader.h src/core/json/json_writer.h src/core/profiling/timers.h src/core/profiling/timers_preciseclock.h src/core/surface/byte_buffer_queue.h src/core/surface/call.h src/core/surface/channel.h src/core/surface/client.h src/core/surface/completion_queue.h src/core/surface/event_string.h src/core/surface/init.h src/core/surface/server.h src/core/surface/surface_trace.h src/core/transport/chttp2/alpn.h src/core/transport/chttp2/bin_encoder.h src/core/transport/chttp2/frame.h src/core/transport/chttp2/frame_data.h src/core/transport/chttp2/frame_goaway.h src/core/transport/chttp2/frame_ping.h src/core/transport/chttp2/frame_rst_stream.h src/core/transport/chttp2/frame_settings.h src/core/transport/chttp2/frame_window_update.h src/core/transport/chttp2/hpack_parser.h src/core/transport/chttp2/hpack_table.h src/core/transport/chttp2/http2_errors.h src/core/transport/chttp2/huffsyms.h src/core/transport/chttp2/status_conversion.h src/core/transport/chttp2/stream_encoder.h src/core/transport/chttp2/stream_map.h src/core/transport/chttp2/timeout_encoding.h src/core/transport/chttp2/varint.h src/core/transport/chttp2_transport.h src/core/transport/metadata.h src/core/transport/stream_op.h src/core/transport/transport.h src/core/transport/transport_impl.h src/core/census/context.h src/core/httpcli/format_request.c src/core/httpcli/httpcli.c src/core/httpcli/httpcli_security_connector.c src/core/httpcli/parser.c src/core/security/base64.c src/core/security/client_auth_filter.c src/core/security/credentials.c src/core/security/credentials_metadata.c src/core/security/credentials_posix.c src/core/security/credentials_win32.c src/core/security/google_default_credentials.c src/core/security/json_token.c src/core/security/secure_endpoint.c src/core/security/secure_transport_setup.c src/core/security/security_connector.c src/core/security/security_context.c src/core/security/server_auth_filter.c src/core/security/server_secure_chttp2.c src/core/surface/init_secure.c src/core/surface/secure_channel_create.c src/core/tsi/fake_transport_security.c src/core/tsi/ssl_transport_security.c src/core/tsi/transport_security.c src/core/census/grpc_context.c src/core/channel/channel_args.c src/core/channel/channel_stack.c src/core/channel/child_channel.c src/core/channel/client_channel.c src/core/channel/client_setup.c src/core/channel/connected_channel.c src/core/channel/http_client_filter.c src/core/channel/http_server_filter.c src/core/channel/noop_filter.c src/core/compression/algorithm.c src/core/compression/message_compress.c src/core/debug/trace.c src/core/iomgr/alarm.c src/core/iomgr/alarm_heap.c src/core/iomgr/endpoint.c src/core/iomgr/endpoint_pair_posix.c src/core/iomgr/endpoint_pair_windows.c src/core/iomgr/fd_posix.c src/core/iomgr/iocp_windows.c src/core/iomgr/iomgr.c src/core/iomgr/iomgr_posix.c src/core/iomgr/iomgr_windows.c src/core/iomgr/pollset_kick_posix.c src/core/iomgr/pollset_multipoller_with_epoll.c src/core/iomgr/pollset_multipoller_with_poll_posix.c src/core/iomgr/pollset_posix.c src/core/iomgr/pollset_set_posix.c src/core/iomgr/pollset_set_windows.c src/core/iomgr/pollset_windows.c src/core/iomgr/resolve_address_posix.c src/core/iomgr/resolve_address_windows.c src/core/iomgr/sockaddr_utils.c src/core/iomgr/socket_utils_common_posix.c src/core/iomgr/socket_utils_linux.c src/core/iomgr/socket_utils_posix.c src/core/iomgr/socket_windows.c src/core/iomgr/tcp_client_posix.c src/core/iomgr/tcp_client_windows.c src/core/iomgr/tcp_posix.c src/core/iomgr/tcp_server_posix.c src/core/iomgr/tcp_server_windows.c src/core/iomgr/tcp_windows.c src/core/iomgr/time_averaged_stats.c src/core/iomgr/wakeup_fd_eventfd.c src/core/iomgr/wakeup_fd_nospecial.c src/core/iomgr/wakeup_fd_pipe.c src/core/iomgr/wakeup_fd_posix.c src/core/json/json.c src/core/json/json_reader.c src/core/json/json_string.c src/core/json/json_writer.c src/core/profiling/basic_timers.c src/core/profiling/stap_timers.c src/core/surface/byte_buffer.c src/core/surface/byte_buffer_queue.c src/core/surface/byte_buffer_reader.c src/core/surface/call.c src/core/surface/call_details.c src/core/surface/call_log_batch.c src/core/surface/channel.c src/core/surface/channel_create.c src/core/surface/client.c src/core/surface/completion_queue.c src/core/surface/event_string.c src/core/surface/init.c src/core/surface/lame_client.c src/core/surface/metadata_array.c src/core/surface/server.c src/core/surface/server_chttp2.c src/core/surface/server_create.c src/core/surface/surface_trace.c src/core/transport/chttp2/alpn.c src/core/transport/chttp2/bin_encoder.c src/core/transport/chttp2/frame_data.c src/core/transport/chttp2/frame_goaway.c src/core/transport/chttp2/frame_ping.c src/core/transport/chttp2/frame_rst_stream.c src/core/transport/chttp2/frame_settings.c src/core/transport/chttp2/frame_window_update.c src/core/transport/chttp2/hpack_parser.c src/core/transport/chttp2/hpack_table.c src/core/transport/chttp2/huffsyms.c src/core/transport/chttp2/status_conversion.c src/core/transport/chttp2/stream_encoder.c src/core/transport/chttp2/stream_map.c src/core/transport/chttp2/timeout_encoding.c src/core/transport/chttp2/varint.c src/core/transport/chttp2_transport.c src/core/transport/metadata.c src/core/transport/stream_op.c src/core/transport/transport.c src/core/transport/transport_op_string.c src/core/census/context.c src/core/census/initialize.c include/grpc/support/alloc.h include/grpc/support/atm.h include/grpc/support/atm_gcc_atomic.h include/grpc/support/atm_gcc_sync.h include/grpc/support/atm_win32.h include/grpc/support/cancellable_platform.h include/grpc/support/cmdline.h include/grpc/support/cpu.h include/grpc/support/histogram.h include/grpc/support/host_port.h include/grpc/support/log.h include/grpc/support/log_win32.h include/grpc/support/port_platform.h include/grpc/support/slice.h include/grpc/support/slice_buffer.h include/grpc/support/string_util.h include/grpc/support/subprocess.h include/grpc/support/sync.h include/grpc/support/sync_generic.h include/grpc/support/sync_posix.h include/grpc/support/sync_win32.h include/grpc/support/thd.h include/grpc/support/time.h include/grpc/support/tls.h include/grpc/support/tls_gcc.h include/grpc/support/tls_msvc.h include/grpc/support/tls_pthread.h include/grpc/support/useful.h src/core/support/env.h src/core/support/file.h src/core/support/murmur_hash.h src/core/support/string.h src/core/support/string_win32.h src/core/support/thd_internal.h src/core/support/alloc.c src/core/support/cancellable.c src/core/support/cmdline.c src/core/support/cpu_iphone.c src/core/support/cpu_linux.c src/core/support/cpu_posix.c src/core/support/cpu_windows.c src/core/support/env_linux.c src/core/support/env_posix.c src/core/support/env_win32.c src/core/support/file.c src/core/support/file_posix.c src/core/support/file_win32.c src/core/support/histogram.c src/core/support/host_port.c src/core/support/log.c src/core/support/log_android.c src/core/support/log_linux.c src/core/support/log_posix.c src/core/support/log_win32.c src/core/support/murmur_hash.c src/core/support/slice.c src/core/support/slice_buffer.c src/core/support/string.c src/core/support/string_posix.c src/core/support/string_win32.c src/core/support/subprocess_posix.c src/core/support/sync.c src/core/support/sync_posix.c src/core/support/sync_win32.c src/core/support/thd.c src/core/support/thd_posix.c src/core/support/thd_win32.c src/core/support/time.c src/core/support/time_posix.c src/core/support/time_win32.c src/core/support/tls_pthread.c # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index 123d0615b4..b53aa98aab 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -1274,8 +1274,7 @@ grpc_cloud_prod_gen_go_cmd() { # cmd=$($grpc_gen_test_cmd $flags) grpc_interop_gen_java_cmd() { local cmd_prefix="sudo docker run grpc/java"; - local test_script="/var/local/git/grpc-java/run-test-client.sh"; - local test_script+=" --use_test_ca=true --use_tls=true" + local test_script="/var/local/git/grpc-java/run-test-client.sh --use_test_ca=true --use_tls=true"; local the_cmd="$cmd_prefix $test_script $@"; echo $the_cmd } @@ -1287,13 +1286,40 @@ grpc_interop_gen_java_cmd() { # cmd=$($grpc_gen_test_cmd $flags) grpc_cloud_prod_gen_java_cmd() { local cmd_prefix="sudo docker run grpc/java"; - local test_script="/var/local/git/grpc-java/run-test-client.sh"; - local test_script+=" --use_tls=true" + local test_script="/var/local/git/grpc-java/run-test-client.sh --use_tls=true"; local gfe_flags=$(_grpc_prod_gfe_flags) local the_cmd="$cmd_prefix $test_script $gfe_flags $@"; echo $the_cmd } +# constructs the full dockerized java service_account auth interop test cmd. +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_cloud_prod_auth_service_account_creds_gen_java_cmd() { + local cmd_prefix="sudo docker run grpc/java"; + local test_script="/var/local/git/grpc-java/run-test-client.sh --use_tls=true"; + local gfe_flags=$(_grpc_prod_gfe_flags) + local added_gfe_flags=$(_grpc_svc_acc_test_flags) + local the_cmd="$cmd_prefix $test_script $gfe_flags $added_gfe_flags $@"; + echo $the_cmd +} + +# constructs the full dockerized java gce auth interop test cmd. +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_cloud_prod_auth_compute_engine_creds_gen_java_cmd() { + local cmd_prefix="sudo docker run grpc/java"; + local test_script="/var/local/git/grpc-java/run-test-client.sh --use_tls=true"; + local gfe_flags=$(_grpc_prod_gfe_flags) + local added_gfe_flags=$(_grpc_gce_test_flags) + local the_cmd="$cmd_prefix $test_script $gfe_flags $added_gfe_flags $@"; + echo $the_cmd +} + # constructs the full dockerized php interop test cmd. # # TODO(mlumish): update this to use the script once that's on git diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh index a3a3b59c8a..60cd9f6897 100755 --- a/tools/gce_setup/shared_startup_funcs.sh +++ b/tools/gce_setup/shared_startup_funcs.sh @@ -413,6 +413,9 @@ grpc_dockerfile_install() { [[ $image_label == "grpc/java_base" ]] && { grpc_docker_sync_github_key $dockerfile_dir/.ssh 'java_base_ssh_key' || return 1; } + [[ $image_label == "grpc/java" ]] && { + grpc_docker_sync_service_account $dockerfile_dir/service_account || return 1; + } [[ $image_label == "grpc/ruby" ]] && { grpc_docker_sync_roots_pem $dockerfile_dir/cacerts || return 1; grpc_docker_sync_service_account $dockerfile_dir/service_account || return 1; diff --git a/tools/jenkins/grpc_jenkins_slave/Dockerfile b/tools/jenkins/grpc_jenkins_slave/Dockerfile new file mode 100644 index 0000000000..16b076cbbc --- /dev/null +++ b/tools/jenkins/grpc_jenkins_slave/Dockerfile @@ -0,0 +1,140 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# A work-in-progress Dockerfile that allows running gRPC test suites +# inside a docker container. + +FROM debian:jessie + +# Install Git. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + strace \ + python-dev \ + python-setuptools \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +################## +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang + +################# +# C# dependencies + +# Update to a newer version of mono +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list + +# Install dependencies +RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ + mono-devel \ + nunit \ + nunit-console \ + monodevelop + +# Download NuGet +RUN cd /var/local && wget www.nuget.org/NuGet.exe +ENV NUGET mono /var/local/NuGet.exe + +# TODO(jtattermusch): add dependencies for other languages + +################## +# Node dependencies + +# Install nvm +RUN touch .profile +RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +RUN /bin/bash -l -c "nvm install 0.12" + +################## +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm use --default ruby-2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +################## +# Python dependencies + +# Install dependencies + +RUN apt-get update && apt-get install -y \ + python-all-dev \ + python3-all-dev \ + python-pip \ + python-virtualenv + +# Install Python packages from PyPI +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 + +# For sanity test +RUN pip install simplejson mako + +################## +# PHP dependencies + +# Install dependencies + +RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \ + >> /etc/apt/sources.list.d/dotdeb.list" +RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \ + >> /etc/apt/sources.list.d/dotdeb.list" +RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add - + +RUN apt-get update && apt-get install -y \ + git php5 php5-dev phpunit unzip + +# Define the default command. +CMD ["bash"] diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh new file mode 100755 index 0000000000..534ed306ef --- /dev/null +++ b/tools/jenkins/run_jenkins.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and triggers a test run based on +# env variable settings. +# +# To prevent cygwin bash complaining about empty lines ending with \r +# we set the igncr option. The option doesn't exist on Linux, so we fallback +# to just 'set -ex' there. +# NOTE: No empty lines should appear in this file before igncr is set! +set -ex -o igncr || set -ex + +if [ "$platform" == "linux" ] +then + echo "building $language on Linux" + + # Use image name based on Dockerfile checksum + DOCKER_IMAGE_NAME=grpc_jenkins_slave_`sha1sum tools/jenkins/grpc_jenkins_slave/Dockerfile | cut -f1 -d\ ` + + # Make sure docker image has been built. Should be instantaneous if so. + docker build -t $DOCKER_IMAGE_NAME tools/jenkins/grpc_jenkins_slave + + if [ "$ghprbPullId" != "" ] + then + # if we are building a pull request, grab corresponding refs. + FETCH_PULL_REQUEST_CMD="&& git fetch $GIT_URL refs/pull/$ghprbPullId/merge refs/pull/$ghprbPullId/head" + fi + + # Make sure the CID file is gone. + rm -f docker.cid + + # Run tests inside docker + docker run --cidfile=docker.cid $DOCKER_IMAGE_NAME bash -c -l "git clone --recursive $GIT_URL /var/local/git/grpc \ + && cd /var/local/git/grpc \ + $FETCH_PULL_REQUEST_CMD \ + && git checkout -f $GIT_COMMIT \ + && git submodule update \ + && nvm use 0.12 \ + && rvm use ruby-2.1 \ + && CONFIG=$config tools/run_tests/prepare_travis.sh \ + && CPPFLAGS=-I/tmp/prebuilt/include tools/run_tests/run_tests.py -t -c $config -l $language" || DOCKER_FAILED="true" + + DOCKER_CID=`cat docker.cid` + if [ "$DOCKER_FAILED" == "" ] + then + echo "Docker finished successfully, deleting the container $DOCKER_CID" + docker rm $DOCKER_CID + else + echo "Docker exited with failure, keeping container $DOCKER_CID." + echo "You can SSH to the worker and use 'docker commit CID YOUR_IMAGE_NAME' and 'docker run -i -t YOUR_IMAGE_NAME bash' to debug the problem." + exit 1 + fi + +elif [ "$platform" == "windows" ] +then + echo "building $language on Windows" + + # Prevent msbuild from picking up "platform" env variable, which would break the build + unset platform + + # TODO(jtattermusch): integrate nuget restore in a nicer way. + /cygdrive/c/nuget/nuget.exe restore vsprojects/grpc.sln + /cygdrive/c/nuget/nuget.exe restore src/csharp/Grpc.sln + + python tools/run_tests/run_tests.py -t -l $language +else + echo "Unknown platform $platform" + exit 1 +fi diff --git a/tools/run_tests/build_node.sh b/tools/run_tests/build_node.sh index c85ecf1b25..95ffb94c6e 100755 --- a/tools/run_tests/build_node.sh +++ b/tools/run_tests/build_node.sh @@ -41,4 +41,4 @@ export LDFLAGS=-L`pwd`/libs/$CONFIG cd src/node -npm install +npm install --unsafe-perm diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index dc5baccf9f..d9b7644f44 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -37,6 +37,6 @@ root=`pwd` rm -rf python2.7_virtual_environment virtualenv -p /usr/bin/python2.7 python2.7_virtual_environment source python2.7_virtual_environment/bin/activate -pip install enum34==1.0.4 futures==2.2.0 protobuf==3.0.0a2 -CFLAGS=-I$root/include LDFLAGS=-L$root/libs/$CONFIG pip install src/python/src +pip install -r src/python/requirements.txt +CFLAGS="-I$root/include -std=c89 -Werror" LDFLAGS=-L$root/libs/$CONFIG pip install src/python/src pip install src/python/interop diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 51d61db7f6..058a30d1ce 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -33,7 +33,6 @@ import hashlib import multiprocessing import os import platform -import random import signal import subprocess import sys @@ -59,40 +58,6 @@ else: signal.signal(signal.SIGALRM, alarm_handler) -def shuffle_iteratable(it): - """Return an iterable that randomly walks it""" - # take a random sampling from the passed in iterable - # we take an element with probability 1/p and rapidly increase - # p as we take elements - this gives us a somewhat random set of values before - # we've seen all the values, but starts producing values without having to - # compute ALL of them at once, allowing tests to start a little earlier - LARGE_THRESHOLD = 1000 - nextit = [] - p = 1 - for val in it: - if random.randint(0, p) == 0: - p = min(p*2, 100) - yield val - else: - nextit.append(val) - # if the input iterates over a large number of values (potentially - # infinite, we'd be in the loop for a while (again, potentially forever). - # We need to reset "nextit" every so often to, in the case of an infinite - # iterator, avoid growing "nextit" without ever freeing it. - if len(nextit) > LARGE_THRESHOLD: - random.shuffle(nextit) - for val in nextit: - yield val - nextit = [] - p = 1 - - # after taking a random sampling, we shuffle the rest of the elements and - # yield them - random.shuffle(nextit) - for val in nextit: - yield val - - _SUCCESS = object() _FAILURE = object() _RUNNING = object() @@ -258,6 +223,7 @@ class Jobset(object): self._travis = travis self._cache = cache self._stop_on_failure = stop_on_failure + self._hashes = {} def start(self, spec): """Start a job. Return True on success, False on failure.""" @@ -266,11 +232,15 @@ class Jobset(object): self.reap() if self.cancelled(): return False if spec.hash_targets: - bin_hash = hashlib.sha1() - for fn in spec.hash_targets: - with open(which(fn)) as f: - bin_hash.update(f.read()) - bin_hash = bin_hash.hexdigest() + if spec.identity() in self._hashes: + bin_hash = self._hashes[spec.identity()] + else: + bin_hash = hashlib.sha1() + for fn in spec.hash_targets: + with open(which(fn)) as f: + bin_hash.update(f.read()) + bin_hash = bin_hash.hexdigest() + self._hashes[spec.identity()] = bin_hash should_run = self._cache.should_run(spec.identity(), bin_hash) else: bin_hash = None @@ -301,6 +271,7 @@ class Jobset(object): for job in self._running: job.kill() dead.add(job) + break for job in dead: self._completed += 1 self._running.remove(job) @@ -358,11 +329,6 @@ def run(cmdlines, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, cache if cache is not None else NoCache()) - # We can't sort an infinite sequence of runs. - if not travis or infinite_runs: - cmdlines = shuffle_iteratable(cmdlines) - else: - cmdlines = sorted(cmdlines, key=lambda x: x.shortname) for cmdline in cmdlines: if not js.start(cmdline): break diff --git a/tools/run_tests/prepare_travis.sh b/tools/run_tests/prepare_travis.sh index 34a058f2a8..10546535e8 100755 --- a/tools/run_tests/prepare_travis.sh +++ b/tools/run_tests/prepare_travis.sh @@ -32,17 +32,17 @@ cd `dirname $0`/../.. grpc_dir=`pwd` distrib=`md5sum /etc/issue | cut -f1 -d\ ` -echo "Configuring for disbribution $distrib" +echo "Configuring for distribution $distrib" git submodule | while read sha path extra ; do cd /tmp name=`basename $path` file=$name-$sha-$CONFIG-prebuilt-$distrib.tar.gz - echo -n "$file ..." + echo -n "Looking for $file ..." url=http://storage.googleapis.com/grpc-prebuilt-packages/$file wget -q $url && ( echo " Found." tar xfz $file - ) || true + ) || echo " Not found." done mkdir -p bins/$CONFIG/protobuf diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 85885dd79f..4235edc1a2 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -36,20 +36,23 @@ import itertools import json import multiprocessing import os +import platform +import random import re +import subprocess import sys import time -import platform -import subprocess import jobset import watch_dirs - ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(ROOT) +_FORCE_ENVIRON_FOR_WRAPPERS = {} + + # SimpleConfig: just compile with CONFIG=config, and run the binary to test class SimpleConfig(object): @@ -123,7 +126,7 @@ class CLanguage(object): if travis and target['flaky']: continue if self.platform == 'windows': - binary = 'vsprojects\\test_bin\\%s.exe' % (target['name']) + binary = 'vsprojects/test_bin/%s.exe' % (target['name']) else: binary = 'bins/%s/%s' % (config.build_config, target['name']) out.append(config.job_spec([binary], [binary])) @@ -146,7 +149,7 @@ class NodeLanguage(object): def test_specs(self, config, travis): return [config.job_spec(['tools/run_tests/run_node.sh'], None, - environ={'GRPC_TRACE': 'surface,batch'})] + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def make_targets(self): return ['static_c', 'shared_c'] @@ -165,7 +168,7 @@ class PhpLanguage(object): def test_specs(self, config, travis): return [config.job_spec(['src/php/bin/run_tests.sh'], None, - environ={'GRPC_TRACE': 'surface,batch'})] + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def make_targets(self): return ['static_c', 'shared_c'] @@ -190,13 +193,13 @@ class PythonLanguage(object): modules = [config.job_spec(['tools/run_tests/run_python.sh', '-m', test['module']], None, - environ={'GRPC_TRACE': 'surface,batch'}, + environ=_FORCE_ENVIRON_FOR_WRAPPERS, shortname=test['module']) for test in self._tests if 'module' in test] files = [config.job_spec(['tools/run_tests/run_python.sh', test['file']], None, - environ={'GRPC_TRACE': 'surface,batch'}, + environ=_FORCE_ENVIRON_FOR_WRAPPERS, shortname=test['file']) for test in self._tests if 'file' in test] return files + modules @@ -218,7 +221,7 @@ class RubyLanguage(object): def test_specs(self, config, travis): return [config.job_spec(['tools/run_tests/run_ruby.sh'], None, - environ={'GRPC_TRACE': 'surface,batch'})] + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def make_targets(self): return ['run_dep_checks'] @@ -251,7 +254,7 @@ class CSharpLanguage(object): cmd = 'tools/run_tests/run_csharp.sh' return [config.job_spec([cmd, assembly], None, shortname=assembly, - environ={'GRPC_TRACE': 'surface,batch'}) + environ=_FORCE_ENVIRON_FOR_WRAPPERS) for assembly in assemblies ] def make_targets(self): @@ -313,7 +316,7 @@ _CONFIGS = { 'dbg': SimpleConfig('dbg'), 'opt': SimpleConfig('opt'), 'tsan': SimpleConfig('tsan', environ={ - 'TSAN_OPTIONS': 'suppressions=tools/tsan_suppressions.txt'}), + 'TSAN_OPTIONS': 'suppressions=tools/tsan_suppressions.txt:halt_on_error=1'}), 'msan': SimpleConfig('msan'), 'ubsan': SimpleConfig('ubsan'), 'asan': SimpleConfig('asan', environ={ @@ -402,6 +405,9 @@ run_configs = set(_CONFIGS[cfg] for x in args.config)) build_configs = set(cfg.build_config for cfg in run_configs) +if args.travis: + _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'surface,batch'} + make_targets = [] languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( @@ -452,6 +458,7 @@ class TestCache(object): def __init__(self, use_cache_results): self._last_successful_run = {} self._use_cache_results = use_cache_results + self._last_save = time.time() def should_run(self, cmdline, bin_hash): if cmdline not in self._last_successful_run: @@ -464,7 +471,8 @@ class TestCache(object): def finished(self, cmdline, bin_hash): self._last_successful_run[cmdline] = bin_hash - self.save() + if time.time() - self._last_save > 1: + self.save() def dump(self): return [{'cmdline': k, 'hash': v} @@ -476,6 +484,7 @@ class TestCache(object): def save(self): with open('.run_tests_cache', 'w') as f: f.write(json.dumps(self.dump())) + self._last_save = time.time() def maybe_load(self): if os.path.exists('.run_tests_cache'): @@ -491,13 +500,21 @@ def _build_and_run(check_cancelled, newline_on_success, travis, cache): return 1 # start antagonists - antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) + antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) for _ in range(0, args.antagonists)] try: infinite_runs = runs_per_test == 0 - # run all the tests - runs_sequence = (itertools.repeat(one_run) if infinite_runs - else itertools.repeat(one_run, runs_per_test)) + # When running on travis, we want out test runs to be as similar as possible + # for reproducibility purposes. + if travis: + massaged_one_run = sorted(one_run, key=lambda x: x.shortname) + else: + # whereas otherwise, we want to shuffle things up to give all tests a + # chance to run. + massaged_one_run = list(one_run) # random.shuffle needs an indexable seq. + random.shuffle(massaged_one_run) # which it modifies in-place. + runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs + else itertools.repeat(massaged_one_run, runs_per_test)) all_runs = itertools.chain.from_iterable(runs_sequence) if not jobset.run(all_runs, check_cancelled, newline_on_success=newline_on_success, travis=travis, @@ -510,6 +527,8 @@ def _build_and_run(check_cancelled, newline_on_success, travis, cache): for antagonist in antagonists: antagonist.kill() + if cache: cache.save() + return 0 diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 81d32d3edb..9fc942df2e 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -351,6 +351,14 @@ { "flaky": false, "language": "c", + "name": "httpcli_test", + "platforms": [ + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "json_rewrite_test", "platforms": [ "windows", @@ -635,6 +643,24 @@ { "flaky": false, "language": "c++", + "name": "qps_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c++", + "name": "qps_test_openloop", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c++", "name": "server_crash_test", "platforms": [ "windows", @@ -896,6 +922,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_fake_security_request_with_flags_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_fake_security_request_with_large_metadata_test", "platforms": [ "windows", @@ -1157,6 +1192,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_fullstack_request_with_flags_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_fullstack_request_with_large_metadata_test", "platforms": [ "windows", @@ -1395,6 +1439,14 @@ { "flaky": false, "language": "c", + "name": "chttp2_fullstack_uds_posix_request_with_flags_test", + "platforms": [ + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_fullstack_uds_posix_request_with_large_metadata_test", "platforms": [ "posix" @@ -1627,6 +1679,14 @@ { "flaky": false, "language": "c", + "name": "chttp2_fullstack_with_poll_request_with_flags_test", + "platforms": [ + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_fullstack_with_poll_request_with_large_metadata_test", "platforms": [ "posix" @@ -1882,6 +1942,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_simple_ssl_fullstack_request_with_flags_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_simple_ssl_fullstack_request_with_large_metadata_test", "platforms": [ "windows", @@ -2120,6 +2189,14 @@ { "flaky": false, "language": "c", + "name": "chttp2_simple_ssl_fullstack_with_poll_request_with_flags_test", + "platforms": [ + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_simple_ssl_fullstack_with_poll_request_with_large_metadata_test", "platforms": [ "posix" @@ -2375,6 +2452,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_simple_ssl_with_oauth2_fullstack_request_with_flags_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_simple_ssl_with_oauth2_fullstack_request_with_large_metadata_test", "platforms": [ "windows", @@ -2636,6 +2722,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_socket_pair_request_with_flags_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_socket_pair_request_with_large_metadata_test", "platforms": [ "windows", @@ -2897,6 +2992,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_socket_pair_one_byte_at_a_time_request_with_flags_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_socket_pair_one_byte_at_a_time_request_with_large_metadata_test", "platforms": [ "windows", @@ -3158,6 +3262,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_socket_pair_with_grpc_trace_request_with_flags_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_socket_pair_with_grpc_trace_request_with_large_metadata_test", "platforms": [ "windows", @@ -3410,6 +3523,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_fullstack_request_with_flags_unsecure_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_fullstack_request_with_large_metadata_unsecure_test", "platforms": [ "windows", @@ -3640,6 +3762,14 @@ { "flaky": false, "language": "c", + "name": "chttp2_fullstack_uds_posix_request_with_flags_unsecure_test", + "platforms": [ + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_fullstack_uds_posix_request_with_large_metadata_unsecure_test", "platforms": [ "posix" @@ -3864,6 +3994,14 @@ { "flaky": false, "language": "c", + "name": "chttp2_fullstack_with_poll_request_with_flags_unsecure_test", + "platforms": [ + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_fullstack_with_poll_request_with_large_metadata_unsecure_test", "platforms": [ "posix" @@ -4110,6 +4248,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_socket_pair_request_with_flags_unsecure_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_socket_pair_request_with_large_metadata_unsecure_test", "platforms": [ "windows", @@ -4362,6 +4509,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_socket_pair_one_byte_at_a_time_request_with_flags_unsecure_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_socket_pair_one_byte_at_a_time_request_with_large_metadata_unsecure_test", "platforms": [ "windows", @@ -4614,6 +4770,15 @@ { "flaky": false, "language": "c", + "name": "chttp2_socket_pair_with_grpc_trace_request_with_flags_unsecure_test", + "platforms": [ + "windows", + "posix" + ] + }, + { + "flaky": false, + "language": "c", "name": "chttp2_socket_pair_with_grpc_trace_request_with_large_metadata_unsecure_test", "platforms": [ "windows", @@ -4673,6 +4838,15 @@ "windows", "posix" ] + }, + { + "flaky": false, + "language": "c", + "name": "initial_settings_frame_bad_client_test", + "platforms": [ + "windows", + "posix" + ] } ] |