diff options
| -rw-r--r-- | .travis.yml | 8 | ||||
| -rw-r--r-- | Makefile | 14 | ||||
| -rw-r--r-- | src/compiler/cpp_generator_helpers.h | 40 | ||||
| -rw-r--r-- | src/compiler/cpp_plugin.cc | 2 | ||||
| -rw-r--r-- | src/compiler/generator_helpers.h | 79 | ||||
| -rw-r--r-- | src/compiler/python_generator.cc | 17 | ||||
| -rw-r--r-- | templates/Makefile.template | 14 | ||||
| -rw-r--r-- | test/build/c++11.cc | 2 | ||||
| -rw-r--r-- | test/compiler/python_plugin_test.py | 136 | ||||
| -rw-r--r-- | test/core/util/port_posix.c | 5 | ||||
| -rw-r--r-- | test/core/util/test_config.c | 4 | ||||
| -rw-r--r-- | test/core/util/test_config.h | 13 | ||||
| -rw-r--r-- | tools/dockerfile/grpc_java_base/Dockerfile | 5 | ||||
| -rwxr-xr-x | tools/run_tests/jobset.py | 1 | ||||
| -rwxr-xr-x | tools/run_tests/run_python.sh | 2 | ||||
| -rwxr-xr-x | tools/run_tests/run_tests.py | 6 |
16 files changed, 204 insertions, 144 deletions
diff --git a/.travis.yml b/.travis.yml index d8c9e385bb..d770e7261f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,11 @@ before_install: - sudo apt-get update -qq - sudo apt-get install -qq libgtest-dev libgflags-dev python-virtualenv script: - - ./tools/run_tests/run_tests.py -l c -t -j 16 -c dbg - - ./tools/run_tests/run_tests.py -l c++ -t -j 16 -c dbg + - ./tools/run_tests/run_tests.py -l c -t -j 16 -c dbg -s 2.0 + - ./tools/run_tests/run_tests.py -l c++ -t -j 16 -c dbg -s 2.0 - make clean - - ./tools/run_tests/run_tests.py -l c -t -j 16 -c opt - - ./tools/run_tests/run_tests.py -l c++ -t -j 16 -c opt + - ./tools/run_tests/run_tests.py -l c -t -j 16 -c opt -s 2.0 + - ./tools/run_tests/run_tests.py -l c++ -t -j 16 -c opt -s 2.0 - ./tools/run_tests/run_tests.py -l node -t -j 16 -c opt notifications: email: false @@ -77,7 +77,7 @@ LDXX_valgrind = g++ CPPFLAGS_valgrind = -O0 OPENSSL_CFLAGS_valgrind = -DPURIFY LDFLAGS_valgrind = -DEFINES_valgrind = _DEBUG DEBUG GRPC_TEST_SLOWDOWN_FACTOR=20 +DEFINES_valgrind = _DEBUG DEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20 VALID_CONFIG_tsan = 1 REQUIRE_CUSTOM_LIBRARIES_tsan = 1 @@ -87,7 +87,7 @@ LD_tsan = clang LDXX_tsan = clang++ CPPFLAGS_tsan = -O1 -fsanitize=thread -fno-omit-frame-pointer LDFLAGS_tsan = -fsanitize=thread -DEFINES_tsan = NDEBUG GRPC_TEST_SLOWDOWN_FACTOR=10 +DEFINES_tsan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=10 VALID_CONFIG_asan = 1 REQUIRE_CUSTOM_LIBRARIES_asan = 1 @@ -97,7 +97,7 @@ LD_asan = clang LDXX_asan = clang++ CPPFLAGS_asan = -O1 -fsanitize=address -fno-omit-frame-pointer LDFLAGS_asan = -fsanitize=address -DEFINES_asan = NDEBUG GRPC_TEST_SLOWDOWN_FACTOR=5 +DEFINES_asan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=5 VALID_CONFIG_msan = 1 REQUIRE_CUSTOM_LIBRARIES_msan = 1 @@ -108,7 +108,7 @@ LDXX_msan = clang++-libc++ CPPFLAGS_msan = -O1 -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 OPENSSL_CFLAGS_msan = -DPURIFY LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -DEFINES_msan = NDEBUG GRPC_TEST_SLOWDOWN_FACTOR=20 +DEFINES_msan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20 VALID_CONFIG_ubsan = 1 REQUIRE_CUSTOM_LIBRARIES_ubsan = 1 @@ -119,7 +119,7 @@ LDXX_ubsan = clang++ CPPFLAGS_ubsan = -O1 -fsanitize=undefined -fno-omit-frame-pointer OPENSSL_CFLAGS_ubsan = -DPURIFY LDFLAGS_ubsan = -fsanitize=undefined -DEFINES_ubsan = NDEBUG GRPC_TEST_SLOWDOWN_FACTOR=10 +DEFINES_ubsan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=10 VALID_CONFIG_gcov = 1 CC_gcov = gcc @@ -178,6 +178,10 @@ CPPFLAGS += $(CPPFLAGS_$(CONFIG)) DEFINES += $(DEFINES_$(CONFIG)) INSTALL_PREFIX=\"$(prefix)\" LDFLAGS += $(LDFLAGS_$(CONFIG)) +ifdef EXTRA_DEFINES +DEFINES += $(EXTRA_DEFINES) +endif + CFLAGS += -std=c89 -pedantic ifeq ($(HAS_CXX11),true) CXXFLAGS += -std=c++11 diff --git a/src/compiler/cpp_generator_helpers.h b/src/compiler/cpp_generator_helpers.h index e3c76e0291..632ff3c8cf 100644 --- a/src/compiler/cpp_generator_helpers.h +++ b/src/compiler/cpp_generator_helpers.h @@ -38,50 +38,16 @@ #include <string> #include <google/protobuf/descriptor.h> #include <google/protobuf/descriptor.pb.h> +#include "src/compiler/generator_helpers.h" namespace grpc_cpp_generator { -inline bool StripSuffix(std::string *filename, const std::string &suffix) { - if (filename->length() >= suffix.length()) { - size_t suffix_pos = filename->length() - suffix.length(); - if (filename->compare(suffix_pos, std::string::npos, suffix) == 0) { - filename->resize(filename->size() - suffix.size()); - return true; - } - } - - return false; -} - -inline std::string StripProto(std::string filename) { - if (!StripSuffix(&filename, ".protodevel")) { - StripSuffix(&filename, ".proto"); - } - return filename; -} - -inline std::string StringReplace(std::string str, const std::string &from, - const std::string &to) { - size_t pos = 0; - - for (;;) { - pos = str.find(from, pos); - if (pos == std::string::npos) { - break; - } - str.replace(pos, from.length(), to); - pos += to.length(); - } - - return str; -} - inline std::string DotsToColons(const std::string &name) { - return StringReplace(name, ".", "::"); + return grpc_generator::StringReplace(name, ".", "::"); } inline std::string DotsToUnderscores(const std::string &name) { - return StringReplace(name, ".", "_"); + return grpc_generator::StringReplace(name, ".", "_"); } inline std::string ClassName(const google::protobuf::Descriptor *descriptor, diff --git a/src/compiler/cpp_plugin.cc b/src/compiler/cpp_plugin.cc index a421e51b78..feb158f89e 100644 --- a/src/compiler/cpp_plugin.cc +++ b/src/compiler/cpp_plugin.cc @@ -63,7 +63,7 @@ class CppGrpcGenerator : public google::protobuf::compiler::CodeGenerator { return false; } - std::string file_name = grpc_cpp_generator::StripProto(file->name()); + std::string file_name = grpc_generator::StripProto(file->name()); // Generate .pb.h Insert(context, file_name + ".pb.h", "includes", diff --git a/src/compiler/generator_helpers.h b/src/compiler/generator_helpers.h new file mode 100644 index 0000000000..0c14bb8bcf --- /dev/null +++ b/src/compiler/generator_helpers.h @@ -0,0 +1,79 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef NET_GRPC_COMPILER_GENERATOR_HELPERS_H__ +#define NET_GRPC_COMPILER_GENERATOR_HELPERS_H__ + +#include <map> +#include <string> + +namespace grpc_generator { + +inline bool StripSuffix(std::string *filename, const std::string &suffix) { + if (filename->length() >= suffix.length()) { + size_t suffix_pos = filename->length() - suffix.length(); + if (filename->compare(suffix_pos, std::string::npos, suffix) == 0) { + filename->resize(filename->size() - suffix.size()); + return true; + } + } + + return false; +} + +inline std::string StripProto(std::string filename) { + if (!StripSuffix(&filename, ".protodevel")) { + StripSuffix(&filename, ".proto"); + } + return filename; +} + +inline std::string StringReplace(std::string str, const std::string &from, + const std::string &to) { + size_t pos = 0; + + for (;;) { + pos = str.find(from, pos); + if (pos == std::string::npos) { + break; + } + str.replace(pos, from.length(), to); + pos += to.length(); + } + + return str; +} + +} // namespace grpc_generator + +#endif // NET_GRPC_COMPILER_GENERATOR_HELPERS_H__ diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index ae4d65df4c..b8d4aa509b 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -40,20 +40,19 @@ #include <sstream> #include <vector> +#include "src/compiler/generator_helpers.h" #include "src/compiler/python_generator.h" #include <google/protobuf/io/printer.h> #include <google/protobuf/io/zero_copy_stream_impl_lite.h> #include <google/protobuf/descriptor.pb.h> #include <google/protobuf/descriptor.h> -#include <google/protobuf/stubs/strutil.h> +using grpc_generator::StringReplace; +using grpc_generator::StripProto; using google::protobuf::Descriptor; using google::protobuf::FileDescriptor; -using google::protobuf::HasSuffixString; using google::protobuf::MethodDescriptor; using google::protobuf::ServiceDescriptor; -using google::protobuf::StripString; -using google::protobuf::StripSuffixString; using google::protobuf::io::Printer; using google::protobuf::io::StringOutputStream; using std::initializer_list; @@ -197,18 +196,12 @@ bool PrintStub(const ServiceDescriptor* service, return true; } -// TODO(protobuf team): See TODO for `ModuleName`. -string StripProto(const string& filename) { - const char* suffix = HasSuffixString(filename, ".protodevel") - ? ".protodevel" : ".proto"; - return StripSuffixString(filename, suffix); -} // TODO(protobuf team): Export `ModuleName` from protobuf's // `src/google/protobuf/compiler/python/python_generator.cc` file. string ModuleName(const string& filename) { string basename = StripProto(filename); - StripString(&basename, "-", '_'); - StripString(&basename, "/", '.'); + basename = StringReplace(basename, "-", "_"); + basename = StringReplace(basename, "/", "."); return basename + "_pb2"; } diff --git a/templates/Makefile.template b/templates/Makefile.template index 491f142142..0413f19e44 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -94,7 +94,7 @@ LDXX_valgrind = g++ CPPFLAGS_valgrind = -O0 OPENSSL_CFLAGS_valgrind = -DPURIFY LDFLAGS_valgrind = -DEFINES_valgrind = _DEBUG DEBUG GRPC_TEST_SLOWDOWN_FACTOR=20 +DEFINES_valgrind = _DEBUG DEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20 VALID_CONFIG_tsan = 1 REQUIRE_CUSTOM_LIBRARIES_tsan = 1 @@ -104,7 +104,7 @@ LD_tsan = clang LDXX_tsan = clang++ CPPFLAGS_tsan = -O1 -fsanitize=thread -fno-omit-frame-pointer LDFLAGS_tsan = -fsanitize=thread -DEFINES_tsan = NDEBUG GRPC_TEST_SLOWDOWN_FACTOR=10 +DEFINES_tsan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=10 VALID_CONFIG_asan = 1 REQUIRE_CUSTOM_LIBRARIES_asan = 1 @@ -114,7 +114,7 @@ LD_asan = clang LDXX_asan = clang++ CPPFLAGS_asan = -O1 -fsanitize=address -fno-omit-frame-pointer LDFLAGS_asan = -fsanitize=address -DEFINES_asan = NDEBUG GRPC_TEST_SLOWDOWN_FACTOR=5 +DEFINES_asan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=5 VALID_CONFIG_msan = 1 REQUIRE_CUSTOM_LIBRARIES_msan = 1 @@ -125,7 +125,7 @@ LDXX_msan = clang++-libc++ CPPFLAGS_msan = -O1 -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 OPENSSL_CFLAGS_msan = -DPURIFY LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -DEFINES_msan = NDEBUG GRPC_TEST_SLOWDOWN_FACTOR=20 +DEFINES_msan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20 VALID_CONFIG_ubsan = 1 REQUIRE_CUSTOM_LIBRARIES_ubsan = 1 @@ -136,7 +136,7 @@ LDXX_ubsan = clang++ CPPFLAGS_ubsan = -O1 -fsanitize=undefined -fno-omit-frame-pointer OPENSSL_CFLAGS_ubsan = -DPURIFY LDFLAGS_ubsan = -fsanitize=undefined -DEFINES_ubsan = NDEBUG GRPC_TEST_SLOWDOWN_FACTOR=10 +DEFINES_ubsan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=10 VALID_CONFIG_gcov = 1 CC_gcov = gcc @@ -195,6 +195,10 @@ CPPFLAGS += $(CPPFLAGS_$(CONFIG)) DEFINES += $(DEFINES_$(CONFIG)) INSTALL_PREFIX=\"$(prefix)\" LDFLAGS += $(LDFLAGS_$(CONFIG)) +ifdef EXTRA_DEFINES +DEFINES += EXTRA_DEFINES +endif + CFLAGS += -std=c89 -pedantic ifeq ($(HAS_CXX11),true) CXXFLAGS += -std=c++11 diff --git a/test/build/c++11.cc b/test/build/c++11.cc index 519395f20a..4822a20e7f 100644 --- a/test/build/c++11.cc +++ b/test/build/c++11.cc @@ -31,7 +31,7 @@ * */ -/* This is just a compilation test, to see if we have zlib installed. */ +/* This is just a compilation test, to see if we have C++11. */ #include <stdlib.h> #include <zlib.h> diff --git a/test/compiler/python_plugin_test.py b/test/compiler/python_plugin_test.py index 3919de1450..1981f49fbb 100644 --- a/test/compiler/python_plugin_test.py +++ b/test/compiler/python_plugin_test.py @@ -57,7 +57,6 @@ LONG_DELAY = 1 # Assigned in __main__. _build_mode = None -_port = None class _ServicerMethods(object): @@ -87,14 +86,14 @@ class _ServicerMethods(object): while self._paused: time.sleep(0) - def UnaryCall(self, request, context): + def UnaryCall(self, request, unused_context): response = self.test_pb2.SimpleResponse() response.payload.payload_type = self.test_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * request.response_size self._control() return response - def StreamingOutputCall(self, request, context): + def StreamingOutputCall(self, request, unused_context): for parameter in request.response_parameters: response = self.test_pb2.StreamingOutputCallResponse() response.payload.payload_type = self.test_pb2.COMPRESSABLE @@ -102,7 +101,7 @@ class _ServicerMethods(object): self._control() yield response - def StreamingInputCall(self, request_iter, context): + def StreamingInputCall(self, request_iter, unused_context): response = self.test_pb2.StreamingInputCallResponse() aggregated_payload_size = 0 for request in request_iter: @@ -111,7 +110,7 @@ class _ServicerMethods(object): self._control() return response - def FullDuplexCall(self, request_iter, context): + def FullDuplexCall(self, request_iter, unused_context): for request in request_iter: for parameter in request.response_parameters: response = self.test_pb2.StreamingOutputCallResponse() @@ -120,7 +119,7 @@ class _ServicerMethods(object): self._control() yield response - def HalfDuplexCall(self, request_iter, context): + def HalfDuplexCall(self, request_iter, unused_context): responses = [] for request in request_iter: for parameter in request.response_parameters: @@ -133,6 +132,7 @@ class _ServicerMethods(object): yield response +@contextlib.contextmanager def _CreateService(test_pb2, delay): """Provides a servicer backend and a stub. @@ -148,9 +148,11 @@ def _CreateService(test_pb2, delay): test_pb2: the test_pb2 module generated by this test delay: delay in seconds per response from the servicer timeout: how long the stub will wait for the servicer by default. - Returns: - A two-tuple (servicer, stub), where the servicer is the back-end of the - service bound to the stub. + + Yields: + A three-tuple (servicer_methods, servicer, stub), where the servicer is + the back-end of the service bound to the stub and the server and stub + are both activated and ready for use. """ servicer_methods = _ServicerMethods(test_pb2, delay) @@ -172,10 +174,13 @@ def _CreateService(test_pb2, delay): return servicer_methods.HalfDuplexCall(request_iter, context) servicer = Servicer() - server = getattr(test_pb2, SERVER_FACTORY_IDENTIFIER)(servicer, _port, - None, None) - stub = getattr(test_pb2, STUB_FACTORY_IDENTIFIER)('localhost', _port) - return servicer_methods, stub, server + server = getattr( + test_pb2, SERVER_FACTORY_IDENTIFIER)(servicer, 0, None, None) + with server: + port = server.port() + stub = getattr(test_pb2, STUB_FACTORY_IDENTIFIER)('localhost', port) + with stub: + yield servicer_methods, stub, server def StreamingInputRequest(test_pb2): @@ -255,25 +260,23 @@ class PythonPluginTest(unittest.TestCase): def testUpDown(self): import test_pb2 - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) - request = test_pb2.SimpleRequest(response_size=13) - with server, stub: - pass + with _CreateService( + test_pb2, DOES_NOT_MATTER_DELAY) as (servicer, stub, unused_server): + request = test_pb2.SimpleRequest(response_size=13) def testUnaryCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, NO_DELAY) - request = test_pb2.SimpleRequest(response_size=13) - with server, stub: + with _CreateService(test_pb2, NO_DELAY) as (servicer, stub, unused_server): + request = test_pb2.SimpleRequest(response_size=13) response = stub.UnaryCall(request, NORMAL_TIMEOUT) expected_response = servicer.UnaryCall(request, None) self.assertEqual(expected_response, response) def testUnaryCallAsync(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, LONG_DELAY) request = test_pb2.SimpleRequest(response_size=13) - with server, stub: + with _CreateService(test_pb2, LONG_DELAY) as ( + servicer, stub, unused_server): start_time = time.clock() response_future = stub.UnaryCall.async(request, LONG_TIMEOUT) # Check that we didn't block on the asynchronous call. @@ -285,10 +288,9 @@ class PythonPluginTest(unittest.TestCase): def testUnaryCallAsyncExpired(self): import test_pb2 # pylint: disable=g-import-not-at-top # set the timeout super low... - servicer, stub, server = _CreateService(test_pb2, - delay=DOES_NOT_MATTER_DELAY) - request = test_pb2.SimpleRequest(response_size=13) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): + request = test_pb2.SimpleRequest(response_size=13) with servicer.pause(): response_future = stub.UnaryCall.async(request, SHORT_TIMEOUT) with self.assertRaises(exceptions.ExpirationError): @@ -296,9 +298,9 @@ class PythonPluginTest(unittest.TestCase): def testUnaryCallAsyncCancelled(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = test_pb2.SimpleRequest(response_size=13) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): with servicer.pause(): response_future = stub.UnaryCall.async(request, 1) response_future.cancel() @@ -306,18 +308,17 @@ class PythonPluginTest(unittest.TestCase): def testUnaryCallAsyncFailed(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = test_pb2.SimpleRequest(response_size=13) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): with servicer.fail(): response_future = stub.UnaryCall.async(request, NORMAL_TIMEOUT) self.assertIsNotNone(response_future.exception()) def testStreamingOutputCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, NO_DELAY) request = StreamingOutputRequest(test_pb2) - with server, stub: + with _CreateService(test_pb2, NO_DELAY) as (servicer, stub, unused_server): responses = stub.StreamingOutputCall(request, NORMAL_TIMEOUT) expected_responses = servicer.StreamingOutputCall(request, None) for check in itertools.izip_longest(expected_responses, responses): @@ -326,9 +327,9 @@ class PythonPluginTest(unittest.TestCase): def testStreamingOutputCallExpired(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = StreamingOutputRequest(test_pb2) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): with servicer.pause(): responses = stub.StreamingOutputCall(request, SHORT_TIMEOUT) with self.assertRaises(exceptions.ExpirationError): @@ -336,10 +337,9 @@ class PythonPluginTest(unittest.TestCase): def testStreamingOutputCallCancelled(self): import test_pb2 # pylint: disable=g-import-not-at-top - unused_servicer, stub, server = _CreateService(test_pb2, - DOES_NOT_MATTER_DELAY) request = StreamingOutputRequest(test_pb2) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + unused_servicer, stub, unused_server): responses = stub.StreamingOutputCall(request, SHORT_TIMEOUT) next(responses) responses.cancel() @@ -350,9 +350,9 @@ class PythonPluginTest(unittest.TestCase): 'instead of raising the proper error.') def testStreamingOutputCallFailed(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = StreamingOutputRequest(test_pb2) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): with servicer.fail(): responses = stub.StreamingOutputCall(request, 1) self.assertIsNotNone(responses) @@ -361,8 +361,7 @@ class PythonPluginTest(unittest.TestCase): def testStreamingInputCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, NO_DELAY) - with server, stub: + with _CreateService(test_pb2, NO_DELAY) as (servicer, stub, unused_server): response = stub.StreamingInputCall(StreamingInputRequest(test_pb2), NORMAL_TIMEOUT) expected_response = servicer.StreamingInputCall( @@ -371,9 +370,8 @@ class PythonPluginTest(unittest.TestCase): def testStreamingInputCallAsync(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService( - test_pb2, LONG_DELAY) - with server, stub: + with _CreateService(test_pb2, LONG_DELAY) as ( + servicer, stub, unused_server): start_time = time.clock() response_future = stub.StreamingInputCall.async( StreamingInputRequest(test_pb2), LONG_TIMEOUT) @@ -386,8 +384,8 @@ class PythonPluginTest(unittest.TestCase): def testStreamingInputCallAsyncExpired(self): import test_pb2 # pylint: disable=g-import-not-at-top # set the timeout super low... - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): with servicer.pause(): response_future = stub.StreamingInputCall.async( StreamingInputRequest(test_pb2), SHORT_TIMEOUT) @@ -398,8 +396,8 @@ class PythonPluginTest(unittest.TestCase): def testStreamingInputCallAsyncCancelled(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): with servicer.pause(): response_future = stub.StreamingInputCall.async( StreamingInputRequest(test_pb2), NORMAL_TIMEOUT) @@ -410,8 +408,8 @@ class PythonPluginTest(unittest.TestCase): def testStreamingInputCallAsyncFailed(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): with servicer.fail(): response_future = stub.StreamingInputCall.async( StreamingInputRequest(test_pb2), SHORT_TIMEOUT) @@ -419,8 +417,7 @@ class PythonPluginTest(unittest.TestCase): def testFullDuplexCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, NO_DELAY) - with server, stub: + with _CreateService(test_pb2, NO_DELAY) as (servicer, stub, unused_server): responses = stub.FullDuplexCall(FullDuplexRequest(test_pb2), NORMAL_TIMEOUT) expected_responses = servicer.FullDuplexCall(FullDuplexRequest(test_pb2), @@ -431,9 +428,9 @@ class PythonPluginTest(unittest.TestCase): def testFullDuplexCallExpired(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = FullDuplexRequest(test_pb2) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): with servicer.pause(): responses = stub.FullDuplexCall(request, SHORT_TIMEOUT) with self.assertRaises(exceptions.ExpirationError): @@ -441,8 +438,7 @@ class PythonPluginTest(unittest.TestCase): def testFullDuplexCallCancelled(self): import test_pb2 # pylint: disable=g-import-not-at-top - unused_servicer, stub, server = _CreateService(test_pb2, NO_DELAY) - with server, stub: + with _CreateService(test_pb2, NO_DELAY) as (servicer, stub, unused_server): request = FullDuplexRequest(test_pb2) responses = stub.FullDuplexCall(request, NORMAL_TIMEOUT) next(responses) @@ -454,9 +450,9 @@ class PythonPluginTest(unittest.TestCase): 'and fix.') def testFullDuplexCallFailed(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = FullDuplexRequest(test_pb2) - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): with servicer.fail(): responses = stub.FullDuplexCall(request, NORMAL_TIMEOUT) self.assertIsNotNone(responses) @@ -465,16 +461,16 @@ class PythonPluginTest(unittest.TestCase): def testHalfDuplexCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub, server = _CreateService(test_pb2, NO_DELAY) - def HalfDuplexRequest(): - request = test_pb2.StreamingOutputCallRequest() - request.response_parameters.add(size=1, interval_us=0) - yield request - request = test_pb2.StreamingOutputCallRequest() - request.response_parameters.add(size=2, interval_us=0) - request.response_parameters.add(size=3, interval_us=0) - yield request - with server, stub: + with _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) as ( + servicer, stub, unused_server): + def HalfDuplexRequest(): + request = test_pb2.StreamingOutputCallRequest() + request.response_parameters.add(size=1, interval_us=0) + yield request + request = test_pb2.StreamingOutputCallRequest() + request.response_parameters.add(size=2, interval_us=0) + request.response_parameters.add(size=3, interval_us=0) + yield request responses = stub.HalfDuplexCall(HalfDuplexRequest(), NORMAL_TIMEOUT) expected_responses = servicer.HalfDuplexCall(HalfDuplexRequest(), None) for check in itertools.izip_longest(expected_responses, responses): @@ -483,7 +479,6 @@ class PythonPluginTest(unittest.TestCase): def testHalfDuplexCallWedged(self): import test_pb2 # pylint: disable=g-import-not-at-top - _, stub, server = _CreateService(test_pb2, NO_DELAY) wait_flag = [False] @contextlib.contextmanager def wait(): # pylint: disable=invalid-name @@ -497,7 +492,7 @@ class PythonPluginTest(unittest.TestCase): yield request while wait_flag[0]: time.sleep(0.1) - with server, stub: + with _CreateService(test_pb2, NO_DELAY) as (servicer, stub, unused_server): with wait(): responses = stub.HalfDuplexCall(HalfDuplexRequest(), NORMAL_TIMEOUT) # half-duplex waits for the client to send all info @@ -516,6 +511,5 @@ if __name__ == '__main__': parser.add_argument('--port', dest='port', type=int, default=0) args, remainder = parser.parse_known_args() _build_mode = args.build_mode - _port = args.port sys.argv[1:] = remainder unittest.main() diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c index 55150a7ced..36f13e1b51 100644 --- a/test/core/util/port_posix.c +++ b/test/core/util/port_posix.c @@ -117,9 +117,10 @@ int grpc_pick_unused_port(void) { for (;;) { int port; - if (try == 0) { + try++; + if (try == 1) { port = getpid() % (65536 - 30000) + 30000; - } else if (try < NUM_RANDOM_PORTS_TO_PICK) { + } else if (try <= NUM_RANDOM_PORTS_TO_PICK) { port = rand() % (65536 - 30000) + 30000; } else { port = 0; diff --git a/test/core/util/test_config.c b/test/core/util/test_config.c index 1c46407311..1f0f0175b1 100644 --- a/test/core/util/test_config.c +++ b/test/core/util/test_config.c @@ -34,6 +34,7 @@ #include "test/core/util/test_config.h" #include <grpc/support/port_platform.h> +#include <grpc/support/log.h> #include <stdlib.h> #include <signal.h> @@ -52,6 +53,9 @@ void grpc_test_init(int argc, char **argv) { /* disable SIGPIPE */ signal(SIGPIPE, SIG_IGN); #endif + gpr_log(GPR_DEBUG, "test slowdown: machine=%f build=%f total=%f", + GRPC_TEST_SLOWDOWN_MACHINE_FACTOR, GRPC_TEST_SLOWDOWN_BUILD_FACTOR, + GRPC_TEST_SLOWDOWN_FACTOR); /* seed rng with pid, so we don't end up with the same random numbers as a concurrently running test binary */ srand(seed()); diff --git a/test/core/util/test_config.h b/test/core/util/test_config.h index 74c2a3cf1b..5292c7b525 100644 --- a/test/core/util/test_config.h +++ b/test/core/util/test_config.h @@ -40,16 +40,23 @@ extern "C" { #endif /* __cplusplus */ -#ifndef GRPC_TEST_SLOWDOWN_FACTOR -#define GRPC_TEST_SLOWDOWN_FACTOR 1.0 +#ifndef GRPC_TEST_SLOWDOWN_BUILD_FACTOR +#define GRPC_TEST_SLOWDOWN_BUILD_FACTOR 1.0 #endif +#ifndef GRPC_TEST_SLOWDOWN_MACHINE_FACTOR +#define GRPC_TEST_SLOWDOWN_MACHINE_FACTOR 1.0 +#endif + +#define GRPC_TEST_SLOWDOWN_FACTOR \ + (GRPC_TEST_SLOWDOWN_BUILD_FACTOR * GRPC_TEST_SLOWDOWN_MACHINE_FACTOR) + #define GRPC_TIMEOUT_SECONDS_TO_DEADLINE(x) \ gpr_time_add(gpr_now(), \ gpr_time_from_micros(GRPC_TEST_SLOWDOWN_FACTOR * 1e6 * (x))) #define GRPC_TIMEOUT_MILLIS_TO_DEADLINE(x) \ - gpr_time_add(gpr_now(), \ + gpr_time_add(gpr_now(), \ gpr_time_from_micros(GRPC_TEST_SLOWDOWN_FACTOR * 1e3 * (x))) void grpc_test_init(int argc, char **argv); diff --git a/tools/dockerfile/grpc_java_base/Dockerfile b/tools/dockerfile/grpc_java_base/Dockerfile index feac5e745e..3eebc2bb93 100644 --- a/tools/dockerfile/grpc_java_base/Dockerfile +++ b/tools/dockerfile/grpc_java_base/Dockerfile @@ -51,13 +51,14 @@ ENV PATH $PATH:$JAVA_HOME/bin:$M2_HOME/bin ENV LD_LIBRARY_PATH /usr/local/lib # Get the protobuf source from GitHub and install it -RUN wget -O - https://github.com/google/protobuf/archive/master.tar.gz | \ +RUN wget -O - https://github.com/google/protobuf/archive/v3.0.0-alpha-2.tar.gz | \ tar xz && \ - cd protobuf-master && \ + cd protobuf-3.0.0-alpha-2 && \ ./autogen.sh && \ ./configure --prefix=/usr && \ make -j12 && make check && make install && \ cd java && mvn install && cd .. && \ + cd javanano && mvn install && cd .. && \ rm -r "$(pwd)" # Install a GitHub SSH service credential that gives access to the GitHub repo while it's private diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index b8178ffebf..ad65da535b 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -99,6 +99,7 @@ _CLEAR_LINE = '\x1b[2K' _TAG_COLOR = { 'FAILED': 'red', + 'TIMEOUT': 'red', 'PASSED': 'green', 'START': 'gray', 'WAITING': 'yellow', diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index 19f1458fab..06ddb8e41a 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -38,7 +38,7 @@ export LD_LIBRARY_PATH=$root/libs/opt source python2.7_virtual_environment/bin/activate # TODO(issue 215): Properly itemize these in run_tests.py so that they can be parallelized. # TODO(atash): Enable dynamic unused port discovery for this test. -python2.7 -B test/compiler/python_plugin_test.py --build_mode=opt --port=40987 +python2.7 -B test/compiler/python_plugin_test.py --build_mode=opt python2.7 -B -m grpc._adapter._blocking_invocation_inline_service_test python2.7 -B -m grpc._adapter._c_test python2.7 -B -m grpc._adapter._event_invocation_synchronous_event_service_test diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 22f12b019a..372321c5e3 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -44,6 +44,10 @@ import jobset import watch_dirs +ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(ROOT) + + # SimpleConfig: just compile with CONFIG=config, and run the binary to test class SimpleConfig(object): @@ -171,6 +175,7 @@ argp.add_argument('-c', '--config', argp.add_argument('-n', '--runs_per_test', default=1, type=int) argp.add_argument('-r', '--regex', default='.*', type=str) argp.add_argument('-j', '--jobs', default=1000, type=int) +argp.add_argument('-s', '--slowdown', default=1.0, type=float) argp.add_argument('-f', '--forever', default=False, action='store_const', @@ -200,6 +205,7 @@ make_targets = [] languages = set(_LANGUAGES[l] for l in args.language) build_steps = [jobset.JobSpec(['make', '-j', '%d' % (multiprocessing.cpu_count() + 1), + 'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' % args.slowdown, 'CONFIG=%s' % cfg] + list(set( itertools.chain.from_iterable( l.make_targets() for l in languages)))) |