diff options
author | Masood Malekghassemi <soltanmm@users.noreply.github.com> | 2015-02-23 15:28:07 -0800 |
---|---|---|
committer | Masood Malekghassemi <soltanmm@users.noreply.github.com> | 2015-02-24 19:11:08 -0800 |
commit | 59d9ff4d9fd596f7cf77cf4285c3519363df4af8 (patch) | |
tree | 542c4f27001461938a42337986cf2365889e68ec | |
parent | a7552b9d4cfdb2a4a36dbe87cb83eeb863aa38d9 (diff) |
Updated Python protoc plugin testing.
-rw-r--r-- | src/compiler/python_generator.cc | 367 | ||||
-rw-r--r-- | src/compiler/python_generator.h | 3 | ||||
-rw-r--r-- | src/compiler/python_plugin.cc | 16 | ||||
-rw-r--r-- | test/compiler/python_plugin_test.py | 395 | ||||
-rw-r--r-- | test/compiler/test.proto | 3 | ||||
-rwxr-xr-x | tools/run_tests/run_python.sh | 3 |
6 files changed, 443 insertions, 344 deletions
diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index cdd3d8a98a..34d5332d03 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -33,9 +33,11 @@ #include <cassert> #include <cctype> +#include <cstring> #include <map> #include <ostream> #include <sstream> +#include <vector> #include "src/compiler/python_generator.h" #include <google/protobuf/io/printer.h> @@ -43,14 +45,19 @@ #include <google/protobuf/descriptor.pb.h> #include <google/protobuf/descriptor.h> +using google::protobuf::Descriptor; using google::protobuf::FileDescriptor; using google::protobuf::ServiceDescriptor; using google::protobuf::MethodDescriptor; using google::protobuf::io::Printer; using google::protobuf::io::StringOutputStream; using std::initializer_list; +using std::make_pair; using std::map; +using std::pair; using std::string; +using std::strlen; +using std::vector; namespace grpc_python_generator { namespace { @@ -99,62 +106,81 @@ class IndentScope { // END FORMATTING BOILERPLATE // //////////////////////////////// -void PrintService(const ServiceDescriptor* service, - Printer* out) { +bool PrintServicer(const ServiceDescriptor* service, + Printer* out) { string doc = "<fill me in later!>"; map<string, string> dict = ListToDict({ "Service", service->name(), "Documentation", doc, }); - out->Print(dict, "class $Service$Service(object):\n"); + out->Print(dict, "class EarlyAdopter$Service$Servicer(object):\n"); { IndentScope raii_class_indent(out); out->Print(dict, "\"\"\"$Documentation$\"\"\"\n"); - out->Print("def __init__(self):\n"); - { - IndentScope raii_method_indent(out); - out->Print("pass\n"); + out->Print("__metaclass__ = abc.ABCMeta\n"); + for (int i = 0; i < service->method_count(); ++i) { + auto meth = service->method(i); + string arg_name = meth->client_streaming() ? + "request_iterator" : "request"; + out->Print("@abc.abstractmethod\n"); + out->Print("def $Method$(self, $ArgName$):\n", + "Method", meth->name(), "ArgName", arg_name); + { + IndentScope raii_method_indent(out); + out->Print("raise NotImplementedError()\n"); + } } } + return true; } -void PrintServicer(const ServiceDescriptor* service, - Printer* out) { +bool PrintServer(const ServiceDescriptor* service, Printer* out) { string doc = "<fill me in later!>"; map<string, string> dict = ListToDict({ "Service", service->name(), "Documentation", doc, }); - out->Print(dict, "class $Service$Servicer(object):\n"); + out->Print(dict, "class EarlyAdopter$Service$Server(object):\n"); { IndentScope raii_class_indent(out); out->Print(dict, "\"\"\"$Documentation$\"\"\"\n"); - for (int i = 0; i < service->method_count(); ++i) { - auto meth = service->method(i); - out->Print("def $Method$(self, arg):\n", "Method", meth->name()); - { - IndentScope raii_method_indent(out); - out->Print("raise NotImplementedError()\n"); - } + out->Print("__metaclass__ = abc.ABCMeta\n"); + out->Print("@abc.abstractmethod\n"); + out->Print("def start(self):\n"); + { + IndentScope raii_method_indent(out); + out->Print("raise NotImplementedError()\n"); + } + + out->Print("@abc.abstractmethod\n"); + out->Print("def stop(self):\n"); + { + IndentScope raii_method_indent(out); + out->Print("raise NotImplementedError()\n"); } } + return true; } -void PrintStub(const ServiceDescriptor* service, +bool PrintStub(const ServiceDescriptor* service, Printer* out) { string doc = "<fill me in later!>"; map<string, string> dict = ListToDict({ "Service", service->name(), "Documentation", doc, }); - out->Print(dict, "class $Service$Stub(object):\n"); + out->Print(dict, "class EarlyAdopter$Service$Stub(object):\n"); { IndentScope raii_class_indent(out); out->Print(dict, "\"\"\"$Documentation$\"\"\"\n"); + out->Print("__metaclass__ = abc.ABCMeta\n"); for (int i = 0; i < service->method_count(); ++i) { const MethodDescriptor* meth = service->method(i); - auto methdict = ListToDict({"Method", meth->name()}); - out->Print(methdict, "def $Method$(self, arg):\n"); + string arg_name = meth->client_streaming() ? + "request_iterator" : "request"; + auto methdict = ListToDict({"Method", meth->name(), "ArgName", arg_name}); + out->Print("@abc.abstractmethod\n"); + out->Print(methdict, "def $Method$(self, $ArgName$):\n"); { IndentScope raii_method_indent(out); out->Print("raise NotImplementedError()\n"); @@ -162,169 +188,190 @@ void PrintStub(const ServiceDescriptor* service, out->Print(methdict, "$Method$.async = None\n"); } } + return true; } -void PrintStubImpl(const ServiceDescriptor* service, - Printer* out) { - map<string, string> dict = ListToDict({ - "Service", service->name(), - }); - out->Print(dict, "class _$Service$Stub($Service$Stub):\n"); - { - IndentScope raii_class_indent(out); - out->Print("def __init__(self, face_stub, default_timeout):\n"); - { - IndentScope raii_method_indent(out); - out->Print("self._face_stub = face_stub\n" - "self._default_timeout = default_timeout\n" - "stub_self = self\n"); +bool GetModuleAndMessagePath(const Descriptor* type, + pair<string, string>* out) { + const Descriptor* path_elem_type = type; + vector<const Descriptor*> message_path; + do { + message_path.push_back(path_elem_type); + path_elem_type = path_elem_type->containing_type(); + } while (path_elem_type != nullptr); + string file_name = type->file()->name(); + string module_name; + static const int proto_suffix_length = strlen(".proto"); + if (!(file_name.size() > static_cast<size_t>(proto_suffix_length) && + file_name.find_last_of(".proto") == file_name.size() - 1)) { + return false; + } + module_name = file_name.substr( + 0, file_name.size() - proto_suffix_length) + "_pb2"; + string package = type->file()->package(); + string module = (package.empty() ? "" : package + ".") + + module_name; + string message_type; + for (auto path_iter = message_path.rbegin(); + path_iter != message_path.rend(); ++path_iter) { + message_type += (*path_iter)->name() + "."; + } + message_type.pop_back(); + *out = make_pair(module, message_type); + return true; +} - for (int i = 0; i < service->method_count(); ++i) { - const MethodDescriptor* meth = service->method(i); - bool server_streaming = meth->server_streaming(); - bool client_streaming = meth->client_streaming(); - std::string blocking_call, future_call; - if (server_streaming) { - if (client_streaming) { - blocking_call = "stub_self._face_stub.inline_stream_in_stream_out"; - future_call = blocking_call; - } else { - blocking_call = "stub_self._face_stub.inline_value_in_stream_out"; - future_call = blocking_call; - } - } else { - if (client_streaming) { - blocking_call = "stub_self._face_stub.blocking_stream_in_value_out"; - future_call = "stub_self._face_stub.future_stream_in_value_out"; - } else { - blocking_call = "stub_self._face_stub.blocking_value_in_value_out"; - future_call = "stub_self._face_stub.future_value_in_value_out"; - } - } - // TODO(atash): use the solution described at - // http://stackoverflow.com/a/2982 to bind 'async' attribute - // functions to def'd functions instead of using callable attributes. - auto methdict = ListToDict({ - "Method", meth->name(), - "BlockingCall", blocking_call, - "FutureCall", future_call - }); - out->Print(methdict, "class $Method$(object):\n"); - { - IndentScope raii_callable_indent(out); - out->Print("def __call__(self, arg):\n"); - { - IndentScope raii_callable_call_indent(out); - out->Print(methdict, - "return $BlockingCall$(\"$Method$\", arg, " - "stub_self._default_timeout)\n"); - } - out->Print("def async(self, arg):\n"); - { - IndentScope raii_callable_async_indent(out); - out->Print(methdict, - "return $FutureCall$(\"$Method$\", arg, " - "stub_self._default_timeout)\n"); - } - } - out->Print(methdict, "self.$Method$ = $Method$()\n"); +bool PrintServerFactory(const ServiceDescriptor* service, Printer* out) { + out->Print("def early_adopter_create_$Service$_server(servicer, port, " + "root_certificates, key_chain_pairs):\n", + "Service", service->name()); + { + IndentScope raii_create_server_indent(out); + map<string, pair<string, string>> method_to_module_and_message; + out->Print("method_implementations = {\n"); + for (int i = 0; i < service->method_count(); ++i) { + IndentScope raii_implementations_indent(out); + const MethodDescriptor* meth = service->method(i); + string meth_type = + string(meth->client_streaming() ? "stream" : "unary") + + string(meth->server_streaming() ? "_stream" : "_unary") + "_inline"; + out->Print("\"$Method$\": utilities.$Type$(servicer.$Method$),\n", + "Method", meth->name(), + "Type", meth_type); + // Maintain information on the input type of the service method for later + // use in constructing the service assembly's activated fore link. + const Descriptor* input_type = meth->input_type(); + pair<string, string> module_and_message; + if (!GetModuleAndMessagePath(input_type, &module_and_message)) { + return false; } + method_to_module_and_message.emplace( + meth->name(), module_and_message); + } + out->Print("}\n"); + // Ensure that we've imported all of the relevant messages. + for (auto& meth_vals : method_to_module_and_message) { + out->Print("import $Module$\n", + "Module", meth_vals.second.first); } + out->Print("request_deserializers = {\n"); + for (auto& meth_vals : method_to_module_and_message) { + IndentScope raii_serializers_indent(out); + string full_input_type_path = meth_vals.second.first + "." + + meth_vals.second.second; + out->Print("\"$Method$\": $Type$.FromString,\n", + "Method", meth_vals.first, + "Type", full_input_type_path); + } + out->Print("}\n"); + out->Print("response_serializers = {\n"); + for (auto& meth_vals : method_to_module_and_message) { + IndentScope raii_serializers_indent(out); + out->Print("\"$Method$\": lambda x: x.SerializeToString(),\n", + "Method", meth_vals.first); + } + out->Print("}\n"); + out->Print("link = fore.activated_fore_link(port, request_deserializers, " + "response_serializers, root_certificates, key_chain_pairs)\n"); + out->Print("return implementations.assemble_service(" + "method_implementations, link)\n"); } + return true; } -void PrintStubGenerators(const ServiceDescriptor* service, Printer* out) { +bool PrintStubFactory(const ServiceDescriptor* service, Printer* out) { map<string, string> dict = ListToDict({ "Service", service->name(), }); - // Write out a generator of linked pairs of Server/Stub - out->Print(dict, "def mock_$Service$(servicer, default_timeout):\n"); + out->Print(dict, "def early_adopter_create_$Service$_stub(host, port):\n"); { - IndentScope raii_mock_indent(out); - out->Print("value_in_value_out = {}\n" - "value_in_stream_out = {}\n" - "stream_in_value_out = {}\n" - "stream_in_stream_out = {}\n"); + IndentScope raii_create_server_indent(out); + map<string, pair<string, string>> method_to_module_and_message; + out->Print("method_implementations = {\n"); for (int i = 0; i < service->method_count(); ++i) { + IndentScope raii_implementations_indent(out); const MethodDescriptor* meth = service->method(i); - std::string super_interface, meth_dict; - bool server_streaming = meth->server_streaming(); - bool client_streaming = meth->client_streaming(); - if (server_streaming) { - if (client_streaming) { - super_interface = "InlineStreamInStreamOutMethod"; - meth_dict = "stream_in_stream_out"; - } else { - super_interface = "InlineValueInStreamOutMethod"; - meth_dict = "value_in_stream_out"; - } - } else { - if (client_streaming) { - super_interface = "InlineStreamInValueOutMethod"; - meth_dict = "stream_in_value_out"; - } else { - super_interface = "InlineValueInValueOutMethod"; - meth_dict = "value_in_value_out"; - } - } - map<string, string> methdict = ListToDict({ - "Method", meth->name(), - "SuperInterface", super_interface, - "MethodDict", meth_dict - }); - out->Print( - methdict, "class $Method$(_face_interfaces.$SuperInterface$):\n"); - { - IndentScope raii_inline_class_indent(out); - out->Print("def service(self, request, context):\n"); - { - IndentScope raii_inline_class_fn_indent(out); - out->Print(methdict, "return servicer.$Method$(request)\n"); - } + string meth_type = + string(meth->client_streaming() ? "stream" : "unary") + + string(meth->server_streaming() ? "_stream" : "_unary") + "_inline"; + // TODO(atash): once the expected input to assemble_dynamic_inline_stub is + // cleaned up, change this to the expected argument's dictionary values. + out->Print("\"$Method$\": utilities.$Type$(None),\n", + "Method", meth->name(), + "Type", meth_type); + // Maintain information on the input type of the service method for later + // use in constructing the service assembly's activated fore link. + const Descriptor* output_type = meth->output_type(); + pair<string, string> module_and_message; + if (!GetModuleAndMessagePath(output_type, &module_and_message)) { + return false; } - out->Print(methdict, "$MethodDict$['$Method$'] = $Method$()\n"); + method_to_module_and_message.emplace( + meth->name(), module_and_message); } - out->Print( - "face_linked_pair = _face_testing.server_and_stub(default_timeout," - "inline_value_in_value_out_methods=value_in_value_out," - "inline_value_in_stream_out_methods=value_in_stream_out," - "inline_stream_in_value_out_methods=stream_in_value_out," - "inline_stream_in_stream_out_methods=stream_in_stream_out)\n"); - out->Print("class LinkedPair(object):\n"); - { - IndentScope raii_linked_pair(out); - out->Print("def __init__(self, server, stub):\n"); - { - IndentScope raii_linked_pair_init(out); - out->Print("self.server = server\n" - "self.stub = stub\n"); - } + out->Print("}\n"); + // Ensure that we've imported all of the relevant messages. + for (auto& meth_vals : method_to_module_and_message) { + out->Print("import $Module$\n", + "Module", meth_vals.second.first); } - out->Print( - dict, - "stub = _$Service$Stub(face_linked_pair.stub, default_timeout)\n"); - out->Print("return LinkedPair(None, stub)\n"); + out->Print("response_deserializers = {\n"); + for (auto& meth_vals : method_to_module_and_message) { + IndentScope raii_serializers_indent(out); + string full_output_type_path = meth_vals.second.first + "." + + meth_vals.second.second; + out->Print("\"$Method$\": $Type$.FromString,\n", + "Method", meth_vals.first, + "Type", full_output_type_path); + } + out->Print("}\n"); + out->Print("request_serializers = {\n"); + for (auto& meth_vals : method_to_module_and_message) { + IndentScope raii_serializers_indent(out); + out->Print("\"$Method$\": lambda x: x.SerializeToString(),\n", + "Method", meth_vals.first); + } + out->Print("}\n"); + out->Print("link = rear.activated_rear_link(" + "host, port, request_serializers, response_deserializers)\n"); + out->Print("return implementations.assemble_dynamic_inline_stub(" + "method_implementations, link)\n"); } + return true; +} + +bool PrintPreamble(const FileDescriptor* file, Printer* out) { + out->Print("import abc\n"); + out->Print("from grpc._adapter import fore\n"); + out->Print("from grpc._adapter import rear\n"); + out->Print("from grpc.framework.assembly import implementations\n"); + out->Print("from grpc.framework.assembly import utilities\n"); + return true; } } // namespace -string GetServices(const FileDescriptor* file) { +pair<bool, string> GetServices(const FileDescriptor* file) { string output; - StringOutputStream output_stream(&output); - Printer out(&output_stream, '$'); - out.Print("from grpc.framework.face import demonstration as _face_testing\n"); - out.Print("from grpc.framework.face import interfaces as _face_interfaces\n"); - - for (int i = 0; i < file->service_count(); ++i) { - auto service = file->service(i); - PrintService(service, &out); - PrintServicer(service, &out); - PrintStub(service, &out); - PrintStubImpl(service, &out); - PrintStubGenerators(service, &out); + { + // Scope the output stream so it closes and finalizes output to the string. + StringOutputStream output_stream(&output); + Printer out(&output_stream, '$'); + if (!PrintPreamble(file, &out)) { + return make_pair(false, ""); + } + for (int i = 0; i < file->service_count(); ++i) { + auto service = file->service(i); + if (!(PrintServicer(service, &out) && + PrintServer(service, &out) && + PrintStub(service, &out) && + PrintServerFactory(service, &out) && + PrintStubFactory(service, &out))) { + return make_pair(false, ""); + } + } } - return output; + return make_pair(true, std::move(output)); } } // namespace grpc_python_generator diff --git a/src/compiler/python_generator.h b/src/compiler/python_generator.h index 673ef7b23b..773dfa3513 100644 --- a/src/compiler/python_generator.h +++ b/src/compiler/python_generator.h @@ -35,6 +35,7 @@ #define __GRPC_COMPILER_PYTHON_GENERATOR_H__ #include <string> +#include <utility> namespace google { namespace protobuf { @@ -44,7 +45,7 @@ class FileDescriptor; namespace grpc_python_generator { -std::string GetServices(const google::protobuf::FileDescriptor* file); +std::pair<bool, std::string> GetServices(const google::protobuf::FileDescriptor* file); } // namespace grpc_python_generator diff --git a/src/compiler/python_plugin.cc b/src/compiler/python_plugin.cc index 05c6b095d8..ed1e0494fb 100644 --- a/src/compiler/python_plugin.cc +++ b/src/compiler/python_plugin.cc @@ -33,6 +33,7 @@ // Generates a Python gRPC service interface out of Protobuf IDL. +#include <cstring> #include <memory> #include <string> @@ -50,6 +51,7 @@ using google::protobuf::compiler::PluginMain; using google::protobuf::io::CodedOutputStream; using google::protobuf::io::ZeroCopyOutputStream; using std::string; +using std::strlen; class PythonGrpcGenerator : public CodeGenerator { public: @@ -62,7 +64,7 @@ class PythonGrpcGenerator : public CodeGenerator { string* error) const override { // Get output file name. string file_name; - static const int proto_suffix_length = 6; // length of ".proto" + static const int proto_suffix_length = strlen(".proto"); if (file->name().size() > static_cast<size_t>(proto_suffix_length) && file->name().find_last_of(".proto") == file->name().size() - 1) { file_name = file->name().substr( @@ -75,9 +77,15 @@ class PythonGrpcGenerator : public CodeGenerator { std::unique_ptr<ZeroCopyOutputStream> output( context->OpenForInsert(file_name, "module_scope")); CodedOutputStream coded_out(output.get()); - string code = grpc_python_generator::GetServices(file); - coded_out.WriteRaw(code.data(), code.size()); - return true; + bool success = false; + string code = ""; + tie(success, code) = grpc_python_generator::GetServices(file); + if (success) { + coded_out.WriteRaw(code.data(), code.size()); + return true; + } else { + return false; + } } }; diff --git a/test/compiler/python_plugin_test.py b/test/compiler/python_plugin_test.py index b0c9ec62d0..3919de1450 100644 --- a/test/compiler/python_plugin_test.py +++ b/test/compiler/python_plugin_test.py @@ -40,8 +40,24 @@ import unittest from grpc.framework.face import exceptions from grpc.framework.foundation import future +# Identifiers of entities we expect to find in the generated module. +SERVICER_IDENTIFIER = 'EarlyAdopterTestServiceServicer' +SERVER_IDENTIFIER = 'EarlyAdopterTestServiceServer' +STUB_IDENTIFIER = 'EarlyAdopterTestServiceStub' +SERVER_FACTORY_IDENTIFIER = 'early_adopter_create_TestService_server' +STUB_FACTORY_IDENTIFIER = 'early_adopter_create_TestService_stub' + +# Timeouts and delays. +SHORT_TIMEOUT = 0.1 +NORMAL_TIMEOUT = 1 +LONG_TIMEOUT = 2 +DOES_NOT_MATTER_DELAY = 0 +NO_DELAY = 0 +LONG_DELAY = 1 + # Assigned in __main__. _build_mode = None +_port = None class _ServicerMethods(object): @@ -71,14 +87,14 @@ class _ServicerMethods(object): while self._paused: time.sleep(0) - def UnaryCall(self, request): + def UnaryCall(self, request, context): response = self.test_pb2.SimpleResponse() response.payload.payload_type = self.test_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * request.response_size self._control() return response - def StreamingOutputCall(self, request): + def StreamingOutputCall(self, request, context): for parameter in request.response_parameters: response = self.test_pb2.StreamingOutputCallResponse() response.payload.payload_type = self.test_pb2.COMPRESSABLE @@ -86,7 +102,7 @@ class _ServicerMethods(object): self._control() yield response - def StreamingInputCall(self, request_iter): + def StreamingInputCall(self, request_iter, context): response = self.test_pb2.StreamingInputCallResponse() aggregated_payload_size = 0 for request in request_iter: @@ -95,7 +111,7 @@ class _ServicerMethods(object): self._control() return response - def FullDuplexCall(self, request_iter): + def FullDuplexCall(self, request_iter, context): for request in request_iter: for parameter in request.response_parameters: response = self.test_pb2.StreamingOutputCallResponse() @@ -104,7 +120,7 @@ class _ServicerMethods(object): self._control() yield response - def HalfDuplexCall(self, request_iter): + def HalfDuplexCall(self, request_iter, context): responses = [] for request in request_iter: for parameter in request.response_parameters: @@ -117,7 +133,7 @@ class _ServicerMethods(object): yield response -def CreateService(test_pb2, delay=0, timeout=1): +def _CreateService(test_pb2, delay): """Provides a servicer backend and a stub. The servicer is just the implementation @@ -136,28 +152,30 @@ def CreateService(test_pb2, delay=0, timeout=1): A two-tuple (servicer, stub), where the servicer is the back-end of the service bound to the stub. """ - class Servicer(test_pb2.TestServiceServicer): + servicer_methods = _ServicerMethods(test_pb2, delay) - def UnaryCall(self, request): - return servicer_methods.UnaryCall(request) + class Servicer(getattr(test_pb2, SERVICER_IDENTIFIER)): - def StreamingOutputCall(self, request): - return servicer_methods.StreamingOutputCall(request) + def UnaryCall(self, request, context): + return servicer_methods.UnaryCall(request, context) - def StreamingInputCall(self, request_iter): - return servicer_methods.StreamingInputCall(request_iter) + def StreamingOutputCall(self, request, context): + return servicer_methods.StreamingOutputCall(request, context) - def FullDuplexCall(self, request_iter): - return servicer_methods.FullDuplexCall(request_iter) + def StreamingInputCall(self, request_iter, context): + return servicer_methods.StreamingInputCall(request_iter, context) - def HalfDuplexCall(self, request_iter): - return servicer_methods.HalfDuplexCall(request_iter) + def FullDuplexCall(self, request_iter, context): + return servicer_methods.FullDuplexCall(request_iter, context) + + def HalfDuplexCall(self, request_iter, context): + return servicer_methods.HalfDuplexCall(request_iter, context) - servicer_methods = _ServicerMethods(test_pb2, delay) servicer = Servicer() - linked_pair = test_pb2.mock_TestService(servicer, timeout) - stub = linked_pair.stub - return servicer_methods, stub + server = getattr(test_pb2, SERVER_FACTORY_IDENTIFIER)(servicer, _port, + None, None) + stub = getattr(test_pb2, STUB_FACTORY_IDENTIFIER)('localhost', _port) + return servicer_methods, stub, server def StreamingInputRequest(test_pb2): @@ -198,19 +216,20 @@ class PythonPluginTest(unittest.TestCase): def setUp(self): protoc_command = '../../bins/%s/protobuf/protoc' % _build_mode protoc_plugin_filename = '../../bins/%s/grpc_python_plugin' % _build_mode - test_proto_filename = '../cpp/interop/test.proto' + test_proto_filename = './test.proto' if not os.path.isfile(protoc_command): # Assume that if we haven't built protoc that it's on the system. protoc_command = 'protoc' - # ensure that the output directory exists - outdir = '../../gens/test/compiler/python/' + # Ensure that the output directory exists. + outdir = '../../gens/test/compiler/python' try: os.makedirs(outdir) except OSError as exception: if exception.errno != errno.EEXIST: raise + # Invoke protoc with the plugin. cmd = [ protoc_command, '--plugin=protoc-gen-python-grpc=%s' % protoc_plugin_filename, @@ -222,215 +241,231 @@ class PythonPluginTest(unittest.TestCase): subprocess.call(' '.join(cmd), shell=True) sys.path.append(outdir) - self.delay = 1 # seconds - self.timeout = 2 # seconds + # TODO(atash): Figure out which of theses tests is hanging flakily with small + # probability. def testImportAttributes(self): - # check that we can access the members + # check that we can access the generated module and its members. import test_pb2 # pylint: disable=g-import-not-at-top - self.assertIsNotNone(getattr(test_pb2, 'TestServiceServicer', None)) - self.assertIsNotNone(getattr(test_pb2, 'TestServiceService', None)) - self.assertIsNotNone(getattr(test_pb2, 'TestServiceStub', None)) + self.assertIsNotNone(getattr(test_pb2, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone(getattr(test_pb2, SERVER_IDENTIFIER, None)) + self.assertIsNotNone(getattr(test_pb2, STUB_IDENTIFIER, None)) + self.assertIsNotNone(getattr(test_pb2, SERVER_FACTORY_IDENTIFIER, None)) + self.assertIsNotNone(getattr(test_pb2, STUB_FACTORY_IDENTIFIER, None)) + + def testUpDown(self): + import test_pb2 + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) + request = test_pb2.SimpleRequest(response_size=13) + with server, stub: + pass def testUnaryCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2) + servicer, stub, server = _CreateService(test_pb2, NO_DELAY) request = test_pb2.SimpleRequest(response_size=13) - response = stub.UnaryCall(request) - expected_response = servicer.UnaryCall(request) + with server, stub: + response = stub.UnaryCall(request, NORMAL_TIMEOUT) + expected_response = servicer.UnaryCall(request, None) self.assertEqual(expected_response, response) def testUnaryCallAsync(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService( - test_pb2, delay=self.delay, timeout=self.timeout) + servicer, stub, server = _CreateService(test_pb2, LONG_DELAY) request = test_pb2.SimpleRequest(response_size=13) - # TODO(atash): consider using the 'profile' module? Does it even work here? - start_time = time.clock() - response_future = stub.UnaryCall.async(request) - self.assertGreater(self.delay, time.clock() - start_time) - response = response_future.result() - expected_response = servicer.UnaryCall(request) + with server, stub: + start_time = time.clock() + response_future = stub.UnaryCall.async(request, LONG_TIMEOUT) + # Check that we didn't block on the asynchronous call. + self.assertGreater(LONG_DELAY, time.clock() - start_time) + response = response_future.result() + expected_response = servicer.UnaryCall(request, None) self.assertEqual(expected_response, response) def testUnaryCallAsyncExpired(self): import test_pb2 # pylint: disable=g-import-not-at-top # set the timeout super low... - servicer, stub = CreateService(test_pb2, delay=1, timeout=0.1) + servicer, stub, server = _CreateService(test_pb2, + delay=DOES_NOT_MATTER_DELAY) request = test_pb2.SimpleRequest(response_size=13) - with servicer.pause(): - response_future = stub.UnaryCall.async(request) - with self.assertRaises(exceptions.ExpirationError): - response_future.result() + with server, stub: + with servicer.pause(): + response_future = stub.UnaryCall.async(request, SHORT_TIMEOUT) + with self.assertRaises(exceptions.ExpirationError): + response_future.result() def testUnaryCallAsyncCancelled(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2) + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = test_pb2.SimpleRequest(response_size=13) - with servicer.pause(): - response_future = stub.UnaryCall.async(request) - response_future.cancel() - self.assertTrue(response_future.cancelled()) + with server, stub: + with servicer.pause(): + response_future = stub.UnaryCall.async(request, 1) + response_future.cancel() + self.assertTrue(response_future.cancelled()) def testUnaryCallAsyncFailed(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2) + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = test_pb2.SimpleRequest(response_size=13) - with servicer.fail(): - response_future = stub.UnaryCall.async(request) - self.assertIsNotNone(response_future.exception()) + with server, stub: + with servicer.fail(): + response_future = stub.UnaryCall.async(request, NORMAL_TIMEOUT) + self.assertIsNotNone(response_future.exception()) def testStreamingOutputCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2) + servicer, stub, server = _CreateService(test_pb2, NO_DELAY) request = StreamingOutputRequest(test_pb2) - responses = stub.StreamingOutputCall(request) - expected_responses = servicer.StreamingOutputCall(request) - for check in itertools.izip_longest(expected_responses, responses): - expected_response, response = check - self.assertEqual(expected_response, response) - - def testStreamingOutputCallAsync(self): - import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2, timeout=self.timeout) - request = StreamingOutputRequest(test_pb2) - responses = stub.StreamingOutputCall.async(request) - expected_responses = servicer.StreamingOutputCall(request) - for check in itertools.izip_longest(expected_responses, responses): - expected_response, response = check - self.assertEqual(expected_response, response) - - def testStreamingOutputCallAsyncExpired(self): + with server, stub: + responses = stub.StreamingOutputCall(request, NORMAL_TIMEOUT) + expected_responses = servicer.StreamingOutputCall(request, None) + for check in itertools.izip_longest(expected_responses, responses): + expected_response, response = check + self.assertEqual(expected_response, response) + + def testStreamingOutputCallExpired(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2, timeout=0.1) + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = StreamingOutputRequest(test_pb2) - with servicer.pause(): - responses = stub.StreamingOutputCall.async(request) - with self.assertRaises(exceptions.ExpirationError): - list(responses) + with server, stub: + with servicer.pause(): + responses = stub.StreamingOutputCall(request, SHORT_TIMEOUT) + with self.assertRaises(exceptions.ExpirationError): + list(responses) - def testStreamingOutputCallAsyncCancelled(self): + def testStreamingOutputCallCancelled(self): import test_pb2 # pylint: disable=g-import-not-at-top - _, stub = CreateService(test_pb2, timeout=0.1) + unused_servicer, stub, server = _CreateService(test_pb2, + DOES_NOT_MATTER_DELAY) request = StreamingOutputRequest(test_pb2) - responses = stub.StreamingOutputCall.async(request) - next(responses) - responses.cancel() - with self.assertRaises(future.CancelledError): + with server, stub: + responses = stub.StreamingOutputCall(request, SHORT_TIMEOUT) next(responses) + responses.cancel() + with self.assertRaises(future.CancelledError): + next(responses) - def testStreamingOutputCallAsyncFailed(self): + @unittest.skip('TODO(atash,nathaniel): figure out why this times out ' + 'instead of raising the proper error.') + def testStreamingOutputCallFailed(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2, timeout=0.1) + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = StreamingOutputRequest(test_pb2) - with servicer.fail(): - responses = stub.StreamingOutputCall.async(request) - self.assertIsNotNone(responses) - with self.assertRaises(exceptions.ServicerError): - next(responses) + with server, stub: + with servicer.fail(): + responses = stub.StreamingOutputCall(request, 1) + self.assertIsNotNone(responses) + with self.assertRaises(exceptions.ServicerError): + next(responses) def testStreamingInputCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2) - response = stub.StreamingInputCall(StreamingInputRequest(test_pb2)) + servicer, stub, server = _CreateService(test_pb2, NO_DELAY) + with server, stub: + response = stub.StreamingInputCall(StreamingInputRequest(test_pb2), + NORMAL_TIMEOUT) expected_response = servicer.StreamingInputCall( - StreamingInputRequest(test_pb2)) + StreamingInputRequest(test_pb2), None) self.assertEqual(expected_response, response) def testStreamingInputCallAsync(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService( - test_pb2, delay=self.delay, timeout=self.timeout) - start_time = time.clock() - response_future = stub.StreamingInputCall.async( - StreamingInputRequest(test_pb2)) - self.assertGreater(self.delay, time.clock() - start_time) - response = response_future.result() + servicer, stub, server = _CreateService( + test_pb2, LONG_DELAY) + with server, stub: + start_time = time.clock() + response_future = stub.StreamingInputCall.async( + StreamingInputRequest(test_pb2), LONG_TIMEOUT) + self.assertGreater(LONG_DELAY, time.clock() - start_time) + response = response_future.result() expected_response = servicer.StreamingInputCall( - StreamingInputRequest(test_pb2)) + StreamingInputRequest(test_pb2), None) self.assertEqual(expected_response, response) def testStreamingInputCallAsyncExpired(self): import test_pb2 # pylint: disable=g-import-not-at-top # set the timeout super low... - servicer, stub = CreateService(test_pb2, delay=1, timeout=0.1) - with servicer.pause(): - response_future = stub.StreamingInputCall.async( - StreamingInputRequest(test_pb2)) - with self.assertRaises(exceptions.ExpirationError): - response_future.result() - self.assertIsInstance( - response_future.exception(), exceptions.ExpirationError) + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) + with server, stub: + with servicer.pause(): + response_future = stub.StreamingInputCall.async( + StreamingInputRequest(test_pb2), SHORT_TIMEOUT) + with self.assertRaises(exceptions.ExpirationError): + response_future.result() + self.assertIsInstance( + response_future.exception(), exceptions.ExpirationError) def testStreamingInputCallAsyncCancelled(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2) - with servicer.pause(): - response_future = stub.StreamingInputCall.async( - StreamingInputRequest(test_pb2)) - response_future.cancel() - self.assertTrue(response_future.cancelled()) - with self.assertRaises(future.CancelledError): - response_future.result() + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) + with server, stub: + with servicer.pause(): + response_future = stub.StreamingInputCall.async( + StreamingInputRequest(test_pb2), NORMAL_TIMEOUT) + response_future.cancel() + self.assertTrue(response_future.cancelled()) + with self.assertRaises(future.CancelledError): + response_future.result() def testStreamingInputCallAsyncFailed(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2) - with servicer.fail(): - response_future = stub.StreamingInputCall.async( - StreamingInputRequest(test_pb2)) - self.assertIsNotNone(response_future.exception()) + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) + with server, stub: + with servicer.fail(): + response_future = stub.StreamingInputCall.async( + StreamingInputRequest(test_pb2), SHORT_TIMEOUT) + self.assertIsNotNone(response_future.exception()) def testFullDuplexCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2) - responses = stub.FullDuplexCall(FullDuplexRequest(test_pb2)) - expected_responses = servicer.FullDuplexCall(FullDuplexRequest(test_pb2)) - for check in itertools.izip_longest(expected_responses, responses): - expected_response, response = check - self.assertEqual(expected_response, response) - - def testFullDuplexCallAsync(self): + servicer, stub, server = _CreateService(test_pb2, NO_DELAY) + with server, stub: + responses = stub.FullDuplexCall(FullDuplexRequest(test_pb2), + NORMAL_TIMEOUT) + expected_responses = servicer.FullDuplexCall(FullDuplexRequest(test_pb2), + None) + for check in itertools.izip_longest(expected_responses, responses): + expected_response, response = check + self.assertEqual(expected_response, response) + + def testFullDuplexCallExpired(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2, timeout=self.timeout) - responses = stub.FullDuplexCall.async(FullDuplexRequest(test_pb2)) - expected_responses = servicer.FullDuplexCall(FullDuplexRequest(test_pb2)) - for check in itertools.izip_longest(expected_responses, responses): - expected_response, response = check - self.assertEqual(expected_response, response) - - def testFullDuplexCallAsyncExpired(self): - import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2, timeout=0.1) + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = FullDuplexRequest(test_pb2) - with servicer.pause(): - responses = stub.FullDuplexCall.async(request) - with self.assertRaises(exceptions.ExpirationError): - list(responses) + with server, stub: + with servicer.pause(): + responses = stub.FullDuplexCall(request, SHORT_TIMEOUT) + with self.assertRaises(exceptions.ExpirationError): + list(responses) - def testFullDuplexCallAsyncCancelled(self): + def testFullDuplexCallCancelled(self): import test_pb2 # pylint: disable=g-import-not-at-top - _, stub = CreateService(test_pb2, timeout=0.1) - request = FullDuplexRequest(test_pb2) - responses = stub.FullDuplexCall.async(request) - next(responses) - responses.cancel() - with self.assertRaises(future.CancelledError): + unused_servicer, stub, server = _CreateService(test_pb2, NO_DELAY) + with server, stub: + request = FullDuplexRequest(test_pb2) + responses = stub.FullDuplexCall(request, NORMAL_TIMEOUT) next(responses) + responses.cancel() + with self.assertRaises(future.CancelledError): + next(responses) - def testFullDuplexCallAsyncFailed(self): + @unittest.skip('TODO(atash,nathaniel): figure out why this hangs forever ' + 'and fix.') + def testFullDuplexCallFailed(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2, timeout=0.1) + servicer, stub, server = _CreateService(test_pb2, DOES_NOT_MATTER_DELAY) request = FullDuplexRequest(test_pb2) - with servicer.fail(): - responses = stub.FullDuplexCall.async(request) - self.assertIsNotNone(responses) - with self.assertRaises(exceptions.ServicerError): - next(responses) + with server, stub: + with servicer.fail(): + responses = stub.FullDuplexCall(request, NORMAL_TIMEOUT) + self.assertIsNotNone(responses) + with self.assertRaises(exceptions.ServicerError): + next(responses) def testHalfDuplexCall(self): import test_pb2 # pylint: disable=g-import-not-at-top - servicer, stub = CreateService(test_pb2) + servicer, stub, server = _CreateService(test_pb2, NO_DELAY) def HalfDuplexRequest(): request = test_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) @@ -439,15 +474,16 @@ class PythonPluginTest(unittest.TestCase): request.response_parameters.add(size=2, interval_us=0) request.response_parameters.add(size=3, interval_us=0) yield request - responses = stub.HalfDuplexCall(HalfDuplexRequest()) - expected_responses = servicer.HalfDuplexCall(HalfDuplexRequest()) - for check in itertools.izip_longest(expected_responses, responses): - expected_response, response = check - self.assertEqual(expected_response, response) - - def testHalfDuplexCallAsyncWedged(self): + with server, stub: + responses = stub.HalfDuplexCall(HalfDuplexRequest(), NORMAL_TIMEOUT) + expected_responses = servicer.HalfDuplexCall(HalfDuplexRequest(), None) + for check in itertools.izip_longest(expected_responses, responses): + expected_response, response = check + self.assertEqual(expected_response, response) + + def testHalfDuplexCallWedged(self): import test_pb2 # pylint: disable=g-import-not-at-top - _, stub = CreateService(test_pb2, timeout=1) + _, stub, server = _CreateService(test_pb2, NO_DELAY) wait_flag = [False] @contextlib.contextmanager def wait(): # pylint: disable=invalid-name @@ -461,20 +497,25 @@ class PythonPluginTest(unittest.TestCase): yield request while wait_flag[0]: time.sleep(0.1) - with wait(): - responses = stub.HalfDuplexCall.async(HalfDuplexRequest()) - # half-duplex waits for the client to send all info - with self.assertRaises(exceptions.ExpirationError): - next(responses) + with server, stub: + with wait(): + responses = stub.HalfDuplexCall(HalfDuplexRequest(), NORMAL_TIMEOUT) + # half-duplex waits for the client to send all info + with self.assertRaises(exceptions.ExpirationError): + next(responses) if __name__ == '__main__': os.chdir(os.path.dirname(sys.argv[0])) - parser = argparse.ArgumentParser(description='Run Python compiler plugin test.') - parser.add_argument('--build_mode', dest='build_mode', type=str, default='dbg', - help='The build mode of the targets to test, e.g. ' - '"dbg", "opt", "asan", etc.') + parser = argparse.ArgumentParser( + description='Run Python compiler plugin test.') + parser.add_argument( + '--build_mode', dest='build_mode', type=str, default='dbg', + help='The build mode of the targets to test, e.g. "dbg", "opt", "asan", ' + 'etc.') + parser.add_argument('--port', dest='port', type=int, default=0) args, remainder = parser.parse_known_args() _build_mode = args.build_mode + _port = args.port sys.argv[1:] = remainder unittest.main() diff --git a/test/compiler/test.proto b/test/compiler/test.proto index ed7c6a7b79..1714de7c11 100644 --- a/test/compiler/test.proto +++ b/test/compiler/test.proto @@ -32,7 +32,8 @@ // This file is duplicated around the code base. See GitHub issue #526. syntax = "proto2"; -package grpc.testing; +// TODO(atash): Investigate this statement's utility. +// package grpc.testing; enum PayloadType { // Compressable text format. diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index fe40b51186..2475c37b0b 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -37,7 +37,8 @@ root=`pwd` export LD_LIBRARY_PATH=$root/libs/opt source python2.7_virtual_environment/bin/activate # TODO(issue 215): Properly itemize these in run_tests.py so that they can be parallelized. -python2.7 -B test/compiler/python_plugin_test.py +# TODO(atash): Enable dynamic unused port discovery for this test. +python2.7 -B test/compiler/python_plugin_test.py --build_mode=opt --port=40987 python2.7 -B -m grpc._adapter._blocking_invocation_inline_service_test python2.7 -B -m grpc._adapter._c_test python2.7 -B -m grpc._adapter._event_invocation_synchronous_event_service_test |