diff options
48 files changed, 691 insertions, 390 deletions
@@ -2705,6 +2705,7 @@ LIBGRPC_TEST_UTIL_SRC = \ test/core/end2end/fixtures/proxy.c \ test/core/iomgr/endpoint_tests.c \ test/core/util/grpc_profiler.c \ + test/core/util/memory_counters.c \ test/core/util/mock_endpoint.c \ test/core/util/parse_hexstring.c \ test/core/util/passthru_endpoint.c \ @@ -2754,6 +2755,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ test/core/end2end/fixtures/proxy.c \ test/core/iomgr/endpoint_tests.c \ test/core/util/grpc_profiler.c \ + test/core/util/memory_counters.c \ test/core/util/mock_endpoint.c \ test/core/util/parse_hexstring.c \ test/core/util/passthru_endpoint.c \ diff --git a/build.yaml b/build.yaml index 74119a37d2..e54282dd29 100644 --- a/build.yaml +++ b/build.yaml @@ -567,6 +567,7 @@ filegroups: - test/core/end2end/fixtures/proxy.h - test/core/iomgr/endpoint_tests.h - test/core/util/grpc_profiler.h + - test/core/util/memory_counters.h - test/core/util/mock_endpoint.h - test/core/util/parse_hexstring.h - test/core/util/passthru_endpoint.h @@ -578,6 +579,7 @@ filegroups: - test/core/end2end/fixtures/proxy.c - test/core/iomgr/endpoint_tests.c - test/core/util/grpc_profiler.c + - test/core/util/memory_counters.c - test/core/util/mock_endpoint.c - test/core/util/parse_hexstring.c - test/core/util/passthru_endpoint.c diff --git a/grpc.gemspec b/grpc.gemspec index b05f238c43..b8cfc4e6c4 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -35,6 +35,7 @@ Gem::Specification.new do |s| s.add_dependency 'googleauth', '~> 0.5.1' s.add_development_dependency 'bundler', '~> 1.9' + s.add_development_dependency 'facter', '~> 2.4' s.add_development_dependency 'logging', '~> 2.0' s.add_development_dependency 'simplecov', '~> 0.9' s.add_development_dependency 'rake', '~> 10.4' diff --git a/include/grpc/support/string_util.h b/include/grpc/support/string_util.h index 0abdb4a8c8..f981bc0db0 100644 --- a/include/grpc/support/string_util.h +++ b/include/grpc/support/string_util.h @@ -34,6 +34,8 @@ #ifndef GRPC_SUPPORT_STRING_UTIL_H #define GRPC_SUPPORT_STRING_UTIL_H +#include <grpc/support/port_platform.h> + #ifdef __cplusplus extern "C" { #endif diff --git a/src/compiler/cpp_generator.cc b/src/compiler/cpp_generator.cc index 08b1123a51..b133699306 100644 --- a/src/compiler/cpp_generator.cc +++ b/src/compiler/cpp_generator.cc @@ -34,9 +34,6 @@ #include <map> #include "src/compiler/cpp_generator.h" -#include "src/compiler/cpp_generator_helpers.h" - -#include "src/compiler/config.h" #include <sstream> @@ -50,22 +47,6 @@ grpc::string as_string(T x) { return out.str(); } -bool NoStreaming(const grpc::protobuf::MethodDescriptor *method) { - return !method->client_streaming() && !method->server_streaming(); -} - -bool ClientOnlyStreaming(const grpc::protobuf::MethodDescriptor *method) { - return method->client_streaming() && !method->server_streaming(); -} - -bool ServerOnlyStreaming(const grpc::protobuf::MethodDescriptor *method) { - return !method->client_streaming() && method->server_streaming(); -} - -bool BidiStreaming(const grpc::protobuf::MethodDescriptor *method) { - return method->client_streaming() && method->server_streaming(); -} - grpc::string FilenameIdentifier(const grpc::string &filename) { grpc::string result; for (unsigned i = 0; i < filename.size(); i++) { @@ -86,7 +67,7 @@ grpc::string FilenameIdentifier(const grpc::string &filename) { template<class T, size_t N> T *array_end(T (&array)[N]) { return array + N; } -void PrintIncludes(grpc::protobuf::io::Printer *printer, const std::vector<grpc::string>& headers, const Parameters ¶ms) { +void PrintIncludes(Printer *printer, const std::vector<grpc::string>& headers, const Parameters ¶ms) { std::map<grpc::string, grpc::string> vars; vars["l"] = params.use_system_headers ? '<' : '"'; @@ -105,39 +86,36 @@ void PrintIncludes(grpc::protobuf::io::Printer *printer, const std::vector<grpc: } } -grpc::string GetHeaderPrologue(const grpc::protobuf::FileDescriptor *file, - const Parameters ¶ms) { +grpc::string GetHeaderPrologue(File *file, const Parameters ¶ms) { grpc::string output; { // Scope the output stream so it closes and finalizes output to the string. - grpc::protobuf::io::StringOutputStream output_stream(&output); - grpc::protobuf::io::Printer printer(&output_stream, '$'); + auto printer = file->CreatePrinter(&output); std::map<grpc::string, grpc::string> vars; - vars["filename"] = file->name(); - vars["filename_identifier"] = FilenameIdentifier(file->name()); - vars["filename_base"] = grpc_generator::StripProto(file->name()); + vars["filename"] = file->filename(); + vars["filename_identifier"] = FilenameIdentifier(file->filename()); + vars["filename_base"] = file->filename_without_ext(); - printer.Print(vars, "// Generated by the gRPC protobuf plugin.\n"); - printer.Print(vars, + printer->Print(vars, "// Generated by the gRPC protobuf plugin.\n"); + printer->Print(vars, "// If you make any local change, they will be lost.\n"); - printer.Print(vars, "// source: $filename$\n"); - printer.Print(vars, "#ifndef GRPC_$filename_identifier$__INCLUDED\n"); - printer.Print(vars, "#define GRPC_$filename_identifier$__INCLUDED\n"); - printer.Print(vars, "\n"); - printer.Print(vars, "#include \"$filename_base$.pb.h\"\n"); - printer.Print(vars, "\n"); + printer->Print(vars, "// source: $filename$\n"); + printer->Print(vars, "#ifndef GRPC_$filename_identifier$__INCLUDED\n"); + printer->Print(vars, "#define GRPC_$filename_identifier$__INCLUDED\n"); + printer->Print(vars, "\n"); + printer->Print(vars, "#include \"$filename_base$.pb.h\"\n"); + printer->Print(vars, "\n"); } return output; } -grpc::string GetHeaderIncludes(const grpc::protobuf::FileDescriptor *file, +grpc::string GetHeaderIncludes(File *file, const Parameters ¶ms) { grpc::string output; { // Scope the output stream so it closes and finalizes output to the string. - grpc::protobuf::io::StringOutputStream output_stream(&output); - grpc::protobuf::io::Printer printer(&output_stream, '$'); + auto printer = file->CreatePrinter(&output); std::map<grpc::string, grpc::string> vars; static const char *headers_strs[] = { @@ -151,42 +129,38 @@ grpc::string GetHeaderIncludes(const grpc::protobuf::FileDescriptor *file, "grpc++/impl/codegen/sync_stream.h" }; std::vector<grpc::string> headers(headers_strs, array_end(headers_strs)); - PrintIncludes(&printer, headers, params); - printer.Print(vars, "\n"); - printer.Print(vars, "namespace grpc {\n"); - printer.Print(vars, "class CompletionQueue;\n"); - printer.Print(vars, "class Channel;\n"); - printer.Print(vars, "class RpcService;\n"); - printer.Print(vars, "class ServerCompletionQueue;\n"); - printer.Print(vars, "class ServerContext;\n"); - printer.Print(vars, "} // namespace grpc\n\n"); + PrintIncludes(printer.get(), headers, params); + printer->Print(vars, "\n"); + printer->Print(vars, "namespace grpc {\n"); + printer->Print(vars, "class CompletionQueue;\n"); + printer->Print(vars, "class Channel;\n"); + printer->Print(vars, "class RpcService;\n"); + printer->Print(vars, "class ServerCompletionQueue;\n"); + printer->Print(vars, "class ServerContext;\n"); + printer->Print(vars, "} // namespace grpc\n\n"); if (!file->package().empty()) { - std::vector<grpc::string> parts = - grpc_generator::tokenize(file->package(), "."); + std::vector<grpc::string> parts = file->package_parts(); for (auto part = parts.begin(); part != parts.end(); part++) { vars["part"] = *part; - printer.Print(vars, "namespace $part$ {\n"); + printer->Print(vars, "namespace $part$ {\n"); } - printer.Print(vars, "\n"); + printer->Print(vars, "\n"); } } return output; } void PrintHeaderClientMethodInterfaces( - grpc::protobuf::io::Printer *printer, - const grpc::protobuf::MethodDescriptor *method, + Printer *printer, const Method *method, std::map<grpc::string, grpc::string> *vars, bool is_public) { (*vars)["Method"] = method->name(); - (*vars)["Request"] = - grpc_cpp_generator::ClassName(method->input_type(), true); - (*vars)["Response"] = - grpc_cpp_generator::ClassName(method->output_type(), true); + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); if (is_public) { - if (NoStreaming(method)) { + if (method->NoStreaming()) { printer->Print( *vars, "virtual ::grpc::Status $Method$(::grpc::ClientContext* context, " @@ -204,7 +178,7 @@ void PrintHeaderClientMethodInterfaces( "Async$Method$Raw(context, request, cq));\n"); printer->Outdent(); printer->Print("}\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print( *vars, "std::unique_ptr< ::grpc::ClientWriterInterface< $Request$>>" @@ -230,7 +204,7 @@ void PrintHeaderClientMethodInterfaces( "Async$Method$Raw(context, response, cq, tag));\n"); printer->Outdent(); printer->Print("}\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print( *vars, "std::unique_ptr< ::grpc::ClientReaderInterface< $Response$>>" @@ -256,7 +230,7 @@ void PrintHeaderClientMethodInterfaces( "Async$Method$Raw(context, request, cq, tag));\n"); printer->Outdent(); printer->Print("}\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print(*vars, "std::unique_ptr< ::grpc::ClientReaderWriterInterface< " "$Request$, $Response$>> " @@ -285,14 +259,14 @@ void PrintHeaderClientMethodInterfaces( printer->Print("}\n"); } } else { - if (NoStreaming(method)) { + if (method->NoStreaming()) { printer->Print( *vars, "virtual ::grpc::ClientAsyncResponseReaderInterface< $Response$>* " "Async$Method$Raw(::grpc::ClientContext* context, " "const $Request$& request, " "::grpc::CompletionQueue* cq) = 0;\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print( *vars, "virtual ::grpc::ClientWriterInterface< $Request$>*" @@ -303,7 +277,7 @@ void PrintHeaderClientMethodInterfaces( " Async$Method$Raw(::grpc::ClientContext* context, " "$Response$* response, " "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print( *vars, "virtual ::grpc::ClientReaderInterface< $Response$>* $Method$Raw(" @@ -314,7 +288,7 @@ void PrintHeaderClientMethodInterfaces( "Async$Method$Raw(" "::grpc::ClientContext* context, const $Request$& request, " "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print(*vars, "virtual ::grpc::ClientReaderWriterInterface< $Request$, " "$Response$>* " @@ -328,17 +302,15 @@ void PrintHeaderClientMethodInterfaces( } } -void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer, - const grpc::protobuf::MethodDescriptor *method, +void PrintHeaderClientMethod(Printer *printer, + const Method *method, std::map<grpc::string, grpc::string> *vars, bool is_public) { (*vars)["Method"] = method->name(); - (*vars)["Request"] = - grpc_cpp_generator::ClassName(method->input_type(), true); - (*vars)["Response"] = - grpc_cpp_generator::ClassName(method->output_type(), true); + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); if (is_public) { - if (NoStreaming(method)) { + if (method->NoStreaming()) { printer->Print( *vars, "::grpc::Status $Method$(::grpc::ClientContext* context, " @@ -356,7 +328,7 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer, "Async$Method$Raw(context, request, cq));\n"); printer->Outdent(); printer->Print("}\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print( *vars, "std::unique_ptr< ::grpc::ClientWriter< $Request$>>" @@ -380,7 +352,7 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer, "Async$Method$Raw(context, response, cq, tag));\n"); printer->Outdent(); printer->Print("}\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print( *vars, "std::unique_ptr< ::grpc::ClientReader< $Response$>>" @@ -406,7 +378,7 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer, "Async$Method$Raw(context, request, cq, tag));\n"); printer->Outdent(); printer->Print("}\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print( *vars, "std::unique_ptr< ::grpc::ClientReaderWriter< $Request$, $Response$>>" @@ -432,13 +404,13 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer, printer->Print("}\n"); } } else { - if (NoStreaming(method)) { + if (method->NoStreaming()) { printer->Print(*vars, "::grpc::ClientAsyncResponseReader< $Response$>* " "Async$Method$Raw(::grpc::ClientContext* context, " "const $Request$& request, " "::grpc::CompletionQueue* cq) GRPC_OVERRIDE;\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print(*vars, "::grpc::ClientWriter< $Request$>* $Method$Raw(" "::grpc::ClientContext* context, $Response$* response) " @@ -448,7 +420,7 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer, "::grpc::ClientAsyncWriter< $Request$>* Async$Method$Raw(" "::grpc::ClientContext* context, $Response$* response, " "::grpc::CompletionQueue* cq, void* tag) GRPC_OVERRIDE;\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print(*vars, "::grpc::ClientReader< $Response$>* $Method$Raw(" "::grpc::ClientContext* context, const $Request$& request)" @@ -458,7 +430,7 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer, "::grpc::ClientAsyncReader< $Response$>* Async$Method$Raw(" "::grpc::ClientContext* context, const $Request$& request, " "::grpc::CompletionQueue* cq, void* tag) GRPC_OVERRIDE;\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print( *vars, "::grpc::ClientReaderWriter< $Request$, $Response$>* " @@ -472,38 +444,34 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer, } } -void PrintHeaderClientMethodData(grpc::protobuf::io::Printer *printer, - const grpc::protobuf::MethodDescriptor *method, +void PrintHeaderClientMethodData(Printer *printer, const Method *method, std::map<grpc::string, grpc::string> *vars) { (*vars)["Method"] = method->name(); printer->Print(*vars, "const ::grpc::RpcMethod rpcmethod_$Method$_;\n"); } -void PrintHeaderServerMethodSync(grpc::protobuf::io::Printer *printer, - const grpc::protobuf::MethodDescriptor *method, +void PrintHeaderServerMethodSync(Printer *printer, const Method *method, std::map<grpc::string, grpc::string> *vars) { (*vars)["Method"] = method->name(); - (*vars)["Request"] = - grpc_cpp_generator::ClassName(method->input_type(), true); - (*vars)["Response"] = - grpc_cpp_generator::ClassName(method->output_type(), true); - if (NoStreaming(method)) { + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); + if (method->NoStreaming()) { printer->Print(*vars, "virtual ::grpc::Status $Method$(" "::grpc::ServerContext* context, const $Request$* request, " "$Response$* response);\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print(*vars, "virtual ::grpc::Status $Method$(" "::grpc::ServerContext* context, " "::grpc::ServerReader< $Request$>* reader, " "$Response$* response);\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print(*vars, "virtual ::grpc::Status $Method$(" "::grpc::ServerContext* context, const $Request$* request, " "::grpc::ServerWriter< $Response$>* writer);\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print( *vars, "virtual ::grpc::Status $Method$(" @@ -514,20 +482,18 @@ void PrintHeaderServerMethodSync(grpc::protobuf::io::Printer *printer, } void PrintHeaderServerMethodAsync( - grpc::protobuf::io::Printer *printer, - const grpc::protobuf::MethodDescriptor *method, + Printer *printer, + const Method *method, std::map<grpc::string, grpc::string> *vars) { (*vars)["Method"] = method->name(); - (*vars)["Request"] = - grpc_cpp_generator::ClassName(method->input_type(), true); - (*vars)["Response"] = - grpc_cpp_generator::ClassName(method->output_type(), true); + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); printer->Print(*vars, "template <class BaseClass>\n"); printer->Print(*vars, "class WithAsyncMethod_$Method$ : public BaseClass {\n"); printer->Print( " private:\n" - " void BaseClassMustBeDerivedFromService(Service *service) {}\n"); + " void BaseClassMustBeDerivedFromService(const Service *service) {}\n"); printer->Print(" public:\n"); printer->Indent(); printer->Print(*vars, @@ -538,7 +504,7 @@ void PrintHeaderServerMethodAsync( "~WithAsyncMethod_$Method$() GRPC_OVERRIDE {\n" " BaseClassMustBeDerivedFromService(this);\n" "}\n"); - if (NoStreaming(method)) { + if (method->NoStreaming()) { printer->Print( *vars, "// disable synchronous version of this method\n" @@ -559,7 +525,7 @@ void PrintHeaderServerMethodAsync( " ::grpc::Service::RequestAsyncUnary($Idx$, context, " "request, response, new_call_cq, notification_cq, tag);\n"); printer->Print("}\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print( *vars, "// disable synchronous version of this method\n" @@ -581,7 +547,7 @@ void PrintHeaderServerMethodAsync( " ::grpc::Service::RequestAsyncClientStreaming($Idx$, " "context, reader, new_call_cq, notification_cq, tag);\n"); printer->Print("}\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print( *vars, "// disable synchronous version of this method\n" @@ -604,7 +570,7 @@ void PrintHeaderServerMethodAsync( " ::grpc::Service::RequestAsyncServerStreaming($Idx$, " "context, request, writer, new_call_cq, notification_cq, tag);\n"); printer->Print("}\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print( *vars, "// disable synchronous version of this method\n" @@ -632,20 +598,18 @@ void PrintHeaderServerMethodAsync( } void PrintHeaderServerMethodGeneric( - grpc::protobuf::io::Printer *printer, - const grpc::protobuf::MethodDescriptor *method, + Printer *printer, + const Method *method, std::map<grpc::string, grpc::string> *vars) { (*vars)["Method"] = method->name(); - (*vars)["Request"] = - grpc_cpp_generator::ClassName(method->input_type(), true); - (*vars)["Response"] = - grpc_cpp_generator::ClassName(method->output_type(), true); + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); printer->Print(*vars, "template <class BaseClass>\n"); printer->Print(*vars, "class WithGenericMethod_$Method$ : public BaseClass {\n"); printer->Print( " private:\n" - " void BaseClassMustBeDerivedFromService(Service *service) {}\n"); + " void BaseClassMustBeDerivedFromService(const Service *service) {}\n"); printer->Print(" public:\n"); printer->Indent(); printer->Print(*vars, @@ -656,7 +620,7 @@ void PrintHeaderServerMethodGeneric( "~WithGenericMethod_$Method$() GRPC_OVERRIDE {\n" " BaseClassMustBeDerivedFromService(this);\n" "}\n"); - if (NoStreaming(method)) { + if (method->NoStreaming()) { printer->Print( *vars, "// disable synchronous version of this method\n" @@ -666,7 +630,7 @@ void PrintHeaderServerMethodGeneric( " abort();\n" " return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n" "}\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print( *vars, "// disable synchronous version of this method\n" @@ -677,7 +641,7 @@ void PrintHeaderServerMethodGeneric( " abort();\n" " return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n" "}\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print( *vars, "// disable synchronous version of this method\n" @@ -688,7 +652,7 @@ void PrintHeaderServerMethodGeneric( " abort();\n" " return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, \"\");\n" "}\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print( *vars, "// disable synchronous version of this method\n" @@ -704,8 +668,8 @@ void PrintHeaderServerMethodGeneric( printer->Print(*vars, "};\n"); } -void PrintHeaderService(grpc::protobuf::io::Printer *printer, - const grpc::protobuf::ServiceDescriptor *service, +void PrintHeaderService(Printer *printer, + const Service *service, std::map<grpc::string, grpc::string> *vars) { (*vars)["Service"] = service->name(); @@ -721,13 +685,13 @@ void PrintHeaderService(grpc::protobuf::io::Printer *printer, printer->Indent(); printer->Print("virtual ~StubInterface() {}\n"); for (int i = 0; i < service->method_count(); ++i) { - PrintHeaderClientMethodInterfaces(printer, service->method(i), vars, true); + PrintHeaderClientMethodInterfaces(printer, service->method(i).get(), vars, true); } printer->Outdent(); printer->Print("private:\n"); printer->Indent(); for (int i = 0; i < service->method_count(); ++i) { - PrintHeaderClientMethodInterfaces(printer, service->method(i), vars, false); + PrintHeaderClientMethodInterfaces(printer, service->method(i).get(), vars, false); } printer->Outdent(); printer->Print("};\n"); @@ -737,17 +701,17 @@ void PrintHeaderService(grpc::protobuf::io::Printer *printer, printer->Indent(); printer->Print("Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel);\n"); for (int i = 0; i < service->method_count(); ++i) { - PrintHeaderClientMethod(printer, service->method(i), vars, true); + PrintHeaderClientMethod(printer, service->method(i).get(), vars, true); } printer->Outdent(); printer->Print("\n private:\n"); printer->Indent(); printer->Print("std::shared_ptr< ::grpc::ChannelInterface> channel_;\n"); for (int i = 0; i < service->method_count(); ++i) { - PrintHeaderClientMethod(printer, service->method(i), vars, false); + PrintHeaderClientMethod(printer, service->method(i).get(), vars, false); } for (int i = 0; i < service->method_count(); ++i) { - PrintHeaderClientMethodData(printer, service->method(i), vars); + PrintHeaderClientMethodData(printer, service->method(i).get(), vars); } printer->Outdent(); printer->Print("};\n"); @@ -766,7 +730,7 @@ void PrintHeaderService(grpc::protobuf::io::Printer *printer, printer->Print("Service();\n"); printer->Print("virtual ~Service();\n"); for (int i = 0; i < service->method_count(); ++i) { - PrintHeaderServerMethodSync(printer, service->method(i), vars); + PrintHeaderServerMethodSync(printer, service->method(i).get(), vars); } printer->Outdent(); printer->Print("};\n"); @@ -774,13 +738,13 @@ void PrintHeaderService(grpc::protobuf::io::Printer *printer, // Server side - Asynchronous for (int i = 0; i < service->method_count(); ++i) { (*vars)["Idx"] = as_string(i); - PrintHeaderServerMethodAsync(printer, service->method(i), vars); + PrintHeaderServerMethodAsync(printer, service->method(i).get(), vars); } printer->Print("typedef "); for (int i = 0; i < service->method_count(); ++i) { - (*vars)["method_name"] = service->method(i)->name(); + (*vars)["method_name"] = service->method(i).get()->name(); printer->Print(*vars, "WithAsyncMethod_$method_name$<"); } printer->Print("Service"); @@ -792,20 +756,19 @@ void PrintHeaderService(grpc::protobuf::io::Printer *printer, // Server side - Generic for (int i = 0; i < service->method_count(); ++i) { (*vars)["Idx"] = as_string(i); - PrintHeaderServerMethodGeneric(printer, service->method(i), vars); + PrintHeaderServerMethodGeneric(printer, service->method(i).get(), vars); } printer->Outdent(); printer->Print("};\n"); } -grpc::string GetHeaderServices(const grpc::protobuf::FileDescriptor *file, +grpc::string GetHeaderServices(File *file, const Parameters ¶ms) { grpc::string output; { // Scope the output stream so it closes and finalizes output to the string. - grpc::protobuf::io::StringOutputStream output_stream(&output); - grpc::protobuf::io::Printer printer(&output_stream, '$'); + auto printer = file->CreatePrinter(&output); std::map<grpc::string, grpc::string> vars; // Package string is empty or ends with a dot. It is used to fully qualify // method names. @@ -816,80 +779,76 @@ grpc::string GetHeaderServices(const grpc::protobuf::FileDescriptor *file, if (!params.services_namespace.empty()) { vars["services_namespace"] = params.services_namespace; - printer.Print(vars, "\nnamespace $services_namespace$ {\n\n"); + printer->Print(vars, "\nnamespace $services_namespace$ {\n\n"); } for (int i = 0; i < file->service_count(); ++i) { - PrintHeaderService(&printer, file->service(i), &vars); - printer.Print("\n"); + PrintHeaderService(printer.get(), file->service(i).get(), &vars); + printer->Print("\n"); } if (!params.services_namespace.empty()) { - printer.Print(vars, "} // namespace $services_namespace$\n\n"); + printer->Print(vars, "} // namespace $services_namespace$\n\n"); } } return output; } -grpc::string GetHeaderEpilogue(const grpc::protobuf::FileDescriptor *file, +grpc::string GetHeaderEpilogue(File *file, const Parameters ¶ms) { grpc::string output; { // Scope the output stream so it closes and finalizes output to the string. - grpc::protobuf::io::StringOutputStream output_stream(&output); - grpc::protobuf::io::Printer printer(&output_stream, '$'); + auto printer = file->CreatePrinter(&output); std::map<grpc::string, grpc::string> vars; - vars["filename"] = file->name(); - vars["filename_identifier"] = FilenameIdentifier(file->name()); + vars["filename"] = file->filename(); + vars["filename_identifier"] = FilenameIdentifier(file->filename()); if (!file->package().empty()) { - std::vector<grpc::string> parts = - grpc_generator::tokenize(file->package(), "."); + std::vector<grpc::string> parts = file->package_parts(); for (auto part = parts.rbegin(); part != parts.rend(); part++) { vars["part"] = *part; - printer.Print(vars, "} // namespace $part$\n"); + printer->Print(vars, "} // namespace $part$\n"); } - printer.Print(vars, "\n"); + printer->Print(vars, "\n"); } - printer.Print(vars, "\n"); - printer.Print(vars, "#endif // GRPC_$filename_identifier$__INCLUDED\n"); + printer->Print(vars, "\n"); + printer->Print(vars, "#endif // GRPC_$filename_identifier$__INCLUDED\n"); } return output; } -grpc::string GetSourcePrologue(const grpc::protobuf::FileDescriptor *file, +grpc::string GetSourcePrologue(File *file, const Parameters ¶ms) { grpc::string output; { // Scope the output stream so it closes and finalizes output to the string. - grpc::protobuf::io::StringOutputStream output_stream(&output); - grpc::protobuf::io::Printer printer(&output_stream, '$'); + auto printer = file->CreatePrinter(&output); std::map<grpc::string, grpc::string> vars; - vars["filename"] = file->name(); - vars["filename_base"] = grpc_generator::StripProto(file->name()); + vars["filename"] = file->filename(); + vars["filename_base"] = file->filename_without_ext(); - printer.Print(vars, "// Generated by the gRPC protobuf plugin.\n"); - printer.Print(vars, + printer->Print(vars, "// Generated by the gRPC protobuf plugin.\n"); + printer->Print(vars, "// If you make any local change, they will be lost.\n"); - printer.Print(vars, "// source: $filename$\n\n"); - printer.Print(vars, "#include \"$filename_base$.pb.h\"\n"); - printer.Print(vars, "#include \"$filename_base$.grpc.pb.h\"\n"); - printer.Print(vars, "\n"); + printer->Print(vars, "// source: $filename$\n\n"); + printer->Print(vars, "#include \"$filename_base$.pb.h\"\n"); + printer->Print(vars, "#include \"$filename_base$.grpc.pb.h\"\n"); + printer->Print(vars, "\n"); } return output; } -grpc::string GetSourceIncludes(const grpc::protobuf::FileDescriptor *file, +grpc::string GetSourceIncludes(File *file, const Parameters ¶ms) { grpc::string output; { // Scope the output stream so it closes and finalizes output to the string. - grpc::protobuf::io::StringOutputStream output_stream(&output); - grpc::protobuf::io::Printer printer(&output_stream, '$'); + auto printer = file->CreatePrinter(&output); std::map<grpc::string, grpc::string> vars; static const char *headers_strs[] = { @@ -903,32 +862,29 @@ grpc::string GetSourceIncludes(const grpc::protobuf::FileDescriptor *file, "grpc++/impl/codegen/sync_stream.h" }; std::vector<grpc::string> headers(headers_strs, array_end(headers_strs)); - PrintIncludes(&printer, headers, params); + PrintIncludes(printer.get(), headers, params); if (!file->package().empty()) { - std::vector<grpc::string> parts = - grpc_generator::tokenize(file->package(), "."); + std::vector<grpc::string> parts = file->package_parts(); for (auto part = parts.begin(); part != parts.end(); part++) { vars["part"] = *part; - printer.Print(vars, "namespace $part$ {\n"); + printer->Print(vars, "namespace $part$ {\n"); } } - printer.Print(vars, "\n"); + printer->Print(vars, "\n"); } return output; } -void PrintSourceClientMethod(grpc::protobuf::io::Printer *printer, - const grpc::protobuf::MethodDescriptor *method, +void PrintSourceClientMethod(Printer *printer, + const Method *method, std::map<grpc::string, grpc::string> *vars) { (*vars)["Method"] = method->name(); - (*vars)["Request"] = - grpc_cpp_generator::ClassName(method->input_type(), true); - (*vars)["Response"] = - grpc_cpp_generator::ClassName(method->output_type(), true); - if (NoStreaming(method)) { + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); + if (method->NoStreaming()) { printer->Print(*vars, "::grpc::Status $ns$$Service$::Stub::$Method$(" "::grpc::ClientContext* context, " @@ -951,7 +907,7 @@ void PrintSourceClientMethod(grpc::protobuf::io::Printer *printer, "rpcmethod_$Method$_, " "context, request);\n" "}\n\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print(*vars, "::grpc::ClientWriter< $Request$>* " "$ns$$Service$::Stub::$Method$Raw(" @@ -973,7 +929,7 @@ void PrintSourceClientMethod(grpc::protobuf::io::Printer *printer, "rpcmethod_$Method$_, " "context, response, tag);\n" "}\n\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print( *vars, "::grpc::ClientReader< $Response$>* " @@ -996,7 +952,7 @@ void PrintSourceClientMethod(grpc::protobuf::io::Printer *printer, "rpcmethod_$Method$_, " "context, request, tag);\n" "}\n\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print( *vars, "::grpc::ClientReaderWriter< $Request$, $Response$>* " @@ -1023,15 +979,13 @@ void PrintSourceClientMethod(grpc::protobuf::io::Printer *printer, } } -void PrintSourceServerMethod(grpc::protobuf::io::Printer *printer, - const grpc::protobuf::MethodDescriptor *method, +void PrintSourceServerMethod(Printer *printer, + const Method *method, std::map<grpc::string, grpc::string> *vars) { (*vars)["Method"] = method->name(); - (*vars)["Request"] = - grpc_cpp_generator::ClassName(method->input_type(), true); - (*vars)["Response"] = - grpc_cpp_generator::ClassName(method->output_type(), true); - if (NoStreaming(method)) { + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); + if (method->NoStreaming()) { printer->Print(*vars, "::grpc::Status $ns$$Service$::Service::$Method$(" "::grpc::ServerContext* context, " @@ -1043,7 +997,7 @@ void PrintSourceServerMethod(grpc::protobuf::io::Printer *printer, " return ::grpc::Status(" "::grpc::StatusCode::UNIMPLEMENTED, \"\");\n"); printer->Print("}\n\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print(*vars, "::grpc::Status $ns$$Service$::Service::$Method$(" "::grpc::ServerContext* context, " @@ -1056,7 +1010,7 @@ void PrintSourceServerMethod(grpc::protobuf::io::Printer *printer, " return ::grpc::Status(" "::grpc::StatusCode::UNIMPLEMENTED, \"\");\n"); printer->Print("}\n\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print(*vars, "::grpc::Status $ns$$Service$::Service::$Method$(" "::grpc::ServerContext* context, " @@ -1069,7 +1023,7 @@ void PrintSourceServerMethod(grpc::protobuf::io::Printer *printer, " return ::grpc::Status(" "::grpc::StatusCode::UNIMPLEMENTED, \"\");\n"); printer->Print("}\n\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print(*vars, "::grpc::Status $ns$$Service$::Service::$Method$(" "::grpc::ServerContext* context, " @@ -1084,15 +1038,15 @@ void PrintSourceServerMethod(grpc::protobuf::io::Printer *printer, } } -void PrintSourceService(grpc::protobuf::io::Printer *printer, - const grpc::protobuf::ServiceDescriptor *service, +void PrintSourceService(Printer *printer, + const Service *service, std::map<grpc::string, grpc::string> *vars) { (*vars)["Service"] = service->name(); printer->Print(*vars, "static const char* $prefix$$Service$_method_names[] = {\n"); for (int i = 0; i < service->method_count(); ++i) { - (*vars)["Method"] = service->method(i)->name(); + (*vars)["Method"] = service->method(i).get()->name(); printer->Print(*vars, " \"/$Package$$Service$/$Method$\",\n"); } printer->Print(*vars, "};\n\n"); @@ -1111,14 +1065,14 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer, printer->Indent(); printer->Print(": channel_(channel)"); for (int i = 0; i < service->method_count(); ++i) { - const grpc::protobuf::MethodDescriptor *method = service->method(i); + auto method = service->method(i); (*vars)["Method"] = method->name(); (*vars)["Idx"] = as_string(i); - if (NoStreaming(method)) { + if (method->NoStreaming()) { (*vars)["StreamingType"] = "NORMAL_RPC"; - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { (*vars)["StreamingType"] = "CLIENT_STREAMING"; - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { (*vars)["StreamingType"] = "SERVER_STREAMING"; } else { (*vars)["StreamingType"] = "BIDI_STREAMING"; @@ -1135,21 +1089,19 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer, for (int i = 0; i < service->method_count(); ++i) { (*vars)["Idx"] = as_string(i); - PrintSourceClientMethod(printer, service->method(i), vars); + PrintSourceClientMethod(printer, service->method(i).get(), vars); } printer->Print(*vars, "$ns$$Service$::Service::Service() {\n"); printer->Indent(); printer->Print(*vars, "(void)$prefix$$Service$_method_names;\n"); for (int i = 0; i < service->method_count(); ++i) { - const grpc::protobuf::MethodDescriptor *method = service->method(i); + auto method = service->method(i); (*vars)["Idx"] = as_string(i); (*vars)["Method"] = method->name(); - (*vars)["Request"] = - grpc_cpp_generator::ClassName(method->input_type(), true); - (*vars)["Response"] = - grpc_cpp_generator::ClassName(method->output_type(), true); - if (NoStreaming(method)) { + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); + if (method->NoStreaming()) { printer->Print( *vars, "AddMethod(new ::grpc::RpcServiceMethod(\n" @@ -1159,7 +1111,7 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer, "$Request$, " "$Response$>(\n" " std::mem_fn(&$ns$$Service$::Service::$Method$), this)));\n"); - } else if (ClientOnlyStreaming(method)) { + } else if (method->ClientOnlyStreaming()) { printer->Print( *vars, "AddMethod(new ::grpc::RpcServiceMethod(\n" @@ -1168,7 +1120,7 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer, " new ::grpc::ClientStreamingHandler< " "$ns$$Service$::Service, $Request$, $Response$>(\n" " std::mem_fn(&$ns$$Service$::Service::$Method$), this)));\n"); - } else if (ServerOnlyStreaming(method)) { + } else if (method->ServerOnlyStreaming()) { printer->Print( *vars, "AddMethod(new ::grpc::RpcServiceMethod(\n" @@ -1177,7 +1129,7 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer, " new ::grpc::ServerStreamingHandler< " "$ns$$Service$::Service, $Request$, $Response$>(\n" " std::mem_fn(&$ns$$Service$::Service::$Method$), this)));\n"); - } else if (BidiStreaming(method)) { + } else if (method->BidiStreaming()) { printer->Print( *vars, "AddMethod(new ::grpc::RpcServiceMethod(\n" @@ -1195,17 +1147,16 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer, "}\n\n"); for (int i = 0; i < service->method_count(); ++i) { (*vars)["Idx"] = as_string(i); - PrintSourceServerMethod(printer, service->method(i), vars); + PrintSourceServerMethod(printer, service->method(i).get(), vars); } } -grpc::string GetSourceServices(const grpc::protobuf::FileDescriptor *file, +grpc::string GetSourceServices(File *file, const Parameters ¶ms) { grpc::string output; { // Scope the output stream so it closes and finalizes output to the string. - grpc::protobuf::io::StringOutputStream output_stream(&output); - grpc::protobuf::io::Printer printer(&output_stream, '$'); + auto printer = file->CreatePrinter(&output); std::map<grpc::string, grpc::string> vars; // Package string is empty or ends with a dot. It is used to fully qualify // method names. @@ -1222,20 +1173,19 @@ grpc::string GetSourceServices(const grpc::protobuf::FileDescriptor *file, } for (int i = 0; i < file->service_count(); ++i) { - PrintSourceService(&printer, file->service(i), &vars); - printer.Print("\n"); + PrintSourceService(printer.get(), file->service(i).get(), &vars); + printer->Print("\n"); } } return output; } -grpc::string GetSourceEpilogue(const grpc::protobuf::FileDescriptor *file, +grpc::string GetSourceEpilogue(File *file, const Parameters ¶ms) { grpc::string temp; if (!file->package().empty()) { - std::vector<grpc::string> parts = - grpc_generator::tokenize(file->package(), "."); + std::vector<grpc::string> parts = file->package_parts(); for (auto part = parts.begin(); part != parts.end(); part++) { temp.append("} // namespace "); diff --git a/src/compiler/cpp_generator.h b/src/compiler/cpp_generator.h index 03621c8794..99a60a2eae 100644 --- a/src/compiler/cpp_generator.h +++ b/src/compiler/cpp_generator.h @@ -34,7 +34,23 @@ #ifndef GRPC_INTERNAL_COMPILER_CPP_GENERATOR_H #define GRPC_INTERNAL_COMPILER_CPP_GENERATOR_H -#include "src/compiler/config.h" +// cpp_generator.h/.cc do not directly depend on GRPC/ProtoBuf, such that they +// can be used to generate code for other serialization systems, such as +// FlatBuffers. + +#include <memory> +#include <vector> + +#ifndef GRPC_CUSTOM_STRING +#include <string> +#define GRPC_CUSTOM_STRING std::string +#endif + +namespace grpc { + +typedef GRPC_CUSTOM_STRING string; + +} // namespace grpc namespace grpc_cpp_generator { @@ -48,37 +64,80 @@ struct Parameters { grpc::string grpc_search_path; }; +// An abstract interface representing a method. +struct Method { + virtual ~Method() {} + + virtual grpc::string name() const = 0; + + virtual grpc::string input_type_name() const = 0; + virtual grpc::string output_type_name() const = 0; + + virtual bool NoStreaming() const = 0; + virtual bool ClientOnlyStreaming() const = 0; + virtual bool ServerOnlyStreaming() const = 0; + virtual bool BidiStreaming() const = 0; +}; + +// An abstract interface representing a service. +struct Service { + virtual ~Service() {} + + virtual grpc::string name() const = 0; + + virtual int method_count() const = 0; + virtual std::unique_ptr<const Method> method(int i) const = 0; +}; + +struct Printer { + virtual ~Printer() {} + + virtual void Print(const std::map<grpc::string, grpc::string> &vars, + const char *template_string) = 0; + virtual void Print(const char *string) = 0; + virtual void Indent() = 0; + virtual void Outdent() = 0; +}; + +// An interface that allows the source generated to be output using various +// libraries/idls/serializers. +struct File { + virtual ~File() {} + + virtual grpc::string filename() const = 0; + virtual grpc::string filename_without_ext() const = 0; + virtual grpc::string package() const = 0; + virtual std::vector<grpc::string> package_parts() const = 0; + + virtual int service_count() const = 0; + virtual std::unique_ptr<const Service> service(int i) const = 0; + + virtual std::unique_ptr<Printer> CreatePrinter(grpc::string *str) const = 0; +}; + // Return the prologue of the generated header file. -grpc::string GetHeaderPrologue(const grpc::protobuf::FileDescriptor *file, - const Parameters ¶ms); +grpc::string GetHeaderPrologue(File *file, const Parameters ¶ms); // Return the includes needed for generated header file. -grpc::string GetHeaderIncludes(const grpc::protobuf::FileDescriptor *file, - const Parameters ¶ms); +grpc::string GetHeaderIncludes(File *file, const Parameters ¶ms); // Return the includes needed for generated source file. -grpc::string GetSourceIncludes(const grpc::protobuf::FileDescriptor *file, - const Parameters ¶ms); +grpc::string GetSourceIncludes(File *file, const Parameters ¶ms); // Return the epilogue of the generated header file. -grpc::string GetHeaderEpilogue(const grpc::protobuf::FileDescriptor *file, - const Parameters ¶ms); +grpc::string GetHeaderEpilogue(File *file, const Parameters ¶ms); // Return the prologue of the generated source file. -grpc::string GetSourcePrologue(const grpc::protobuf::FileDescriptor *file, - const Parameters ¶ms); +grpc::string GetSourcePrologue(File *file, const Parameters ¶ms); // Return the services for generated header file. -grpc::string GetHeaderServices(const grpc::protobuf::FileDescriptor *file, - const Parameters ¶ms); +grpc::string GetHeaderServices(File *file, const Parameters ¶ms); // Return the services for generated source file. -grpc::string GetSourceServices(const grpc::protobuf::FileDescriptor *file, - const Parameters ¶ms); +grpc::string GetSourceServices(File *file, const Parameters ¶ms); // Return the epilogue of the generated source file. -grpc::string GetSourceEpilogue(const grpc::protobuf::FileDescriptor *file, - const Parameters ¶ms); +grpc::string GetSourceEpilogue(File *file, const Parameters ¶ms); } // namespace grpc_cpp_generator diff --git a/src/compiler/cpp_plugin.cc b/src/compiler/cpp_plugin.cc index 92a9ba7549..f703c6453d 100644 --- a/src/compiler/cpp_plugin.cc +++ b/src/compiler/cpp_plugin.cc @@ -41,6 +41,105 @@ #include "src/compiler/cpp_generator.h" #include "src/compiler/cpp_generator_helpers.h" +class ProtoBufMethod : public grpc_cpp_generator::Method { + public: + ProtoBufMethod(const grpc::protobuf::MethodDescriptor *method) + : method_(method) {} + + grpc::string name() const { return method_->name(); } + + grpc::string input_type_name() const { + return grpc_cpp_generator::ClassName(method_->input_type(), true); + } + grpc::string output_type_name() const { + return grpc_cpp_generator::ClassName(method_->output_type(), true); + } + + bool NoStreaming() const { + return !method_->client_streaming() && !method_->server_streaming(); + } + + bool ClientOnlyStreaming() const { + return method_->client_streaming() && !method_->server_streaming(); + } + + bool ServerOnlyStreaming() const { + return !method_->client_streaming() && method_->server_streaming(); + } + + bool BidiStreaming() const { + return method_->client_streaming() && method_->server_streaming(); + } + + private: + const grpc::protobuf::MethodDescriptor *method_; +}; + +class ProtoBufService : public grpc_cpp_generator::Service { + public: + ProtoBufService(const grpc::protobuf::ServiceDescriptor *service) + : service_(service) {} + + grpc::string name() const { return service_->name(); } + + int method_count() const { return service_->method_count(); }; + std::unique_ptr<const grpc_cpp_generator::Method> method(int i) const { + return std::unique_ptr<const grpc_cpp_generator::Method>( + new ProtoBufMethod(service_->method(i))); + }; + + private: + const grpc::protobuf::ServiceDescriptor *service_; +}; + +class ProtoBufPrinter : public grpc_cpp_generator::Printer { + public: + ProtoBufPrinter(grpc::string *str) + : output_stream_(str), printer_(&output_stream_, '$') {} + + void Print(const std::map<grpc::string, grpc::string> &vars, + const char *string_template) { + printer_.Print(vars, string_template); + } + + void Print(const char *string) { printer_.Print(string); } + void Indent() { printer_.Indent(); } + void Outdent() { printer_.Outdent(); } + + private: + grpc::protobuf::io::StringOutputStream output_stream_; + grpc::protobuf::io::Printer printer_; +}; + +class ProtoBufFile : public grpc_cpp_generator::File { + public: + ProtoBufFile(const grpc::protobuf::FileDescriptor *file) : file_(file) {} + + grpc::string filename() const { return file_->name(); } + grpc::string filename_without_ext() const { + return grpc_generator::StripProto(filename()); + } + + grpc::string package() const { return file_->package(); } + std::vector<grpc::string> package_parts() const { + return grpc_generator::tokenize(package(), "."); + } + + int service_count() const { return file_->service_count(); }; + std::unique_ptr<const grpc_cpp_generator::Service> service(int i) const { + return std::unique_ptr<const grpc_cpp_generator::Service> ( + new ProtoBufService(file_->service(i))); + } + + std::unique_ptr<grpc_cpp_generator::Printer> CreatePrinter(grpc::string *str) const { + return std::unique_ptr<grpc_cpp_generator::Printer>( + new ProtoBufPrinter(str)); + } + + private: + const grpc::protobuf::FileDescriptor *file_; +}; + class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { public: CppGrpcGenerator() {} @@ -61,6 +160,8 @@ class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { grpc_cpp_generator::Parameters generator_parameters; generator_parameters.use_system_headers = true; + ProtoBufFile pbfile(file); + if (!parameter.empty()) { std::vector<grpc::string> parameters_list = grpc_generator::tokenize(parameter, ","); @@ -92,10 +193,10 @@ class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { grpc::string file_name = grpc_generator::StripProto(file->name()); grpc::string header_code = - grpc_cpp_generator::GetHeaderPrologue(file, generator_parameters) + - grpc_cpp_generator::GetHeaderIncludes(file, generator_parameters) + - grpc_cpp_generator::GetHeaderServices(file, generator_parameters) + - grpc_cpp_generator::GetHeaderEpilogue(file, generator_parameters); + grpc_cpp_generator::GetHeaderPrologue(&pbfile, generator_parameters) + + grpc_cpp_generator::GetHeaderIncludes(&pbfile, generator_parameters) + + grpc_cpp_generator::GetHeaderServices(&pbfile, generator_parameters) + + grpc_cpp_generator::GetHeaderEpilogue(&pbfile, generator_parameters); std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> header_output( context->Open(file_name + ".grpc.pb.h")); grpc::protobuf::io::CodedOutputStream header_coded_out( @@ -103,10 +204,10 @@ class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { header_coded_out.WriteRaw(header_code.data(), header_code.size()); grpc::string source_code = - grpc_cpp_generator::GetSourcePrologue(file, generator_parameters) + - grpc_cpp_generator::GetSourceIncludes(file, generator_parameters) + - grpc_cpp_generator::GetSourceServices(file, generator_parameters) + - grpc_cpp_generator::GetSourceEpilogue(file, generator_parameters); + grpc_cpp_generator::GetSourcePrologue(&pbfile, generator_parameters) + + grpc_cpp_generator::GetSourceIncludes(&pbfile, generator_parameters) + + grpc_cpp_generator::GetSourceServices(&pbfile, generator_parameters) + + grpc_cpp_generator::GetSourceEpilogue(&pbfile, generator_parameters); std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> source_output( context->Open(file_name + ".grpc.pb.cc")); grpc::protobuf::io::CodedOutputStream source_coded_out( diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index cfb5251684..aaeb384f6e 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -450,7 +450,7 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, &sockname_len)) { port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp); if (port > 0) { - allocated_addr = malloc(addr_len); + allocated_addr = gpr_malloc(addr_len); memcpy(allocated_addr, addr, addr_len); grpc_sockaddr_set_port(allocated_addr, port); addr = allocated_addr; diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c index 9f9d7dcb54..6940dec7b0 100644 --- a/src/core/lib/iomgr/tcp_server_windows.c +++ b/src/core/lib/iomgr/tcp_server_windows.c @@ -467,7 +467,7 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, (struct sockaddr *)&sockname_temp, &sockname_len)) { port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp); if (port > 0) { - allocated_addr = malloc(addr_len); + allocated_addr = gpr_malloc(addr_len); memcpy(allocated_addr, addr, addr_len); grpc_sockaddr_set_port(allocated_addr, port); addr = allocated_addr; diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index a0b9709be5..24131179af 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -328,7 +328,7 @@ int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr, &sockname_len)) { port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp); if (port > 0) { - allocated_addr = malloc(addr_len); + allocated_addr = gpr_malloc(addr_len); memcpy(allocated_addr, addr, addr_len); grpc_sockaddr_set_port(allocated_addr, port); addr = allocated_addr; diff --git a/src/core/lib/profiling/basic_timers.c b/src/core/lib/profiling/basic_timers.c index 93e0b99e90..50082cd7ee 100644 --- a/src/core/lib/profiling/basic_timers.c +++ b/src/core/lib/profiling/basic_timers.c @@ -208,6 +208,7 @@ static void init_output() { } static void rotate_log() { + /* Using malloc here, as this code could end up being called by gpr_malloc */ gpr_timer_log *new = malloc(sizeof(*new)); gpr_once_init(&g_once_init, init_output); new->num_entries = 0; diff --git a/src/core/lib/support/alloc.c b/src/core/lib/support/alloc.c index 020917f79c..618c3f6acd 100644 --- a/src/core/lib/support/alloc.c +++ b/src/core/lib/support/alloc.c @@ -53,6 +53,7 @@ void gpr_set_allocation_functions(gpr_allocation_functions functions) { void *gpr_malloc(size_t size) { void *p; + if (size == 0) return NULL; GPR_TIMER_BEGIN("gpr_malloc", 0); p = g_alloc_functions.malloc_fn(size); if (!p) { @@ -69,6 +70,7 @@ void gpr_free(void *p) { } void *gpr_realloc(void *p, size_t size) { + if ((size == 0) && (p == NULL)) return NULL; GPR_TIMER_BEGIN("gpr_realloc", 0); p = g_alloc_functions.realloc_fn(p, size); if (!p) { diff --git a/src/core/lib/support/log_linux.c b/src/core/lib/support/log_linux.c index 6d4b63bbe0..ff3febb38e 100644 --- a/src/core/lib/support/log_linux.c +++ b/src/core/lib/support/log_linux.c @@ -68,6 +68,7 @@ void gpr_log(const char *file, int line, gpr_log_severity severity, } va_end(args); gpr_log_message(file, line, severity, message); + /* message has been allocated by vasprintf above, and needs free */ free(message); } diff --git a/src/core/lib/support/thd_posix.c b/src/core/lib/support/thd_posix.c index 89832e3ce1..2fc23bffaf 100644 --- a/src/core/lib/support/thd_posix.c +++ b/src/core/lib/support/thd_posix.c @@ -81,6 +81,7 @@ int gpr_thd_new(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg, thread_started = (pthread_create(&p, &attr, &thread_body, a) == 0); GPR_ASSERT(pthread_attr_destroy(&attr) == 0); if (!thread_started) { + /* don't use gpr_free, as this was allocated using malloc (see above) */ free(a); } *t = (gpr_thd_id)p; diff --git a/src/core/lib/surface/byte_buffer.c b/src/core/lib/surface/byte_buffer.c index fb39c4531d..a093a37af3 100644 --- a/src/core/lib/surface/byte_buffer.c +++ b/src/core/lib/surface/byte_buffer.c @@ -44,7 +44,7 @@ grpc_byte_buffer *grpc_raw_byte_buffer_create(gpr_slice *slices, grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( gpr_slice *slices, size_t nslices, grpc_compression_algorithm compression) { size_t i; - grpc_byte_buffer *bb = malloc(sizeof(grpc_byte_buffer)); + grpc_byte_buffer *bb = gpr_malloc(sizeof(grpc_byte_buffer)); bb->type = GRPC_BB_RAW; bb->data.raw.compression = compression; gpr_slice_buffer_init(&bb->data.raw.slice_buffer); @@ -57,7 +57,7 @@ grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( grpc_byte_buffer_reader *reader) { - grpc_byte_buffer *bb = malloc(sizeof(grpc_byte_buffer)); + grpc_byte_buffer *bb = gpr_malloc(sizeof(grpc_byte_buffer)); gpr_slice slice; bb->type = GRPC_BB_RAW; bb->data.raw.compression = GRPC_COMPRESS_NONE; @@ -85,7 +85,7 @@ void grpc_byte_buffer_destroy(grpc_byte_buffer *bb) { gpr_slice_buffer_destroy(&bb->data.raw.slice_buffer); break; } - free(bb); + gpr_free(bb); } size_t grpc_byte_buffer_length(grpc_byte_buffer *bb) { diff --git a/src/core/lib/tsi/fake_transport_security.c b/src/core/lib/tsi/fake_transport_security.c index 4b045b8cd9..0e20d6fd71 100644 --- a/src/core/lib/tsi/fake_transport_security.c +++ b/src/core/lib/tsi/fake_transport_security.c @@ -36,6 +36,7 @@ #include <stdlib.h> #include <string.h> +#include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/port_platform.h> #include <grpc/support/useful.h> @@ -134,12 +135,12 @@ static void tsi_fake_frame_reset(tsi_fake_frame *frame, int needs_draining) { static int tsi_fake_frame_ensure_size(tsi_fake_frame *frame) { if (frame->data == NULL) { frame->allocated_size = frame->size; - frame->data = malloc(frame->allocated_size); + frame->data = gpr_malloc(frame->allocated_size); if (frame->data == NULL) return 0; } else if (frame->size > frame->allocated_size) { - unsigned char *new_data = realloc(frame->data, frame->size); + unsigned char *new_data = gpr_realloc(frame->data, frame->size); if (new_data == NULL) { - free(frame->data); + gpr_free(frame->data); frame->data = NULL; return 0; } @@ -160,7 +161,7 @@ static tsi_result fill_frame_from_bytes(const unsigned char *incoming_bytes, if (frame->needs_draining) return TSI_INTERNAL_ERROR; if (frame->data == NULL) { frame->allocated_size = TSI_FAKE_FRAME_INITIAL_ALLOCATED_SIZE; - frame->data = malloc(frame->allocated_size); + frame->data = gpr_malloc(frame->allocated_size); if (frame->data == NULL) return TSI_OUT_OF_RESOURCES; } @@ -226,7 +227,7 @@ static tsi_result bytes_to_frame(unsigned char *bytes, size_t bytes_size, } static void tsi_fake_frame_destruct(tsi_fake_frame *frame) { - if (frame->data != NULL) free(frame->data); + if (frame->data != NULL) gpr_free(frame->data); } /* --- tsi_frame_protector methods implementation. ---*/ @@ -366,7 +367,7 @@ static void fake_protector_destroy(tsi_frame_protector *self) { tsi_fake_frame_protector *impl = (tsi_fake_frame_protector *)self; tsi_fake_frame_destruct(&impl->protect_frame); tsi_fake_frame_destruct(&impl->unprotect_frame); - free(self); + gpr_free(self); } static const tsi_frame_protector_vtable frame_protector_vtable = { @@ -488,7 +489,7 @@ static void fake_handshaker_destroy(tsi_handshaker *self) { tsi_fake_handshaker *impl = (tsi_fake_handshaker *)self; tsi_fake_frame_destruct(&impl->incoming); tsi_fake_frame_destruct(&impl->outgoing); - free(self); + gpr_free(self); } static const tsi_handshaker_vtable handshaker_vtable = { @@ -501,7 +502,8 @@ static const tsi_handshaker_vtable handshaker_vtable = { }; tsi_handshaker *tsi_create_fake_handshaker(int is_client) { - tsi_fake_handshaker *impl = calloc(1, sizeof(tsi_fake_handshaker)); + tsi_fake_handshaker *impl = gpr_malloc(sizeof(*impl)); + memset(impl, 0, sizeof(*impl)); impl->base.vtable = &handshaker_vtable; impl->is_client = is_client; impl->result = TSI_HANDSHAKE_IN_PROGRESS; @@ -517,8 +519,8 @@ tsi_handshaker *tsi_create_fake_handshaker(int is_client) { tsi_frame_protector *tsi_create_fake_protector( size_t *max_protected_frame_size) { - tsi_fake_frame_protector *impl = calloc(1, sizeof(tsi_fake_frame_protector)); - if (impl == NULL) return NULL; + tsi_fake_frame_protector *impl = gpr_malloc(sizeof(*impl)); + memset(impl, 0, sizeof(*impl)); impl->max_frame_size = (max_protected_frame_size == NULL) ? TSI_FAKE_DEFAULT_FRAME_SIZE : *max_protected_frame_size; diff --git a/src/core/lib/tsi/ssl_transport_security.c b/src/core/lib/tsi/ssl_transport_security.c index d98b3e1558..045901cc72 100644 --- a/src/core/lib/tsi/ssl_transport_security.c +++ b/src/core/lib/tsi/ssl_transport_security.c @@ -45,6 +45,7 @@ #include <arpa/inet.h> #endif +#include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/sync.h> #include <grpc/support/thd.h> @@ -148,8 +149,7 @@ static void init_openssl(void) { OpenSSL_add_all_algorithms(); num_locks = CRYPTO_num_locks(); GPR_ASSERT(num_locks > 0); - openssl_mutexes = malloc((size_t)num_locks * sizeof(gpr_mu)); - GPR_ASSERT(openssl_mutexes != NULL); + openssl_mutexes = gpr_malloc((size_t)num_locks * sizeof(gpr_mu)); for (i = 0; i < CRYPTO_num_locks(); i++) { gpr_mu_init(&openssl_mutexes[i]); } @@ -701,7 +701,7 @@ static tsi_result build_alpn_protocol_name_list( } *protocol_name_list_length += (size_t)alpn_protocols_lengths[i] + 1; } - *protocol_name_list = malloc(*protocol_name_list_length); + *protocol_name_list = gpr_malloc(*protocol_name_list_length); if (*protocol_name_list == NULL) return TSI_OUT_OF_RESOURCES; current = *protocol_name_list; for (i = 0; i < num_alpn_protocols; i++) { @@ -852,9 +852,9 @@ static tsi_result ssl_protector_unprotect( static void ssl_protector_destroy(tsi_frame_protector *self) { tsi_ssl_frame_protector *impl = (tsi_ssl_frame_protector *)self; - if (impl->buffer != NULL) free(impl->buffer); + if (impl->buffer != NULL) gpr_free(impl->buffer); if (impl->ssl != NULL) SSL_free(impl->ssl); - free(self); + gpr_free(self); } static const tsi_frame_protector_vtable frame_protector_vtable = { @@ -966,8 +966,9 @@ static tsi_result ssl_handshaker_extract_peer(tsi_handshaker *self, if (alpn_selected != NULL) { size_t i; tsi_peer_property *new_properties = - calloc(1, sizeof(tsi_peer_property) * (peer->property_count + 1)); - if (new_properties == NULL) return TSI_OUT_OF_RESOURCES; + gpr_malloc(sizeof(*new_properties) * (peer->property_count + 1)); + memset(new_properties, 0, + sizeof(*new_properties) * (peer->property_count + 1)); for (i = 0; i < peer->property_count; i++) { new_properties[i] = peer->properties[i]; } @@ -975,10 +976,10 @@ static tsi_result ssl_handshaker_extract_peer(tsi_handshaker *self, TSI_SSL_ALPN_SELECTED_PROTOCOL, (const char *)alpn_selected, alpn_selected_len, &new_properties[peer->property_count]); if (result != TSI_OK) { - free(new_properties); + gpr_free(new_properties); return result; } - if (peer->properties != NULL) free(peer->properties); + if (peer->properties != NULL) gpr_free(peer->properties); peer->property_count++; peer->properties = new_properties; } @@ -991,11 +992,8 @@ static tsi_result ssl_handshaker_create_frame_protector( size_t actual_max_output_protected_frame_size = TSI_SSL_MAX_PROTECTED_FRAME_SIZE_UPPER_BOUND; tsi_ssl_handshaker *impl = (tsi_ssl_handshaker *)self; - tsi_ssl_frame_protector *protector_impl = - calloc(1, sizeof(tsi_ssl_frame_protector)); - if (protector_impl == NULL) { - return TSI_OUT_OF_RESOURCES; - } + tsi_ssl_frame_protector *protector_impl = gpr_malloc(sizeof(*protector_impl)); + memset(protector_impl, 0, sizeof(*protector_impl)); if (max_output_protected_frame_size != NULL) { if (*max_output_protected_frame_size > @@ -1011,11 +1009,11 @@ static tsi_result ssl_handshaker_create_frame_protector( } protector_impl->buffer_size = actual_max_output_protected_frame_size - TSI_SSL_MAX_PROTECTION_OVERHEAD; - protector_impl->buffer = malloc(protector_impl->buffer_size); + protector_impl->buffer = gpr_malloc(protector_impl->buffer_size); if (protector_impl->buffer == NULL) { gpr_log(GPR_ERROR, "Could not allocated buffer for tsi_ssl_frame_protector."); - free(protector_impl); + gpr_free(protector_impl); return TSI_INTERNAL_ERROR; } @@ -1034,7 +1032,7 @@ static tsi_result ssl_handshaker_create_frame_protector( static void ssl_handshaker_destroy(tsi_handshaker *self) { tsi_ssl_handshaker *impl = (tsi_ssl_handshaker *)self; SSL_free(impl->ssl); /* The BIO objects are owned by ssl */ - free(impl); + gpr_free(impl); } static const tsi_handshaker_vtable handshaker_vtable = { @@ -1111,11 +1109,8 @@ static tsi_result create_tsi_ssl_handshaker(SSL_CTX *ctx, int is_client, SSL_set_accept_state(ssl); } - impl = calloc(1, sizeof(tsi_ssl_handshaker)); - if (impl == NULL) { - SSL_free(ssl); - return TSI_OUT_OF_RESOURCES; - } + impl = gpr_malloc(sizeof(*impl)); + memset(impl, 0, sizeof(*impl)); impl->ssl = ssl; impl->into_ssl = into_ssl; impl->from_ssl = from_ssl; @@ -1167,8 +1162,8 @@ static void ssl_client_handshaker_factory_destroy( tsi_ssl_client_handshaker_factory *impl = (tsi_ssl_client_handshaker_factory *)self; if (impl->ssl_context != NULL) SSL_CTX_free(impl->ssl_context); - if (impl->alpn_protocol_list != NULL) free(impl->alpn_protocol_list); - free(impl); + if (impl->alpn_protocol_list != NULL) gpr_free(impl->alpn_protocol_list); + gpr_free(impl); } static int client_handshaker_factory_npn_callback(SSL *ssl, unsigned char **out, @@ -1209,12 +1204,12 @@ static void ssl_server_handshaker_factory_destroy( tsi_peer_destruct(&impl->ssl_context_x509_subject_names[i]); } } - if (impl->ssl_contexts != NULL) free(impl->ssl_contexts); + if (impl->ssl_contexts != NULL) gpr_free(impl->ssl_contexts); if (impl->ssl_context_x509_subject_names != NULL) { - free(impl->ssl_context_x509_subject_names); + gpr_free(impl->ssl_context_x509_subject_names); } - if (impl->alpn_protocol_list != NULL) free(impl->alpn_protocol_list); - free(impl); + if (impl->alpn_protocol_list != NULL) gpr_free(impl->alpn_protocol_list); + gpr_free(impl); } static int does_entry_match_name(const char *entry, size_t entry_length, @@ -1333,11 +1328,8 @@ tsi_result tsi_create_ssl_client_handshaker_factory( return TSI_INVALID_ARGUMENT; } - impl = calloc(1, sizeof(tsi_ssl_client_handshaker_factory)); - if (impl == NULL) { - SSL_CTX_free(ssl_context); - return TSI_OUT_OF_RESOURCES; - } + impl = gpr_malloc(sizeof(*impl)); + memset(impl, 0, sizeof(*impl)); impl->ssl_context = ssl_context; do { @@ -1411,14 +1403,17 @@ tsi_result tsi_create_ssl_server_handshaker_factory( return TSI_INVALID_ARGUMENT; } - impl = calloc(1, sizeof(tsi_ssl_server_handshaker_factory)); - if (impl == NULL) return TSI_OUT_OF_RESOURCES; + impl = gpr_malloc(sizeof(*impl)); + memset(impl, 0, sizeof(*impl)); impl->base.create_handshaker = ssl_server_handshaker_factory_create_handshaker; impl->base.destroy = ssl_server_handshaker_factory_destroy; - impl->ssl_contexts = calloc(key_cert_pair_count, sizeof(SSL_CTX *)); + impl->ssl_contexts = gpr_malloc(key_cert_pair_count * sizeof(SSL_CTX *)); + memset(impl->ssl_contexts, 0, key_cert_pair_count * sizeof(SSL_CTX *)); impl->ssl_context_x509_subject_names = - calloc(key_cert_pair_count, sizeof(tsi_peer)); + gpr_malloc(key_cert_pair_count * sizeof(tsi_peer)); + memset(impl->ssl_context_x509_subject_names, 0, + key_cert_pair_count * sizeof(tsi_peer)); if (impl->ssl_contexts == NULL || impl->ssl_context_x509_subject_names == NULL) { tsi_ssl_handshaker_factory_destroy(&impl->base); diff --git a/src/core/lib/tsi/transport_security.c b/src/core/lib/tsi/transport_security.c index 861fc791bc..830cf09584 100644 --- a/src/core/lib/tsi/transport_security.c +++ b/src/core/lib/tsi/transport_security.c @@ -33,6 +33,9 @@ #include "src/core/lib/tsi/transport_security.h" +#include <grpc/support/alloc.h> +#include <grpc/support/string_util.h> + #include <stdlib.h> #include <string.h> @@ -40,19 +43,6 @@ int tsi_tracing_enabled = 0; -/* --- Utils. --- */ - -char *tsi_strdup(const char *src) { - char *dst; - size_t len; - if (!src) return NULL; - len = strlen(src) + 1; - dst = malloc(len); - if (!dst) return NULL; - memcpy(dst, src, len); - return dst; -} - /* --- tsi_result common implementation. --- */ const char *tsi_result_to_string(tsi_result result) { @@ -214,15 +204,15 @@ static void tsi_peer_destroy_list_property(tsi_peer_property *children, for (i = 0; i < child_count; i++) { tsi_peer_property_destruct(&children[i]); } - free(children); + gpr_free(children); } void tsi_peer_property_destruct(tsi_peer_property *property) { if (property->name != NULL) { - free(property->name); + gpr_free(property->name); } if (property->value.data != NULL) { - free(property->value.data); + gpr_free(property->value.data); } *property = tsi_init_peer_property(); /* Reset everything to 0. */ } @@ -239,16 +229,10 @@ void tsi_peer_destruct(tsi_peer *self) { tsi_result tsi_construct_allocated_string_peer_property( const char *name, size_t value_length, tsi_peer_property *property) { *property = tsi_init_peer_property(); - if (name != NULL) { - property->name = tsi_strdup(name); - if (property->name == NULL) return TSI_OUT_OF_RESOURCES; - } + if (name != NULL) property->name = gpr_strdup(name); if (value_length > 0) { - property->value.data = calloc(1, value_length); - if (property->value.data == NULL) { - tsi_peer_property_destruct(property); - return TSI_OUT_OF_RESOURCES; - } + property->value.data = gpr_malloc(value_length); + memset(property->value.data, 0, value_length); property->value.length = value_length; } return TSI_OK; @@ -276,8 +260,8 @@ tsi_result tsi_construct_string_peer_property(const char *name, tsi_result tsi_construct_peer(size_t property_count, tsi_peer *peer) { memset(peer, 0, sizeof(tsi_peer)); if (property_count > 0) { - peer->properties = calloc(property_count, sizeof(tsi_peer_property)); - if (peer->properties == NULL) return TSI_OUT_OF_RESOURCES; + peer->properties = gpr_malloc(property_count * sizeof(tsi_peer_property)); + memset(peer->properties, 0, property_count * sizeof(tsi_peer_property)); peer->property_count = property_count; } return TSI_OK; diff --git a/src/ruby/lib/grpc/generic/client_stub.rb b/src/ruby/lib/grpc/generic/client_stub.rb index 98e83a8396..a6bb92d72c 100644 --- a/src/ruby/lib/grpc/generic/client_stub.rb +++ b/src/ruby/lib/grpc/generic/client_stub.rb @@ -85,7 +85,8 @@ module GRPC # when present, this is the default timeout used for calls # # @param host [String] the host the stub connects to - # @param q [Core::CompletionQueue] used to wait for events + # @param q [Core::CompletionQueue] used to wait for events - now deprecated + # since each new active call gets its own separately # @param creds [Core::ChannelCredentials|Symbol] the channel credentials, or # :this_channel_is_insecure # @param channel_override [Core::Channel] a pre-created channel @@ -97,7 +98,6 @@ module GRPC propagate_mask: nil, **kw) fail(TypeError, '!CompletionQueue') unless q.is_a?(Core::CompletionQueue) - @queue = q @ch = ClientStub.setup_channel(channel_override, host, creds, **kw) alt_host = kw[Core::Channel::SSL_TARGET] @host = alt_host.nil? ? host : alt_host @@ -458,14 +458,17 @@ module GRPC if deadline.nil? deadline = from_relative_time(timeout.nil? ? @timeout : timeout) end - call = @ch.create_call(@queue, + # Provide each new client call with its own completion queue + call_queue = Core::CompletionQueue.new + call = @ch.create_call(call_queue, parent, # parent call @propagate_mask, # propagation options method, nil, # host use nil, deadline) call.set_credentials! credentials unless credentials.nil? - ActiveCall.new(call, @queue, marshal, unmarshal, deadline, started: false) + ActiveCall.new(call, call_queue, marshal, unmarshal, deadline, + started: false) end end end diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index 701e1c7485..6f8d1fb9e6 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -37,6 +37,7 @@ s.add_dependency 'googleauth', '~> 0.5.1' s.add_development_dependency 'bundler', '~> 1.9' + s.add_development_dependency 'facter', '~> 2.4' s.add_development_dependency 'logging', '~> 2.0' s.add_development_dependency 'simplecov', '~> 0.9' s.add_development_dependency 'rake', '~> 10.4' diff --git a/templates/tools/fuzzer/runners.template b/templates/tools/fuzzer/runners.template index 72d0b363f2..4b8d4f3c68 100644 --- a/templates/tools/fuzzer/runners.template +++ b/templates/tools/fuzzer/runners.template @@ -39,7 +39,7 @@ template: | if [ "$jobs" != "1" ] then - flags="-jobs=$jobs -workers=$jobs" + flags="-jobs=$jobs -workers=$jobs $flags" fi if [ "$config" == "asan-trace-cmp" ] diff --git a/test/core/end2end/fuzzers/client_fuzzer.c b/test/core/end2end/fuzzers/client_fuzzer.c index 4e22e268f0..afcf7638f7 100644 --- a/test/core/end2end/fuzzers/client_fuzzer.c +++ b/test/core/end2end/fuzzers/client_fuzzer.c @@ -36,6 +36,7 @@ #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" #include "src/core/lib/surface/channel.h" +#include "test/core/util/memory_counters.h" #include "test/core/util/mock_endpoint.h" static const bool squelch = true; @@ -48,7 +49,9 @@ static void dont_log(gpr_log_func_args *args) {} int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_test_only_set_metadata_hash_seed(0); + struct grpc_memory_counters counters; if (squelch) gpr_set_log_function(dont_log); + grpc_memory_counters_init(); grpc_init(); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -150,5 +153,8 @@ done: grpc_byte_buffer_destroy(response_payload_recv); } grpc_shutdown(); + counters = grpc_memory_counters_snapshot(); + grpc_memory_counters_destroy(); + GPR_ASSERT(counters.total_size_relative == 0); return 0; } diff --git a/test/core/end2end/fuzzers/server_fuzzer.c b/test/core/end2end/fuzzers/server_fuzzer.c index 393d33033b..40273711ab 100644 --- a/test/core/end2end/fuzzers/server_fuzzer.c +++ b/test/core/end2end/fuzzers/server_fuzzer.c @@ -35,6 +35,7 @@ #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" #include "src/core/lib/surface/server.h" +#include "test/core/util/memory_counters.h" #include "test/core/util/mock_endpoint.h" static const bool squelch = true; @@ -48,7 +49,9 @@ static void dont_log(gpr_log_func_args *args) {} int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_test_only_set_metadata_hash_seed(0); + struct grpc_memory_counters counters; if (squelch) gpr_set_log_function(dont_log); + grpc_memory_counters_init(); grpc_init(); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -117,5 +120,8 @@ done: grpc_server_destroy(server); grpc_completion_queue_destroy(cq); grpc_shutdown(); + counters = grpc_memory_counters_snapshot(); + grpc_memory_counters_destroy(); + GPR_ASSERT(counters.total_size_relative == 0); return 0; } diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c index 9b3b033624..52082c3c6b 100644 --- a/test/core/iomgr/endpoint_tests.c +++ b/test/core/iomgr/endpoint_tests.c @@ -89,7 +89,7 @@ static void end_test(grpc_endpoint_test_config config) { config.clean_up(); } static gpr_slice *allocate_blocks(size_t num_bytes, size_t slice_size, size_t *num_blocks, uint8_t *current_data) { size_t nslices = num_bytes / slice_size + (num_bytes % slice_size ? 1 : 0); - gpr_slice *slices = malloc(sizeof(gpr_slice) * nslices); + gpr_slice *slices = gpr_malloc(sizeof(gpr_slice) * nslices); size_t num_bytes_left = num_bytes; size_t i; size_t j; @@ -164,7 +164,7 @@ static void read_and_write_test_write_handler(grpc_exec_ctx *exec_ctx, gpr_slice_buffer_addn(&state->outgoing, slices, nslices); grpc_endpoint_write(exec_ctx, state->write_ep, &state->outgoing, &state->done_write); - free(slices); + gpr_free(slices); return; } } diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index d51b6b7597..7a98fa0e50 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -97,7 +97,7 @@ static ssize_t fill_socket(int fd) { static size_t fill_socket_partial(int fd, size_t bytes) { ssize_t write_bytes; size_t total_bytes = 0; - unsigned char *buf = malloc(bytes); + unsigned char *buf = gpr_malloc(bytes); unsigned i; for (i = 0; i < bytes; ++i) { buf[i] = (uint8_t)(i % 256); @@ -292,7 +292,7 @@ static void write_done(grpc_exec_ctx *exec_ctx, } void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { - unsigned char *buf = malloc(read_size); + unsigned char *buf = gpr_malloc(read_size); ssize_t bytes_read; size_t bytes_left = num_bytes; int flags; diff --git a/test/core/json/fuzzer.c b/test/core/json/fuzzer.c index 044db973ab..e94b41ca99 100644 --- a/test/core/json/fuzzer.c +++ b/test/core/json/fuzzer.c @@ -38,42 +38,12 @@ #include <grpc/support/log.h> #include "src/core/lib/json/json.h" - -static size_t g_total_size = 0; -static gpr_allocation_functions g_old_allocs; - -void *guard_malloc(size_t size) { - size_t *ptr; - g_total_size += size; - ptr = g_old_allocs.malloc_fn(size + sizeof(size)); - *ptr++ = size; - return ptr; -} - -void *guard_realloc(void *vptr, size_t size) { - size_t *ptr = vptr; - --ptr; - g_total_size -= *ptr; - ptr = g_old_allocs.realloc_fn(ptr, size + sizeof(size)); - g_total_size += size; - *ptr++ = size; - return ptr; -} - -void guard_free(void *vptr) { - size_t *ptr = vptr; - --ptr; - g_total_size -= *ptr; - g_old_allocs.free_fn(ptr); -} - -struct gpr_allocation_functions g_guard_allocs = {guard_malloc, guard_realloc, - guard_free}; +#include "test/core/util/memory_counters.h" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { char *s; - g_old_allocs = gpr_get_allocation_functions(); - gpr_set_allocation_functions(g_guard_allocs); + struct grpc_memory_counters counters; + grpc_memory_counters_init(); s = gpr_malloc(size); memcpy(s, data, size); grpc_json *x; @@ -81,7 +51,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_json_destroy(x); } gpr_free(s); - gpr_set_allocation_functions(g_old_allocs); - GPR_ASSERT(g_total_size == 0); + counters = grpc_memory_counters_snapshot(); + grpc_memory_counters_destroy(); + GPR_ASSERT(counters.total_size_relative == 0); return 0; } diff --git a/test/core/network_benchmarks/low_level_ping_pong.c b/test/core/network_benchmarks/low_level_ping_pong.c index ac5ee72daa..b3a38a55ad 100644 --- a/test/core/network_benchmarks/low_level_ping_pong.c +++ b/test/core/network_benchmarks/low_level_ping_pong.c @@ -265,19 +265,19 @@ static int epoll_setup(thread_args *args) { #endif static void server_thread(thread_args *args) { - char *buf = malloc(args->msg_size); + char *buf = gpr_malloc(args->msg_size); if (args->setup(args) < 0) { gpr_log(GPR_ERROR, "Setup failed"); } for (;;) { if (args->read_bytes(args, buf) < 0) { gpr_log(GPR_ERROR, "Server read failed"); - free(buf); + gpr_free(buf); return; } if (args->write_bytes(args, buf) < 0) { gpr_log(GPR_ERROR, "Server write failed"); - free(buf); + gpr_free(buf); return; } } @@ -304,7 +304,8 @@ static double now(void) { } static void client_thread(thread_args *args) { - char *buf = calloc(args->msg_size, sizeof(char)); + char *buf = gpr_malloc(args->msg_size * sizeof(char)); + memset(buf, 0, args->msg_size * sizeof(char)); gpr_histogram *histogram = gpr_histogram_create(0.01, 60e9); double start_time; double end_time; @@ -333,7 +334,7 @@ static void client_thread(thread_args *args) { } print_histogram(histogram); error: - free(buf); + gpr_free(buf); gpr_histogram_destroy(histogram); } @@ -596,8 +597,8 @@ static int run_all_benchmarks(size_t msg_size) { test_strategy *strategy = &test_strategies[i]; size_t j; for (j = 0; j < GPR_ARRAY_SIZE(socket_types); ++j) { - thread_args *client_args = malloc(sizeof(thread_args)); - thread_args *server_args = malloc(sizeof(thread_args)); + thread_args *client_args = gpr_malloc(sizeof(thread_args)); + thread_args *server_args = gpr_malloc(sizeof(thread_args)); char *socket_type = socket_types[j]; client_args->read_bytes = strategy->read_strategy; @@ -620,8 +621,8 @@ static int run_all_benchmarks(size_t msg_size) { } int main(int argc, char **argv) { - thread_args *client_args = malloc(sizeof(thread_args)); - thread_args *server_args = malloc(sizeof(thread_args)); + thread_args *client_args = gpr_malloc(sizeof(thread_args)); + thread_args *server_args = gpr_malloc(sizeof(thread_args)); int msg_size = -1; char *read_strategy = NULL; char *socket_type = NULL; diff --git a/test/core/support/alloc_test.c b/test/core/support/alloc_test.c index a7df1977da..451c580054 100644 --- a/test/core/support/alloc_test.c +++ b/test/core/support/alloc_test.c @@ -46,7 +46,7 @@ static void fake_free(void *addr) { static void test_custom_allocs() { const gpr_allocation_functions default_fns = gpr_get_allocation_functions(); intptr_t addr_to_free = 0; - int *i; + char *i; gpr_allocation_functions fns = {fake_malloc, fake_realloc, fake_free}; gpr_set_allocation_functions(fns); @@ -58,7 +58,7 @@ static void test_custom_allocs() { /* Restore and check we don't get funky values and that we don't leak */ gpr_set_allocation_functions(default_fns); - GPR_ASSERT((void *)1 != (i = gpr_malloc(sizeof(*i)))); + GPR_ASSERT((void *)sizeof(*i) != (i = gpr_malloc(sizeof(*i)))); GPR_ASSERT((void *)2 != (i = gpr_realloc(i, 2))); gpr_free(i); } diff --git a/test/core/util/memory_counters.c b/test/core/util/memory_counters.c new file mode 100644 index 0000000000..bebe94e582 --- /dev/null +++ b/test/core/util/memory_counters.c @@ -0,0 +1,117 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <stdint.h> +#include <string.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/sync.h> + +#include "test/core/util/memory_counters.h" + +static gpr_mu g_memory_mutex; +static struct grpc_memory_counters g_memory_counters; +static gpr_allocation_functions g_old_allocs; + +static void *guard_malloc(size_t size); +static void *guard_realloc(void *vptr, size_t size); +static void guard_free(void *vptr); + +static void *guard_malloc(size_t size) { + size_t *ptr; + if (!size) return NULL; + gpr_mu_lock(&g_memory_mutex); + g_memory_counters.total_size_absolute += size; + g_memory_counters.total_size_relative += size; + g_memory_counters.total_allocs_absolute++; + g_memory_counters.total_allocs_relative++; + gpr_mu_unlock(&g_memory_mutex); + ptr = g_old_allocs.malloc_fn(size + sizeof(size)); + *ptr++ = size; + return ptr; +} + +static void *guard_realloc(void *vptr, size_t size) { + size_t *ptr = vptr; + if (vptr == NULL) { + return guard_malloc(size); + } + if (size == 0) { + guard_free(vptr); + return NULL; + } + --ptr; + gpr_mu_lock(&g_memory_mutex); + g_memory_counters.total_size_absolute += size; + g_memory_counters.total_size_relative -= *ptr; + g_memory_counters.total_size_relative += size; + g_memory_counters.total_allocs_absolute++; + gpr_mu_unlock(&g_memory_mutex); + ptr = g_old_allocs.realloc_fn(ptr, size + sizeof(size)); + *ptr++ = size; + return ptr; +} + +static void guard_free(void *vptr) { + size_t *ptr = vptr; + if (!vptr) return; + --ptr; + gpr_mu_lock(&g_memory_mutex); + g_memory_counters.total_size_relative -= *ptr; + g_memory_counters.total_allocs_relative--; + gpr_mu_unlock(&g_memory_mutex); + g_old_allocs.free_fn(ptr); +} + +struct gpr_allocation_functions g_guard_allocs = {guard_malloc, guard_realloc, + guard_free}; + +void grpc_memory_counters_init() { + memset(&g_memory_counters, 0, sizeof(g_memory_counters)); + gpr_mu_init(&g_memory_mutex); + g_old_allocs = gpr_get_allocation_functions(); + gpr_set_allocation_functions(g_guard_allocs); +} + +void grpc_memory_counters_destroy() { + gpr_set_allocation_functions(g_old_allocs); + gpr_mu_destroy(&g_memory_mutex); +} + +struct grpc_memory_counters grpc_memory_counters_snapshot() { + struct grpc_memory_counters counters; + gpr_mu_lock(&g_memory_mutex); + counters = g_memory_counters; + gpr_mu_unlock(&g_memory_mutex); + return counters; +} diff --git a/test/core/util/memory_counters.h b/test/core/util/memory_counters.h new file mode 100644 index 0000000000..f332816501 --- /dev/null +++ b/test/core/util/memory_counters.h @@ -0,0 +1,48 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_TEST_CORE_UTIL_MEMORY_COUNTERS_H +#define GRPC_TEST_CORE_UTIL_MEMORY_COUNTERS_H + +struct grpc_memory_counters { + size_t total_size_relative; + size_t total_size_absolute; + size_t total_allocs_relative; + size_t total_allocs_absolute; +}; + +void grpc_memory_counters_init(); +void grpc_memory_counters_destroy(); +struct grpc_memory_counters grpc_memory_counters_snapshot(); + +#endif diff --git a/tools/fuzzer/runners/client_fuzzer.sh b/tools/fuzzer/runners/client_fuzzer.sh index 239d552c57..97d4e60d90 100644 --- a/tools/fuzzer/runners/client_fuzzer.sh +++ b/tools/fuzzer/runners/client_fuzzer.sh @@ -33,7 +33,7 @@ flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=2048" if [ "$jobs" != "1" ] then - flags="-jobs=$jobs -workers=$jobs" + flags="-jobs=$jobs -workers=$jobs $flags" fi if [ "$config" == "asan-trace-cmp" ] diff --git a/tools/fuzzer/runners/hpack_parser_fuzzer_test.sh b/tools/fuzzer/runners/hpack_parser_fuzzer_test.sh index e69b4b4dfe..c6f70a623d 100644 --- a/tools/fuzzer/runners/hpack_parser_fuzzer_test.sh +++ b/tools/fuzzer/runners/hpack_parser_fuzzer_test.sh @@ -33,7 +33,7 @@ flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=512" if [ "$jobs" != "1" ] then - flags="-jobs=$jobs -workers=$jobs" + flags="-jobs=$jobs -workers=$jobs $flags" fi if [ "$config" == "asan-trace-cmp" ] diff --git a/tools/fuzzer/runners/http_fuzzer_test.sh b/tools/fuzzer/runners/http_fuzzer_test.sh index c190ba40b6..bb54a23814 100644 --- a/tools/fuzzer/runners/http_fuzzer_test.sh +++ b/tools/fuzzer/runners/http_fuzzer_test.sh @@ -33,7 +33,7 @@ flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=2048" if [ "$jobs" != "1" ] then - flags="-jobs=$jobs -workers=$jobs" + flags="-jobs=$jobs -workers=$jobs $flags" fi if [ "$config" == "asan-trace-cmp" ] diff --git a/tools/fuzzer/runners/json_fuzzer_test.sh b/tools/fuzzer/runners/json_fuzzer_test.sh index 9fc6271976..e11e25dc09 100644 --- a/tools/fuzzer/runners/json_fuzzer_test.sh +++ b/tools/fuzzer/runners/json_fuzzer_test.sh @@ -33,7 +33,7 @@ flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=512" if [ "$jobs" != "1" ] then - flags="-jobs=$jobs -workers=$jobs" + flags="-jobs=$jobs -workers=$jobs $flags" fi if [ "$config" == "asan-trace-cmp" ] diff --git a/tools/fuzzer/runners/nanopb_fuzzer_response_test.sh b/tools/fuzzer/runners/nanopb_fuzzer_response_test.sh index bbcebf11cc..97359277ce 100644 --- a/tools/fuzzer/runners/nanopb_fuzzer_response_test.sh +++ b/tools/fuzzer/runners/nanopb_fuzzer_response_test.sh @@ -33,7 +33,7 @@ flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=128" if [ "$jobs" != "1" ] then - flags="-jobs=$jobs -workers=$jobs" + flags="-jobs=$jobs -workers=$jobs $flags" fi if [ "$config" == "asan-trace-cmp" ] diff --git a/tools/fuzzer/runners/nanopb_fuzzer_serverlist_test.sh b/tools/fuzzer/runners/nanopb_fuzzer_serverlist_test.sh index e9099bac04..2dfaa2372f 100644 --- a/tools/fuzzer/runners/nanopb_fuzzer_serverlist_test.sh +++ b/tools/fuzzer/runners/nanopb_fuzzer_serverlist_test.sh @@ -33,7 +33,7 @@ flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=128" if [ "$jobs" != "1" ] then - flags="-jobs=$jobs -workers=$jobs" + flags="-jobs=$jobs -workers=$jobs $flags" fi if [ "$config" == "asan-trace-cmp" ] diff --git a/tools/fuzzer/runners/server_fuzzer.sh b/tools/fuzzer/runners/server_fuzzer.sh index 28ca8b3271..fc0567f670 100644 --- a/tools/fuzzer/runners/server_fuzzer.sh +++ b/tools/fuzzer/runners/server_fuzzer.sh @@ -33,7 +33,7 @@ flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=2048" if [ "$jobs" != "1" ] then - flags="-jobs=$jobs -workers=$jobs" + flags="-jobs=$jobs -workers=$jobs $flags" fi if [ "$config" == "asan-trace-cmp" ] diff --git a/tools/fuzzer/runners/uri_fuzzer_test.sh b/tools/fuzzer/runners/uri_fuzzer_test.sh index 7dac54ec51..5f33e73465 100644 --- a/tools/fuzzer/runners/uri_fuzzer_test.sh +++ b/tools/fuzzer/runners/uri_fuzzer_test.sh @@ -33,7 +33,7 @@ flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=128" if [ "$jobs" != "1" ] then - flags="-jobs=$jobs -workers=$jobs" + flags="-jobs=$jobs -workers=$jobs $flags" fi if [ "$config" == "asan-trace-cmp" ] diff --git a/tools/gcp/stress_test/stress_test_utils.py b/tools/gcp/stress_test/stress_test_utils.py index 6c7fe44dc1..19d59c0df1 100755 --- a/tools/gcp/stress_test/stress_test_utils.py +++ b/tools/gcp/stress_test/stress_test_utils.py @@ -103,23 +103,29 @@ class BigQueryHelper: return bq_utils.insert_rows(self.bq, self.project_id, self.dataset_id, self.qps_table_id, [row]) - def check_if_any_tests_failed(self, num_query_retries=3): + def check_if_any_tests_failed(self, num_query_retries=3, timeout_msec=30000): query = ('SELECT event_type FROM %s.%s WHERE run_id = \'%s\' AND ' 'event_type="%s"') % (self.dataset_id, self.summary_table_id, self.run_id, EventType.FAILURE) + page = None try: query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) + job_id = query_job['jobReference']['jobId'] + project_id = query_job['jobReference']['projectId'] page = self.bq.jobs().getQueryResults( - **query_job['jobReference']).execute(num_retries=num_query_retries) + projectId=project_id, + jobId=job_id, + timeoutMs=timeout_msec).execute(num_retries=num_query_retries) + + if not page['jobComplete']: + print('TIMEOUT ERROR: The query %s timed out. Current timeout value is' + ' %d msec. Returning False (i.e assuming there are no failures)' + ) % (query, timeoout_msec) + return False + num_failures = int(page['totalRows']) print 'num rows: ', num_failures return num_failures > 0 - # TODO (sreek): Cleanup the following lines once we have a better idea of - # why we sometimes get KeyError exceptions in long running test cases - except KeyError: - print 'KeyError in check_if_any_tests_failed()' - print 'Query:', query - print 'Query result page:', page except: print 'Exception in check_if_any_tests_failed(). Info: ', sys.exc_info() print 'Query: ', query diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 8d805fd889..5a426aa7f5 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -6259,6 +6259,7 @@ "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.h", "test/core/util/grpc_profiler.h", + "test/core/util/memory_counters.h", "test/core/util/mock_endpoint.h", "test/core/util/parse_hexstring.h", "test/core/util/passthru_endpoint.h", @@ -6277,6 +6278,8 @@ "test/core/iomgr/endpoint_tests.h", "test/core/util/grpc_profiler.c", "test/core/util/grpc_profiler.h", + "test/core/util/memory_counters.c", + "test/core/util/memory_counters.h", "test/core/util/mock_endpoint.c", "test/core/util/mock_endpoint.h", "test/core/util/parse_hexstring.c", diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json index 768088d93d..c558855314 100644 --- a/tools/run_tests/stress_test/configs/asan.json +++ b/tools/run_tests/stress_test/configs/asan.json @@ -11,13 +11,13 @@ "baseTemplates": { "default": { "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, + "pollIntervalSecs": 120, "clientArgs": { "num_channels_per_server":5, "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":60 + "metrics_collection_interval_secs":120 }, "metricsPort": 8081, "metricsArgs": { @@ -66,7 +66,7 @@ "stress-client-asan": { "clientTemplate": "cxx_client_asan", "dockerImage": "grpc_stress_cxx_asan", - "numInstances": 20, + "numInstances": 5, "serverPodSpec": "stress-server-asan" } } diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json index ffd4a704c3..75505186f2 100644 --- a/tools/run_tests/stress_test/configs/opt.json +++ b/tools/run_tests/stress_test/configs/opt.json @@ -66,7 +66,7 @@ "stress-client-opt": { "clientTemplate": "cxx_client_opt", "dockerImage": "grpc_stress_cxx_opt", - "numInstances": 10, + "numInstances": 15, "serverPodSpec": "stress-server-opt" } } diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json index f8d3f878e1..a7ec08313d 100644 --- a/tools/run_tests/stress_test/configs/tsan.json +++ b/tools/run_tests/stress_test/configs/tsan.json @@ -11,13 +11,13 @@ "baseTemplates": { "default": { "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, + "pollIntervalSecs": 120, "clientArgs": { "num_channels_per_server":5, "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":60 + "metrics_collection_interval_secs":120 }, "metricsPort": 8081, "metricsArgs": { @@ -66,7 +66,7 @@ "stress-client-tsan": { "clientTemplate": "cxx_client_tsan", "dockerImage": "grpc_stress_cxx_tsan", - "numInstances": 20, + "numInstances": 5, "serverPodSpec": "stress-server-tsan" } } diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index db3ba28346..916c890cbd 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -604,6 +604,17 @@ def run_tests(config): return is_success +def tear_down(config): + gke = Gke(config.global_settings.gcp_project_id, '', '', + config.global_settings.summary_table_id, + config.global_settings.qps_table_id, + config.global_settings.kubernetes_proxy_port) + for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): + gke.delete_servers(server_pod_spec) + for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): + gke.delete_clients(client_pod_spec) + + argp = argparse.ArgumentParser( description='Launch stress tests in GKE', formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -614,6 +625,7 @@ argp.add_argument('--config_file', required=True, type=str, help='The test config file') +argp.add_argument('--tear_down', action='store_true', default=False) if __name__ == '__main__': args = argp.parse_args() @@ -636,5 +648,11 @@ if __name__ == '__main__': os.path.dirname(sys.argv[0]), '../../..')) os.chdir(grpc_root) + # Note that tear_down is only in cases where we want to manually tear down a + # test that for some reason run_tests() could not cleanup + if args.tear_down: + tear_down(config) + sys.exit(1) + if not run_tests(config): sys.exit(1) diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 44e9f8034a..e361e24c60 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -153,6 +153,7 @@ <ClInclude Include="$(SolutionDir)\..\test\core\end2end\fixtures\proxy.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\iomgr\endpoint_tests.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\util\grpc_profiler.h" /> + <ClInclude Include="$(SolutionDir)\..\test\core\util\memory_counters.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\util\mock_endpoint.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\util\parse_hexstring.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\util\passthru_endpoint.h" /> @@ -177,6 +178,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\grpc_profiler.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\test\core\util\memory_counters.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\mock_endpoint.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\parse_hexstring.c"> diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 1cae9fb1fd..ae30396e67 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -25,6 +25,9 @@ <ClCompile Include="$(SolutionDir)\..\test\core\util\grpc_profiler.c"> <Filter>test\core\util</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\test\core\util\memory_counters.c"> + <Filter>test\core\util</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\mock_endpoint.c"> <Filter>test\core\util</Filter> </ClCompile> @@ -66,6 +69,9 @@ <ClInclude Include="$(SolutionDir)\..\test\core\util\grpc_profiler.h"> <Filter>test\core\util</Filter> </ClInclude> + <ClInclude Include="$(SolutionDir)\..\test\core\util\memory_counters.h"> + <Filter>test\core\util</Filter> + </ClInclude> <ClInclude Include="$(SolutionDir)\..\test\core\util\mock_endpoint.h"> <Filter>test\core\util</Filter> </ClInclude> diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj index e681622071..33860c48a4 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj @@ -151,6 +151,7 @@ <ClInclude Include="$(SolutionDir)\..\test\core\end2end\fixtures\proxy.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\iomgr\endpoint_tests.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\util\grpc_profiler.h" /> + <ClInclude Include="$(SolutionDir)\..\test\core\util\memory_counters.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\util\mock_endpoint.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\util\parse_hexstring.h" /> <ClInclude Include="$(SolutionDir)\..\test\core\util\passthru_endpoint.h" /> @@ -167,6 +168,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\grpc_profiler.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\test\core\util\memory_counters.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\mock_endpoint.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\parse_hexstring.c"> diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters index b40fe48409..372bb2af67 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters @@ -13,6 +13,9 @@ <ClCompile Include="$(SolutionDir)\..\test\core\util\grpc_profiler.c"> <Filter>test\core\util</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\test\core\util\memory_counters.c"> + <Filter>test\core\util</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\mock_endpoint.c"> <Filter>test\core\util</Filter> </ClCompile> @@ -48,6 +51,9 @@ <ClInclude Include="$(SolutionDir)\..\test\core\util\grpc_profiler.h"> <Filter>test\core\util</Filter> </ClInclude> + <ClInclude Include="$(SolutionDir)\..\test\core\util\memory_counters.h"> + <Filter>test\core\util</Filter> + </ClInclude> <ClInclude Include="$(SolutionDir)\..\test\core\util\mock_endpoint.h"> <Filter>test\core\util</Filter> </ClInclude> |