From 9086d9643903c608ab015b0b7d903547a4e7b6f3 Mon Sep 17 00:00:00 2001 From: Feng Xiao Date: Wed, 13 Jul 2016 13:47:51 -0700 Subject: Integrate from internal code base. --- src/google/protobuf/compiler/cpp/cpp_file.cc | 26 +-- src/google/protobuf/compiler/cpp/cpp_map_field.cc | 205 ++++++++++++--------- src/google/protobuf/compiler/cpp/cpp_map_field.h | 4 + src/google/protobuf/compiler/cpp/cpp_message.cc | 8 +- src/google/protobuf/compiler/java/java_file.cc | 18 +- .../protobuf/compiler/java/java_generator.cc | 2 - src/google/protobuf/compiler/js/js_generator.cc | 24 ++- .../protobuf/compiler/python/python_generator.cc | 60 ++++-- .../compiler/python/python_plugin_unittest.cc | 48 +++++ src/google/protobuf/descriptor.proto | 1 + src/google/protobuf/extension_set_heavy.cc | 2 +- src/google/protobuf/generated_message_util.cc | 6 + src/google/protobuf/generated_message_util.h | 6 +- src/google/protobuf/io/coded_stream.cc | 8 +- src/google/protobuf/io/coded_stream.h | 47 +++++ src/google/protobuf/map_entry_lite.h | 26 +++ src/google/protobuf/map_proto2_unittest.proto | 20 ++ src/google/protobuf/map_test.cc | 77 ++++++++ src/google/protobuf/map_type_handler.h | 18 +- src/google/protobuf/message.cc | 2 - src/google/protobuf/message.h | 2 +- src/google/protobuf/message_lite.cc | 2 - src/google/protobuf/message_lite.h | 2 +- src/google/protobuf/testdata/golden_message_maps | Bin 0 -> 13619 bytes src/google/protobuf/text_format.cc | 14 ++ src/google/protobuf/unknown_field_set.cc | 28 +-- src/google/protobuf/unknown_field_set.h | 8 +- .../util/internal/default_value_objectwriter.cc | 29 +-- .../util/internal/default_value_objectwriter.h | 12 +- .../internal/default_value_objectwriter_test.cc | 33 ++++ src/google/protobuf/util/internal/proto_writer.cc | 34 +++- src/google/protobuf/util/internal/proto_writer.h | 9 +- src/google/protobuf/util/json_util.cc | 60 ++++++ src/google/protobuf/util/json_util.h | 25 +++ src/google/protobuf/util/json_util_test.cc | 60 ++++-- src/google/protobuf/wire_format_lite.cc | 6 +- 36 files changed, 719 insertions(+), 213 deletions(-) create mode 100644 src/google/protobuf/testdata/golden_message_maps (limited to 'src') diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc index 385b973e..b3eca660 100644 --- a/src/google/protobuf/compiler/cpp/cpp_file.cc +++ b/src/google/protobuf/compiler/cpp/cpp_file.cc @@ -336,19 +336,6 @@ void FileGenerator::GenerateSource(io::Printer* printer) { // Generate classes. for (int i = 0; i < file_->message_type_count(); i++) { - if (i == 0 && HasGeneratedMethods(file_, options_)) { - printer->Print( - "\n" - "namespace {\n" - "\n" - "static void MergeFromFail(int line) GOOGLE_ATTRIBUTE_COLD;\n" - "static void MergeFromFail(int line) {\n" - " GOOGLE_CHECK(false) << __FILE__ << \":\" << line;\n" - "}\n" - "\n" - "} // namespace\n" - "\n"); - } printer->Print("\n"); printer->Print(kThickSeparator); printer->Print("\n"); @@ -464,9 +451,10 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) { // and we only use AddDescriptors() to allocate default instances. if (HasDescriptorMethods(file_, options_)) { printer->Print( - "\n" - "void $assigndescriptorsname$() {\n", - "assigndescriptorsname", GlobalAssignDescriptorsName(file_->name())); + "\n" + "void $assigndescriptorsname$() GOOGLE_ATTRIBUTE_COLD;\n" + "void $assigndescriptorsname$() {\n", + "assigndescriptorsname", GlobalAssignDescriptorsName(file_->name())); printer->Indent(); // Make sure the file has found its way into the pool. If a descriptor @@ -525,8 +513,9 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) { // protobuf_RegisterTypes(): Calls // MessageFactory::InternalRegisterGeneratedType() for each message type. printer->Print( - "void protobuf_RegisterTypes(const ::std::string&) {\n" - " protobuf_AssignDescriptorsOnce();\n"); + "void protobuf_RegisterTypes(const ::std::string&) GOOGLE_ATTRIBUTE_COLD;\n" + "void protobuf_RegisterTypes(const ::std::string&) {\n" + " protobuf_AssignDescriptorsOnce();\n"); printer->Indent(); for (int i = 0; i < file_->message_type_count(); i++) { @@ -566,6 +555,7 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) { // Note that we don't need any special synchronization in the following // code // because it is called at static init time before any threads exist. + "void $adddescriptorsname$() GOOGLE_ATTRIBUTE_COLD;\n" "void $adddescriptorsname$() {\n" " static bool already_here = false;\n" " if (already_here) return;\n" diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.cc b/src/google/protobuf/compiler/cpp/cpp_map_field.cc index dd9f1887..0588e34e 100644 --- a/src/google/protobuf/compiler/cpp/cpp_map_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_map_field.cc @@ -251,117 +251,148 @@ GenerateMergeFromCodedStream(io::Printer* printer) const { } } -void MapFieldGenerator:: -GenerateSerializeWithCachedSizes(io::Printer* printer) const { - printer->Print(variables_, - "{\n" - " ::google::protobuf::scoped_ptr<$map_classname$> entry;\n" - " for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n" - " it = this->$name$().begin();\n" - " it != this->$name$().end(); ++it) {\n"); +static void GenerateSerializationLoop(io::Printer* printer, + const map& variables, + bool supports_arenas, + const string& utf8_check, + const string& loop_header, + const string& ptr, + bool loop_via_iterators) { + printer->Print(variables, + StrCat("::google::protobuf::scoped_ptr<$map_classname$> entry;\n", + loop_header, " {\n").c_str()); + printer->Indent(); + + printer->Print(variables, StrCat( + "entry.reset($name$_.New$wrapper$(\n" + " ", ptr, "->first, ", ptr, "->second));\n" + "$write_entry$;\n").c_str()); // If entry is allocated by arena, its desctructor should be avoided. - if (SupportsArenas(descriptor_)) { - printer->Print(variables_, - " if (entry.get() != NULL && entry->GetArena() != NULL) {\n" - " entry.release();\n" - " }\n"); + if (supports_arenas) { + printer->Print( + "if (entry->GetArena() != NULL) {\n" + " entry.release();\n" + "}\n"); } - printer->Print(variables_, - " entry.reset($name$_.New$wrapper$(it->first, it->second));\n" - " ::google::protobuf::internal::WireFormatLite::Write$stream_writer$(\n" - " $number$, *entry, output);\n"); - - printer->Indent(); - printer->Indent(); - - const FieldDescriptor* key_field = - descriptor_->message_type()->FindFieldByName("key"); - const FieldDescriptor* value_field = - descriptor_->message_type()->FindFieldByName("value"); - if (key_field->type() == FieldDescriptor::TYPE_STRING) { - GenerateUtf8CheckCodeForString(key_field, options_, false, variables_, - "it->first.data(), it->first.length(),\n", - printer); - } - if (value_field->type() == FieldDescriptor::TYPE_STRING) { - GenerateUtf8CheckCodeForString(value_field, options_, false, variables_, - "it->second.data(), it->second.length(),\n", - printer); + if (!utf8_check.empty()) { + // If loop_via_iterators is true then ptr is actually an iterator, and we + // create a pointer by prefixing it with "&*". + printer->Print( + StrCat(utf8_check, "(", (loop_via_iterators ? "&*" : ""), ptr, ");\n") + .c_str()); } printer->Outdent(); - printer->Outdent(); - printer->Print( - " }\n"); - - // If entry is allocated by arena, its desctructor should be avoided. - if (SupportsArenas(descriptor_)) { - printer->Print(variables_, - " if (entry.get() != NULL && entry->GetArena() != NULL) {\n" - " entry.release();\n" - " }\n"); - } + "}\n"); +} - printer->Print("}\n"); +void MapFieldGenerator:: +GenerateSerializeWithCachedSizes(io::Printer* printer) const { + map variables(variables_); + variables["write_entry"] = "::google::protobuf::internal::WireFormatLite::Write" + + variables["stream_writer"] + "(\n " + + variables["number"] + ", *entry, output)"; + variables["deterministic"] = "output->IsSerializationDeterminstic()"; + GenerateSerializeWithCachedSizes(printer, variables); } void MapFieldGenerator:: GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const { - printer->Print(variables_, - "{\n" - " ::google::protobuf::scoped_ptr<$map_classname$> entry;\n" - " for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n" - " it = this->$name$().begin();\n" - " it != this->$name$().end(); ++it) {\n"); - - // If entry is allocated by arena, its desctructor should be avoided. - if (SupportsArenas(descriptor_)) { - printer->Print(variables_, - " if (entry.get() != NULL && entry->GetArena() != NULL) {\n" - " entry.release();\n" - " }\n"); - } - - printer->Print(variables_, - " entry.reset($name$_.New$wrapper$(it->first, it->second));\n" - " target = ::google::protobuf::internal::WireFormatLite::\n" - " InternalWrite$declared_type$NoVirtualToArray(\n" - " $number$, *entry, false, target);\n"); + map variables(variables_); + variables["write_entry"] = + "target = ::google::protobuf::internal::WireFormatLite::\n" + " InternalWrite" + variables["declared_type"] + + "NoVirtualToArray(\n " + variables["number"] + + ", *entry, deterministic, target);\n"; + variables["deterministic"] = "deterministic"; + GenerateSerializeWithCachedSizes(printer, variables); +} +void MapFieldGenerator::GenerateSerializeWithCachedSizes( + io::Printer* printer, const map& variables) const { + printer->Print(variables, + "if (!this->$name$().empty()) {\n"); printer->Indent(); - printer->Indent(); - const FieldDescriptor* key_field = descriptor_->message_type()->FindFieldByName("key"); const FieldDescriptor* value_field = descriptor_->message_type()->FindFieldByName("value"); - if (key_field->type() == FieldDescriptor::TYPE_STRING) { - GenerateUtf8CheckCodeForString(key_field, options_, false, variables_, - "it->first.data(), it->first.length(),\n", - printer); + const bool string_key = key_field->type() == FieldDescriptor::TYPE_STRING; + const bool string_value = value_field->type() == FieldDescriptor::TYPE_STRING; + + printer->Print(variables, + "typedef ::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_pointer\n" + " ConstPtr;\n"); + if (string_key) { + printer->Print(variables, + "typedef ConstPtr SortItem;\n" + "typedef ::google::protobuf::internal::" + "CompareByDerefFirst Less;\n"); + } else { + printer->Print(variables, + "typedef ::google::protobuf::internal::SortItem< $key_cpp$, ConstPtr > " + "SortItem;\n" + "typedef ::google::protobuf::internal::CompareByFirstField Less;\n"); } - if (value_field->type() == FieldDescriptor::TYPE_STRING) { - GenerateUtf8CheckCodeForString(value_field, options_, false, variables_, - "it->second.data(), it->second.length(),\n", - printer); + string utf8_check; + if (string_key || string_value) { + printer->Print( + "struct Utf8Check {\n" + " static void Check(ConstPtr p) {\n"); + printer->Indent(); + printer->Indent(); + if (string_key) { + GenerateUtf8CheckCodeForString(key_field, options_, false, variables, + "p->first.data(), p->first.length(),\n", + printer); + } + if (string_value) { + GenerateUtf8CheckCodeForString(value_field, options_, false, variables, + "p->second.data(), p->second.length(),\n", + printer); + } + printer->Outdent(); + printer->Outdent(); + printer->Print( + " }\n" + "};\n"); + utf8_check = "Utf8Check::Check"; } - printer->Outdent(); + printer->Print(variables, + "\n" + "if ($deterministic$ &&\n" + " this->$name$().size() > 1) {\n" + " ::google::protobuf::scoped_array items(\n" + " new SortItem[this->$name$().size()]);\n" + " typedef ::google::protobuf::Map< $key_cpp$, $val_cpp$ >::size_type size_type;\n" + " size_type n = 0;\n" + " for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n" + " it = this->$name$().begin();\n" + " it != this->$name$().end(); ++it, ++n) {\n" + " items[n] = SortItem(&*it);\n" + " }\n" + " ::std::sort(&items[0], &items[n], Less());\n"); + printer->Indent(); + GenerateSerializationLoop(printer, variables, SupportsArenas(descriptor_), + utf8_check, "for (size_type i = 0; i < n; i++)", + string_key ? "items[i]" : "items[i].second", false); printer->Outdent(); printer->Print( - " }\n"); - - // If entry is allocated by arena, its desctructor should be avoided. - if (SupportsArenas(descriptor_)) { - printer->Print(variables_, - " if (entry.get() != NULL && entry->GetArena() != NULL) {\n" - " entry.release();\n" - " }\n"); - } - + "} else {\n"); + printer->Indent(); + GenerateSerializationLoop( + printer, variables, SupportsArenas(descriptor_), utf8_check, + "for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n" + " it = this->$name$().begin();\n" + " it != this->$name$().end(); ++it)", + "it", true); + printer->Outdent(); + printer->Print("}\n"); + printer->Outdent(); printer->Print("}\n"); } diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.h b/src/google/protobuf/compiler/cpp/cpp_map_field.h index 087dcde0..2930fe59 100644 --- a/src/google/protobuf/compiler/cpp/cpp_map_field.h +++ b/src/google/protobuf/compiler/cpp/cpp_map_field.h @@ -61,6 +61,10 @@ class MapFieldGenerator : public FieldGenerator { void GenerateByteSize(io::Printer* printer) const; private: + // A helper for GenerateSerializeWithCachedSizes{,ToArray}. + void GenerateSerializeWithCachedSizes( + io::Printer* printer, const map& variables) const; + const FieldDescriptor* descriptor_; const bool dependent_field_; map variables_; diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc index 32f63152..405a5fed 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message.cc @@ -2715,7 +2715,9 @@ GenerateMergeFrom(io::Printer* printer) { "void $classname$::MergeFrom(const ::google::protobuf::Message& from) {\n" "// @@protoc_insertion_point(generalized_merge_from_start:" "$full_name$)\n" - " if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);\n", + " if (GOOGLE_PREDICT_FALSE(&from == this)) {\n" + " ::google::protobuf::internal::MergeFromFail(__FILE__, __LINE__);\n" + " }\n", "classname", classname_, "full_name", descriptor_->full_name()); printer->Indent(); @@ -2756,7 +2758,9 @@ GenerateMergeFrom(io::Printer* printer) { "void $classname$::MergeFrom(const $classname$& from) {\n" "// @@protoc_insertion_point(class_specific_merge_from_start:" "$full_name$)\n" - " if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);\n", + " if (GOOGLE_PREDICT_FALSE(&from == this)) {\n" + " ::google::protobuf::internal::MergeFromFail(__FILE__, __LINE__);\n" + " }\n", "classname", classname_, "full_name", descriptor_->full_name()); printer->Indent(); diff --git a/src/google/protobuf/compiler/java/java_file.cc b/src/google/protobuf/compiler/java/java_file.cc index a06c5f6d..5e387285 100644 --- a/src/google/protobuf/compiler/java/java_file.cc +++ b/src/google/protobuf/compiler/java/java_file.cc @@ -266,9 +266,7 @@ void FileGenerator::Generate(io::Printer* printer) { printer->Print( "public static void registerAllExtensions(\n" - " com.google.protobuf.ExtensionRegistry$lite$ registry) {\n", - "lite", - HasDescriptorMethods(file_, context_->EnforceLite()) ? "" : "Lite"); + " com.google.protobuf.ExtensionRegistryLite registry) {\n"); printer->Indent(); @@ -283,6 +281,20 @@ void FileGenerator::Generate(io::Printer* printer) { printer->Outdent(); printer->Print( "}\n"); + if (HasDescriptorMethods(file_, context_->EnforceLite())) { + // Overload registerAllExtensions for the non-lite usage to + // redundantly maintain the original signature (this is + // redundant because ExtensionRegistryLite now invokes + // ExtensionRegistry in the non-lite usage). Intent is + // to remove this in the future. + printer->Print( + "\n" + "public static void registerAllExtensions(\n" + " com.google.protobuf.ExtensionRegistry registry) {\n" + " registerAllExtensions(\n" + " (com.google.protobuf.ExtensionRegistryLite) registry);\n" + "}\n"); + } // ----------------------------------------------------------------- diff --git a/src/google/protobuf/compiler/java/java_generator.cc b/src/google/protobuf/compiler/java/java_generator.cc index 3c545e15..b1ab4043 100644 --- a/src/google/protobuf/compiler/java/java_generator.cc +++ b/src/google/protobuf/compiler/java/java_generator.cc @@ -79,8 +79,6 @@ bool JavaGenerator::Generate(const FileDescriptor* file, file_options.generate_mutable_code = true; } else if (options[i].first == "shared") { file_options.generate_shared_code = true; - } else if (options[i].first == "lite") { - file_options.enforce_lite = true; } else if (options[i].first == "annotate_code") { file_options.annotate_code = true; } else if (options[i].first == "annotation_list_file") { diff --git a/src/google/protobuf/compiler/js/js_generator.cc b/src/google/protobuf/compiler/js/js_generator.cc index 8a2633d7..ad8cb166 100755 --- a/src/google/protobuf/compiler/js/js_generator.cc +++ b/src/google/protobuf/compiler/js/js_generator.cc @@ -2031,9 +2031,8 @@ void Generator::GenerateClassFieldToObject(const GeneratorOptions& options, "getter", JSGetterName(options, field, BYTES_B64)); } else { if (field->has_default_value()) { - printer->Print("jspb.Message.getField(msg, $index$) == null ? " - "$defaultValue$ : ", - "index", JSFieldIndex(field), + printer->Print("!msg.has$name$() ? $defaultValue$ : ", + "name", JSGetterName(options, field), "defaultValue", JSFieldDefault(field)); } if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT || @@ -2408,9 +2407,8 @@ void Generator::GenerateClassField(const GeneratorOptions& options, "default", Proto3PrimitiveFieldDefault(field)); } else { if (field->has_default_value()) { - printer->Print("jspb.Message.getField(this, $index$) == null ? " - "$defaultValue$ : ", - "index", JSFieldIndex(field), + printer->Print("!this.has$name$() ? $defaultValue$ : ", + "name", JSGetterName(options, field), "defaultValue", JSFieldDefault(field)); } if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT || @@ -2515,6 +2513,20 @@ void Generator::GenerateClassField(const GeneratorOptions& options, "\n", "clearedvalue", (field->is_repeated() ? "[]" : "undefined"), "returnvalue", JSReturnClause(field)); + + printer->Print( + "/**\n" + " * Returns whether this field is set.\n" + " * @return{!boolean}\n" + " */\n" + "$class$.prototype.has$name$ = function() {\n" + " return jspb.Message.getField(this, $index$) != null;\n" + "};\n" + "\n" + "\n", + "class", GetPath(options, field->containing_type()), + "name", JSGetterName(options, field), + "index", JSFieldIndex(field)); } } } diff --git a/src/google/protobuf/compiler/python/python_generator.cc b/src/google/protobuf/compiler/python/python_generator.cc index cc54a8ab..d5468a0c 100644 --- a/src/google/protobuf/compiler/python/python_generator.cc +++ b/src/google/protobuf/compiler/python/python_generator.cc @@ -44,6 +44,7 @@ // performance-minded Python code leverage the fast C++ implementation // directly. +#include #include #include #include @@ -107,20 +108,25 @@ string ModuleAlias(const string& filename) { return module_name; } - -// Returns an import statement of form "from X.Y.Z import T" for the given -// .proto filename. -string ModuleImportStatement(const string& filename) { - string module_name = ModuleName(filename); - int last_dot_pos = module_name.rfind('.'); - if (last_dot_pos == string::npos) { - // NOTE(petya): this is not tested as it would require a protocol buffer - // outside of any package, and I don't think that is easily achievable. - return "import " + module_name; - } else { - return "from " + module_name.substr(0, last_dot_pos) + " import " + - module_name.substr(last_dot_pos + 1); +// Keywords reserved by the Python language. +const char* const kKeywords[] = { + "False", "None", "True", "and", "as", "assert", "break", + "class", "continue", "def", "del", "elif", "else", "except", + "finally", "for", "from", "global", "if", "import", "in", + "is", "lambda", "nonlocal", "not", "or", "pass", "raise", + "return", "try", "while", "with", "yield", +}; +const char* const* kKeywordsEnd = + kKeywords + (sizeof(kKeywords) / sizeof(kKeywords[0])); + +bool ContainsPythonKeyword(const string& module_name) { + vector tokens = Split(module_name, "."); + for (int i = 0; i < tokens.size(); ++i) { + if (std::find(kKeywords, kKeywordsEnd, tokens[i]) != kKeywordsEnd) { + return true; + } } + return false; } @@ -359,10 +365,32 @@ bool Generator::Generate(const FileDescriptor* file, void Generator::PrintImports() const { for (int i = 0; i < file_->dependency_count(); ++i) { const string& filename = file_->dependency(i)->name(); - string import_statement = ModuleImportStatement(filename); + + string module_name = ModuleName(filename); string module_alias = ModuleAlias(filename); - printer_->Print("$statement$ as $alias$\n", "statement", - import_statement, "alias", module_alias); + if (ContainsPythonKeyword(module_name)) { + // If the module path contains a Python keyword, we have to quote the + // module name and import it using importlib. Otherwise the usual kind of + // import statement would result in a syntax error from the presence of + // the keyword. + printer_->Print("import importlib\n"); + printer_->Print("$alias$ = importlib.import_module('$name$')\n", "alias", + module_alias, "name", module_name); + } else { + int last_dot_pos = module_name.rfind('.'); + string import_statement; + if (last_dot_pos == string::npos) { + // NOTE(petya): this is not tested as it would require a protocol buffer + // outside of any package, and I don't think that is easily achievable. + import_statement = "import " + module_name; + } else { + import_statement = "from " + module_name.substr(0, last_dot_pos) + + " import " + module_name.substr(last_dot_pos + 1); + } + printer_->Print("$statement$ as $alias$\n", "statement", import_statement, + "alias", module_alias); + } + CopyPublicDependenciesAliases(module_alias, file_->dependency(i)); } printer_->Print("\n"); diff --git a/src/google/protobuf/compiler/python/python_plugin_unittest.cc b/src/google/protobuf/compiler/python/python_plugin_unittest.cc index 23f2449c..34f857fd 100644 --- a/src/google/protobuf/compiler/python/python_plugin_unittest.cc +++ b/src/google/protobuf/compiler/python/python_plugin_unittest.cc @@ -46,6 +46,7 @@ #include #include +#include #include #include @@ -115,6 +116,53 @@ TEST(PythonPluginTest, PluginTest) { EXPECT_EQ(0, cli.Run(5, argv)); } +// This test verifies that the generated Python output uses regular imports (as +// opposed to importlib) in the usual case where the .proto file paths do not +// not contain any Python keywords. +TEST(PythonPluginTest, ImportTest) { + // Create files test1.proto and test2.proto with the former importing the + // latter. + GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/test1.proto", + "syntax = \"proto3\";\n" + "package foo;\n" + "import \"test2.proto\";" + "message Message1 {\n" + " Message2 message_2 = 1;\n" + "}\n", + true)); + GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/test2.proto", + "syntax = \"proto3\";\n" + "package foo;\n" + "message Message2 {}\n", + true)); + + google::protobuf::compiler::CommandLineInterface cli; + cli.SetInputsAreProtoPathRelative(true); + python::Generator python_generator; + cli.RegisterGenerator("--python_out", &python_generator, ""); + string proto_path = "-I" + TestTempDir(); + string python_out = "--python_out=" + TestTempDir(); + const char* argv[] = {"protoc", proto_path.c_str(), "-I.", python_out.c_str(), + "test1.proto"}; + ASSERT_EQ(0, cli.Run(5, argv)); + + // Loop over the lines of the generated code and verify that we find an + // ordinary Python import but do not find the string "importlib". + string output; + GOOGLE_CHECK_OK(File::GetContents(TestTempDir() + "/test1_pb2.py", &output, + true)); + std::vector lines = Split(output, "\n"); + string expected_import = "import test2_pb2"; + bool found_expected_import = false; + for (int i = 0; i < lines.size(); ++i) { + if (lines[i].find(expected_import) != string::npos) { + found_expected_import = true; + } + EXPECT_EQ(string::npos, lines[i].find("importlib")); + } + EXPECT_TRUE(found_expected_import); +} + } // namespace } // namespace python } // namespace compiler diff --git a/src/google/protobuf/descriptor.proto b/src/google/protobuf/descriptor.proto index da853dbc..28410d4a 100644 --- a/src/google/protobuf/descriptor.proto +++ b/src/google/protobuf/descriptor.proto @@ -45,6 +45,7 @@ option java_package = "com.google.protobuf"; option java_outer_classname = "DescriptorProtos"; option csharp_namespace = "Google.Protobuf.Reflection"; option objc_class_prefix = "GPB"; +option java_generate_equals_and_hash = true; // descriptor.proto must be optimized for speed because reflection-based // algorithms don't work during bootstrapping. diff --git a/src/google/protobuf/extension_set_heavy.cc b/src/google/protobuf/extension_set_heavy.cc index 5dd171ed..b26a246c 100644 --- a/src/google/protobuf/extension_set_heavy.cc +++ b/src/google/protobuf/extension_set_heavy.cc @@ -341,7 +341,7 @@ bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input, int ExtensionSet::SpaceUsedExcludingSelf() const { int total_size = - extensions_.size() * sizeof(map::value_type); + extensions_.size() * sizeof(ExtensionMap::value_type); for (ExtensionMap::const_iterator iter = extensions_.begin(), end = extensions_.end(); iter != end; diff --git a/src/google/protobuf/generated_message_util.cc b/src/google/protobuf/generated_message_util.cc index 7b813f8a..7ad6d61c 100644 --- a/src/google/protobuf/generated_message_util.cc +++ b/src/google/protobuf/generated_message_util.cc @@ -73,6 +73,12 @@ int StringSpaceUsedExcludingSelf(const string& str) { +void MergeFromFail(const char* file, int line) { + GOOGLE_CHECK(false) << file << ":" << line; + // Open-source GOOGLE_CHECK(false) is not NORETURN. + exit(1); +} + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/generated_message_util.h b/src/google/protobuf/generated_message_util.h index 56f5afd2..dc815189 100644 --- a/src/google/protobuf/generated_message_util.h +++ b/src/google/protobuf/generated_message_util.h @@ -41,8 +41,8 @@ #include #include -#include #include +#include namespace google { @@ -115,6 +115,10 @@ class ArenaString; ArenaString* ReadArenaString(::google::protobuf::io::CodedInputStream* input, ::google::protobuf::Arena* arena); +// Helper function to crash on merge failure. +// Moved out of generated code to reduce binary size. +void MergeFromFail(const char* file, int line) GOOGLE_ATTRIBUTE_NORETURN; + } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/io/coded_stream.cc b/src/google/protobuf/io/coded_stream.cc index 148eee0e..a5675e79 100644 --- a/src/google/protobuf/io/coded_stream.cc +++ b/src/google/protobuf/io/coded_stream.cc @@ -649,13 +649,16 @@ bool CodedInputStream::Refresh() { // CodedOutputStream ================================================= +bool CodedOutputStream::default_serialization_deterministic_ = false; + CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output) : output_(output), buffer_(NULL), buffer_size_(0), total_bytes_(0), had_error_(false), - aliasing_enabled_(false) { + aliasing_enabled_(false), + serialization_deterministic_is_overridden_(false) { // Eagerly Refresh() so buffer space is immediately available. Refresh(); // The Refresh() may have failed. If the client doesn't write any data, @@ -671,7 +674,8 @@ CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output, buffer_size_(0), total_bytes_(0), had_error_(false), - aliasing_enabled_(false) { + aliasing_enabled_(false), + serialization_deterministic_is_overridden_(false) { if (do_eager_refresh) { // Eagerly Refresh() so buffer space is immediately available. Refresh(); diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index eb320745..316da765 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -813,6 +813,44 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { // created. bool HadError() const { return had_error_; } + // Deterministic serialization, if requested, guarantees that for a given + // binary, equal messages will always be serialized to the same bytes. This + // implies: + // . repeated serialization of a message will return the same bytes + // . different processes of the same binary (which may be executing on + // different machines) will serialize equal messages to the same bytes. + // + // Note the deterministic serialization is NOT canonical across languages; it + // is also unstable across different builds with schema changes due to unknown + // fields. Users who need canonical serialization, e.g., persistent storage in + // a canonical form, fingerprinting, etc., should define their own + // canonicalization specification and implement the serializer using + // reflection APIs rather than relying on this API. + // + // If determinisitc serialization is requested, the serializer will + // sort map entries by keys in lexicographical order or numerical order. + // (This is an implementation detail and may subject to change.) + // + // There are two ways to determine whether serialization should be + // deterministic for this CodedOutputStream. If SetSerializationDeterministic + // has not yet been called, then the default comes from the global default, + // which is false, until SetDefaultSerializationDeterministic has been called. + // Otherwise, SetSerializationDeterministic has been called, and the last + // value passed to it is all that matters. + void SetSerializationDeterministic(bool value) { + serialization_deterministic_is_overridden_ = true; + serialization_deterministic_override_ = value; + } + // See above. Also, note that users of this CodedOutputStream may need to + // call IsSerializationDeterminstic() to serialize in the intended way. This + // CodedOutputStream cannot enforce a desire for deterministic serialization + // by itself. + bool IsSerializationDeterminstic() const { + return serialization_deterministic_is_overridden_ ? + serialization_deterministic_override_ : + default_serialization_deterministic_; + } + private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedOutputStream); @@ -822,6 +860,10 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { int total_bytes_; // Sum of sizes of all buffers seen so far. bool had_error_; // Whether an error occurred during output. bool aliasing_enabled_; // See EnableAliasing(). + // See SetSerializationDeterministic() regarding these three fields. + bool serialization_deterministic_is_overridden_; + bool serialization_deterministic_override_; + static bool default_serialization_deterministic_; // Advance the buffer by a given number of bytes. void Advance(int amount); @@ -849,6 +891,11 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { uint64 value, uint8* target); static int VarintSize32Fallback(uint32 value); + + // See above. Other projects may use "friend" to allow them to call this. + static void SetDefaultSerializationDeterministic() { + default_serialization_deterministic_ = true; + } }; // inline methods ==================================================== diff --git a/src/google/protobuf/map_entry_lite.h b/src/google/protobuf/map_entry_lite.h index 23ac7b8a..4dedfd57 100644 --- a/src/google/protobuf/map_entry_lite.h +++ b/src/google/protobuf/map_entry_lite.h @@ -535,6 +535,32 @@ class MapEntryLite : public MessageLite { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MapEntryLite); }; +// Helpers for deterministic serialization ============================= + +// This struct can be used with any generic sorting algorithm. If the Key +// type is relatively small and easy to copy then copying Keys into an +// array of SortItems can be beneficial. Then all the data the sorting +// algorithm needs to touch is in that one array. +template struct SortItem { + SortItem() {} + explicit SortItem(PtrToKeyValuePair p) : first(p->first), second(p) {} + + Key first; + PtrToKeyValuePair second; +}; + +template struct CompareByFirstField { + bool operator()(const T& a, const T& b) const { + return a.first < b.first; + } +}; + +template struct CompareByDerefFirst { + bool operator()(const T& a, const T& b) const { + return a->first < b->first; + } +}; + } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/map_proto2_unittest.proto b/src/google/protobuf/map_proto2_unittest.proto index 916cc546..ddc2a582 100644 --- a/src/google/protobuf/map_proto2_unittest.proto +++ b/src/google/protobuf/map_proto2_unittest.proto @@ -64,3 +64,23 @@ message TestEnumMapPlusExtra { message TestImportEnumMap { map import_enum_amp = 1; } + +message TestIntIntMap { + map m = 1; +} + +// Test all key types: string, plus the non-floating-point scalars. +message TestMaps { + map m_int32 = 1; + map m_int64 = 2; + map m_uint32 = 3; + map m_uint64 = 4; + map m_sint32 = 5; + map m_sint64 = 6; + map m_fixed32 = 7; + map m_fixed64 = 8; + map m_sfixed32 = 9; + map m_sfixed64 = 10; + map m_bool = 11; + map m_string = 12; +} diff --git a/src/google/protobuf/map_test.cc b/src/google/protobuf/map_test.cc index cdd1ccd5..03954e75 100644 --- a/src/google/protobuf/map_test.cc +++ b/src/google/protobuf/map_test.cc @@ -74,6 +74,7 @@ #include #include #include +#include #include #include #include @@ -2869,6 +2870,82 @@ TEST(WireFormatForMapFieldTest, MapParseHelpers) { } } +// Deterministic Serialization Test ========================================== + +template +static string DeterministicSerialization(const T& t) { + const int size = t.ByteSize(); + string result(size, '\0'); + io::ArrayOutputStream array_stream(string_as_array(&result), size); + io::CodedOutputStream output_stream(&array_stream); + output_stream.SetSerializationDeterministic(true); + t.SerializeWithCachedSizes(&output_stream); + EXPECT_FALSE(output_stream.HadError()); + EXPECT_EQ(size, output_stream.ByteCount()); + return result; +} + +// Helper to test the serialization of the first arg against a golden file. +static void TestDeterministicSerialization(const protobuf_unittest::TestMaps& t, + const string& filename) { + string expected; + GOOGLE_CHECK_OK(File::GetContents( + TestSourceDir() + "/google/protobuf/testdata/" + filename, + &expected, true)); + const string actual = DeterministicSerialization(t); + EXPECT_EQ(expected, actual); + protobuf_unittest::TestMaps u; + EXPECT_TRUE(u.ParseFromString(actual)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(u, t)); +} + +// Helper for MapSerializationTest. Return a 7-bit ASCII string. +static string ConstructKey(uint64 n) { + string s(n % static_cast(9), '\0'); + if (s.empty()) { + return StrCat(n); + } else { + while (n != 0) { + s[n % s.size()] = (n >> 10) & 0x7f; + n /= 888; + } + return s; + } +} + +TEST(MapSerializationTest, Deterministic) { + const int kIters = 25; + protobuf_unittest::TestMaps t; + protobuf_unittest::TestIntIntMap inner; + (*inner.mutable_m())[0] = (*inner.mutable_m())[10] = + (*inner.mutable_m())[-200] = 0; + uint64 frog = 9; + const uint64 multiplier = 0xa29cd16f; + for (int i = 0; i < kIters; i++) { + const int32 i32 = static_cast(frog & 0xffffffff); + const uint32 u32 = static_cast(i32) * 91919; + const int64 i64 = static_cast(frog); + const uint64 u64 = frog * static_cast(187321); + const bool b = i32 > 0; + const string s = ConstructKey(frog); + (*inner.mutable_m())[i] = i32; + (*t.mutable_m_int32())[i32] = (*t.mutable_m_sint32())[i32] = + (*t.mutable_m_sfixed32())[i32] = inner; + (*t.mutable_m_uint32())[u32] = (*t.mutable_m_fixed32())[u32] = inner; + (*t.mutable_m_int64())[i64] = (*t.mutable_m_sint64())[i64] = + (*t.mutable_m_sfixed64())[i64] = inner; + (*t.mutable_m_uint64())[u64] = (*t.mutable_m_fixed64())[u64] = inner; + (*t.mutable_m_bool())[b] = inner; + (*t.mutable_m_string())[s] = inner; + (*t.mutable_m_string())[s + string(1 << (u32 % static_cast(9)), + b)] = inner; + inner.mutable_m()->erase(i); + frog = frog * multiplier + i; + frog ^= (frog >> 41); + } + TestDeterministicSerialization(t, "golden_message_maps"); +} + // Text Format Test ================================================= TEST(TextFormatMapTest, SerializeAndParse) { diff --git a/src/google/protobuf/map_type_handler.h b/src/google/protobuf/map_type_handler.h index 74e8bb50..685a770f 100644 --- a/src/google/protobuf/map_type_handler.h +++ b/src/google/protobuf/map_type_handler.h @@ -166,10 +166,10 @@ class MapTypeHandler { io::CodedOutputStream* output); static inline uint8* InternalWriteToArray(int field, const MapEntryAccessorType& value, - bool deterministic, uint8* output); + bool deterministic, uint8* target); static inline uint8* WriteToArray(int field, const MapEntryAccessorType& value, - uint8* output); + uint8* target); // Functions to manipulate data on memory. ======================== static inline const Type& GetExternalReference(const Type* value); @@ -227,11 +227,11 @@ class MapTypeHandler { int field, \ const MapEntryAccessorType& value, \ bool deterministic, \ - uint8* output); \ + uint8* target); \ static inline uint8* WriteToArray(int field, \ const MapEntryAccessorType& value, \ - uint8* output) { \ - return InternalWriteToArray(field, value, false, output); \ + uint8* target) { \ + return InternalWriteToArray(field, value, false, target); \ } \ static inline const MapEntryAccessorType& GetExternalReference( \ const TypeOnMemory& value); \ @@ -374,9 +374,9 @@ template inline uint8* MapTypeHandler::InternalWriteToArray( int field, const MapEntryAccessorType& value, bool deterministic, - uint8* output) { + uint8* target) { return WireFormatLite::InternalWriteMessageToArray(field, value, - deterministic, output); + deterministic, target); } #define WRITE_METHOD(FieldType, DeclaredType) \ @@ -390,8 +390,8 @@ MapTypeHandler::InternalWriteToArray( inline uint8* \ MapTypeHandler::InternalWriteToArray( \ - int field, const MapEntryAccessorType& value, bool, uint8* output) { \ - return WireFormatLite::Write##DeclaredType##ToArray(field, value, output); \ + int field, const MapEntryAccessorType& value, bool, uint8* target) { \ + return WireFormatLite::Write##DeclaredType##ToArray(field, value, target); \ } WRITE_METHOD(STRING , String) diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index d62ca79c..f18077dd 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -62,8 +62,6 @@ namespace protobuf { using internal::WireFormat; using internal::ReflectionOps; -Message::~Message() {} - void Message::MergeFrom(const Message& from) { const Descriptor* descriptor = GetDescriptor(); GOOGLE_CHECK_EQ(from.GetDescriptor(), descriptor) diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index dcdffe1c..9705e97e 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -179,7 +179,7 @@ struct Metadata { class LIBPROTOBUF_EXPORT Message : public MessageLite { public: inline Message() {} - virtual ~Message(); + virtual ~Message() {} // Basic Operations ------------------------------------------------ diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 3913be1b..ba56db95 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -46,8 +46,6 @@ namespace google { namespace protobuf { -MessageLite::~MessageLite() {} - string MessageLite::InitializationErrorString() const { return "(cannot determine missing fields for lite message)"; } diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index f606aeec..2bdfe496 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -81,7 +81,7 @@ namespace internal { class LIBPROTOBUF_EXPORT MessageLite { public: inline MessageLite() {} - virtual ~MessageLite(); + virtual ~MessageLite() {} // Basic Operations ------------------------------------------------ diff --git a/src/google/protobuf/testdata/golden_message_maps b/src/google/protobuf/testdata/golden_message_maps new file mode 100644 index 00000000..c70a4d7c Binary files /dev/null and b/src/google/protobuf/testdata/golden_message_maps differ diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index d49d8588..66b2648b 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -759,6 +759,20 @@ class TextFormat::Parser::ParserImpl { } return true; } + if (TryConsume("[")) { + while (true) { + if (!LookingAt("{") && !LookingAt("<")) { + DO(SkipFieldValue()); + } else { + DO(SkipFieldMessage()); + } + if (TryConsume("]")) { + break; + } + DO(Consume(",")); + } + return true; + } // Possible field values other than string: // 12345 => TYPE_INTEGER // -12345 => TYPE_SYMBOL + TYPE_INTEGER diff --git a/src/google/protobuf/unknown_field_set.cc b/src/google/protobuf/unknown_field_set.cc index d4e383da..8ee99d48 100644 --- a/src/google/protobuf/unknown_field_set.cc +++ b/src/google/protobuf/unknown_field_set.cc @@ -69,28 +69,14 @@ const UnknownFieldSet* UnknownFieldSet::default_instance() { return default_unknown_field_set_instance_; } -UnknownFieldSet::UnknownFieldSet() - : fields_(NULL) {} - -UnknownFieldSet::~UnknownFieldSet() { - Clear(); - delete fields_; -} - void UnknownFieldSet::ClearFallback() { - if (fields_ != NULL) { - for (int i = 0; i < fields_->size(); i++) { - (*fields_)[i].Delete(); - } - delete fields_; - fields_ = NULL; - } -} - -void UnknownFieldSet::ClearAndFreeMemory() { - if (fields_ != NULL) { - Clear(); - } + GOOGLE_DCHECK(fields_ != NULL && fields_->size() > 0); + int n = fields_->size(); + do { + (*fields_)[--n].Delete(); + } while (n > 0); + delete fields_; + fields_ = NULL; } void UnknownFieldSet::InternalMergeFrom(const UnknownFieldSet& other) { diff --git a/src/google/protobuf/unknown_field_set.h b/src/google/protobuf/unknown_field_set.h index 612a942a..aa752916 100644 --- a/src/google/protobuf/unknown_field_set.h +++ b/src/google/protobuf/unknown_field_set.h @@ -241,8 +241,14 @@ class LIBPROTOBUF_EXPORT UnknownField { // =================================================================== // inline implementations +inline UnknownFieldSet::UnknownFieldSet() : fields_(NULL) {} + +inline UnknownFieldSet::~UnknownFieldSet() { Clear(); } + +inline void UnknownFieldSet::ClearAndFreeMemory() { Clear(); } + inline void UnknownFieldSet::Clear() { - if (fields_) { + if (fields_ != NULL) { ClearFallback(); } } diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.cc b/src/google/protobuf/util/internal/default_value_objectwriter.cc index 21d7a2e4..1e8dab70 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter.cc @@ -64,6 +64,7 @@ DefaultValueObjectWriter::DefaultValueObjectWriter( type_(type), current_(NULL), root_(NULL), + suppress_empty_list_(false), field_scrub_callback_(NULL), ow_(ow) {} @@ -184,12 +185,10 @@ void DefaultValueObjectWriter::RegisterFieldScrubCallBack( field_scrub_callback_.reset(field_scrub_callback.release()); } -DefaultValueObjectWriter::Node::Node(const string& name, - const google::protobuf::Type* type, - NodeKind kind, const DataPiece& data, - bool is_placeholder, - const vector& path, - FieldScrubCallBack* field_scrub_callback) +DefaultValueObjectWriter::Node::Node( + const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, const vector& path, + bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback) : name_(name), type_(type), kind_(kind), @@ -197,6 +196,7 @@ DefaultValueObjectWriter::Node::Node(const string& name, data_(data), is_placeholder_(is_placeholder), path_(path), + suppress_empty_list_(suppress_empty_list), field_scrub_callback_(field_scrub_callback) {} DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild( @@ -230,6 +230,9 @@ void DefaultValueObjectWriter::Node::WriteTo(ObjectWriter* ow) { // Write out lists. If we didn't have any list in response, write out empty // list. if (kind_ == LIST) { + // Suppress empty lists if requested. + if (suppress_empty_list_ && is_placeholder_) return; + ow->StartList(name_); WriteChildren(ow); ow->EndList(); @@ -366,7 +369,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren( field.json_name(), field_type, kind, kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo) : DataPiece::NullData(), - true, path, field_scrub_callback_)); + true, path, suppress_empty_list_, field_scrub_callback_)); new_children.push_back(child.release()); } // Adds all leftover nodes in children_ to the beginning of new_child. @@ -462,7 +465,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject( if (current_ == NULL) { vector path; root_.reset(new Node(name.ToString(), &type_, OBJECT, DataPiece::NullData(), - false, path, field_scrub_callback_.get())); + false, path, suppress_empty_list_, + field_scrub_callback_.get())); root_->PopulateChildren(typeinfo_); current_ = root_.get(); return this; @@ -478,7 +482,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject( : NULL), OBJECT, DataPiece::NullData(), false, child == NULL ? current_->path() : child->path(), - field_scrub_callback_.get())); + suppress_empty_list_, field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } @@ -509,7 +513,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList( if (current_ == NULL) { vector path; root_.reset(new Node(name.ToString(), &type_, LIST, DataPiece::NullData(), - false, path, field_scrub_callback_.get())); + false, path, suppress_empty_list_, + field_scrub_callback_.get())); current_ = root_.get(); return this; } @@ -519,7 +524,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList( google::protobuf::scoped_ptr node( new Node(name.ToString(), NULL, LIST, DataPiece::NullData(), false, child == NULL ? current_->path() : child->path(), - field_scrub_callback_.get())); + suppress_empty_list_, field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } @@ -577,7 +582,7 @@ void DefaultValueObjectWriter::RenderDataPiece(StringPiece name, google::protobuf::scoped_ptr node( new Node(name.ToString(), NULL, PRIMITIVE, data, false, child == NULL ? current_->path() : child->path(), - field_scrub_callback_.get())); + suppress_empty_list_, field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } else { diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.h b/src/google/protobuf/util/internal/default_value_objectwriter.h index 1d85bed8..5f3b25f3 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.h +++ b/src/google/protobuf/util/internal/default_value_objectwriter.h @@ -122,6 +122,10 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // field_scrub_callback pointer is also transferred to this class void RegisterFieldScrubCallBack(FieldScrubCallBackPtr field_scrub_callback); + // If set to true, empty lists are suppressed from output when default values + // are written. + void set_suppress_empty_list(bool value) { suppress_empty_list_ = value; } + private: enum NodeKind { PRIMITIVE = 0, @@ -136,7 +140,7 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { public: Node(const string& name, const google::protobuf::Type* type, NodeKind kind, const DataPiece& data, bool is_placeholder, const vector& path, - FieldScrubCallBack* field_scrub_callback); + bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback); virtual ~Node() { for (int i = 0; i < children_.size(); ++i) { delete children_[i]; @@ -212,6 +216,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // Path of the field of this node std::vector path_; + // Whether to suppress empty list output. + bool suppress_empty_list_; + // Pointer to function for determining whether a field needs to be scrubbed // or not. This callback is owned by the creator of this node. FieldScrubCallBack* field_scrub_callback_; @@ -257,6 +264,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // The stack to hold the path of Nodes from current_ to root_; std::stack stack_; + // Whether to suppress output of empty lists. + bool suppress_empty_list_; + // Unique Pointer to function for determining whether a field needs to be // scrubbed or not. FieldScrubCallBackPtr field_scrub_callback_; diff --git a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc index 8254c0fa..e1dd697a 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc @@ -149,6 +149,39 @@ TEST_P(DefaultValueObjectWriterTest, ShouldRetainUnknownField) { } +class DefaultValueObjectWriterSuppressListTest + : public BaseDefaultValueObjectWriterTest { + protected: + DefaultValueObjectWriterSuppressListTest() + : BaseDefaultValueObjectWriterTest(DefaultValueTest::descriptor()) { + testing_->set_suppress_empty_list(true); + } + ~DefaultValueObjectWriterSuppressListTest() {} +}; + +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + DefaultValueObjectWriterSuppressListTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + +TEST_P(DefaultValueObjectWriterSuppressListTest, Empty) { + // Set expectation. Emtpy lists should be suppressed. + expects_.StartObject("") + ->RenderDouble("doubleValue", 0.0) + ->RenderFloat("floatValue", 0.0) + ->RenderInt64("int64Value", 0) + ->RenderUint64("uint64Value", 0) + ->RenderInt32("int32Value", 0) + ->RenderUint32("uint32Value", 0) + ->RenderBool("boolValue", false) + ->RenderString("stringValue", "") + ->RenderBytes("bytesValue", "") + ->RenderString("enumValue", "ENUM_FIRST") + ->EndObject(); + + // Actual testing + testing_->StartObject("")->EndObject(); +} } // namespace testing } // namespace converter } // namespace util diff --git a/src/google/protobuf/util/internal/proto_writer.cc b/src/google/protobuf/util/internal/proto_writer.cc index 7a1a6cbd..0c38aeb9 100644 --- a/src/google/protobuf/util/internal/proto_writer.cc +++ b/src/google/protobuf/util/internal/proto_writer.cc @@ -298,7 +298,9 @@ ProtoWriter::ProtoElement::ProtoElement(const TypeInfo* typeinfo, proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3), type_(type), size_index_(-1), - array_index_(-1) { + array_index_(-1), + // oneof_indices_ values are 1-indexed (0 means not present). + oneof_indices_(type.oneofs_size() + 1) { if (!proto3_) { required_fields_ = GetRequiredFields(type_); } @@ -312,13 +314,15 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent, ow_(this->parent()->ow_), parent_field_(field), typeinfo_(this->parent()->typeinfo_), - proto3_(this->parent()->proto3_), + proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3), type_(type), size_index_( !is_list && field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE ? ow_->size_insert_.size() : -1), - array_index_(is_list ? 0 : -1) { + array_index_(is_list ? 0 : -1), + // oneof_indices_ values are 1-indexed (0 means not present). + oneof_indices_(type_.oneofs_size() + 1) { if (!is_list) { if (ow_->IsRepeated(*field)) { // Update array_index_ if it is an explicit list. @@ -411,11 +415,11 @@ string ProtoWriter::ProtoElement::ToString() const { } bool ProtoWriter::ProtoElement::IsOneofIndexTaken(int32 index) { - return ContainsKey(oneof_indices_, index); + return oneof_indices_[index]; } void ProtoWriter::ProtoElement::TakeOneofIndex(int32 index) { - InsertIfNotPresent(&oneof_indices_, index); + oneof_indices_[index] = true; } void ProtoWriter::InvalidName(StringPiece unknown_name, StringPiece message) { @@ -573,10 +577,19 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField( // Pushing a ProtoElement and then pop it off at the end for 2 purposes: // error location reporting and required field accounting. - element_.reset(new ProtoElement(element_.release(), &field, type, false)); + // + // For proto3, since there is no required field tracking, we only need to push + // ProtoElement for error cases. + if (!element_->proto3()) { + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + } if (field.kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN || field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { + // Push a ProtoElement for location reporting purposes. + if (element_->proto3()) { + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + } InvalidValue(field.type_url().empty() ? google::protobuf::Field_Kind_Name(field.kind()) : field.type_url(), @@ -657,11 +670,18 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField( } if (!status.ok()) { + // Push a ProtoElement for location reporting purposes. + if (element_->proto3()) { + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + } InvalidValue(google::protobuf::Field_Kind_Name(field.kind()), status.error_message()); + element_.reset(element()->pop()); + return this; } - element_.reset(element()->pop()); + if (!element_->proto3()) element_.reset(element()->pop()); + return this; } diff --git a/src/google/protobuf/util/internal/proto_writer.h b/src/google/protobuf/util/internal/proto_writer.h index 8b7c6c34..7f1108ab 100644 --- a/src/google/protobuf/util/internal/proto_writer.h +++ b/src/google/protobuf/util/internal/proto_writer.h @@ -32,8 +32,8 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_PROTO_WRITER_H__ #include -#include #include +#include #include #include @@ -45,6 +45,7 @@ #include #include #include +#include namespace google { namespace protobuf { @@ -191,6 +192,8 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // generate an error. void TakeOneofIndex(int32 index); + bool proto3() { return proto3_; } + private: // Used for access to variables of the enclosing instance of // ProtoWriter. @@ -203,7 +206,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // TypeInfo to lookup types. const TypeInfo* typeinfo_; - // Whether the root type is a proto3 or not. + // Whether the type_ is proto3 or not. bool proto3_; // Additional variables if this element is a message: @@ -221,7 +224,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // Set of oneof indices already seen for the type_. Used to validate // incoming messages so no more than one oneof is set. - hash_set oneof_indices_; + std::vector oneof_indices_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement); }; diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc index 6d45a4f9..d7ac2dba 100644 --- a/src/google/protobuf/util/json_util.cc +++ b/src/google/protobuf/util/json_util.cc @@ -177,6 +177,66 @@ util::Status JsonToBinaryString(TypeResolver* resolver, resolver, type_url, &input_stream, &output_stream, options); } +namespace { +const char* kTypeUrlPrefix = "type.googleapis.com"; +TypeResolver* generated_type_resolver_ = NULL; +GOOGLE_PROTOBUF_DECLARE_ONCE(generated_type_resolver_init_); + +string GetTypeUrl(const Message& message) { + return string(kTypeUrlPrefix) + "/" + message.GetDescriptor()->full_name(); +} + +void DeleteGeneratedTypeResolver() { delete generated_type_resolver_; } + +void InitGeneratedTypeResolver() { + generated_type_resolver_ = NewTypeResolverForDescriptorPool( + kTypeUrlPrefix, DescriptorPool::generated_pool()); + ::google::protobuf::internal::OnShutdown(&DeleteGeneratedTypeResolver); +} + +TypeResolver* GetGeneratedTypeResolver() { + ::google::protobuf::GoogleOnceInit(&generated_type_resolver_init_, &InitGeneratedTypeResolver); + return generated_type_resolver_; +} +} // namespace + +util::Status MessageToJsonString(const Message& message, string* output, + const JsonOptions& options) { + const DescriptorPool* pool = message.GetDescriptor()->file()->pool(); + TypeResolver* resolver = + pool == DescriptorPool::generated_pool() + ? GetGeneratedTypeResolver() + : NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool); + util::Status result = + BinaryToJsonString(resolver, GetTypeUrl(message), + message.SerializeAsString(), output, options); + if (pool != DescriptorPool::generated_pool()) { + delete resolver; + } + return result; +} + +util::Status JsonStringToMessage(const string& input, Message* message, + const JsonParseOptions& options) { + const DescriptorPool* pool = message->GetDescriptor()->file()->pool(); + TypeResolver* resolver = + pool == DescriptorPool::generated_pool() + ? GetGeneratedTypeResolver() + : NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool); + string binary; + util::Status result = JsonToBinaryString( + resolver, GetTypeUrl(*message), input, &binary, options); + if (result.ok() && !message->ParseFromString(binary)) { + result = + util::Status(util::error::INVALID_ARGUMENT, + "JSON transcoder produced invalid protobuf output."); + } + if (pool != DescriptorPool::generated_pool()) { + delete resolver; + } + return result; +} + } // namespace util } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/util/json_util.h b/src/google/protobuf/util/json_util.h index b4c2579b..170ae91b 100644 --- a/src/google/protobuf/util/json_util.h +++ b/src/google/protobuf/util/json_util.h @@ -33,6 +33,7 @@ #ifndef GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__ #define GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__ +#include #include #include @@ -69,6 +70,30 @@ struct JsonPrintOptions { // DEPRECATED. Use JsonPrintOptions instead. typedef JsonPrintOptions JsonOptions; +// Converts from protobuf message to JSON. This is a simple wrapper of +// BinaryToJsonString(). It will use the DescriptorPool of the passed-in +// message to resolve Any types. +util::Status MessageToJsonString(const Message& message, + string* output, + const JsonOptions& options); + +inline util::Status MessageToJsonString(const Message& message, + string* output) { + return MessageToJsonString(message, output, JsonOptions()); +} + +// Converts from JSON to protobuf message. This is a simple wrapper of +// JsonStringToBinary(). It will use the DescriptorPool of the passed-in +// message to resolve Any types. +util::Status JsonStringToMessage(const string& input, + Message* message, + const JsonParseOptions& options); + +inline util::Status JsonStringToMessage(const string& input, + Message* message) { + return JsonStringToMessage(input, message, JsonParseOptions()); +} + // Converts protobuf binary data to JSON. // The conversion will fail if: // 1. TypeResolver fails to resolve a type. diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index c7d5c59e..dacac5e0 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -34,6 +34,8 @@ #include #include +#include +#include #include #include #include @@ -63,28 +65,21 @@ static string GetTypeUrl(const Descriptor* message) { class JsonUtilTest : public testing::Test { protected: JsonUtilTest() { - resolver_.reset(NewTypeResolverForDescriptorPool( - kTypeUrlPrefix, DescriptorPool::generated_pool())); } string ToJson(const Message& message, const JsonPrintOptions& options) { string result; - GOOGLE_CHECK_OK(BinaryToJsonString(resolver_.get(), - GetTypeUrl(message.GetDescriptor()), - message.SerializeAsString(), &result, options)); + GOOGLE_CHECK_OK(MessageToJsonString(message, &result, options)); return result; } bool FromJson(const string& json, Message* message, const JsonParseOptions& options) { - string binary; - if (!JsonToBinaryString(resolver_.get(), - GetTypeUrl(message->GetDescriptor()), json, &binary, - options) - .ok()) { - return false; - } - return message->ParseFromString(binary); + return JsonStringToMessage(json, message, options).ok(); + } + + bool FromJson(const string& json, Message* message) { + return FromJson(json, message, JsonParseOptions()); } google::protobuf::scoped_ptr resolver_; @@ -189,6 +184,45 @@ TEST_F(JsonUtilTest, TestParseErrors) { EXPECT_FALSE(FromJson("{\"int32Value\":2147483648}", &m, options)); } +TEST_F(JsonUtilTest, TestDynamicMessage) { + // Some random message but good enough to test the wrapper functions. + string input = + "{\n" + " \"int32Value\": 1024,\n" + " \"repeatedInt32Value\": [1, 2],\n" + " \"messageValue\": {\n" + " \"value\": 2048\n" + " },\n" + " \"repeatedMessageValue\": [\n" + " {\"value\": 40}, {\"value\": 96}\n" + " ]\n" + "}\n"; + + // Create a new DescriptorPool with the same protos as the generated one. + DescriptorPoolDatabase database(*DescriptorPool::generated_pool()); + DescriptorPool pool(&database); + // A dynamic version of the test proto. + DynamicMessageFactory factory; + google::protobuf::scoped_ptr message(factory.GetPrototype( + pool.FindMessageTypeByName("proto3.TestMessage"))->New()); + EXPECT_TRUE(FromJson(input, message.get())); + + // Convert to generated message for easy inspection. + TestMessage generated; + EXPECT_TRUE(generated.ParseFromString(message->SerializeAsString())); + EXPECT_EQ(1024, generated.int32_value()); + ASSERT_EQ(2, generated.repeated_int32_value_size()); + EXPECT_EQ(1, generated.repeated_int32_value(0)); + EXPECT_EQ(2, generated.repeated_int32_value(1)); + EXPECT_EQ(2048, generated.message_value().value()); + ASSERT_EQ(2, generated.repeated_message_value_size()); + EXPECT_EQ(40, generated.repeated_message_value(0).value()); + EXPECT_EQ(96, generated.repeated_message_value(1).value()); + + JsonOptions options; + EXPECT_EQ(ToJson(generated, options), ToJson(*message, options)); +} + typedef pair Segment; // A ZeroCopyOutputStream that writes to multiple buffers. class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream { diff --git a/src/google/protobuf/wire_format_lite.cc b/src/google/protobuf/wire_format_lite.cc index f2517074..05cc0854 100644 --- a/src/google/protobuf/wire_format_lite.cc +++ b/src/google/protobuf/wire_format_lite.cc @@ -466,7 +466,8 @@ void WireFormatLite::WriteGroupMaybeToArray(int field_number, const int size = value.GetCachedSize(); uint8* target = output->GetDirectBufferForNBytesAndAdvance(size); if (target != NULL) { - uint8* end = value.SerializeWithCachedSizesToArray(target); + uint8* end = value.InternalSerializeWithCachedSizesToArray( + output->IsSerializationDeterminstic(), target); GOOGLE_DCHECK_EQ(end - target, size); } else { value.SerializeWithCachedSizes(output); @@ -482,7 +483,8 @@ void WireFormatLite::WriteMessageMaybeToArray(int field_number, output->WriteVarint32(size); uint8* target = output->GetDirectBufferForNBytesAndAdvance(size); if (target != NULL) { - uint8* end = value.SerializeWithCachedSizesToArray(target); + uint8* end = value.InternalSerializeWithCachedSizesToArray( + output->IsSerializationDeterminstic(), target); GOOGLE_DCHECK_EQ(end - target, size); } else { value.SerializeWithCachedSizes(output); -- cgit v1.2.3