Down-integrate from internal code base.

1 files changed, 606 insertions, 113 deletions

diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc
index 21dda598..ab8691fd 100644
--- a/src/google/protobuf/descriptor.cc
+++ b/src/google/protobuf/descriptor.cc
@@ -54,6 +54,7 @@
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/stubs/once.h>
+#include <google/protobuf/stubs/stringprintf.h>
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/stubs/substitute.h>
 #include <google/protobuf/stubs/map_util.h>
@@ -134,6 +135,17 @@ const char * const FieldDescriptor::kLabelToName[MAX_LABEL + 1] = {
   "repeated",  // LABEL_REPEATED
 };
 
+const char* FileDescriptor::SyntaxName(FileDescriptor::Syntax syntax) {
+  switch (syntax) {
+    case SYNTAX_PROTO2:
+      return "proto2";
+    case SYNTAX_PROTO3:
+      return "proto3";
+    case SYNTAX_UNKNOWN:
+      return "unknown";
+  }
+}
+
 static const char * const kNonLinkedWeakMessageReplacementName = "google.protobuf.Empty";
 
 #ifndef _MSC_VER  // MSVC doesn't need these and won't even accept them.
@@ -144,8 +156,8 @@ const int FieldDescriptor::kLastReservedNumber;
 
 namespace {
 
-string ToCamelCase(const string& input) {
-  bool capitalize_next = false;
+string ToCamelCase(const string& input, bool lower_first) {
+  bool capitalize_next = !lower_first;
   string result;
   result.reserve(input.size());
 
@@ -166,8 +178,8 @@ string ToCamelCase(const string& input) {
   }
 
   // Lower-case the first letter.
-  if (!result.empty() && 'A' <= result[0] && result[0] <= 'Z') {
-    result[0] = result[0] - 'A' + 'a';
+  if (lower_first && !result.empty() && 'A' <= result[0] && result[0] <= 'Z') {
+      result[0] = result[0] - 'A' + 'a';
   }
 
   return result;
@@ -327,6 +339,31 @@ typedef hash_map<EnumIntPair, const EnumValueDescriptor*,
 typedef map<DescriptorIntPair, const FieldDescriptor*>
   ExtensionsGroupedByDescriptorMap;
 typedef hash_map<string, const SourceCodeInfo_Location*> LocationsByPathMap;
+
+set<string>* allowed_proto3_extendees_ = NULL;
+GOOGLE_PROTOBUF_DECLARE_ONCE(allowed_proto3_extendees_init_);
+
+void InitAllowedProto3Extendee() {
+  allowed_proto3_extendees_ = new set<string>;
+  allowed_proto3_extendees_->insert("google.protobuf.FileOptions");
+  allowed_proto3_extendees_->insert("google.protobuf.MessageOptions");
+  allowed_proto3_extendees_->insert("google.protobuf.FieldOptions");
+  allowed_proto3_extendees_->insert("google.protobuf.EnumOptions");
+  allowed_proto3_extendees_->insert("google.protobuf.EnumValueOptions");
+  allowed_proto3_extendees_->insert("google.protobuf.ServiceOptions");
+  allowed_proto3_extendees_->insert("google.protobuf.MethodOptions");
+}
+
+// Checks whether the extendee type is allowed in proto3.
+// Only extensions to descriptor options are allowed. We use name comparison
+// instead of comparing the descriptor directly because the extensions may be
+// defined in a different pool.
+bool AllowedExtendeeInProto3(const string& name) {
+  ::google::protobuf::GoogleOnceInit(&allowed_proto3_extendees_init_, &InitAllowedProto3Extendee);
+  return allowed_proto3_extendees_->find(name) !=
+         allowed_proto3_extendees_->end();
+}
+
 }  // anonymous namespace
 
 // ===================================================================
@@ -532,6 +569,9 @@ class FileDescriptorTables {
     const void* parent, const string& camelcase_name) const;
   inline const EnumValueDescriptor* FindEnumValueByNumber(
     const EnumDescriptor* parent, int number) const;
+  // This creates a new EnumValueDescriptor if not found, in a thread-safe way.
+  inline const EnumValueDescriptor* FindEnumValueByNumberCreatingIfUnknown(
+      const EnumDescriptor* parent, int number) const;
 
   // -----------------------------------------------------------------
   // Adding items.
@@ -567,10 +607,16 @@ class FileDescriptorTables {
   FieldsByNameMap       fields_by_camelcase_name_;
   FieldsByNumberMap     fields_by_number_;       // Not including extensions.
   EnumValuesByNumberMap enum_values_by_number_;
+  mutable EnumValuesByNumberMap unknown_enum_values_by_number_
+      GOOGLE_GUARDED_BY(unknown_enum_values_mu_);
 
   // Populated on first request to save space, hence constness games.
   mutable GoogleOnceDynamic locations_by_path_once_;
   mutable LocationsByPathMap locations_by_path_;
+
+  // Mutex to protect the unknown-enum-value map due to dynamic
+  // EnumValueDescriptor creation on unknown values.
+  mutable Mutex unknown_enum_values_mu_;
 };
 
 DescriptorPool::Tables::Tables()
@@ -600,7 +646,8 @@ FileDescriptorTables::FileDescriptorTables()
       fields_by_lowercase_name_(3),
       fields_by_camelcase_name_(3),
       fields_by_number_(3),
-      enum_values_by_number_(3) {
+      enum_values_by_number_(3),
+      unknown_enum_values_by_number_(3) {
 }
 
 FileDescriptorTables::~FileDescriptorTables() {}
@@ -749,6 +796,60 @@ inline const EnumValueDescriptor* FileDescriptorTables::FindEnumValueByNumber(
   return FindPtrOrNull(enum_values_by_number_, make_pair(parent, number));
 }
 
+inline const EnumValueDescriptor*
+FileDescriptorTables::FindEnumValueByNumberCreatingIfUnknown(
+    const EnumDescriptor* parent, int number) const {
+  // First try, with map of compiled-in values.
+  {
+    const EnumValueDescriptor* desc = FindPtrOrNull(
+        enum_values_by_number_, make_pair(parent, number));
+    if (desc != NULL) {
+      return desc;
+    }
+  }
+  // Second try, with reader lock held on unknown enum values: common case.
+  {
+    ReaderMutexLock l(&unknown_enum_values_mu_);
+    const EnumValueDescriptor* desc = FindPtrOrNull(
+        unknown_enum_values_by_number_, make_pair(parent, number));
+    if (desc != NULL) {
+      return desc;
+    }
+  }
+  // If not found, try again with writer lock held, and create new descriptor if
+  // necessary.
+  {
+    WriterMutexLock l(&unknown_enum_values_mu_);
+    const EnumValueDescriptor* desc = FindPtrOrNull(
+        unknown_enum_values_by_number_, make_pair(parent, number));
+    if (desc != NULL) {
+      return desc;
+    }
+
+    // Create an EnumValueDescriptor dynamically. We don't insert it into the
+    // EnumDescriptor (it's not a part of the enum as originally defined), but
+    // we do insert it into the table so that we can return the same pointer
+    // later.
+    string enum_value_name = StringPrintf(
+        "UNKNOWN_ENUM_VALUE_%s_%d", parent->name().c_str(), number);
+    DescriptorPool::Tables* tables =
+        const_cast<DescriptorPool::Tables*>(DescriptorPool::generated_pool()->
+                                            tables_.get());
+    EnumValueDescriptor* result = tables->Allocate<EnumValueDescriptor>();
+    result->name_ = tables->AllocateString(enum_value_name);
+    result->full_name_ = tables->AllocateString(parent->full_name() +
+                                                "." + enum_value_name);
+    result->number_ = number;
+    result->type_ = parent;
+    result->options_ = &EnumValueOptions::default_instance();
+    InsertIfNotPresent(&unknown_enum_values_by_number_,
+                       make_pair(parent, number),
+                       result);
+    return result;
+  }
+}
+
+
 inline const FieldDescriptor* DescriptorPool::Tables::FindExtension(
     const Descriptor* extendee, int number) {
   return FindPtrOrNull(extensions_, make_pair(extendee, number));
@@ -1311,6 +1412,11 @@ EnumDescriptor::FindValueByNumber(int key) const {
   return file()->tables_->FindEnumValueByNumber(this, key);
 }
 
+const EnumValueDescriptor*
+EnumDescriptor::FindValueByNumberCreatingIfUnknown(int key) const {
+  return file()->tables_->FindEnumValueByNumberCreatingIfUnknown(this, key);
+}
+
 const MethodDescriptor*
 ServiceDescriptor::FindMethodByName(const string& key) const {
   Symbol result =
@@ -1562,6 +1668,8 @@ string FieldDescriptor::DefaultValueAsString(bool quote_string_type) const {
 void FileDescriptor::CopyTo(FileDescriptorProto* proto) const {
   proto->set_name(name());
   if (!package().empty()) proto->set_package(package());
+  // TODO(liujisi): Also populate when syntax="proto2".
+  if (syntax() == SYNTAX_PROTO3) proto->set_syntax(SyntaxName(syntax()));
 
   for (int i = 0; i < dependency_count(); i++) {
     proto->add_dependency(dependency(i)->name());
@@ -1594,7 +1702,8 @@ void FileDescriptor::CopyTo(FileDescriptorProto* proto) const {
 }
 
 void FileDescriptor::CopySourceCodeInfoTo(FileDescriptorProto* proto) const {
-  if (source_code_info_ != &SourceCodeInfo::default_instance()) {
+  if (source_code_info_ &&
+      source_code_info_ != &SourceCodeInfo::default_instance()) {
     proto->mutable_source_code_info()->CopyFrom(*source_code_info_);
   }
 }
@@ -1802,10 +1911,67 @@ bool FormatLineOptions(int depth, const Message &options, string *output) {
   return !all_options.empty();
 }
 
+template<typename DescType>
+class SourceLocationCommentPrinter {
+ public:
+  SourceLocationCommentPrinter(const DescType* desc,
+                               const string& prefix,
+                               const DebugStringOptions& options)
+      : options_(options), prefix_(prefix) {
+    // Perform the SourceLocation lookup only if we're including user comments,
+    // because the lookup is fairly expensive.
+    have_source_loc_ = options.include_comments &&
+        desc->GetSourceLocation(&source_loc_);
+  }
+  void AddPreComment(string* output) {
+    if (have_source_loc_ && source_loc_.leading_comments.size() > 0) {
+      *output += FormatComment(source_loc_.leading_comments);
+    }
+  }
+  void AddPostComment(string* output) {
+    if (have_source_loc_ && source_loc_.trailing_comments.size() > 0) {
+      *output += FormatComment(source_loc_.trailing_comments);
+    }
+  }
+
+  // Format comment such that each line becomes a full-line C++-style comment in
+  // the DebugString() output.
+  string FormatComment(const string& comment_text) {
+    string stripped_comment = comment_text;
+    StripWhitespace(&stripped_comment);
+    vector<string> lines = Split(stripped_comment, "\n");
+    string output;
+    for (int i = 0; i < lines.size(); ++i) {
+      const string& line = lines[i];
+      strings::SubstituteAndAppend(&output, "$0// $1\n", prefix_, line);
+    }
+    return output;
+  }
+
+ private:
+  const DescType* desc_;
+  bool have_source_loc_;
+  SourceLocation source_loc_;
+  DebugStringOptions options_;
+  string prefix_;
+};
+
 }  // anonymous namespace
 
 string FileDescriptor::DebugString() const {
-  string contents = "syntax = \"proto2\";\n\n";
+  DebugStringOptions options;  // default options
+  return DebugStringWithOptions(options);
+}
+
+string FileDescriptor::DebugStringWithOptions(
+    const DebugStringOptions& debug_string_options) const {
+  string contents;
+  strings::SubstituteAndAppend(&contents, "syntax = \"$0\";\n\n",
+                               SyntaxName(syntax()));
+
+  SourceLocationCommentPrinter<FileDescriptor>
+      comment_printer(this, "", debug_string_options);
+  comment_printer.AddPreComment(&contents);
 
   set<int> public_dependencies;
   set<int> weak_dependencies;
@@ -1836,7 +2002,7 @@ string FileDescriptor::DebugString() const {
   }
 
   for (int i = 0; i < enum_type_count(); i++) {
-    enum_type(i)->DebugString(0, &contents);
+    enum_type(i)->DebugString(0, &contents, debug_string_options);
     contents.append("\n");
   }
 
@@ -1851,15 +2017,14 @@ string FileDescriptor::DebugString() const {
 
   for (int i = 0; i < message_type_count(); i++) {
     if (groups.count(message_type(i)) == 0) {
-      strings::SubstituteAndAppend(&contents, "message $0",
-                                   message_type(i)->name());
-      message_type(i)->DebugString(0, &contents);
+      message_type(i)->DebugString(0, &contents, debug_string_options,
+                                   /* include_opening_clause */ true);
       contents.append("\n");
     }
   }
 
   for (int i = 0; i < service_count(); i++) {
-    service(i)->DebugString(&contents);
+    service(i)->DebugString(&contents, debug_string_options);
     contents.append("\n");
   }
 
@@ -1871,23 +2036,46 @@ string FileDescriptor::DebugString() const {
       strings::SubstituteAndAppend(&contents, "extend .$0 {\n",
                                    containing_type->full_name());
     }
-    extension(i)->DebugString(1, FieldDescriptor::PRINT_LABEL, &contents);
+    extension(i)->DebugString(1, FieldDescriptor::PRINT_LABEL, &contents,
+                              debug_string_options);
   }
   if (extension_count() > 0) contents.append("}\n\n");
 
+  comment_printer.AddPostComment(&contents);
+
   return contents;
 }
 
 string Descriptor::DebugString() const {
+  DebugStringOptions options;  // default options
+  return DebugStringWithOptions(options);
+}
+
+string Descriptor::DebugStringWithOptions(
+    const DebugStringOptions& options) const {
   string contents;
-  strings::SubstituteAndAppend(&contents, "message $0", name());
-  DebugString(0, &contents);
+  DebugString(0, &contents, options, /* include_opening_clause */ true);
   return contents;
 }
 
-void Descriptor::DebugString(int depth, string *contents) const {
+void Descriptor::DebugString(int depth, string *contents,
+                             const DebugStringOptions&
+                             debug_string_options,
+                             bool include_opening_clause) const {
+  if (options().map_entry()) {
+    // Do not generate debug string for auto-generated map-entry type.
+    return;
+  }
   string prefix(depth * 2, ' ');
   ++depth;
+
+  SourceLocationCommentPrinter<Descriptor>
+      comment_printer(this, prefix, debug_string_options);
+  comment_printer.AddPreComment(contents);
+
+  if (include_opening_clause) {
+    strings::SubstituteAndAppend(contents, "$0message $1", prefix, name());
+  }
   contents->append(" {\n");
 
   FormatLineOptions(depth, options(), contents);
@@ -1909,20 +2097,21 @@ void Descriptor::DebugString(int depth, string *contents) const {
 
   for (int i = 0; i < nested_type_count(); i++) {
     if (groups.count(nested_type(i)) == 0) {
-      strings::SubstituteAndAppend(contents, "$0  message $1",
-                                   prefix, nested_type(i)->name());
-      nested_type(i)->DebugString(depth, contents);
+      nested_type(i)->DebugString(depth, contents, debug_string_options,
+                                  /* include_opening_clause */ true);
     }
   }
   for (int i = 0; i < enum_type_count(); i++) {
-    enum_type(i)->DebugString(depth, contents);
+    enum_type(i)->DebugString(depth, contents, debug_string_options);
   }
   for (int i = 0; i < field_count(); i++) {
     if (field(i)->containing_oneof() == NULL) {
-      field(i)->DebugString(depth, FieldDescriptor::PRINT_LABEL, contents);
+      field(i)->DebugString(depth, FieldDescriptor::PRINT_LABEL, contents,
+                            debug_string_options);
     } else if (field(i)->containing_oneof()->field(0) == field(i)) {
       // This is the first field in this oneof, so print the whole oneof.
-      field(i)->containing_oneof()->DebugString(depth, contents);
+      field(i)->containing_oneof()->DebugString(depth, contents,
+                                                debug_string_options);
     }
   }
 
@@ -1943,15 +2132,23 @@ void Descriptor::DebugString(int depth, string *contents) const {
                                    prefix, containing_type->full_name());
     }
     extension(i)->DebugString(
-        depth + 1, FieldDescriptor::PRINT_LABEL, contents);
+        depth + 1, FieldDescriptor::PRINT_LABEL, contents,
+        debug_string_options);
   }
   if (extension_count() > 0)
     strings::SubstituteAndAppend(contents, "$0  }\n", prefix);
 
   strings::SubstituteAndAppend(contents, "$0}\n", prefix);
+  comment_printer.AddPostComment(contents);
 }
 
 string FieldDescriptor::DebugString() const {
+  DebugStringOptions options;  // default options
+  return DebugStringWithOptions(options);
+}
+
+string FieldDescriptor::DebugStringWithOptions(
+    const DebugStringOptions& debug_string_options) const {
   string contents;
   int depth = 0;
   if (is_extension()) {
@@ -1959,35 +2156,55 @@ string FieldDescriptor::DebugString() const {
                                  containing_type()->full_name());
     depth = 1;
   }
-  DebugString(depth, PRINT_LABEL, &contents);
+  DebugString(depth, PRINT_LABEL, &contents, debug_string_options);
   if (is_extension()) {
     contents.append("}\n");
   }
   return contents;
 }
 
+// The field type string used in FieldDescriptor::DebugString()
+string FieldDescriptor::FieldTypeNameDebugString() const {
+  switch(type()) {
+    case TYPE_MESSAGE:
+      return "." + message_type()->full_name();
+    case TYPE_ENUM:
+      return "." + enum_type()->full_name();
+    default:
+      return kTypeToName[type()];
+  }
+}
+
 void FieldDescriptor::DebugString(int depth,
                                   PrintLabelFlag print_label_flag,
-                                  string *contents) const {
+                                  string *contents,
+                                  const DebugStringOptions&
+                                  debug_string_options) const {
   string prefix(depth * 2, ' ');
   string field_type;
-  switch (type()) {
-    case TYPE_MESSAGE:
-      field_type = "." + message_type()->full_name();
-      break;
-    case TYPE_ENUM:
-      field_type = "." + enum_type()->full_name();
-      break;
-    default:
-      field_type = kTypeToName[type()];
+
+  // Special case map fields.
+  bool is_map = false;
+  if (type() == TYPE_MESSAGE && message_type()->options().map_entry()) {
+    is_map = true;
+    strings::SubstituteAndAppend(
+        &field_type, "map<$0, $1>",
+        message_type()->field(0)->FieldTypeNameDebugString(),
+        message_type()->field(1)->FieldTypeNameDebugString());
+  } else {
+    field_type = FieldTypeNameDebugString();
   }
 
   string label;
-  if (print_label_flag == PRINT_LABEL) {
+  if (print_label_flag == PRINT_LABEL && !is_map) {
     label = kLabelToName[this->label()];
     label.push_back(' ');
   }
 
+  SourceLocationCommentPrinter<FieldDescriptor>
+      comment_printer(this, prefix, debug_string_options);
+  comment_printer.AddPreComment(contents);
+
   strings::SubstituteAndAppend(contents, "$0$1$2 $3 = $4",
                                prefix,
                                label,
@@ -2015,57 +2232,101 @@ void FieldDescriptor::DebugString(int depth,
   }
 
   if (type() == TYPE_GROUP) {
-    message_type()->DebugString(depth, contents);
+    message_type()->DebugString(depth, contents, debug_string_options,
+                                /* include_opening_clause */ false);
   } else {
     contents->append(";\n");
   }
+
+  comment_printer.AddPostComment(contents);
 }
 
 string OneofDescriptor::DebugString() const {
+  DebugStringOptions options;  // default values
+  return DebugStringWithOptions(options);
+}
+
+string OneofDescriptor::DebugStringWithOptions(
+    const DebugStringOptions& options) const {
   string contents;
-  DebugString(0, &contents);
+  DebugString(0, &contents, options);
   return contents;
 }
 
-void OneofDescriptor::DebugString(int depth, string* contents) const {
+void OneofDescriptor::DebugString(int depth, string* contents,
+                                  const DebugStringOptions&
+                                  debug_string_options) const {
   string prefix(depth * 2, ' ');
   ++depth;
+  SourceLocationCommentPrinter<OneofDescriptor>
+      comment_printer(this, prefix, debug_string_options);
+  comment_printer.AddPreComment(contents);
   strings::SubstituteAndAppend(
       contents, "$0 oneof $1 {\n", prefix, name());
   for (int i = 0; i < field_count(); i++) {
-    field(i)->DebugString(depth, FieldDescriptor::OMIT_LABEL, contents);
+    field(i)->DebugString(depth, FieldDescriptor::OMIT_LABEL, contents,
+                          debug_string_options);
   }
   strings::SubstituteAndAppend(contents, "$0}\n", prefix);
+  comment_printer.AddPostComment(contents);
 }
 
 string EnumDescriptor::DebugString() const {
+  DebugStringOptions options;  // default values
+  return DebugStringWithOptions(options);
+}
+
+string EnumDescriptor::DebugStringWithOptions(
+    const DebugStringOptions& options) const {
   string contents;
-  DebugString(0, &contents);
+  DebugString(0, &contents, options);
   return contents;
 }
 
-void EnumDescriptor::DebugString(int depth, string *contents) const {
+void EnumDescriptor::DebugString(int depth, string *contents,
+                                 const DebugStringOptions&
+                                 debug_string_options) const {
   string prefix(depth * 2, ' ');
   ++depth;
+
+  SourceLocationCommentPrinter<EnumDescriptor>
+      comment_printer(this, prefix, debug_string_options);
+  comment_printer.AddPreComment(contents);
+
   strings::SubstituteAndAppend(contents, "$0enum $1 {\n",
                                prefix, name());
 
   FormatLineOptions(depth, options(), contents);
 
   for (int i = 0; i < value_count(); i++) {
-    value(i)->DebugString(depth, contents);
+    value(i)->DebugString(depth, contents, debug_string_options);
   }
   strings::SubstituteAndAppend(contents, "$0}\n", prefix);
+
+  comment_printer.AddPostComment(contents);
 }
 
 string EnumValueDescriptor::DebugString() const {
+  DebugStringOptions options;  // default values
+  return DebugStringWithOptions(options);
+}
+
+string EnumValueDescriptor::DebugStringWithOptions(
+    const DebugStringOptions& options) const {
   string contents;
-  DebugString(0, &contents);
+  DebugString(0, &contents, options);
   return contents;
 }
 
-void EnumValueDescriptor::DebugString(int depth, string *contents) const {
+void EnumValueDescriptor::DebugString(int depth, string *contents,
+                                      const DebugStringOptions&
+                                      debug_string_options) const {
   string prefix(depth * 2, ' ');
+
+  SourceLocationCommentPrinter<EnumValueDescriptor>
+      comment_printer(this, prefix, debug_string_options);
+  comment_printer.AddPreComment(contents);
+
   strings::SubstituteAndAppend(contents, "$0$1 = $2",
                                prefix, name(), number());
 
@@ -2074,35 +2335,64 @@ void EnumValueDescriptor::DebugString(int depth, string *contents) const {
     strings::SubstituteAndAppend(contents, " [$0]", formatted_options);
   }
   contents->append(";\n");
+
+  comment_printer.AddPostComment(contents);
 }
 
 string ServiceDescriptor::DebugString() const {
+  DebugStringOptions options;  // default values
+  return DebugStringWithOptions(options);
+}
+
+string ServiceDescriptor::DebugStringWithOptions(
+    const DebugStringOptions& options) const {
   string contents;
-  DebugString(&contents);
+  DebugString(&contents, options);
   return contents;
 }
 
-void ServiceDescriptor::DebugString(string *contents) const {
+void ServiceDescriptor::DebugString(string *contents,
+                                    const DebugStringOptions&
+                                    debug_string_options) const {
+  SourceLocationCommentPrinter<ServiceDescriptor>
+      comment_printer(this, /* prefix */ "", debug_string_options);
+  comment_printer.AddPreComment(contents);
+
   strings::SubstituteAndAppend(contents, "service $0 {\n", name());
 
   FormatLineOptions(1, options(), contents);
 
   for (int i = 0; i < method_count(); i++) {
-    method(i)->DebugString(1, contents);
+    method(i)->DebugString(1, contents, debug_string_options);
   }
 
   contents->append("}\n");
+
+  comment_printer.AddPostComment(contents);
 }
 
 string MethodDescriptor::DebugString() const {
+  DebugStringOptions options;  // default values
+  return DebugStringWithOptions(options);
+}
+
+string MethodDescriptor::DebugStringWithOptions(
+    const DebugStringOptions& options) const {
   string contents;
-  DebugString(0, &contents);
+  DebugString(0, &contents, options);
   return contents;
 }
 
-void MethodDescriptor::DebugString(int depth, string *contents) const {
+void MethodDescriptor::DebugString(int depth, string *contents,
+                                   const DebugStringOptions&
+                                   debug_string_options) const {
   string prefix(depth * 2, ' ');
   ++depth;
+
+  SourceLocationCommentPrinter<MethodDescriptor>
+      comment_printer(this, prefix, debug_string_options);
+  comment_printer.AddPreComment(contents);
+
   strings::SubstituteAndAppend(contents, "$0rpc $1(.$2) returns (.$3)",
                                prefix, name(),
                                input_type()->full_name(),
@@ -2115,6 +2405,8 @@ void MethodDescriptor::DebugString(int depth, string *contents) const {
   } else {
     contents->append(";\n");
   }
+
+  comment_printer.AddPostComment(contents);
 }
 
 
@@ -2142,6 +2434,11 @@ bool FileDescriptor::GetSourceLocation(const vector<int>& path,
   return false;
 }
 
+bool FileDescriptor::GetSourceLocation(SourceLocation* out_location) const {
+  vector<int> path;  // empty path for root FileDescriptor
+  return GetSourceLocation(path, out_location);
+}
+
 bool FieldDescriptor::is_packed() const {
   return is_packable() && (options_ != NULL) && options_->packed();
 }
@@ -2409,7 +2706,7 @@ class DescriptorBuilder {
 
   // Creates a placeholder file.  Never returns NULL.  This is used when an
   // import is not found and AllowUnknownDependencies() is enabled.
-  const FileDescriptor* NewPlaceholderFile(const string& name);
+  FileDescriptor* NewPlaceholderFile(const string& name);
 
   // Calls tables_->AddSymbol() and records an error if it fails.  Returns
   // true if successful or false if failed, though most callers can ignore
@@ -2667,9 +2964,24 @@ class DescriptorBuilder {
                               const ServiceDescriptorProto& proto);
   void ValidateMethodOptions(MethodDescriptor* method,
                              const MethodDescriptorProto& proto);
-
-  void ValidateMapKey(FieldDescriptor* field,
-                      const FieldDescriptorProto& proto);
+  void ValidateProto3(FileDescriptor* file,
+                      const FileDescriptorProto& proto);
+  void ValidateProto3Message(Descriptor* message,
+                             const DescriptorProto& proto);
+  void ValidateProto3Field(FieldDescriptor* field,
+                           const FieldDescriptorProto& proto);
+  void ValidateProto3Enum(EnumDescriptor* enm,
+                          const EnumDescriptorProto& proto);
+
+  // Returns true if the map entry message is compatible with the
+  // auto-generated entry message from map fields syntax.
+  bool ValidateMapEntry(FieldDescriptor* field,
+                        const FieldDescriptorProto& proto);
+
+  // Recursively detects naming conflicts with map entry types for a
+  // better error message.
+  void DetectMapConflicts(const Descriptor* message,
+                          const DescriptorProto& proto);
 
 };
 
@@ -2992,19 +3304,9 @@ Symbol DescriptorBuilder::NewPlaceholder(const string& name,
   }
 
   // Create the placeholders.
-  FileDescriptor* placeholder_file = tables_->Allocate<FileDescriptor>();
-  memset(placeholder_file, 0, sizeof(*placeholder_file));
-
-  placeholder_file->source_code_info_ = &SourceCodeInfo::default_instance();
-
-  placeholder_file->name_ =
-    tables_->AllocateString(*placeholder_full_name + ".placeholder.proto");
+  FileDescriptor* placeholder_file = NewPlaceholderFile(
+      *placeholder_full_name + ".placeholder.proto");
   placeholder_file->package_ = placeholder_package;
-  placeholder_file->pool_ = pool_;
-  placeholder_file->options_ = &FileOptions::default_instance();
-  placeholder_file->tables_ = &FileDescriptorTables::kEmpty;
-  placeholder_file->is_placeholder_ = true;
-  // All other fields are zero or NULL.
 
   if (placeholder_type == PLACEHOLDER_ENUM) {
     placeholder_file->enum_type_count_ = 1;
@@ -3068,7 +3370,7 @@ Symbol DescriptorBuilder::NewPlaceholder(const string& name,
   }
 }
 
-const FileDescriptor* DescriptorBuilder::NewPlaceholderFile(
+FileDescriptor* DescriptorBuilder::NewPlaceholderFile(
     const string& name) {
   FileDescriptor* placeholder = tables_->Allocate<FileDescriptor>();
   memset(placeholder, 0, sizeof(*placeholder));
@@ -3078,7 +3380,9 @@ const FileDescriptor* DescriptorBuilder::NewPlaceholderFile(
   placeholder->pool_ = pool_;
   placeholder->options_ = &FileOptions::default_instance();
   placeholder->tables_ = &FileDescriptorTables::kEmpty;
+  placeholder->source_code_info_ = &SourceCodeInfo::default_instance();
   placeholder->is_placeholder_ = true;
+  placeholder->syntax_ = FileDescriptor::SYNTAX_PROTO2;
   // All other fields are zero or NULL.
 
   return placeholder;
@@ -3363,6 +3667,19 @@ const FileDescriptor* DescriptorBuilder::BuildFile(
              "Missing field: FileDescriptorProto.name.");
   }
 
+  // TODO(liujisi): Report error when the syntax is empty after all the protos
+  // have added the syntax statement.
+  if (proto.syntax().empty() || proto.syntax() == "proto2") {
+    file_->syntax_ = FileDescriptor::SYNTAX_PROTO2;
+  } else if (proto.syntax() == "proto3") {
+    file_->syntax_ = FileDescriptor::SYNTAX_PROTO3;
+  } else {
+    file_->syntax_ = FileDescriptor::SYNTAX_UNKNOWN;
+    AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER,
+             "Unrecognized syntax: " + proto.syntax());
+  }
+
+
   result->name_ = tables_->AllocateString(proto.name());
   if (proto.has_package()) {
     result->package_ = tables_->AllocateString(proto.package());
@@ -3506,6 +3823,14 @@ const FileDescriptor* DescriptorBuilder::BuildFile(
     ValidateFileOptions(result, proto);
   }
 
+  // Additional naming conflict check for map entry types. Only need to check
+  // this if there are already errors.
+  if (had_errors_) {
+    for (int i = 0; i < proto.message_type_size(); ++i) {
+      DetectMapConflicts(result->message_type(i), proto.message_type(i));
+    }
+  }
+
 
   if (!unused_dependency_.empty()) {
     LogUnusedDependency(result);
@@ -3621,7 +3946,9 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto,
   // Don't bother with the above optimization for camel-case names since
   // .proto files that follow the guide shouldn't be using names in this
   // format, so the optimization wouldn't help much.
-  result->camelcase_name_ = tables_->AllocateString(ToCamelCase(proto.name()));
+  result->camelcase_name_ =
+      tables_->AllocateString(ToCamelCase(proto.name(),
+                                          /* lower_first = */ true));
 
   // Some compilers do not allow static_cast directly between two enum types,
   // so we must cast to int first.
@@ -3646,7 +3973,6 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto,
   // Some of these may be filled in when cross-linking.
   result->containing_type_ = NULL;
   result->extension_scope_ = NULL;
-  result->experimental_map_key_ = NULL;
   result->message_type_ = NULL;
   result->enum_type_ = NULL;
 
@@ -4472,8 +4798,98 @@ void DescriptorBuilder::ValidateFileOptions(FileDescriptor* file,
       }
     }
   }
+  if (file->syntax() == FileDescriptor::SYNTAX_PROTO3) {
+    ValidateProto3(file, proto);
+  }
 }
 
+void DescriptorBuilder::ValidateProto3(
+    FileDescriptor* file, const FileDescriptorProto& proto) {
+  for (int i = 0; i < file->extension_count(); ++i) {
+    ValidateProto3Field(file->extensions_ + i, proto.extension(i));
+  }
+  for (int i = 0; i < file->message_type_count(); ++i) {
+    ValidateProto3Message(file->message_types_ + i, proto.message_type(i));
+  }
+  for (int i = 0; i < file->enum_type_count(); ++i) {
+    ValidateProto3Enum(file->enum_types_ + i, proto.enum_type(i));
+  }
+  if (IsLite(file)) {
+    AddError(file->name(), proto,
+             DescriptorPool::ErrorCollector::OTHER,
+             "Lite runtime is not supported in proto3.");
+  }
+}
+
+void DescriptorBuilder::ValidateProto3Message(
+    Descriptor* message, const DescriptorProto& proto) {
+  for (int i = 0; i < message->nested_type_count(); ++i) {
+    ValidateProto3Message(message->nested_types_ + i,
+                                 proto.nested_type(i));
+  }
+  for (int i = 0; i < message->enum_type_count(); ++i) {
+    ValidateProto3Enum(message->enum_types_ + i,
+                              proto.enum_type(i));
+  }
+  for (int i = 0; i < message->field_count(); ++i) {
+    ValidateProto3Field(message->fields_ + i, proto.field(i));
+  }
+  for (int i = 0; i < message->extension_count(); ++i) {
+    ValidateProto3Field(message->extensions_ +i, proto.extension(i));
+  }
+  if (message->extension_range_count() > 0) {
+    AddError(message->full_name(), proto,
+             DescriptorPool::ErrorCollector::OTHER,
+             "Extension ranges are not allowed in proto3.");
+  }
+  if (message->options().message_set_wire_format()) {
+    // Using MessageSet doesn't make sense since we disallow extensions.
+    AddError(message->full_name(), proto,
+             DescriptorPool::ErrorCollector::OTHER,
+             "MessageSet is not supported in proto3.");
+  }
+}
+
+void DescriptorBuilder::ValidateProto3Field(
+    FieldDescriptor* field, const FieldDescriptorProto& proto) {
+  if (field->is_extension() &&
+      !AllowedExtendeeInProto3(field->containing_type()->full_name())) {
+    AddError(field->full_name(), proto,
+             DescriptorPool::ErrorCollector::OTHER,
+             "Extensions in proto3 are only allowed for defining options.");
+  }
+  if (field->is_required()) {
+    AddError(field->full_name(), proto,
+             DescriptorPool::ErrorCollector::OTHER,
+             "Required fields are not allowed in proto3.");
+  }
+  if (field->has_default_value()) {
+    AddError(
+        field->full_name(), proto, DescriptorPool::ErrorCollector::OTHER,
+        "Explicit default values are not allowed in proto3.");
+  }
+  if (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM &&
+      field->enum_type() &&
+      field->enum_type()->file()->syntax() != FileDescriptor::SYNTAX_PROTO3) {
+    // Proto3 messages can only use Proto3 enum types; otherwise we can't
+    // guarantee that the default value is zero.
+    AddError(field->full_name(), proto,
+             DescriptorPool::ErrorCollector::TYPE,
+             "Enum type \"" + field->enum_type()->full_name() +
+             "\" is not a proto3 enum, but is used in \"" +
+             field->containing_type()->full_name() +
+             "\" which is a proto3 message type.");
+  }
+}
+
+void DescriptorBuilder::ValidateProto3Enum(
+    EnumDescriptor* enm, const EnumDescriptorProto& proto) {
+  if (enm->value_count() > 0 && enm->value(0)->number() != 0) {
+    AddError(
+        enm->full_name(), proto, DescriptorPool::ErrorCollector::OTHER,
+        "The first enum value must be zero in proto3.");
+  }
+}
 
 void DescriptorBuilder::ValidateMessageOptions(Descriptor* message,
                                                const DescriptorProto& proto) {
@@ -4499,10 +4915,6 @@ void DescriptorBuilder::ValidateMessageOptions(Descriptor* message,
 
 void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field,
     const FieldDescriptorProto& proto) {
-  if (field->options().has_experimental_map_key()) {
-    ValidateMapKey(field, proto);
-  }
-
   // Only message type fields may be lazy.
   if (field->options().lazy()) {
     if (field->type() != FieldDescriptor::TYPE_MESSAGE) {
@@ -4551,6 +4963,17 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field,
              "a lite type, but the reverse is allowed.");
   }
 
+  // Validate map types.
+  if (field->type() == FieldDescriptor::TYPE_MESSAGE &&
+      field->message_type()->options().map_entry()) {
+    if (!ValidateMapEntry(field, proto)) {
+      AddError(field->full_name(), proto,
+               DescriptorPool::ErrorCollector::OTHER,
+               "map_entry should not be set explicitly. Use map<KeyType, "
+               "ValueType> instead.");
+    }
+  }
+
 }
 
 void DescriptorBuilder::ValidateEnumOptions(EnumDescriptor* enm,
@@ -4608,57 +5031,127 @@ void DescriptorBuilder::ValidateMethodOptions(MethodDescriptor* /* method */,
   // Nothing to do so far.
 }
 
-void DescriptorBuilder::ValidateMapKey(FieldDescriptor* field,
-                                       const FieldDescriptorProto& proto) {
-  if (!field->is_repeated()) {
-    AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE,
-             "map type is only allowed for repeated fields.");
-    return;
+bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field,
+                                         const FieldDescriptorProto& proto) {
+  const Descriptor* message = field->message_type();
+  if (// Must not contain extensions, extension range or nested message or
+      // enums
+      message->extension_count() != 0 ||
+      field->label() != FieldDescriptor::LABEL_REPEATED ||
+      message->extension_range_count() != 0 ||
+      message->nested_type_count() != 0 || message->enum_type_count() != 0 ||
+      // Must contain exactly two fields
+      message->field_count() != 2 ||
+      // Field name and message name must match
+      message->name() != ToCamelCase(field->name(), false) + "Entry" ||
+      // Entry message must be in the same containing type of the field.
+      field->containing_type() != message->containing_type()) {
+    return false;
   }
 
-  if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) {
-    AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE,
-             "map type is only allowed for fields with a message type.");
-    return;
+  const FieldDescriptor* key = message->field(0);
+  const FieldDescriptor* value = message->field(1);
+  if (key->label() != FieldDescriptor::LABEL_OPTIONAL || key->number() != 1 ||
+      key->name() != "key") {
+    return false;
+  }
+  if (value->label() != FieldDescriptor::LABEL_OPTIONAL ||
+      value->number() != 2 || value->name() != "value") {
+    return false;
   }
 
-  const Descriptor* item_type = field->message_type();
-  if (item_type == NULL) {
-    AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE,
-             "Could not find field type.");
-    return;
+  // Check key types are legal.
+  switch (key->type()) {
+    case FieldDescriptor::TYPE_ENUM:
+      AddError(
+          field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE,
+          "Key in map fields cannot be enum types.");
+      break;
+    case FieldDescriptor::TYPE_FLOAT:
+    case FieldDescriptor::TYPE_DOUBLE:
+    case FieldDescriptor::TYPE_MESSAGE:
+    case FieldDescriptor::TYPE_GROUP:
+    case FieldDescriptor::TYPE_BYTES:
+      AddError(
+          field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE,
+          "Key in map fields cannot be float/double, bytes or message types.");
+      break;
+    case FieldDescriptor::TYPE_BOOL:
+    case FieldDescriptor::TYPE_INT32:
+    case FieldDescriptor::TYPE_INT64:
+    case FieldDescriptor::TYPE_SINT32:
+    case FieldDescriptor::TYPE_SINT64:
+    case FieldDescriptor::TYPE_STRING:
+    case FieldDescriptor::TYPE_UINT32:
+    case FieldDescriptor::TYPE_UINT64:
+    case FieldDescriptor::TYPE_FIXED32:
+    case FieldDescriptor::TYPE_FIXED64:
+    case FieldDescriptor::TYPE_SFIXED32:
+    case FieldDescriptor::TYPE_SFIXED64:
+      // Legal cases
+      break;
+    // Do not add a default, so that the compiler will complian when new types
+    // are added.
   }
 
-  // Find the field in item_type named by "experimental_map_key"
-  const string& key_name = field->options().experimental_map_key();
-  const Symbol key_symbol = LookupSymbol(
-      key_name,
-      // We append ".key_name" to the containing type's name since
-      // LookupSymbol() searches for peers of the supplied name, not
-      // children of the supplied name.
-      item_type->full_name() + "." + key_name);
+  return true;
+}
 
-  if (key_symbol.IsNull() || key_symbol.field_descriptor->is_extension()) {
-    AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE,
-             "Could not find field named \"" + key_name + "\" in type \"" +
-             item_type->full_name() + "\".");
-    return;
+void DescriptorBuilder::DetectMapConflicts(const Descriptor* message,
+                                           const DescriptorProto& proto) {
+  map<string, const Descriptor*> seen_types;
+  for (int i = 0; i < message->nested_type_count(); ++i) {
+    const Descriptor* nested = message->nested_type(i);
+    pair<map<string, const Descriptor*>::iterator, bool> result =
+        seen_types.insert(make_pair(nested->name(), nested));
+    if (!result.second) {
+      if (result.first->second->options().map_entry() ||
+          nested->options().map_entry()) {
+        AddError(message->full_name(), proto,
+                 DescriptorPool::ErrorCollector::NAME,
+                 "Expanded map entry type " + nested->name() +
+                 " conflicts with an existing nested message type.");
+      }
+    }
+    // Recursively test on the nested types.
+    DetectMapConflicts(message->nested_type(i), proto.nested_type(i));
+  }
+  // Check for conflicted field names.
+  for (int i = 0; i < message->field_count(); ++i) {
+    const FieldDescriptor* field = message->field(i);
+    map<string, const Descriptor*>::iterator iter =
+        seen_types.find(field->name());
+    if (iter != seen_types.end() && iter->second->options().map_entry()) {
+      AddError(message->full_name(), proto,
+               DescriptorPool::ErrorCollector::NAME,
+               "Expanded map entry type " + iter->second->name() +
+               " conflicts with an existing field.");
+    }
   }
-  const FieldDescriptor* key_field = key_symbol.field_descriptor;
-
-  if (key_field->is_repeated()) {
-    AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE,
-             "map_key must not name a repeated field.");
-    return;
+  // Check for conflicted enum names.
+  for (int i = 0; i < message->enum_type_count(); ++i) {
+    const EnumDescriptor* enum_desc = message->enum_type(i);
+    map<string, const Descriptor*>::iterator iter =
+        seen_types.find(enum_desc->name());
+    if (iter != seen_types.end() && iter->second->options().map_entry()) {
+      AddError(message->full_name(), proto,
+               DescriptorPool::ErrorCollector::NAME,
+               "Expanded map entry type " + iter->second->name() +
+               " conflicts with an existing enum type.");
+    }
   }
-
-  if (key_field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
-    AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE,
-             "map key must name a scalar or string field.");
-    return;
+  // Check for conflicted oneof names.
+  for (int i = 0; i < message->oneof_decl_count(); ++i) {
+    const OneofDescriptor* oneof_desc = message->oneof_decl(i);
+    map<string, const Descriptor*>::iterator iter =
+        seen_types.find(oneof_desc->name());
+    if (iter != seen_types.end() && iter->second->options().map_entry()) {
+      AddError(message->full_name(), proto,
+               DescriptorPool::ErrorCollector::NAME,
+               "Expanded map entry type " + iter->second->name() +
+               " conflicts with an existing oneof type.");
+    }
   }
-
-  field->experimental_map_key_ = key_field;
 }