diff options
Diffstat (limited to 'src/google/protobuf/descriptor.cc')
| -rw-r--r-- | src/google/protobuf/descriptor.cc | 1936 |
1 files changed, 1481 insertions, 455 deletions
diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 9b20946c..dae24f9e 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -32,45 +32,120 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. +#include <algorithm> +#include <functional> #include <google/protobuf/stubs/hash.h> +#include <limits> #include <map> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <set> #include <string> #include <vector> -#include <algorithm> -#include <limits> +#include <google/protobuf/stubs/casts.h> +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/stubs/logging.h> +#include <google/protobuf/stubs/mutex.h> +#include <google/protobuf/stubs/once.h> +#include <google/protobuf/stubs/stringprintf.h> +#include <google/protobuf/stubs/strutil.h> +#include <google/protobuf/descriptor.pb.h> +#include <google/protobuf/io/strtod.h> +#include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/io/tokenizer.h> +#include <google/protobuf/io/zero_copy_stream_impl.h> #include <google/protobuf/descriptor.h> #include <google/protobuf/descriptor_database.h> -#include <google/protobuf/descriptor.pb.h> #include <google/protobuf/dynamic_message.h> #include <google/protobuf/generated_message_util.h> #include <google/protobuf/text_format.h> #include <google/protobuf/unknown_field_set.h> #include <google/protobuf/wire_format.h> -#include <google/protobuf/io/strtod.h> -#include <google/protobuf/io/coded_stream.h> -#include <google/protobuf/io/tokenizer.h> -#include <google/protobuf/io/zero_copy_stream_impl.h> -#include <google/protobuf/stubs/common.h> -#include <google/protobuf/stubs/logging.h> -#include <google/protobuf/stubs/mutex.h> -#include <google/protobuf/stubs/once.h> -#include <google/protobuf/stubs/stringprintf.h> -#include <google/protobuf/stubs/strutil.h> #include <google/protobuf/stubs/substitute.h> + + #include <google/protobuf/stubs/map_util.h> #include <google/protobuf/stubs/stl_util.h> #undef PACKAGE // autoheader #defines this. :( namespace google { + namespace protobuf { +struct Symbol { + enum Type { + NULL_SYMBOL, + MESSAGE, + FIELD, + ONEOF, + ENUM, + ENUM_VALUE, + SERVICE, + METHOD, + PACKAGE + }; + Type type; + union { + const Descriptor* descriptor; + const FieldDescriptor* field_descriptor; + const OneofDescriptor* oneof_descriptor; + const EnumDescriptor* enum_descriptor; + const EnumValueDescriptor* enum_value_descriptor; + const ServiceDescriptor* service_descriptor; + const MethodDescriptor* method_descriptor; + const FileDescriptor* package_file_descriptor; + }; + + inline Symbol() : type(NULL_SYMBOL) { descriptor = NULL; } + inline bool IsNull() const { return type == NULL_SYMBOL; } + inline bool IsType() const { return type == MESSAGE || type == ENUM; } + inline bool IsAggregate() const { + return type == MESSAGE || type == PACKAGE || type == ENUM || + type == SERVICE; + } + +#define CONSTRUCTOR(TYPE, TYPE_CONSTANT, FIELD) \ + inline explicit Symbol(const TYPE* value) { \ + type = TYPE_CONSTANT; \ + this->FIELD = value; \ + } + + CONSTRUCTOR(Descriptor, MESSAGE, descriptor) + CONSTRUCTOR(FieldDescriptor, FIELD, field_descriptor) + CONSTRUCTOR(OneofDescriptor, ONEOF, oneof_descriptor) + CONSTRUCTOR(EnumDescriptor, ENUM, enum_descriptor) + CONSTRUCTOR(EnumValueDescriptor, ENUM_VALUE, enum_value_descriptor) + CONSTRUCTOR(ServiceDescriptor, SERVICE, service_descriptor) + CONSTRUCTOR(MethodDescriptor, METHOD, method_descriptor) + CONSTRUCTOR(FileDescriptor, PACKAGE, package_file_descriptor) +#undef CONSTRUCTOR + + const FileDescriptor* GetFile() const { + switch (type) { + case NULL_SYMBOL: + return NULL; + case MESSAGE: + return descriptor->file(); + case FIELD: + return field_descriptor->file(); + case ONEOF: + return oneof_descriptor->containing_type()->file(); + case ENUM: + return enum_descriptor->file(); + case ENUM_VALUE: + return enum_value_descriptor->type()->file(); + case SERVICE: + return service_descriptor->file(); + case METHOD: + return method_descriptor->service()->file(); + case PACKAGE: + return package_file_descriptor; + } + return NULL; + } +}; + const FieldDescriptor::CppType FieldDescriptor::kTypeToCppTypeMap[MAX_TYPE + 1] = { static_cast<CppType>(0), // 0 is reserved for errors @@ -164,6 +239,15 @@ const int FieldDescriptor::kLastReservedNumber; namespace { +// Note: I distrust ctype.h due to locales. +char ToUpper(char ch) { + return (ch >= 'a' && ch <= 'z') ? (ch - 'a' + 'A') : ch; +} + +char ToLower(char ch) { + return (ch >= 'A' && ch <= 'Z') ? (ch - 'A' + 'a') : ch; +} + string ToCamelCase(const string& input, bool lower_first) { bool capitalize_next = !lower_first; string result; @@ -173,12 +257,7 @@ string ToCamelCase(const string& input, bool lower_first) { if (input[i] == '_') { capitalize_next = true; } else if (capitalize_next) { - // Note: I distrust ctype.h due to locales. - if ('a' <= input[i] && input[i] <= 'z') { - result.push_back(input[i] - 'a' + 'A'); - } else { - result.push_back(input[i]); - } + result.push_back(ToUpper(input[i])); capitalize_next = false; } else { result.push_back(input[i]); @@ -186,28 +265,129 @@ string ToCamelCase(const string& input, bool lower_first) { } // Lower-case the first letter. - if (lower_first && !result.empty() && 'A' <= result[0] && result[0] <= 'Z') { - result[0] = result[0] - 'A' + 'a'; + if (lower_first && !result.empty()) { + result[0] = ToLower(result[0]); } return result; } -// A DescriptorPool contains a bunch of hash_maps to implement the +string ToJsonName(const string& input) { + bool capitalize_next = false; + string result; + result.reserve(input.size()); + + for (int i = 0; i < input.size(); i++) { + if (input[i] == '_') { + capitalize_next = true; + } else if (capitalize_next) { + result.push_back(ToUpper(input[i])); + capitalize_next = false; + } else { + result.push_back(input[i]); + } + } + + return result; +} + +string EnumValueToPascalCase(const string& input) { + bool next_upper = true; + string result; + result.reserve(input.size()); + + for (int i = 0; i < input.size(); i++) { + if (input[i] == '_') { + next_upper = true; + } else { + if (next_upper) { + result.push_back(ToUpper(input[i])); + } else { + result.push_back(ToLower(input[i])); + } + next_upper = false; + } + } + + return result; +} + +// Class to remove an enum prefix from enum values. +class PrefixRemover { + public: + PrefixRemover(StringPiece prefix) { + // Strip underscores and lower-case the prefix. + for (int i = 0; i < prefix.size(); i++) { + if (prefix[i] != '_') { + prefix_ += ascii_tolower(prefix[i]); + } + } + } + + // Tries to remove the enum prefix from this enum value. + // If this is not possible, returns the input verbatim. + string MaybeRemove(StringPiece str) { + // We can't just lowercase and strip str and look for a prefix. + // We need to properly recognize the difference between: + // + // enum Foo { + // FOO_BAR_BAZ = 0; + // FOO_BARBAZ = 1; + // } + // + // This is acceptable (though perhaps not advisable) because even when + // we PascalCase, these two will still be distinct (BarBaz vs. Barbaz). + size_t i, j; + + // Skip past prefix_ in str if we can. + for (i = 0, j = 0; i < str.size() && j < prefix_.size(); i++) { + if (str[i] == '_') { + continue; + } + + if (ascii_tolower(str[i]) != prefix_[j++]) { + return string(str); + } + } + + // If we didn't make it through the prefix, we've failed to strip the + // prefix. + if (j < prefix_.size()) { + return string(str); + } + + // Skip underscores between prefix and further characters. + while (i < str.size() && str[i] == '_') { + i++; + } + + // Enum label can't be the empty string. + if (i == str.size()) { + return string(str); + } + + // We successfully stripped the prefix. + str.remove_prefix(i); + return string(str); + } + + private: + string prefix_; +}; + +// A DescriptorPool contains a bunch of hash-maps to implement the // various Find*By*() methods. Since hashtable lookups are O(1), it's -// most efficient to construct a fixed set of large hash_maps used by +// most efficient to construct a fixed set of large hash-maps used by // all objects in the pool rather than construct one or more small -// hash_maps for each object. +// hash-maps for each object. // -// The keys to these hash_maps are (parent, name) or (parent, number) -// pairs. Unfortunately STL doesn't provide hash functions for pair<>, -// so we must invent our own. +// The keys to these hash-maps are (parent, name) or (parent, number) pairs. // // TODO(kenton): Use StringPiece rather than const char* in keys? It would // be a lot cleaner but we'd just have to convert it back to const char* // for the open source release. -typedef pair<const void*, const char*> PointerStringPair; +typedef std::pair<const void*, const char*> PointerStringPair; struct PointerStringPairEqual { inline bool operator()(const PointerStringPair& a, @@ -216,12 +396,20 @@ struct PointerStringPairEqual { } }; +typedef std::pair<const Descriptor*, int> DescriptorIntPair; +typedef std::pair<const EnumDescriptor*, int> EnumIntPair; + +#define HASH_MAP hash_map +#define HASH_SET hash_set +#define HASH_FXN hash + template<typename PairType> struct PointerIntegerPairHash { size_t operator()(const PairType& p) const { - // FIXME(kenton): What is the best way to compute this hash? I have - // no idea! This seems a bit better than an XOR. - return reinterpret_cast<intptr_t>(p.first) * ((1 << 16) - 1) + p.second; + static const size_t prime1 = 16777499; + static const size_t prime2 = 16777619; + return reinterpret_cast<size_t>(p.first) * prime1 ^ + static_cast<size_t>(p.second) * prime2; } #ifdef _MSC_VER @@ -235,16 +423,12 @@ struct PointerIntegerPairHash { } }; -typedef pair<const Descriptor*, int> DescriptorIntPair; -typedef pair<const EnumDescriptor*, int> EnumIntPair; - struct PointerStringPairHash { size_t operator()(const PointerStringPair& p) const { - // FIXME(kenton): What is the best way to compute this hash? I have - // no idea! This seems a bit better than an XOR. + static const size_t prime = 16777619; hash<const char*> cstring_hash; - return reinterpret_cast<intptr_t>(p.first) * ((1 << 16) - 1) + - cstring_hash(p.second); + return reinterpret_cast<size_t>(p.first) * prime ^ + static_cast<size_t>(cstring_hash(p.second)); } #ifdef _MSC_VER @@ -261,94 +445,41 @@ struct PointerStringPairHash { }; -struct Symbol { - enum Type { - NULL_SYMBOL, MESSAGE, FIELD, ONEOF, ENUM, ENUM_VALUE, SERVICE, METHOD, - PACKAGE - }; - Type type; - union { - const Descriptor* descriptor; - const FieldDescriptor* field_descriptor; - const OneofDescriptor* oneof_descriptor; - const EnumDescriptor* enum_descriptor; - const EnumValueDescriptor* enum_value_descriptor; - const ServiceDescriptor* service_descriptor; - const MethodDescriptor* method_descriptor; - const FileDescriptor* package_file_descriptor; - }; +const Symbol kNullSymbol; - inline Symbol() : type(NULL_SYMBOL) { descriptor = NULL; } - inline bool IsNull() const { return type == NULL_SYMBOL; } - inline bool IsType() const { - return type == MESSAGE || type == ENUM; - } - inline bool IsAggregate() const { - return type == MESSAGE || type == PACKAGE - || type == ENUM || type == SERVICE; - } +typedef HASH_MAP<const char*, Symbol, HASH_FXN<const char*>, streq> + SymbolsByNameMap; -#define CONSTRUCTOR(TYPE, TYPE_CONSTANT, FIELD) \ - inline explicit Symbol(const TYPE* value) { \ - type = TYPE_CONSTANT; \ - this->FIELD = value; \ - } +typedef HASH_MAP<PointerStringPair, Symbol, PointerStringPairHash, + PointerStringPairEqual> + SymbolsByParentMap; - CONSTRUCTOR(Descriptor , MESSAGE , descriptor ) - CONSTRUCTOR(FieldDescriptor , FIELD , field_descriptor ) - CONSTRUCTOR(OneofDescriptor , ONEOF , oneof_descriptor ) - CONSTRUCTOR(EnumDescriptor , ENUM , enum_descriptor ) - CONSTRUCTOR(EnumValueDescriptor, ENUM_VALUE, enum_value_descriptor ) - CONSTRUCTOR(ServiceDescriptor , SERVICE , service_descriptor ) - CONSTRUCTOR(MethodDescriptor , METHOD , method_descriptor ) - CONSTRUCTOR(FileDescriptor , PACKAGE , package_file_descriptor) -#undef CONSTRUCTOR +typedef HASH_MAP<const char*, const FileDescriptor*, HASH_FXN<const char*>, + streq> + FilesByNameMap; - const FileDescriptor* GetFile() const { - switch (type) { - case NULL_SYMBOL: return NULL; - case MESSAGE : return descriptor ->file(); - case FIELD : return field_descriptor ->file(); - case ONEOF : return oneof_descriptor ->containing_type()->file(); - case ENUM : return enum_descriptor ->file(); - case ENUM_VALUE : return enum_value_descriptor->type()->file(); - case SERVICE : return service_descriptor ->file(); - case METHOD : return method_descriptor ->service()->file(); - case PACKAGE : return package_file_descriptor; - } - return NULL; - } -}; - -const Symbol kNullSymbol; - -typedef hash_map<const char*, Symbol, - hash<const char*>, streq> - SymbolsByNameMap; -typedef hash_map<PointerStringPair, Symbol, +typedef HASH_MAP<PointerStringPair, const FieldDescriptor*, PointerStringPairHash, PointerStringPairEqual> - SymbolsByParentMap; -typedef hash_map<const char*, const FileDescriptor*, - hash<const char*>, streq> - FilesByNameMap; -typedef hash_map<PointerStringPair, const FieldDescriptor*, - PointerStringPairHash, PointerStringPairEqual> - FieldsByNameMap; -typedef hash_map<DescriptorIntPair, const FieldDescriptor*, - PointerIntegerPairHash<DescriptorIntPair> > - FieldsByNumberMap; -typedef hash_map<EnumIntPair, const EnumValueDescriptor*, - PointerIntegerPairHash<EnumIntPair> > - EnumValuesByNumberMap; -// This is a map rather than a hash_map, since we use it to iterate + FieldsByNameMap; + +typedef HASH_MAP<DescriptorIntPair, const FieldDescriptor*, + PointerIntegerPairHash<DescriptorIntPair>, + std::equal_to<DescriptorIntPair> > + FieldsByNumberMap; + +typedef HASH_MAP<EnumIntPair, const EnumValueDescriptor*, + PointerIntegerPairHash<EnumIntPair>, + std::equal_to<EnumIntPair> > + EnumValuesByNumberMap; +// This is a map rather than a hash-map, since we use it to iterate // through all the extensions that extend a given Descriptor, and an // ordered data structure that implements lower_bound is convenient // for that. -typedef map<DescriptorIntPair, const FieldDescriptor*> +typedef std::map<DescriptorIntPair, const FieldDescriptor*> ExtensionsGroupedByDescriptorMap; -typedef hash_map<string, const SourceCodeInfo_Location*> LocationsByPathMap; +typedef HASH_MAP<string, const SourceCodeInfo_Location*> LocationsByPathMap; -set<string>* allowed_proto3_extendees_ = NULL; +std::set<string>* allowed_proto3_extendees_ = NULL; GOOGLE_PROTOBUF_DECLARE_ONCE(allowed_proto3_extendees_init_); void DeleteAllowedProto3Extendee() { @@ -356,10 +487,10 @@ void DeleteAllowedProto3Extendee() { } void InitAllowedProto3Extendee() { - allowed_proto3_extendees_ = new set<string>; + allowed_proto3_extendees_ = new std::set<string>; const char* kOptionNames[] = { "FileOptions", "MessageOptions", "FieldOptions", "EnumOptions", - "EnumValueOptions", "ServiceOptions", "MethodOptions"}; + "EnumValueOptions", "ServiceOptions", "MethodOptions", "OneofOptions"}; for (int i = 0; i < GOOGLE_ARRAYSIZE(kOptionNames); ++i) { // descriptor.proto has a different package name in opensource. We allow // both so the opensource protocol compiler can also compile internal @@ -434,24 +565,24 @@ class DescriptorPool::Tables { // The stack of files which are currently being built. Used to detect // cyclic dependencies when loading files from a DescriptorDatabase. Not // used when fallback_database_ == NULL. - vector<string> pending_files_; + std::vector<string> pending_files_; // A set of files which we have tried to load from the fallback database // and encountered errors. We will not attempt to load them again during // execution of the current public API call, but for compatibility with // legacy clients, this is cleared at the beginning of each public API call. // Not used when fallback_database_ == NULL. - hash_set<string> known_bad_files_; + HASH_SET<string> known_bad_files_; // A set of symbols which we have tried to load from the fallback database // and encountered errors. We will not attempt to load them again during // execution of the current public API call, but for compatibility with // legacy clients, this is cleared at the beginning of each public API call. - hash_set<string> known_bad_symbols_; + HASH_SET<string> known_bad_symbols_; // The set of descriptors for which we've already loaded the full // set of extensions numbers from fallback_database_. - hash_set<const Descriptor*> extensions_loaded_from_db_; + HASH_SET<const Descriptor*> extensions_loaded_from_db_; // ----------------------------------------------------------------- // Finding items. @@ -470,9 +601,9 @@ class DescriptorPool::Tables { // These return NULL if not found. inline const FileDescriptor* FindFile(const string& key) const; inline const FieldDescriptor* FindExtension(const Descriptor* extendee, - int number); + int number) const; inline void FindAllExtensions(const Descriptor* extendee, - vector<const FieldDescriptor*>* out) const; + std::vector<const FieldDescriptor*>* out) const; // ----------------------------------------------------------------- // Adding items. @@ -502,6 +633,10 @@ class DescriptorPool::Tables { // The string is initialized to the given value for convenience. string* AllocateString(const string& value); + // Allocate a GoogleOnceDynamic which will be destroyed when the pool is + // destroyed. + GoogleOnceDynamic* AllocateOnceDynamic(); + // Allocate a protocol message object. Some older versions of GCC have // trouble understanding explicit template instantiations in some cases, so // in those cases we have to pass a dummy pointer of the right type as the @@ -512,10 +647,13 @@ class DescriptorPool::Tables { FileDescriptorTables* AllocateFileTables(); private: - vector<string*> strings_; // All strings in the pool. - vector<Message*> messages_; // All messages in the pool. - vector<FileDescriptorTables*> file_tables_; // All file tables in the pool. - vector<void*> allocations_; // All other memory allocated in the pool. + std::vector<string*> strings_; // All strings in the pool. + std::vector<Message*> messages_; // All messages in the pool. + std::vector<GoogleOnceDynamic*> + once_dynamics_; // All GoogleOnceDynamics in the pool. + std::vector<FileDescriptorTables*> + file_tables_; // All file tables in the pool. + std::vector<void*> allocations_; // All other memory allocated in the pool. SymbolsByNameMap symbols_by_name_; FilesByNameMap files_by_name_; @@ -523,29 +661,30 @@ class DescriptorPool::Tables { struct CheckPoint { explicit CheckPoint(const Tables* tables) - : strings_before_checkpoint(tables->strings_.size()), - messages_before_checkpoint(tables->messages_.size()), - file_tables_before_checkpoint(tables->file_tables_.size()), - allocations_before_checkpoint(tables->allocations_.size()), - pending_symbols_before_checkpoint( - tables->symbols_after_checkpoint_.size()), - pending_files_before_checkpoint( - tables->files_after_checkpoint_.size()), - pending_extensions_before_checkpoint( - tables->extensions_after_checkpoint_.size()) { - } + : strings_before_checkpoint(tables->strings_.size()), + messages_before_checkpoint(tables->messages_.size()), + once_dynamics_before_checkpoint(tables->once_dynamics_.size()), + file_tables_before_checkpoint(tables->file_tables_.size()), + allocations_before_checkpoint(tables->allocations_.size()), + pending_symbols_before_checkpoint( + tables->symbols_after_checkpoint_.size()), + pending_files_before_checkpoint( + tables->files_after_checkpoint_.size()), + pending_extensions_before_checkpoint( + tables->extensions_after_checkpoint_.size()) {} int strings_before_checkpoint; int messages_before_checkpoint; + int once_dynamics_before_checkpoint; int file_tables_before_checkpoint; int allocations_before_checkpoint; int pending_symbols_before_checkpoint; int pending_files_before_checkpoint; int pending_extensions_before_checkpoint; }; - vector<CheckPoint> checkpoints_; - vector<const char* > symbols_after_checkpoint_; - vector<const char* > files_after_checkpoint_; - vector<DescriptorIntPair> extensions_after_checkpoint_; + std::vector<CheckPoint> checkpoints_; + std::vector<const char* > symbols_after_checkpoint_; + std::vector<const char* > files_after_checkpoint_; + std::vector<DescriptorIntPair> extensions_after_checkpoint_; // Allocate some bytes which will be reclaimed when the pool is // destroyed. @@ -612,20 +751,36 @@ class FileDescriptorTables { // Populates p->first->locations_by_path_ from p->second. // Unusual signature dictated by GoogleOnceDynamic. static void BuildLocationsByPath( - pair<const FileDescriptorTables*, const SourceCodeInfo*>* p); + std::pair<const FileDescriptorTables*, const SourceCodeInfo*>* p); // Returns the location denoted by the specified path through info, // or NULL if not found. // The value of info must be that of the corresponding FileDescriptor. // (Conceptually a pure function, but stateful as an optimisation.) const SourceCodeInfo_Location* GetSourceLocation( - const vector<int>& path, const SourceCodeInfo* info) const; + const std::vector<int>& path, const SourceCodeInfo* info) const; + + // Must be called after BuildFileImpl(), even if the build failed and + // we are going to roll back to the last checkpoint. + void FinalizeTables(); private: - SymbolsByParentMap symbols_by_parent_; - FieldsByNameMap fields_by_lowercase_name_; - FieldsByNameMap fields_by_camelcase_name_; - FieldsByNumberMap fields_by_number_; // Not including extensions. + const void* FindParentForFieldsByMap(const FieldDescriptor* field) const; + static void FieldsByLowercaseNamesLazyInitStatic( + const FileDescriptorTables* tables); + void FieldsByLowercaseNamesLazyInitInternal() const; + static void FieldsByCamelcaseNamesLazyInitStatic( + const FileDescriptorTables* tables); + void FieldsByCamelcaseNamesLazyInitInternal() const; + + SymbolsByParentMap symbols_by_parent_; + mutable FieldsByNameMap fields_by_lowercase_name_; + mutable FieldsByNameMap* fields_by_lowercase_name_tmp_; + mutable GoogleOnceDynamic fields_by_lowercase_name_once_; + mutable FieldsByNameMap fields_by_camelcase_name_; + mutable FieldsByNameMap* fields_by_camelcase_name_tmp_; + mutable GoogleOnceDynamic fields_by_camelcase_name_once_; + FieldsByNumberMap fields_by_number_; // Not including extensions. EnumValuesByNumberMap enum_values_by_number_; mutable EnumValuesByNumberMap unknown_enum_values_by_number_ GOOGLE_GUARDED_BY(unknown_enum_values_mu_); @@ -640,14 +795,13 @@ class FileDescriptorTables { }; DescriptorPool::Tables::Tables() - // Start some hash_map and hash_set objects with a small # of buckets + // Start some hash-map and hash-set objects with a small # of buckets : known_bad_files_(3), known_bad_symbols_(3), extensions_loaded_from_db_(3), symbols_by_name_(3), files_by_name_(3) {} - DescriptorPool::Tables::~Tables() { GOOGLE_DCHECK(checkpoints_.empty()); // Note that the deletion order is important, since the destructors of some @@ -658,17 +812,20 @@ DescriptorPool::Tables::~Tables() { } STLDeleteElements(&strings_); STLDeleteElements(&file_tables_); + STLDeleteElements(&once_dynamics_); } FileDescriptorTables::FileDescriptorTables() - // Initialize all the hash tables to start out with a small # of buckets + // Initialize all the hash tables to start out with a small # of buckets. : symbols_by_parent_(3), fields_by_lowercase_name_(3), + fields_by_lowercase_name_tmp_(new FieldsByNameMap()), fields_by_camelcase_name_(3), + fields_by_camelcase_name_tmp_(new FieldsByNameMap()), fields_by_number_(3), enum_values_by_number_(3), - unknown_enum_values_by_number_(3) { -} + unknown_enum_values_by_number_(3), + locations_by_path_(3) {} FileDescriptorTables::~FileDescriptorTables() {} @@ -747,6 +904,9 @@ void DescriptorPool::Tables::RollbackToLastCheckpoint() { messages_.begin() + checkpoint.messages_before_checkpoint, messages_.end()); STLDeleteContainerPointers( + once_dynamics_.begin() + checkpoint.once_dynamics_before_checkpoint, + once_dynamics_.end()); + STLDeleteContainerPointers( file_tables_.begin() + checkpoint.file_tables_before_checkpoint, file_tables_.end()); for (int i = checkpoint.allocations_before_checkpoint; @@ -757,6 +917,7 @@ void DescriptorPool::Tables::RollbackToLastCheckpoint() { strings_.resize(checkpoint.strings_before_checkpoint); messages_.resize(checkpoint.messages_before_checkpoint); + once_dynamics_.resize(checkpoint.once_dynamics_before_checkpoint); file_tables_.resize(checkpoint.file_tables_before_checkpoint); allocations_.resize(checkpoint.allocations_before_checkpoint); checkpoints_.pop_back(); @@ -794,8 +955,10 @@ inline Symbol FileDescriptorTables::FindNestedSymbolOfType( Symbol DescriptorPool::Tables::FindByNameHelper( const DescriptorPool* pool, const string& name) { MutexLockMaybe lock(pool->mutex_); - known_bad_symbols_.clear(); - known_bad_files_.clear(); + if (pool->fallback_database_ != NULL) { + known_bad_symbols_.clear(); + known_bad_files_.clear(); + } Symbol result = FindSymbol(name); if (result.IsNull() && pool->underlay_ != NULL) { @@ -824,14 +987,59 @@ inline const FieldDescriptor* FileDescriptorTables::FindFieldByNumber( return FindPtrOrNull(fields_by_number_, std::make_pair(parent, number)); } +const void* FileDescriptorTables::FindParentForFieldsByMap( + const FieldDescriptor* field) const { + if (field->is_extension()) { + if (field->extension_scope() == NULL) { + return field->file(); + } else { + return field->extension_scope(); + } + } else { + return field->containing_type(); + } +} + +void FileDescriptorTables::FieldsByLowercaseNamesLazyInitStatic( + const FileDescriptorTables* tables) { + tables->FieldsByLowercaseNamesLazyInitInternal(); +} + +void FileDescriptorTables::FieldsByLowercaseNamesLazyInitInternal() const { + for (FieldsByNumberMap::const_iterator it = fields_by_number_.begin(); + it != fields_by_number_.end(); it++) { + PointerStringPair lowercase_key(FindParentForFieldsByMap(it->second), + it->second->lowercase_name().c_str()); + InsertIfNotPresent(&fields_by_lowercase_name_, lowercase_key, it->second); + } +} + inline const FieldDescriptor* FileDescriptorTables::FindFieldByLowercaseName( const void* parent, const string& lowercase_name) const { + fields_by_lowercase_name_once_.Init( + &FileDescriptorTables::FieldsByLowercaseNamesLazyInitStatic, this); return FindPtrOrNull(fields_by_lowercase_name_, PointerStringPair(parent, lowercase_name.c_str())); } +void FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic( + const FileDescriptorTables* tables) { + tables->FieldsByCamelcaseNamesLazyInitInternal(); +} + +void FileDescriptorTables::FieldsByCamelcaseNamesLazyInitInternal() const { + for (FieldsByNumberMap::const_iterator it = fields_by_number_.begin(); + it != fields_by_number_.end(); it++) { + PointerStringPair camelcase_key(FindParentForFieldsByMap(it->second), + it->second->camelcase_name().c_str()); + InsertIfNotPresent(&fields_by_camelcase_name_, camelcase_key, it->second); + } +} + inline const FieldDescriptor* FileDescriptorTables::FindFieldByCamelcaseName( const void* parent, const string& camelcase_name) const { + fields_by_camelcase_name_once_.Init( + &FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic, this); return FindPtrOrNull(fields_by_camelcase_name_, PointerStringPair(parent, camelcase_name.c_str())); } @@ -895,12 +1103,13 @@ FileDescriptorTables::FindEnumValueByNumberCreatingIfUnknown( inline const FieldDescriptor* DescriptorPool::Tables::FindExtension( - const Descriptor* extendee, int number) { + const Descriptor* extendee, int number) const { return FindPtrOrNull(extensions_, std::make_pair(extendee, number)); } inline void DescriptorPool::Tables::FindAllExtensions( - const Descriptor* extendee, vector<const FieldDescriptor*>* out) const { + const Descriptor* extendee, + std::vector<const FieldDescriptor*>* out) const { ExtensionsGroupedByDescriptorMap::const_iterator it = extensions_.lower_bound(std::make_pair(extendee, 0)); for (; it != extensions_.end() && it->first.first == extendee; ++it) { @@ -935,24 +1144,40 @@ bool DescriptorPool::Tables::AddFile(const FileDescriptor* file) { } } +void FileDescriptorTables::FinalizeTables() { + // Clean up the temporary maps used by AddFieldByStylizedNames(). + delete fields_by_lowercase_name_tmp_; + fields_by_lowercase_name_tmp_ = NULL; + delete fields_by_camelcase_name_tmp_; + fields_by_camelcase_name_tmp_ = NULL; +} + void FileDescriptorTables::AddFieldByStylizedNames( const FieldDescriptor* field) { - const void* parent; - if (field->is_extension()) { - if (field->extension_scope() == NULL) { - parent = field->file(); - } else { - parent = field->extension_scope(); - } - } else { - parent = field->containing_type(); - } + const void* parent = FindParentForFieldsByMap(field); + + // We want fields_by_{lower,camel}case_name_ to be lazily built, but + // cross-link order determines which entry will be present in the case of a + // conflict. So we use the temporary maps that get destroyed after + // BuildFileImpl() to detect the conflicts, and only store the conflicts in + // the map that will persist. We will then lazily populate the rest of the + // entries from fields_by_number_. PointerStringPair lowercase_key(parent, field->lowercase_name().c_str()); - InsertIfNotPresent(&fields_by_lowercase_name_, lowercase_key, field); + if (!InsertIfNotPresent(fields_by_lowercase_name_tmp_, lowercase_key, + field)) { + InsertIfNotPresent( + &fields_by_lowercase_name_, lowercase_key, + FindPtrOrNull(*fields_by_lowercase_name_tmp_, lowercase_key)); + } PointerStringPair camelcase_key(parent, field->camelcase_name().c_str()); - InsertIfNotPresent(&fields_by_camelcase_name_, camelcase_key, field); + if (!InsertIfNotPresent(fields_by_camelcase_name_tmp_, camelcase_key, + field)) { + InsertIfNotPresent( + &fields_by_camelcase_name_, camelcase_key, + FindPtrOrNull(*fields_by_camelcase_name_tmp_, camelcase_key)); + } } bool FileDescriptorTables::AddFieldByNumber(const FieldDescriptor* field) { @@ -994,6 +1219,12 @@ string* DescriptorPool::Tables::AllocateString(const string& value) { return result; } +GoogleOnceDynamic* DescriptorPool::Tables::AllocateOnceDynamic() { + GoogleOnceDynamic* result = new GoogleOnceDynamic(); + once_dynamics_.push_back(result); + return result; +} + template<typename Type> Type* DescriptorPool::Tables::AllocateMessage(Type* /* dummy */) { Type* result = new Type; @@ -1020,7 +1251,7 @@ void* DescriptorPool::Tables::AllocateBytes(int size) { } void FileDescriptorTables::BuildLocationsByPath( - pair<const FileDescriptorTables*, const SourceCodeInfo*>* p) { + std::pair<const FileDescriptorTables*, const SourceCodeInfo*>* p) { for (int i = 0, len = p->second->location_size(); i < len; ++i) { const SourceCodeInfo_Location* loc = &p->second->location().Get(i); p->first->locations_by_path_[Join(loc->path(), ",")] = loc; @@ -1028,8 +1259,8 @@ void FileDescriptorTables::BuildLocationsByPath( } const SourceCodeInfo_Location* FileDescriptorTables::GetSourceLocation( - const vector<int>& path, const SourceCodeInfo* info) const { - pair<const FileDescriptorTables*, const SourceCodeInfo*> p( + const std::vector<int>& path, const SourceCodeInfo* info) const { + std::pair<const FileDescriptorTables*, const SourceCodeInfo*> p( std::make_pair(this, info)); locations_by_path_once_.Init(&FileDescriptorTables::BuildLocationsByPath, &p); return FindPtrOrNull(locations_by_path_, Join(path, ",")); @@ -1047,8 +1278,10 @@ DescriptorPool::DescriptorPool() underlay_(NULL), tables_(new Tables), enforce_dependencies_(true), + lazily_build_dependencies_(false), allow_unknown_(false), - enforce_weak_(false) {} + enforce_weak_(false), + disallow_enforce_utf8_(false) {} DescriptorPool::DescriptorPool(DescriptorDatabase* fallback_database, ErrorCollector* error_collector) @@ -1058,8 +1291,10 @@ DescriptorPool::DescriptorPool(DescriptorDatabase* fallback_database, underlay_(NULL), tables_(new Tables), enforce_dependencies_(true), + lazily_build_dependencies_(false), allow_unknown_(false), - enforce_weak_(false) { + enforce_weak_(false), + disallow_enforce_utf8_(false) { } DescriptorPool::DescriptorPool(const DescriptorPool* underlay) @@ -1069,8 +1304,10 @@ DescriptorPool::DescriptorPool(const DescriptorPool* underlay) underlay_(underlay), tables_(new Tables), enforce_dependencies_(true), + lazily_build_dependencies_(false), allow_unknown_(false), - enforce_weak_(false) {} + enforce_weak_(false), + disallow_enforce_utf8_(false) {} DescriptorPool::~DescriptorPool() { if (mutex_ != NULL) delete mutex_; @@ -1115,6 +1352,7 @@ void DeleteGeneratedPool() { static void InitGeneratedPool() { generated_database_ = new EncodedDescriptorDatabase; generated_pool_ = new DescriptorPool(generated_database_); + generated_pool_->InternalSetLazilyBuildDependencies(); internal::OnShutdown(&DeleteGeneratedPool); } @@ -1131,6 +1369,7 @@ const DescriptorPool* DescriptorPool::generated_pool() { } + DescriptorPool* DescriptorPool::internal_generated_pool() { InitGeneratedPoolOnce(); return generated_pool_; @@ -1173,8 +1412,10 @@ void DescriptorPool::InternalAddGeneratedFile( const FileDescriptor* DescriptorPool::FindFileByName(const string& name) const { MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != NULL) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } const FileDescriptor* result = tables_->FindFile(name); if (result != NULL) return result; if (underlay_ != NULL) { @@ -1191,8 +1432,10 @@ const FileDescriptor* DescriptorPool::FindFileByName(const string& name) const { const FileDescriptor* DescriptorPool::FindFileContainingSymbol( const string& symbol_name) const { MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != NULL) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } Symbol result = tables_->FindSymbol(symbol_name); if (!result.IsNull()) return result.GetFile(); if (underlay_ != NULL) { @@ -1268,9 +1511,20 @@ const MethodDescriptor* DescriptorPool::FindMethodByName( const FieldDescriptor* DescriptorPool::FindExtensionByNumber( const Descriptor* extendee, int number) const { + // A faster path to reduce lock contention in finding extensions, assuming + // most extensions will be cache hit. + if (mutex_ != NULL) { + ReaderMutexLock lock(mutex_); + const FieldDescriptor* result = tables_->FindExtension(extendee, number); + if (result != NULL) { + return result; + } + } MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != NULL) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } const FieldDescriptor* result = tables_->FindExtension(extendee, number); if (result != NULL) { return result; @@ -1289,16 +1543,19 @@ const FieldDescriptor* DescriptorPool::FindExtensionByNumber( } void DescriptorPool::FindAllExtensions( - const Descriptor* extendee, vector<const FieldDescriptor*>* out) const { + const Descriptor* extendee, + std::vector<const FieldDescriptor*>* out) const { MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != NULL) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } // Initialize tables_->extensions_ from the fallback database first // (but do this only once per descriptor). if (fallback_database_ != NULL && tables_->extensions_loaded_from_db_.count(extendee) == 0) { - vector<int> numbers; + std::vector<int> numbers; if (fallback_database_->FindAllExtensionNumbers(extendee->full_name(), &numbers)) { for (int i = 0; i < numbers.size(); ++i) { @@ -1544,6 +1801,15 @@ FileDescriptor::FindExtensionByCamelcaseName(const string& key) const { } } +void Descriptor::ExtensionRange::CopyTo( + DescriptorProto_ExtensionRange* proto) const { + proto->set_start(this->start); + proto->set_end(this->end); + if (options_ != &google::protobuf::ExtensionRangeOptions::default_instance()) { + *proto->mutable_options() = *options_; + } +} + const Descriptor::ExtensionRange* Descriptor::FindExtensionRangeContainingNumber(int number) const { // Linear search should be fine because we don't expect a message to have @@ -1569,6 +1835,18 @@ Descriptor::FindReservedRangeContainingNumber(int number) const { return NULL; } +const EnumDescriptor::ReservedRange* +EnumDescriptor::FindReservedRangeContainingNumber(int number) const { + // TODO(chrisn): Consider a non-linear search. + for (int i = 0; i < reserved_range_count(); i++) { + if (number >= reserved_range(i)->start && + number <= reserved_range(i)->end) { + return reserved_range(i); + } + } + return NULL; +} + // ------------------------------------------------------------------- bool DescriptorPool::TryFindFileInFallbackDatabase(const string& name) const { @@ -1674,8 +1952,8 @@ bool DescriptorPool::TryFindExtensionInFallbackDatabase( // =================================================================== -bool FieldDescriptor::is_map() const { - return type() == TYPE_MESSAGE && message_type()->options().map_entry(); +bool FieldDescriptor::is_map_message_type() const { + return message_type_->options().map_entry(); } string FieldDescriptor::DefaultValueAsString(bool quote_string_type) const { @@ -1799,9 +2077,7 @@ void Descriptor::CopyTo(DescriptorProto* proto) const { enum_type(i)->CopyTo(proto->add_enum_type()); } for (int i = 0; i < extension_range_count(); i++) { - DescriptorProto::ExtensionRange* range = proto->add_extension_range(); - range->set_start(extension_range(i)->start); - range->set_end(extension_range(i)->end); + extension_range(i)->CopyTo(proto->add_extension_range()); } for (int i = 0; i < extension_count(); i++) { extension(i)->CopyTo(proto->add_extension()); @@ -1907,6 +2183,14 @@ void EnumDescriptor::CopyTo(EnumDescriptorProto* proto) const { for (int i = 0; i < value_count(); i++) { value(i)->CopyTo(proto->add_value()); } + for (int i = 0; i < reserved_range_count(); i++) { + EnumDescriptorProto::EnumReservedRange* range = proto->add_reserved_range(); + range->set_start(reserved_range(i)->start); + range->set_end(reserved_range(i)->end); + } + for (int i = 0; i < reserved_name_count(); i++) { + proto->add_reserved_name(reserved_name(i)); + } if (&options() != &EnumOptions::default_instance()) { proto->mutable_options()->CopyFrom(options()); @@ -1963,13 +2247,11 @@ void MethodDescriptor::CopyTo(MethodDescriptorProto* proto) const { namespace { -// Used by each of the option formatters. -bool RetrieveOptions(int depth, - const Message &options, - vector<string> *option_entries) { +bool RetrieveOptionsAssumingRightPool(int depth, const Message& options, + std::vector<string>* option_entries) { option_entries->clear(); const Reflection* reflection = options.GetReflection(); - vector<const FieldDescriptor*> fields; + std::vector<const FieldDescriptor*> fields; reflection->ListFields(options, &fields); for (int i = 0; i < fields.size(); i++) { int count = 1; @@ -2006,21 +2288,56 @@ bool RetrieveOptions(int depth, return !option_entries->empty(); } +// Used by each of the option formatters. +bool RetrieveOptions(int depth, const Message& options, + const DescriptorPool* pool, + std::vector<string>* option_entries) { + // When printing custom options for a descriptor, we must use an options + // message built on top of the same DescriptorPool where the descriptor + // is coming from. This is to ensure we are interpreting custom options + // against the right pool. + if (options.GetDescriptor()->file()->pool() == pool) { + return RetrieveOptionsAssumingRightPool(depth, options, option_entries); + } else { + const Descriptor* option_descriptor = + pool->FindMessageTypeByName(options.GetDescriptor()->full_name()); + if (option_descriptor == NULL) { + // google/protobuf/descriptor.proto is not in the pool. This means no + // custom options are used so we are safe to proceed with the compiled + // options message type. + return RetrieveOptionsAssumingRightPool(depth, options, option_entries); + } + DynamicMessageFactory factory; + std::unique_ptr<Message> dynamic_options( + factory.GetPrototype(option_descriptor)->New()); + if (dynamic_options->ParseFromString(options.SerializeAsString())) { + return RetrieveOptionsAssumingRightPool(depth, *dynamic_options, + option_entries); + } else { + GOOGLE_LOG(ERROR) << "Found invalid proto option data for: " + << options.GetDescriptor()->full_name(); + return RetrieveOptionsAssumingRightPool(depth, options, option_entries); + } + } +} + // Formats options that all appear together in brackets. Does not include // brackets. -bool FormatBracketedOptions(int depth, const Message &options, string *output) { - vector<string> all_options; - if (RetrieveOptions(depth, options, &all_options)) { +bool FormatBracketedOptions(int depth, const Message& options, + const DescriptorPool* pool, string* output) { + std::vector<string> all_options; + if (RetrieveOptions(depth, options, pool, &all_options)) { output->append(Join(all_options, ", ")); } return !all_options.empty(); } // Formats options one per line -bool FormatLineOptions(int depth, const Message &options, string *output) { +bool FormatLineOptions(int depth, const Message& options, + const DescriptorPool* pool, string* output) { string prefix(depth * 2, ' '); - vector<string> all_options; - if (RetrieveOptions(depth, options, &all_options)) { + std::vector<string> all_options; + if (RetrieveOptions(depth, options, pool, &all_options)) { for (int i = 0; i < all_options.size(); i++) { strings::SubstituteAndAppend(output, "$0option $1;\n", prefix, all_options[i]); @@ -2042,7 +2359,7 @@ class SourceLocationCommentPrinter { desc->GetSourceLocation(&source_loc_); } SourceLocationCommentPrinter(const FileDescriptor* file, - const vector<int>& path, + const std::vector<int>& path, const string& prefix, const DebugStringOptions& options) : options_(options), prefix_(prefix) { @@ -2075,7 +2392,7 @@ class SourceLocationCommentPrinter { string FormatComment(const string& comment_text) { string stripped_comment = comment_text; StripWhitespace(&stripped_comment); - vector<string> lines = Split(stripped_comment, "\n"); + std::vector<string> lines = Split(stripped_comment, "\n"); string output; for (int i = 0; i < lines.size(); ++i) { const string& line = lines[i]; @@ -2103,7 +2420,7 @@ string FileDescriptor::DebugStringWithOptions( const DebugStringOptions& debug_string_options) const { string contents; { - vector<int> path; + std::vector<int> path; path.push_back(FileDescriptorProto::kSyntaxFieldNumber); SourceLocationCommentPrinter syntax_comment( this, path, "", debug_string_options); @@ -2117,8 +2434,8 @@ string FileDescriptor::DebugStringWithOptions( comment_printer(this, "", debug_string_options); comment_printer.AddPreComment(&contents); - set<int> public_dependencies; - set<int> weak_dependencies; + std::set<int> public_dependencies; + std::set<int> weak_dependencies; public_dependencies.insert(public_dependencies_, public_dependencies_ + public_dependency_count_); weak_dependencies.insert(weak_dependencies_, @@ -2138,7 +2455,7 @@ string FileDescriptor::DebugStringWithOptions( } if (!package().empty()) { - vector<int> path; + std::vector<int> path; path.push_back(FileDescriptorProto::kPackageFieldNumber); SourceLocationCommentPrinter package_comment( this, path, "", debug_string_options); @@ -2147,7 +2464,7 @@ string FileDescriptor::DebugStringWithOptions( package_comment.AddPostComment(&contents); } - if (FormatLineOptions(0, options(), &contents)) { + if (FormatLineOptions(0, options(), pool(), &contents)) { contents.append("\n"); // add some space if we had options } @@ -2158,7 +2475,7 @@ string FileDescriptor::DebugStringWithOptions( // Find all the 'group' type extensions; we will not output their nested // definitions (those will be done with their group field descriptor). - set<const Descriptor*> groups; + std::set<const Descriptor*> groups; for (int i = 0; i < extension_count(); i++) { if (extension(i)->type() == FieldDescriptor::TYPE_GROUP) { groups.insert(extension(i)->message_type()); @@ -2228,12 +2545,12 @@ void Descriptor::DebugString(int depth, string *contents, } contents->append(" {\n"); - FormatLineOptions(depth, options(), contents); + FormatLineOptions(depth, options(), file()->pool(), contents); // Find all the 'group' types for fields and extensions; we will not output // their nested definitions (those will be done with their group field // descriptor). - set<const Descriptor*> groups; + std::set<const Descriptor*> groups; for (int i = 0; i < field_count(); i++) { if (field(i)->type() == FieldDescriptor::TYPE_GROUP) { groups.insert(field(i)->message_type()); @@ -2366,8 +2683,18 @@ void FieldDescriptor::DebugString(int depth, field_type = FieldTypeNameDebugString(); } + bool print_label = true; + // Determine whether to omit label: + // 1. For an optional field, omit label if it's in oneof or in proto3. + // 2. For a repeated field, omit label if it's a map. + if (is_optional() && (print_label_flag == OMIT_LABEL || + file()->syntax() == FileDescriptor::SYNTAX_PROTO3)) { + print_label = false; + } else if (is_map()) { + print_label = false; + } string label; - if (print_label_flag == PRINT_LABEL && !is_map()) { + if (print_label) { label = kLabelToName[this->label()]; label.push_back(' '); } @@ -2390,9 +2717,21 @@ void FieldDescriptor::DebugString(int depth, strings::SubstituteAndAppend(contents, " [default = $0", DefaultValueAsString(true)); } + if (has_json_name_) { + if (!bracketed) { + bracketed = true; + contents->append("["); + } else { + contents->append(", "); + } + contents->append("json_name = \""); + contents->append(CEscape(json_name())); + contents->append("\""); + } string formatted_options; - if (FormatBracketedOptions(depth, options(), &formatted_options)) { + if (FormatBracketedOptions(depth, options(), file()->pool(), + &formatted_options)) { contents->append(bracketed ? ", " : " ["); bracketed = true; contents->append(formatted_options); @@ -2436,14 +2775,15 @@ void OneofDescriptor::DebugString(int depth, string* contents, SourceLocationCommentPrinter comment_printer(this, prefix, debug_string_options); comment_printer.AddPreComment(contents); - strings::SubstituteAndAppend( - contents, "$0 oneof $1 {", prefix, name()); + strings::SubstituteAndAppend(contents, "$0oneof $1 {", prefix, name()); - FormatLineOptions(depth, options(), contents); + FormatLineOptions(depth, options(), containing_type()->file()->pool(), + contents); if (debug_string_options.elide_oneof_body) { contents->append(" ... }\n"); } else { + contents->append("\n"); for (int i = 0; i < field_count(); i++) { field(i)->DebugString(depth, FieldDescriptor::OMIT_LABEL, contents, debug_string_options); @@ -2478,11 +2818,35 @@ void EnumDescriptor::DebugString(int depth, string *contents, strings::SubstituteAndAppend(contents, "$0enum $1 {\n", prefix, name()); - FormatLineOptions(depth, options(), contents); + FormatLineOptions(depth, options(), file()->pool(), contents); for (int i = 0; i < value_count(); i++) { value(i)->DebugString(depth, contents, debug_string_options); } + + if (reserved_range_count() > 0) { + strings::SubstituteAndAppend(contents, "$0 reserved ", prefix); + for (int i = 0; i < reserved_range_count(); i++) { + const EnumDescriptor::ReservedRange* range = reserved_range(i); + if (range->end == range->start) { + strings::SubstituteAndAppend(contents, "$0, ", range->start); + } else { + strings::SubstituteAndAppend(contents, "$0 to $1, ", + range->start, range->end); + } + } + contents->replace(contents->size() - 2, 2, ";\n"); + } + + if (reserved_name_count() > 0) { + strings::SubstituteAndAppend(contents, "$0 reserved ", prefix); + for (int i = 0; i < reserved_name_count(); i++) { + strings::SubstituteAndAppend(contents, "\"$0\", ", + CEscape(reserved_name(i))); + } + contents->replace(contents->size() - 2, 2, ";\n"); + } + strings::SubstituteAndAppend(contents, "$0}\n", prefix); comment_printer.AddPostComment(contents); @@ -2513,7 +2877,8 @@ void EnumValueDescriptor::DebugString(int depth, string *contents, prefix, name(), number()); string formatted_options; - if (FormatBracketedOptions(depth, options(), &formatted_options)) { + if (FormatBracketedOptions(depth, options(), type()->file()->pool(), + &formatted_options)) { strings::SubstituteAndAppend(contents, " [$0]", formatted_options); } contents->append(";\n"); @@ -2542,7 +2907,7 @@ void ServiceDescriptor::DebugString(string *contents, strings::SubstituteAndAppend(contents, "service $0 {\n", name()); - FormatLineOptions(1, options(), contents); + FormatLineOptions(1, options(), file()->pool(), contents); for (int i = 0; i < method_count(); i++) { method(i)->DebugString(1, contents, debug_string_options); @@ -2583,7 +2948,8 @@ void MethodDescriptor::DebugString(int depth, string *contents, server_streaming() ? "stream " : ""); string formatted_options; - if (FormatLineOptions(depth, options(), &formatted_options)) { + if (FormatLineOptions(depth, options(), service()->file()->pool(), + &formatted_options)) { strings::SubstituteAndAppend(contents, " {\n$0$1}\n", formatted_options, prefix); } else { @@ -2596,7 +2962,7 @@ void MethodDescriptor::DebugString(int depth, string *contents, // Location methods =============================================== -bool FileDescriptor::GetSourceLocation(const vector<int>& path, +bool FileDescriptor::GetSourceLocation(const std::vector<int>& path, SourceLocation* out_location) const { GOOGLE_CHECK_NOTNULL(out_location); if (source_code_info_) { @@ -2622,7 +2988,7 @@ bool FileDescriptor::GetSourceLocation(const vector<int>& path, } bool FileDescriptor::GetSourceLocation(SourceLocation* out_location) const { - vector<int> path; // empty path for root FileDescriptor + std::vector<int> path; // empty path for root FileDescriptor return GetSourceLocation(path, out_location); } @@ -2636,49 +3002,49 @@ bool FieldDescriptor::is_packed() const { } bool Descriptor::GetSourceLocation(SourceLocation* out_location) const { - vector<int> path; + std::vector<int> path; GetLocationPath(&path); return file()->GetSourceLocation(path, out_location); } bool FieldDescriptor::GetSourceLocation(SourceLocation* out_location) const { - vector<int> path; + std::vector<int> path; GetLocationPath(&path); return file()->GetSourceLocation(path, out_location); } bool OneofDescriptor::GetSourceLocation(SourceLocation* out_location) const { - vector<int> path; + std::vector<int> path; GetLocationPath(&path); return containing_type()->file()->GetSourceLocation(path, out_location); } bool EnumDescriptor::GetSourceLocation(SourceLocation* out_location) const { - vector<int> path; + std::vector<int> path; GetLocationPath(&path); return file()->GetSourceLocation(path, out_location); } bool MethodDescriptor::GetSourceLocation(SourceLocation* out_location) const { - vector<int> path; + std::vector<int> path; GetLocationPath(&path); return service()->file()->GetSourceLocation(path, out_location); } bool ServiceDescriptor::GetSourceLocation(SourceLocation* out_location) const { - vector<int> path; + std::vector<int> path; GetLocationPath(&path); return file()->GetSourceLocation(path, out_location); } bool EnumValueDescriptor::GetSourceLocation( SourceLocation* out_location) const { - vector<int> path; + std::vector<int> path; GetLocationPath(&path); return type()->file()->GetSourceLocation(path, out_location); } -void Descriptor::GetLocationPath(vector<int>* output) const { +void Descriptor::GetLocationPath(std::vector<int>* output) const { if (containing_type()) { containing_type()->GetLocationPath(output); output->push_back(DescriptorProto::kNestedTypeFieldNumber); @@ -2689,7 +3055,7 @@ void Descriptor::GetLocationPath(vector<int>* output) const { } } -void FieldDescriptor::GetLocationPath(vector<int>* output) const { +void FieldDescriptor::GetLocationPath(std::vector<int>* output) const { if (is_extension()) { if (extension_scope() == NULL) { output->push_back(FileDescriptorProto::kExtensionFieldNumber); @@ -2706,13 +3072,13 @@ void FieldDescriptor::GetLocationPath(vector<int>* output) const { } } -void OneofDescriptor::GetLocationPath(vector<int>* output) const { +void OneofDescriptor::GetLocationPath(std::vector<int>* output) const { containing_type()->GetLocationPath(output); output->push_back(DescriptorProto::kOneofDeclFieldNumber); output->push_back(index()); } -void EnumDescriptor::GetLocationPath(vector<int>* output) const { +void EnumDescriptor::GetLocationPath(std::vector<int>* output) const { if (containing_type()) { containing_type()->GetLocationPath(output); output->push_back(DescriptorProto::kEnumTypeFieldNumber); @@ -2723,18 +3089,18 @@ void EnumDescriptor::GetLocationPath(vector<int>* output) const { } } -void EnumValueDescriptor::GetLocationPath(vector<int>* output) const { +void EnumValueDescriptor::GetLocationPath(std::vector<int>* output) const { type()->GetLocationPath(output); output->push_back(EnumDescriptorProto::kValueFieldNumber); output->push_back(index()); } -void ServiceDescriptor::GetLocationPath(vector<int>* output) const { +void ServiceDescriptor::GetLocationPath(std::vector<int>* output) const { output->push_back(FileDescriptorProto::kServiceFieldNumber); output->push_back(index()); } -void MethodDescriptor::GetLocationPath(vector<int>* output) const { +void MethodDescriptor::GetLocationPath(std::vector<int>* output) const { service()->GetLocationPath(output); output->push_back(ServiceDescriptorProto::kMethodFieldNumber); output->push_back(index()); @@ -2752,15 +3118,18 @@ namespace { struct OptionsToInterpret { OptionsToInterpret(const string& ns, const string& el, + std::vector<int>& path, const Message* orig_opt, Message* opt) : name_scope(ns), element_name(el), + element_path(path), original_options(orig_opt), options(opt) { } string name_scope; string element_name; + std::vector<int> element_path; const Message* original_options; Message* options; }; @@ -2780,7 +3149,7 @@ class DescriptorBuilder { friend class OptionInterpreter; // Non-recursive part of BuildFile functionality. - const FileDescriptor* BuildFileImpl(const FileDescriptorProto& proto); + FileDescriptor* BuildFileImpl(const FileDescriptorProto& proto); const DescriptorPool* pool_; DescriptorPool::Tables* tables_; // for convenience @@ -2789,17 +3158,17 @@ class DescriptorBuilder { // As we build descriptors we store copies of the options messages in // them. We put pointers to those copies in this vector, as we build, so we // can later (after cross-linking) interpret those options. - vector<OptionsToInterpret> options_to_interpret_; + std::vector<OptionsToInterpret> options_to_interpret_; bool had_errors_; string filename_; FileDescriptor* file_; FileDescriptorTables* file_tables_; - set<const FileDescriptor*> dependencies_; + std::set<const FileDescriptor*> dependencies_; // unused_dependency_ is used to record the unused imported files. // Note: public import is not considered. - set<const FileDescriptor*> unused_dependency_; + std::set<const FileDescriptor*> unused_dependency_; // If LookupSymbol() finds a symbol that is in a file which is not a declared // dependency of this file, it will fail, but will set @@ -2853,15 +3222,16 @@ class DescriptorBuilder { // - Search the pool's underlay if not found in tables_. // - Insure that the resulting Symbol is from one of the file's declared // dependencies. - Symbol FindSymbol(const string& name); + Symbol FindSymbol(const string& name, bool build_it = true); // Like FindSymbol() but does not require that the symbol is in one of the // file's declared dependencies. - Symbol FindSymbolNotEnforcingDeps(const string& name); + Symbol FindSymbolNotEnforcingDeps(const string& name, bool build_it = true); // This implements the body of FindSymbolNotEnforcingDeps(). Symbol FindSymbolNotEnforcingDepsHelper(const DescriptorPool* pool, - const string& name); + const string& name, + bool build_it = true); // Like FindSymbol(), but looks up the name relative to some other symbol // name. This first searches siblings of relative_to, then siblings of its @@ -2877,31 +3247,21 @@ class DescriptorBuilder { // that LookupSymbol may still return a non-type symbol in LOOKUP_TYPES mode, // if it believes that's all it could refer to. The caller should always // check that it receives the type of symbol it was expecting. - enum PlaceholderType { - PLACEHOLDER_MESSAGE, - PLACEHOLDER_ENUM, - PLACEHOLDER_EXTENDABLE_MESSAGE - }; enum ResolveMode { LOOKUP_ALL, LOOKUP_TYPES }; Symbol LookupSymbol(const string& name, const string& relative_to, - PlaceholderType placeholder_type = PLACEHOLDER_MESSAGE, - ResolveMode resolve_mode = LOOKUP_ALL); + DescriptorPool::PlaceholderType placeholder_type = + DescriptorPool::PLACEHOLDER_MESSAGE, + ResolveMode resolve_mode = LOOKUP_ALL, + bool build_it = true); // Like LookupSymbol() but will not return a placeholder even if // AllowUnknownDependencies() has been used. Symbol LookupSymbolNoPlaceholder(const string& name, const string& relative_to, - ResolveMode resolve_mode = LOOKUP_ALL); - - // Creates a placeholder type suitable for return from LookupSymbol(). May - // return kNullSymbol if the name is not a valid type name. - Symbol NewPlaceholder(const string& name, PlaceholderType placeholder_type); - - // Creates a placeholder file. Never returns NULL. This is used when an - // import is not found and AllowUnknownDependencies() is enabled. - FileDescriptor* NewPlaceholderFile(const string& name); + ResolveMode resolve_mode = LOOKUP_ALL, + bool build_it = true); // Calls tables_->AddSymbol() and records an error if it fails. Returns // true if successful or false if failed, though most callers can ignore @@ -2923,10 +3283,6 @@ class DescriptorBuilder { void ValidateSymbolName(const string& name, const string& full_name, const Message& proto); - // Like ValidateSymbolName(), but the name is allowed to contain periods and - // an error is indicated by returning false (not recording the error). - bool ValidateQualifiedName(const string& name); - // Used by BUILD_ARRAY macro (below) to avoid having to have the type // specified as a macro parameter. template <typename Type> @@ -2940,7 +3296,7 @@ class DescriptorBuilder { // descriptor.proto. template<class DescriptorT> void AllocateOptions( const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor); + DescriptorT* descriptor, int options_field_tag); // Specialization for FileOptions. void AllocateOptions(const FileOptions& orig_options, FileDescriptor* descriptor); @@ -2950,7 +3306,8 @@ class DescriptorBuilder { const string& name_scope, const string& element_name, const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor); + DescriptorT* descriptor, + std::vector<int>& options_path); // These methods all have the same signature for the sake of the BUILD_ARRAY // macro, below. @@ -2977,9 +3334,14 @@ class DescriptorBuilder { void BuildReservedRange(const DescriptorProto::ReservedRange& proto, const Descriptor* parent, Descriptor::ReservedRange* result); + void BuildReservedRange(const EnumDescriptorProto::EnumReservedRange& proto, + const EnumDescriptor* parent, + EnumDescriptor::ReservedRange* result); void BuildOneof(const OneofDescriptorProto& proto, Descriptor* parent, OneofDescriptor* result); + void CheckEnumValueUniqueness(const EnumDescriptorProto& proto, + const EnumDescriptor* result); void BuildEnum(const EnumDescriptorProto& proto, const Descriptor* parent, EnumDescriptor* result); @@ -3005,6 +3367,8 @@ class DescriptorBuilder { void CrossLinkMessage(Descriptor* message, const DescriptorProto& proto); void CrossLinkField(FieldDescriptor* field, const FieldDescriptorProto& proto); + void CrossLinkExtensionRange(Descriptor::ExtensionRange* range, + const DescriptorProto::ExtensionRange& proto); void CrossLinkEnum(EnumDescriptor* enum_type, const EnumDescriptorProto& proto); void CrossLinkEnumValue(EnumValueDescriptor* enum_value, @@ -3032,13 +3396,21 @@ class DescriptorBuilder { // Otherwise returns true. bool InterpretOptions(OptionsToInterpret* options_to_interpret); + // Updates the given source code info by re-writing uninterpreted option + // locations to refer to the corresponding interpreted option. + void UpdateSourceCodeInfo(SourceCodeInfo* info); + class AggregateOptionFinder; private: // Interprets uninterpreted_option_ on the specified message, which // must be the mutable copy of the original options message to which - // uninterpreted_option_ belongs. - bool InterpretSingleOption(Message* options); + // uninterpreted_option_ belongs. The given src_path is the source + // location path to the uninterpreted option, and options_path is the + // source location path to the options message. The location paths are + // recorded and then used in UpdateSourceCodeInfo. + bool InterpretSingleOption(Message* options, std::vector<int>& src_path, + std::vector<int>& options_path); // Adds the uninterpreted_option to the given options message verbatim. // Used when AllowUnknownDependencies() is in effect and we can't find @@ -3050,8 +3422,10 @@ class DescriptorBuilder { // in unknown_fields to check if field innermost_field is set on the // innermost message. Returns false and sets an error if so. bool ExamineIfOptionIsSet( - vector<const FieldDescriptor*>::const_iterator intermediate_fields_iter, - vector<const FieldDescriptor*>::const_iterator intermediate_fields_end, + std::vector<const FieldDescriptor*>::const_iterator + intermediate_fields_iter, + std::vector<const FieldDescriptor*>::const_iterator + intermediate_fields_end, const FieldDescriptor* innermost_field, const string& debug_msg_name, const UnknownFieldSet& unknown_fields); @@ -3111,6 +3485,16 @@ class DescriptorBuilder { // can use it to find locations recorded by the parser. const UninterpretedOption* uninterpreted_option_; + // This maps the element path of uninterpreted options to the element path + // of the resulting interpreted option. This is used to modify a file's + // source code info to account for option interpretation. + std::map<std::vector<int>, std::vector<int>> interpreted_paths_; + + // This maps the path to a repeated option field to the known number of + // elements the field contains. This is used to track the compute the + // index portion of the element path when interpreting a single option. + std::map<std::vector<int>, int> repeated_option_counts_; + // Factory used to create the dynamic messages we need to parse // any aggregate option values we encounter. DynamicMessageFactory dynamic_factory_; @@ -3182,6 +3566,8 @@ class DescriptorBuilder { void DetectMapConflicts(const Descriptor* message, const DescriptorProto& proto); + void ValidateJSType(FieldDescriptor* field, + const FieldDescriptorProto& proto); }; const FileDescriptor* DescriptorPool::BuildFile( @@ -3309,8 +3695,8 @@ void DescriptorBuilder::AddWarning( bool DescriptorBuilder::IsInPackage(const FileDescriptor* file, const string& package_name) { return HasPrefixString(file->package(), package_name) && - (file->package().size() == package_name.size() || - file->package()[package_name.size()] == '.'); + (file->package().size() == package_name.size() || + file->package()[package_name.size()] == '.'); } void DescriptorBuilder::RecordPublicDependencies(const FileDescriptor* file) { @@ -3321,7 +3707,7 @@ void DescriptorBuilder::RecordPublicDependencies(const FileDescriptor* file) { } Symbol DescriptorBuilder::FindSymbolNotEnforcingDepsHelper( - const DescriptorPool* pool, const string& name) { + const DescriptorPool* pool, const string& name, bool build_it) { // If we are looking at an underlay, we must lock its mutex_, since we are // accessing the underlay's tables_ directly. MutexLockMaybe lock((pool == pool_) ? NULL : pool->mutex_); @@ -3333,12 +3719,14 @@ Symbol DescriptorBuilder::FindSymbolNotEnforcingDepsHelper( } if (result.IsNull()) { - // In theory, we shouldn't need to check fallback_database_ because the - // symbol should be in one of its file's direct dependencies, and we have - // already loaded those by the time we get here. But we check anyway so - // that we can generate better error message when dependencies are missing - // (i.e., "missing dependency" rather than "type is not defined"). - if (pool->TryFindSymbolInFallbackDatabase(name)) { + // With lazily_build_dependencies_, a symbol lookup at cross link time is + // not guaranteed to be successful. In most cases, build_it will be false, + // which intentionally prevents us from building an import until it's + // actually needed. In some cases, like registering an extension, we want + // to build the file containing the symbol, and build_it will be set. + // Also, build_it will be true when !lazily_build_dependencies_, to provide + // better error reporting of missing dependencies. + if (build_it && pool->TryFindSymbolInFallbackDatabase(name)) { result = pool->tables_->FindSymbol(name); } } @@ -3346,17 +3734,18 @@ Symbol DescriptorBuilder::FindSymbolNotEnforcingDepsHelper( return result; } -Symbol DescriptorBuilder::FindSymbolNotEnforcingDeps(const string& name) { - return FindSymbolNotEnforcingDepsHelper(pool_, name); +Symbol DescriptorBuilder::FindSymbolNotEnforcingDeps(const string& name, + bool build_it) { + return FindSymbolNotEnforcingDepsHelper(pool_, name, build_it); } -Symbol DescriptorBuilder::FindSymbol(const string& name) { - Symbol result = FindSymbolNotEnforcingDeps(name); +Symbol DescriptorBuilder::FindSymbol(const string& name, bool build_it) { + Symbol result = FindSymbolNotEnforcingDeps(name, build_it); if (result.IsNull()) return result; if (!pool_->enforce_dependencies_) { - // Hack for CompilerUpgrader. + // Hack for CompilerUpgrader, and also used for lazily_build_dependencies_ return result; } @@ -3377,7 +3766,8 @@ Symbol DescriptorBuilder::FindSymbol(const string& name) { // dependency also defines the same package. We can't really rule out this // symbol unless none of the dependencies define it. if (IsInPackage(file_, name)) return result; - for (set<const FileDescriptor*>::const_iterator it = dependencies_.begin(); + for (std::set<const FileDescriptor*>::const_iterator it = + dependencies_.begin(); it != dependencies_.end(); ++it) { // Note: A dependency may be NULL if it was not found or had errors. if (*it != NULL && IsInPackage(*it, name)) return result; @@ -3389,14 +3779,16 @@ Symbol DescriptorBuilder::FindSymbol(const string& name) { return kNullSymbol; } -Symbol DescriptorBuilder::LookupSymbolNoPlaceholder( - const string& name, const string& relative_to, ResolveMode resolve_mode) { +Symbol DescriptorBuilder::LookupSymbolNoPlaceholder(const string& name, + const string& relative_to, + ResolveMode resolve_mode, + bool build_it) { possible_undeclared_dependency_ = NULL; undefine_resolved_name_.clear(); - if (name.size() > 0 && name[0] == '.') { + if (!name.empty() && name[0] == '.') { // Fully-qualified name. - return FindSymbol(name.substr(1)); + return FindSymbol(name.substr(1), build_it); } // If name is something like "Foo.Bar.baz", and symbols named "Foo" are @@ -3424,7 +3816,7 @@ Symbol DescriptorBuilder::LookupSymbolNoPlaceholder( // Chop off the last component of the scope. string::size_type dot_pos = scope_to_try.find_last_of('.'); if (dot_pos == string::npos) { - return FindSymbol(name); + return FindSymbol(name, build_it); } else { scope_to_try.erase(dot_pos); } @@ -3433,7 +3825,7 @@ Symbol DescriptorBuilder::LookupSymbolNoPlaceholder( string::size_type old_size = scope_to_try.size(); scope_to_try.append(1, '.'); scope_to_try.append(first_part_of_name); - Symbol result = FindSymbol(scope_to_try); + Symbol result = FindSymbol(scope_to_try, build_it); if (!result.IsNull()) { if (first_part_of_name.size() < name.size()) { // name is a compound symbol, of which we only found the first part. @@ -3441,7 +3833,7 @@ Symbol DescriptorBuilder::LookupSymbolNoPlaceholder( if (result.IsAggregate()) { scope_to_try.append(name, first_part_of_name.size(), name.size() - first_part_of_name.size()); - result = FindSymbol(scope_to_try); + result = FindSymbol(scope_to_try, build_it); if (result.IsNull()) { undefine_resolved_name_ = scope_to_try; } @@ -3465,19 +3857,49 @@ Symbol DescriptorBuilder::LookupSymbolNoPlaceholder( Symbol DescriptorBuilder::LookupSymbol( const string& name, const string& relative_to, - PlaceholderType placeholder_type, ResolveMode resolve_mode) { - Symbol result = LookupSymbolNoPlaceholder( - name, relative_to, resolve_mode); + DescriptorPool::PlaceholderType placeholder_type, ResolveMode resolve_mode, + bool build_it) { + Symbol result = + LookupSymbolNoPlaceholder(name, relative_to, resolve_mode, build_it); if (result.IsNull() && pool_->allow_unknown_) { // Not found, but AllowUnknownDependencies() is enabled. Return a // placeholder instead. - result = NewPlaceholder(name, placeholder_type); + result = pool_->NewPlaceholderWithMutexHeld(name, placeholder_type); } return result; } -Symbol DescriptorBuilder::NewPlaceholder(const string& name, - PlaceholderType placeholder_type) { +static bool ValidateQualifiedName(const string& name) { + bool last_was_period = false; + + for (int i = 0; i < name.size(); i++) { + // I don't trust isalnum() due to locales. :( + if (('a' <= name[i] && name[i] <= 'z') || + ('A' <= name[i] && name[i] <= 'Z') || + ('0' <= name[i] && name[i] <= '9') || (name[i] == '_')) { + last_was_period = false; + } else if (name[i] == '.') { + if (last_was_period) return false; + last_was_period = true; + } else { + return false; + } + } + + return !name.empty() && !last_was_period; +} + +Symbol DescriptorPool::NewPlaceholder(const string& name, + PlaceholderType placeholder_type) const { + MutexLockMaybe lock(mutex_); + return NewPlaceholderWithMutexHeld(name, placeholder_type); +} + +Symbol DescriptorPool::NewPlaceholderWithMutexHeld( + const string& name, PlaceholderType placeholder_type) const { + if (mutex_) { + mutex_->AssertHeld(); + } // Compute names. const string* placeholder_full_name; const string* placeholder_name; @@ -3503,7 +3925,7 @@ Symbol DescriptorBuilder::NewPlaceholder(const string& name, } // Create the placeholders. - FileDescriptor* placeholder_file = NewPlaceholderFile( + FileDescriptor* placeholder_file = NewPlaceholderFileWithMutexHeld( *placeholder_full_name + ".placeholder.proto"); placeholder_file->package_ = placeholder_package; @@ -3569,19 +3991,28 @@ Symbol DescriptorBuilder::NewPlaceholder(const string& name, } } -FileDescriptor* DescriptorBuilder::NewPlaceholderFile( - const string& name) { +FileDescriptor* DescriptorPool::NewPlaceholderFile(const string& name) const { + MutexLockMaybe lock(mutex_); + return NewPlaceholderFileWithMutexHeld(name); +} + +FileDescriptor* DescriptorPool::NewPlaceholderFileWithMutexHeld( + const string& name) const { + if (mutex_) { + mutex_->AssertHeld(); + } FileDescriptor* placeholder = tables_->Allocate<FileDescriptor>(); memset(placeholder, 0, sizeof(*placeholder)); placeholder->name_ = tables_->AllocateString(name); placeholder->package_ = &internal::GetEmptyString(); - placeholder->pool_ = pool_; + placeholder->pool_ = this; placeholder->options_ = &FileOptions::default_instance(); placeholder->tables_ = &FileDescriptorTables::GetEmptyInstance(); placeholder->source_code_info_ = &SourceCodeInfo::default_instance(); placeholder->is_placeholder_ = true; placeholder->syntax_ = FileDescriptor::SYNTAX_PROTO2; + placeholder->finished_building_ = true; // All other fields are zero or NULL. return placeholder; @@ -3596,9 +4027,13 @@ bool DescriptorBuilder::AddSymbol( if (tables_->AddSymbol(full_name, symbol)) { if (!file_tables_->AddAliasUnderParent(parent, name, symbol)) { - GOOGLE_LOG(DFATAL) << "\"" << full_name << "\" not previously defined in " - "symbols_by_name_, but was defined in symbols_by_parent_; " - "this shouldn't be possible."; + // This is only possible if there was already an error adding something of + // the same name. + if (!had_errors_) { + GOOGLE_LOG(DFATAL) << "\"" << full_name << "\" not previously defined in " + "symbols_by_name_, but was defined in " + "symbols_by_parent_; this shouldn't be possible."; + } return false; } return true; @@ -3671,51 +4106,36 @@ void DescriptorBuilder::ValidateSymbolName( } } -bool DescriptorBuilder::ValidateQualifiedName(const string& name) { - bool last_was_period = false; - - for (int i = 0; i < name.size(); i++) { - // I don't trust isalnum() due to locales. :( - if (('a' <= name[i] && name[i] <= 'z') || - ('A' <= name[i] && name[i] <= 'Z') || - ('0' <= name[i] && name[i] <= '9') || - (name[i] == '_')) { - last_was_period = false; - } else if (name[i] == '.') { - if (last_was_period) return false; - last_was_period = true; - } else { - return false; - } - } - - return !name.empty() && !last_was_period; -} - // ------------------------------------------------------------------- // This generic implementation is good for all descriptors except // FileDescriptor. template<class DescriptorT> void DescriptorBuilder::AllocateOptions( const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor) { + DescriptorT* descriptor, int options_field_tag) { + std::vector<int> options_path; + descriptor->GetLocationPath(&options_path); + options_path.push_back(options_field_tag); AllocateOptionsImpl(descriptor->full_name(), descriptor->full_name(), - orig_options, descriptor); + orig_options, descriptor, options_path); } // We specialize for FileDescriptor. void DescriptorBuilder::AllocateOptions(const FileOptions& orig_options, FileDescriptor* descriptor) { + std::vector<int> options_path; + options_path.push_back(FileDescriptorProto::kOptionsFieldNumber); // We add the dummy token so that LookupSymbol does the right thing. AllocateOptionsImpl(descriptor->package() + ".dummy", descriptor->name(), - orig_options, descriptor); + orig_options, descriptor, options_path); } template<class DescriptorT> void DescriptorBuilder::AllocateOptionsImpl( const string& name_scope, const string& element_name, const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor) { + DescriptorT* descriptor, + std::vector<int>& options_path) { // We need to use a dummy pointer to work around a bug in older versions of // GCC. Otherwise, the following two lines could be replaced with: // typename DescriptorT::OptionsType* options = @@ -3738,7 +4158,8 @@ template<class DescriptorT> void DescriptorBuilder::AllocateOptionsImpl( // we're still trying to build it. if (options->uninterpreted_option_size() > 0) { options_to_interpret_.push_back( - OptionsToInterpret(name_scope, element_name, &orig_options, options)); + OptionsToInterpret(name_scope, element_name, options_path, + &orig_options, options)); } } @@ -3835,35 +4256,50 @@ const FileDescriptor* DescriptorBuilder::BuildFile( } } - // If we have a fallback_database_, attempt to load all dependencies now, - // before checkpointing tables_. This avoids confusion with recursive - // checkpoints. - if (pool_->fallback_database_ != NULL) { - tables_->pending_files_.push_back(proto.name()); - for (int i = 0; i < proto.dependency_size(); i++) { - if (tables_->FindFile(proto.dependency(i)) == NULL && - (pool_->underlay_ == NULL || - pool_->underlay_->FindFileByName(proto.dependency(i)) == NULL)) { - // We don't care what this returns since we'll find out below anyway. - pool_->TryFindFileInFallbackDatabase(proto.dependency(i)); + // If we have a fallback_database_, and we aren't doing lazy import building, + // attempt to load all dependencies now, before checkpointing tables_. This + // avoids confusion with recursive checkpoints. + if (!pool_->lazily_build_dependencies_) { + if (pool_->fallback_database_ != NULL) { + tables_->pending_files_.push_back(proto.name()); + for (int i = 0; i < proto.dependency_size(); i++) { + if (tables_->FindFile(proto.dependency(i)) == NULL && + (pool_->underlay_ == NULL || + pool_->underlay_->FindFileByName(proto.dependency(i)) == NULL)) { + // We don't care what this returns since we'll find out below anyway. + pool_->TryFindFileInFallbackDatabase(proto.dependency(i)); + } } + tables_->pending_files_.pop_back(); } - tables_->pending_files_.pop_back(); } - return BuildFileImpl(proto); -} -const FileDescriptor* DescriptorBuilder::BuildFileImpl( - const FileDescriptorProto& proto) { // Checkpoint the tables so that we can roll back if something goes wrong. tables_->AddCheckpoint(); + FileDescriptor* result = BuildFileImpl(proto); + + file_tables_->FinalizeTables(); + if (result) { + tables_->ClearLastCheckpoint(); + result->finished_building_ = true; + } else { + tables_->RollbackToLastCheckpoint(); + } + + return result; +} + +FileDescriptor* DescriptorBuilder::BuildFileImpl( + const FileDescriptorProto& proto) { FileDescriptor* result = tables_->Allocate<FileDescriptor>(); file_ = result; result->is_placeholder_ = false; + result->finished_building_ = false; + SourceCodeInfo *info = NULL; if (proto.has_source_code_info()) { - SourceCodeInfo *info = tables_->AllocateMessage<SourceCodeInfo>(); + info = tables_->AllocateMessage<SourceCodeInfo>(); info->CopyFrom(proto.source_code_info()); result->source_code_info_ = info; } else { @@ -3908,7 +4344,6 @@ const FileDescriptor* DescriptorBuilder::BuildFileImpl( "A file with this name is already in the pool."); // Bail out early so that if this is actually the exact same file, we // don't end up reporting that every single symbol is already defined. - tables_->RollbackToLastCheckpoint(); return NULL; } if (!result->package().empty()) { @@ -3916,12 +4351,24 @@ const FileDescriptor* DescriptorBuilder::BuildFileImpl( } // Make sure all dependencies are loaded. - set<string> seen_dependencies; + std::set<string> seen_dependencies; result->dependency_count_ = proto.dependency_size(); result->dependencies_ = - tables_->AllocateArray<const FileDescriptor*>(proto.dependency_size()); + tables_->AllocateArray<const FileDescriptor*>(proto.dependency_size()); + if (pool_->lazily_build_dependencies_) { + result->dependencies_once_ = tables_->AllocateOnceDynamic(); + result->dependencies_names_ = + tables_->AllocateArray<const string*>(proto.dependency_size()); + if (proto.dependency_size() > 0) { + memset(result->dependencies_names_, 0, + sizeof(*result->dependencies_names_) * proto.dependency_size()); + } + } else { + result->dependencies_once_ = NULL; + result->dependencies_names_ = NULL; + } unused_dependency_.clear(); - set<int> weak_deps; + std::set<int> weak_deps; for (int i = 0; i < proto.weak_dependency_size(); ++i) { weak_deps.insert(proto.weak_dependency(i)); } @@ -3935,12 +4382,22 @@ const FileDescriptor* DescriptorBuilder::BuildFileImpl( dependency = pool_->underlay_->FindFileByName(proto.dependency(i)); } + if (dependency == result) { + // Recursive import. dependency/result is not fully initialized, and it's + // dangerous to try to do anything with it. The recursive import error + // will be detected and reported in DescriptorBuilder::BuildFile(). + return NULL; + } + if (dependency == NULL) { - if (pool_->allow_unknown_ || - (!pool_->enforce_weak_ && weak_deps.find(i) != weak_deps.end())) { - dependency = NewPlaceholderFile(proto.dependency(i)); - } else { - AddImportError(proto, i); + if (!pool_->lazily_build_dependencies_) { + if (pool_->allow_unknown_ || + (!pool_->enforce_weak_ && weak_deps.find(i) != weak_deps.end())) { + dependency = + pool_->NewPlaceholderFileWithMutexHeld(proto.dependency(i)); + } else { + AddImportError(proto, i); + } } } else { // Add to unused_dependency_ to track unused imported files. @@ -3954,6 +4411,10 @@ const FileDescriptor* DescriptorBuilder::BuildFileImpl( } result->dependencies_[i] = dependency; + if (pool_->lazily_build_dependencies_ && !dependency) { + result->dependencies_names_[i] = + tables_->AllocateString(proto.dependency(i)); + } } // Check public dependencies. @@ -3966,7 +4427,12 @@ const FileDescriptor* DescriptorBuilder::BuildFileImpl( if (index >= 0 && index < proto.dependency_size()) { result->public_dependencies_[public_dependency_count++] = index; // Do not track unused imported files for public import. - unused_dependency_.erase(result->dependency(index)); + // Calling dependency(i) builds that file when doing lazy imports, + // need to avoid doing this. Unused dependency detection isn't done + // when building lazily, anyways. + if (!pool_->lazily_build_dependencies_) { + unused_dependency_.erase(result->dependency(index)); + } } else { AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, @@ -3977,8 +4443,13 @@ const FileDescriptor* DescriptorBuilder::BuildFileImpl( // Build dependency set dependencies_.clear(); - for (int i = 0; i < result->dependency_count(); i++) { - RecordPublicDependencies(result->dependency(i)); + // We don't/can't do proper dependency error checking when + // lazily_build_dependencies_, and calling dependency(i) will force + // a dependency to be built, which we don't want. + if (!pool_->lazily_build_dependencies_) { + for (int i = 0; i < result->dependency_count(); i++) { + RecordPublicDependencies(result->dependency(i)); + } } // Check weak dependencies. @@ -4020,16 +4491,21 @@ const FileDescriptor* DescriptorBuilder::BuildFileImpl( // extension options known, so all interpretations should now succeed. if (!had_errors_) { OptionInterpreter option_interpreter(this); - for (vector<OptionsToInterpret>::iterator iter = + for (std::vector<OptionsToInterpret>::iterator iter = options_to_interpret_.begin(); iter != options_to_interpret_.end(); ++iter) { option_interpreter.InterpretOptions(&(*iter)); } options_to_interpret_.clear(); + + if (info != NULL) { + option_interpreter.UpdateSourceCodeInfo(info); + } } - // Validate options. - if (!had_errors_) { + // Validate options. See comments at InternalSetLazilyBuildDependencies about + // error checking and lazy import building. + if (!had_errors_ && !pool_->lazily_build_dependencies_) { ValidateFileOptions(result, proto); } @@ -4042,15 +4518,15 @@ const FileDescriptor* DescriptorBuilder::BuildFileImpl( } - if (!unused_dependency_.empty()) { + // Again, see comments at InternalSetLazilyBuildDependencies about error + // checking. + if (!unused_dependency_.empty() && !pool_->lazily_build_dependencies_) { LogUnusedDependency(proto, result); } if (had_errors_) { - tables_->RollbackToLastCheckpoint(); return NULL; } else { - tables_->ClearLastCheckpoint(); return result; } } @@ -4096,7 +4572,8 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + DescriptorProto::kOptionsFieldNumber); } AddSymbol(result->full_name(), parent, result->name(), @@ -4117,7 +4594,7 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, } } - hash_set<string> reserved_name_set; + HASH_SET<string> reserved_name_set; for (int i = 0; i < proto.reserved_name_size(); i++) { const string& name = proto.reserved_name(i); if (reserved_name_set.find(name) == reserved_name_set.end()) { @@ -4168,7 +4645,7 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, for (int j = 0; j < result->reserved_range_count(); j++) { const Descriptor::ReservedRange* range2 = result->reserved_range(j); if (range1->end > range2->start && range2->end > range1->start) { - AddError(result->full_name(), proto.extension_range(j), + AddError(result->full_name(), proto.extension_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Extension range $0 to $1 overlaps with " "reserved range $2 to $3.", @@ -4179,7 +4656,7 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, for (int j = i + 1; j < result->extension_range_count(); j++) { const Descriptor::ExtensionRange* range2 = result->extension_range(j); if (range1->end > range2->start && range2->end > range1->start) { - AddError(result->full_name(), proto.extension_range(j), + AddError(result->full_name(), proto.extension_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Extension range $0 to $1 overlaps with " "already-defined range $2 to $3.", @@ -4232,7 +4709,7 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, result->json_name_ = tables_->AllocateString(proto.json_name()); } else { result->has_json_name_ = false; - result->json_name_ = result->camelcase_name_; + result->json_name_ = tables_->AllocateString(ToJsonName(proto.name())); } // Some compilers do not allow static_cast directly between two enum types, @@ -4260,6 +4737,10 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, result->extension_scope_ = NULL; result->message_type_ = NULL; result->enum_type_ = NULL; + result->type_name_ = NULL; + result->type_once_ = NULL; + result->default_value_enum_ = NULL; + result->default_value_enum_name_ = NULL; result->has_default_value_ = proto.has_default_value(); if (proto.has_default_value() && result->is_repeated()) { @@ -4290,11 +4771,14 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, break; case FieldDescriptor::CPPTYPE_FLOAT: if (proto.default_value() == "inf") { - result->default_value_float_ = numeric_limits<float>::infinity(); + result->default_value_float_ = + std::numeric_limits<float>::infinity(); } else if (proto.default_value() == "-inf") { - result->default_value_float_ = -numeric_limits<float>::infinity(); + result->default_value_float_ = + -std::numeric_limits<float>::infinity(); } else if (proto.default_value() == "nan") { - result->default_value_float_ = numeric_limits<float>::quiet_NaN(); + result->default_value_float_ = + std::numeric_limits<float>::quiet_NaN(); } else { result->default_value_float_ = io::SafeDoubleToFloat( io::NoLocaleStrtod(proto.default_value().c_str(), &end_pos)); @@ -4302,11 +4786,14 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, break; case FieldDescriptor::CPPTYPE_DOUBLE: if (proto.default_value() == "inf") { - result->default_value_double_ = numeric_limits<double>::infinity(); + result->default_value_double_ = + std::numeric_limits<double>::infinity(); } else if (proto.default_value() == "-inf") { - result->default_value_double_ = -numeric_limits<double>::infinity(); + result->default_value_double_ = + -std::numeric_limits<double>::infinity(); } else if (proto.default_value() == "nan") { - result->default_value_double_ = numeric_limits<double>::quiet_NaN(); + result->default_value_double_ = + std::numeric_limits<double>::quiet_NaN(); } else { result->default_value_double_ = io::NoLocaleStrtod(proto.default_value().c_str(), &end_pos); @@ -4466,7 +4953,8 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + FieldDescriptorProto::kOptionsFieldNumber); } @@ -4496,6 +4984,21 @@ void DescriptorBuilder::BuildExtensionRange( DescriptorPool::ErrorCollector::NUMBER, "Extension range end number must be greater than start number."); } + + if (!proto.has_options()) { + result->options_ = NULL; // Will set to default_instance later. + } else { + std::vector<int> options_path; + parent->GetLocationPath(&options_path); + options_path.push_back(DescriptorProto::kExtensionRangeFieldNumber); + // find index of this extension range in order to compute path + int index; + for (index = 0; parent->extension_ranges_ + index != result; index++); + options_path.push_back(index); + options_path.push_back(DescriptorProto_ExtensionRange::kOptionsFieldNumber); + AllocateOptionsImpl(parent->full_name(), parent->full_name(), + proto.options(), result, options_path); + } } void DescriptorBuilder::BuildReservedRange( @@ -4511,6 +5014,19 @@ void DescriptorBuilder::BuildReservedRange( } } +void DescriptorBuilder::BuildReservedRange( + const EnumDescriptorProto::EnumReservedRange& proto, + const EnumDescriptor* parent, EnumDescriptor::ReservedRange* result) { + result->start = proto.start(); + result->end = proto.end(); + + if (result->start > result->end) { + AddError(parent->full_name(), proto, + DescriptorPool::ErrorCollector::NUMBER, + "Reserved range end number must be greater than start number."); + } +} + void DescriptorBuilder::BuildOneof(const OneofDescriptorProto& proto, Descriptor* parent, OneofDescriptor* result) { @@ -4533,13 +5049,79 @@ void DescriptorBuilder::BuildOneof(const OneofDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + OneofDescriptorProto::kOptionsFieldNumber); } AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); } +void DescriptorBuilder::CheckEnumValueUniqueness( + const EnumDescriptorProto& proto, const EnumDescriptor* result) { + + // Check that enum labels are still unique when we remove the enum prefix from + // values that have it. + // + // This will fail for something like: + // + // enum MyEnum { + // MY_ENUM_FOO = 0; + // FOO = 1; + // } + // + // By enforcing this reasonable constraint, we allow code generators to strip + // the prefix and/or PascalCase it without creating conflicts. This can lead + // to much nicer language-specific enums like: + // + // enum NameType { + // FirstName = 1, + // LastName = 2, + // } + // + // Instead of: + // + // enum NameType { + // NAME_TYPE_FIRST_NAME = 1, + // NAME_TYPE_LAST_NAME = 2, + // } + PrefixRemover remover(result->name()); + std::map<string, const google::protobuf::EnumValueDescriptor*> values; + for (int i = 0; i < result->value_count(); i++) { + const google::protobuf::EnumValueDescriptor* value = result->value(i); + string stripped = + EnumValueToPascalCase(remover.MaybeRemove(value->name())); + std::pair<std::map<string, const google::protobuf::EnumValueDescriptor*>::iterator, + bool> + insert_result = values.insert(std::make_pair(stripped, value)); + bool inserted = insert_result.second; + + // We don't throw the error if the two conflicting symbols are identical, or + // if they map to the same number. In the former case, the normal symbol + // duplication error will fire so we don't need to (and its error message + // will make more sense). We allow the latter case so users can create + // aliases which add or remove the prefix (code generators that do prefix + // stripping should de-dup the labels in this case). + if (!inserted && insert_result.first->second->name() != value->name() && + insert_result.first->second->number() != value->number()) { + string error_message = + "When enum name is stripped and label is PascalCased (" + stripped + + "), this value label conflicts with " + values[stripped]->name() + + ". This will make the proto fail to compile for some languages, such " + "as C#."; + // There are proto2 enums out there with conflicting names, so to preserve + // compatibility we issue only a warning for proto2. + if (result->file()->syntax() == FileDescriptor::SYNTAX_PROTO2) { + AddWarning(value->full_name(), proto.value(i), + DescriptorPool::ErrorCollector::NAME, error_message); + } else { + AddError(value->full_name(), proto.value(i), + DescriptorPool::ErrorCollector::NAME, error_message); + } + } + } +} + void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, const Descriptor* parent, EnumDescriptor* result) { @@ -4567,16 +5149,80 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, } BUILD_ARRAY(proto, result, value, BuildEnumValue, result); + BUILD_ARRAY(proto, result, reserved_range, BuildReservedRange, result); + + // Copy reserved names. + int reserved_name_count = proto.reserved_name_size(); + result->reserved_name_count_ = reserved_name_count; + result->reserved_names_ = + tables_->AllocateArray<const string*>(reserved_name_count); + for (int i = 0; i < reserved_name_count; ++i) { + result->reserved_names_[i] = + tables_->AllocateString(proto.reserved_name(i)); + } + + CheckEnumValueUniqueness(proto, result); // Copy options. if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + EnumDescriptorProto::kOptionsFieldNumber); } AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); + + for (int i = 0; i < proto.reserved_range_size(); i++) { + const EnumDescriptorProto_EnumReservedRange& range1 = + proto.reserved_range(i); + for (int j = i + 1; j < proto.reserved_range_size(); j++) { + const EnumDescriptorProto_EnumReservedRange& range2 = + proto.reserved_range(j); + if (range1.end() >= range2.start() && range2.end() >= range1.start()) { + AddError(result->full_name(), proto.reserved_range(i), + DescriptorPool::ErrorCollector::NUMBER, + strings::Substitute("Reserved range $0 to $1 overlaps with " + "already-defined range $2 to $3.", + range2.start(), range2.end(), + range1.start(), range1.end())); + } + } + } + + HASH_SET<string> reserved_name_set; + for (int i = 0; i < proto.reserved_name_size(); i++) { + const string& name = proto.reserved_name(i); + if (reserved_name_set.find(name) == reserved_name_set.end()) { + reserved_name_set.insert(name); + } else { + AddError(name, proto, DescriptorPool::ErrorCollector::NAME, + strings::Substitute( + "Enum value \"$0\" is reserved multiple times.", + name)); + } + } + + for (int i = 0; i < result->value_count(); i++) { + const EnumValueDescriptor* value = result->value(i); + for (int j = 0; j < result->reserved_range_count(); j++) { + const EnumDescriptor::ReservedRange* range = result->reserved_range(j); + if (range->start <= value->number() && value->number() <= range->end) { + AddError(value->full_name(), proto.reserved_range(j), + DescriptorPool::ErrorCollector::NUMBER, + strings::Substitute( + "Enum value \"$0\" uses reserved number $1.", + value->name(), value->number())); + } + } + if (reserved_name_set.find(value->name()) != reserved_name_set.end()) { + AddError(value->full_name(), proto.value(i), + DescriptorPool::ErrorCollector::NAME, + strings::Substitute( + "Enum value \"$0\" is reserved.", value->name())); + } + } } void DescriptorBuilder::BuildEnumValue(const EnumValueDescriptorProto& proto, @@ -4599,7 +5245,8 @@ void DescriptorBuilder::BuildEnumValue(const EnumValueDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + EnumValueDescriptorProto::kOptionsFieldNumber); } // Again, enum values are weird because we makes them appear as siblings @@ -4666,7 +5313,8 @@ void DescriptorBuilder::BuildService(const ServiceDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + ServiceDescriptorProto::kOptionsFieldNumber); } AddSymbol(result->full_name(), NULL, result->name(), @@ -4687,14 +5335,15 @@ void DescriptorBuilder::BuildMethod(const MethodDescriptorProto& proto, ValidateSymbolName(proto.name(), *full_name, proto); // These will be filled in when cross-linking. - result->input_type_ = NULL; - result->output_type_ = NULL; + result->input_type_.Init(); + result->output_type_.Init(); // Copy options. if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + MethodDescriptorProto::kOptionsFieldNumber); } result->client_streaming_ = proto.client_streaming(); @@ -4753,6 +5402,11 @@ void DescriptorBuilder::CrossLinkMessage( CrossLinkField(&message->extensions_[i], proto.extension(i)); } + for (int i = 0; i < message->extension_range_count(); i++) { + CrossLinkExtensionRange(&message->extension_ranges_[i], + proto.extension_range(i)); + } + // Set up field array for each oneof. // First count the number of fields per oneof. @@ -4815,15 +5469,27 @@ void DescriptorBuilder::CrossLinkMessage( } } +void DescriptorBuilder::CrossLinkExtensionRange( + Descriptor::ExtensionRange* range, + const DescriptorProto::ExtensionRange& proto) { + if (range->options_ == NULL) { + range->options_ = &ExtensionRangeOptions::default_instance(); + } +} + void DescriptorBuilder::CrossLinkField( FieldDescriptor* field, const FieldDescriptorProto& proto) { if (field->options_ == NULL) { field->options_ = &FieldOptions::default_instance(); } + // Add the field to the lowercase-name and camelcase-name tables. + file_tables_->AddFieldByStylizedNames(field); + if (proto.has_extendee()) { - Symbol extendee = LookupSymbol(proto.extendee(), field->full_name(), - PLACEHOLDER_EXTENDABLE_MESSAGE); + Symbol extendee = + LookupSymbol(proto.extendee(), field->full_name(), + DescriptorPool::PLACEHOLDER_EXTENDABLE_MESSAGE); if (extendee.IsNull()) { AddNotDefinedError(field->full_name(), proto, DescriptorPool::ErrorCollector::EXTENDEE, @@ -4868,21 +5534,56 @@ void DescriptorBuilder::CrossLinkField( bool expecting_enum = (proto.type() == FieldDescriptorProto::TYPE_ENUM) || proto.has_default_value(); - Symbol type = - LookupSymbol(proto.type_name(), field->full_name(), - expecting_enum ? PLACEHOLDER_ENUM : PLACEHOLDER_MESSAGE, - LOOKUP_TYPES); + // In case of weak fields we force building the dependency. We need to know + // if the type exist or not. If it doesnt exist we substitute Empty which + // should only be done if the type can't be found in the generated pool. + // TODO(gerbens) Ideally we should query the database directly to check + // if weak fields exist or not so that we don't need to force building + // weak dependencies. However the name lookup rules for symbols are + // somewhat complicated, so I defer it too another CL. + bool is_weak = !pool_->enforce_weak_ && proto.options().weak(); + bool is_lazy = pool_->lazily_build_dependencies_ && !is_weak; - // If the type is a weak type, we change the type to a google.protobuf.Empty field. - if (type.IsNull() && !pool_->enforce_weak_ && proto.options().weak()) { - type = FindSymbol(kNonLinkedWeakMessageReplacementName); - } + Symbol type = + LookupSymbol(proto.type_name(), field->full_name(), + expecting_enum ? DescriptorPool::PLACEHOLDER_ENUM + : DescriptorPool::PLACEHOLDER_MESSAGE, + LOOKUP_TYPES, !is_lazy); if (type.IsNull()) { - AddNotDefinedError(field->full_name(), proto, - DescriptorPool::ErrorCollector::TYPE, - proto.type_name()); - return; + if (is_lazy) { + // Save the symbol names for later for lookup, and allocate the once + // object needed for the accessors. + string name = proto.type_name(); + field->type_once_ = tables_->AllocateOnceDynamic(); + field->type_name_ = tables_->AllocateString(name); + if (proto.has_default_value()) { + field->default_value_enum_name_ = + tables_->AllocateString(proto.default_value()); + } + // AddFieldByNumber and AddExtension are done later in this function, + // and can/must be done if the field type was not found. The related + // error checking is not necessary when in lazily_build_dependencies_ + // mode, and can't be done without building the type's descriptor, + // which we don't want to do. + file_tables_->AddFieldByNumber(field); + if (field->is_extension()) { + tables_->AddExtension(field); + } + return; + } else { + // If the type is a weak type, we change the type to a google.protobuf.Empty + // field. + if (is_weak) { + type = FindSymbol(kNonLinkedWeakMessageReplacementName); + } + if (type.IsNull()) { + AddNotDefinedError(field->full_name(), proto, + DescriptorPool::ErrorCollector::TYPE, + proto.type_name()); + return; + } + } } if (!proto.has_type()) { @@ -4978,18 +5679,24 @@ void DescriptorBuilder::CrossLinkField( // Add the field to the fields-by-number table. // Note: We have to do this *after* cross-linking because extensions do not - // know their containing type until now. + // know their containing type until now. If we're in + // lazily_build_dependencies_ mode, we're guaranteed there's no errors, so no + // risk to calling containing_type() or other accessors that will build + // dependencies. if (!file_tables_->AddFieldByNumber(field)) { const FieldDescriptor* conflicting_field = file_tables_->FindFieldByNumber(field->containing_type(), field->number()); + string containing_type_name = field->containing_type() == NULL + ? "unknown" + : field->containing_type()->full_name(); if (field->is_extension()) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Extension number $0 has already been used " "in \"$1\" by extension \"$2\".", field->number(), - field->containing_type()->full_name(), + containing_type_name, conflicting_field->full_name())); } else { AddError(field->full_name(), proto, @@ -4997,7 +5704,7 @@ void DescriptorBuilder::CrossLinkField( strings::Substitute("Field number $0 has already been used in " "\"$1\" by field \"$2\".", field->number(), - field->containing_type()->full_name(), + containing_type_name, conflicting_field->name())); } } else { @@ -5005,13 +5712,15 @@ void DescriptorBuilder::CrossLinkField( if (!tables_->AddExtension(field)) { const FieldDescriptor* conflicting_field = tables_->FindExtension(field->containing_type(), field->number()); + string containing_type_name = + field->containing_type() == NULL + ? "unknown" + : field->containing_type()->full_name(); string error_msg = strings::Substitute( "Extension number $0 has already been used in \"$1\" by extension " "\"$2\" defined in $3.", - field->number(), - field->containing_type()->full_name(), - conflicting_field->full_name(), - conflicting_field->file()->name()); + field->number(), containing_type_name, + conflicting_field->full_name(), conflicting_field->file()->name()); // Conflicting extension numbers should be an error. However, before // turning this into an error we need to fix all existing broken // protos first. @@ -5021,9 +5730,6 @@ void DescriptorBuilder::CrossLinkField( } } } - - // Add the field to the lowercase-name and camelcase-name tables. - file_tables_->AddFieldByStylizedNames(field); } void DescriptorBuilder::CrossLinkEnum( @@ -5062,30 +5768,44 @@ void DescriptorBuilder::CrossLinkMethod( method->options_ = &MethodOptions::default_instance(); } - Symbol input_type = LookupSymbol(proto.input_type(), method->full_name()); + Symbol input_type = + LookupSymbol(proto.input_type(), method->full_name(), + DescriptorPool::PLACEHOLDER_MESSAGE, LOOKUP_ALL, + !pool_->lazily_build_dependencies_); if (input_type.IsNull()) { - AddNotDefinedError(method->full_name(), proto, - DescriptorPool::ErrorCollector::INPUT_TYPE, - proto.input_type()); + if (!pool_->lazily_build_dependencies_) { + AddNotDefinedError(method->full_name(), proto, + DescriptorPool::ErrorCollector::INPUT_TYPE, + proto.input_type()); + } else { + method->input_type_.SetLazy(proto.input_type(), file_); + } } else if (input_type.type != Symbol::MESSAGE) { AddError(method->full_name(), proto, DescriptorPool::ErrorCollector::INPUT_TYPE, "\"" + proto.input_type() + "\" is not a message type."); } else { - method->input_type_ = input_type.descriptor; + method->input_type_.Set(input_type.descriptor); } - Symbol output_type = LookupSymbol(proto.output_type(), method->full_name()); + Symbol output_type = + LookupSymbol(proto.output_type(), method->full_name(), + DescriptorPool::PLACEHOLDER_MESSAGE, LOOKUP_ALL, + !pool_->lazily_build_dependencies_); if (output_type.IsNull()) { - AddNotDefinedError(method->full_name(), proto, - DescriptorPool::ErrorCollector::OUTPUT_TYPE, - proto.output_type()); + if (!pool_->lazily_build_dependencies_) { + AddNotDefinedError(method->full_name(), proto, + DescriptorPool::ErrorCollector::OUTPUT_TYPE, + proto.output_type()); + } else { + method->output_type_.SetLazy(proto.output_type(), file_); + } } else if (output_type.type != Symbol::MESSAGE) { AddError(method->full_name(), proto, DescriptorPool::ErrorCollector::OUTPUT_TYPE, "\"" + proto.output_type() + "\" is not a message type."); } else { - method->output_type_ = output_type.descriptor; + method->output_type_.Set(output_type.descriptor); } } @@ -5191,7 +5911,7 @@ void DescriptorBuilder::ValidateProto3Message( // In proto3, we reject field names if they conflict in camelCase. // Note that we currently enforce a stricter rule: Field names must be // unique after being converted to lowercase with underscores removed. - map<string, const FieldDescriptor*> name_to_field; + std::map<string, const FieldDescriptor*> name_to_field; for (int i = 0; i < message->field_count(); ++i) { string lowercase_name = ToLowercaseWithoutUnderscores( message->field(i)->name()); @@ -5254,7 +5974,6 @@ void DescriptorBuilder::ValidateProto3Enum( } } - void DescriptorBuilder::ValidateMessageOptions(Descriptor* message, const DescriptorProto& proto) { VALIDATE_OPTIONS_FROM_ARRAY(message, field, Field); @@ -5275,11 +5994,14 @@ void DescriptorBuilder::ValidateMessageOptions(Descriptor* message, max_extension_range)); } } - } + void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, const FieldDescriptorProto& proto) { + if (pool_->lazily_build_dependencies_ && (!field || !field->message_type())) { + return; + } // Only message type fields may be lazy. if (field->options().lazy()) { if (field->type() != FieldDescriptor::TYPE_MESSAGE) { @@ -5338,13 +6060,15 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, } } + ValidateJSType(field, proto); + } void DescriptorBuilder::ValidateEnumOptions(EnumDescriptor* enm, const EnumDescriptorProto& proto) { VALIDATE_OPTIONS_FROM_ARRAY(enm, value, EnumValue); if (!enm->options().has_allow_alias() || !enm->options().allow_alias()) { - map<int, string> used_values; + std::map<int, string> used_values; for (int i = 0; i < enm->value_count(); ++i) { const EnumValueDescriptor* enum_value = enm->value(i); if (used_values.find(enum_value->number()) != used_values.end()) { @@ -5471,10 +6195,10 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, const DescriptorProto& proto) { - map<string, const Descriptor*> seen_types; + std::map<string, const Descriptor*> seen_types; for (int i = 0; i < message->nested_type_count(); ++i) { const Descriptor* nested = message->nested_type(i); - pair<map<string, const Descriptor*>::iterator, bool> result = + std::pair<std::map<string, const Descriptor*>::iterator, bool> result = seen_types.insert(std::make_pair(nested->name(), nested)); if (!result.second) { if (result.first->second->options().map_entry() || @@ -5491,7 +6215,7 @@ void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, // Check for conflicted field names. for (int i = 0; i < message->field_count(); ++i) { const FieldDescriptor* field = message->field(i); - map<string, const Descriptor*>::iterator iter = + std::map<string, const Descriptor*>::iterator iter = seen_types.find(field->name()); if (iter != seen_types.end() && iter->second->options().map_entry()) { AddError(message->full_name(), proto, @@ -5503,7 +6227,7 @@ void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, // Check for conflicted enum names. for (int i = 0; i < message->enum_type_count(); ++i) { const EnumDescriptor* enum_desc = message->enum_type(i); - map<string, const Descriptor*>::iterator iter = + std::map<string, const Descriptor*>::iterator iter = seen_types.find(enum_desc->name()); if (iter != seen_types.end() && iter->second->options().map_entry()) { AddError(message->full_name(), proto, @@ -5515,7 +6239,7 @@ void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, // Check for conflicted oneof names. for (int i = 0; i < message->oneof_decl_count(); ++i) { const OneofDescriptor* oneof_desc = message->oneof_decl(i); - map<string, const Descriptor*>::iterator iter = + std::map<string, const Descriptor*>::iterator iter = seen_types.find(oneof_desc->name()); if (iter != seen_types.end() && iter->second->options().map_entry()) { AddError(message->full_name(), proto, @@ -5526,6 +6250,40 @@ void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, } } +void DescriptorBuilder::ValidateJSType(FieldDescriptor* field, + const FieldDescriptorProto& proto) { + FieldOptions::JSType jstype = field->options().jstype(); + // The default is always acceptable. + if (jstype == FieldOptions::JS_NORMAL) { + return; + } + + switch (field->type()) { + // Integral 64-bit types may be represented as JavaScript numbers or + // strings. + case FieldDescriptor::TYPE_UINT64: + case FieldDescriptor::TYPE_INT64: + case FieldDescriptor::TYPE_SINT64: + case FieldDescriptor::TYPE_FIXED64: + case FieldDescriptor::TYPE_SFIXED64: + if (jstype == FieldOptions::JS_STRING || + jstype == FieldOptions::JS_NUMBER) { + return; + } + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "Illegal jstype for int64, uint64, sint64, fixed64 " + "or sfixed64 field: " + + FieldOptions_JSType_descriptor()->value(jstype)->name()); + break; + + // No other types permit a jstype option. + default: + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "jstype is only allowed on int64, uint64, sint64, fixed64 " + "or sfixed64 fields."); + break; + } +} #undef VALIDATE_OPTIONS_FROM_ARRAY @@ -5557,6 +6315,9 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( << "No field named \"uninterpreted_option\" in the Options proto."; options->GetReflection()->ClearField(options, uninterpreted_options_field); + std::vector<int> src_path = options_to_interpret->element_path; + src_path.push_back(uninterpreted_options_field->number()); + // Find the uninterpreted_option field in the original options. const FieldDescriptor* original_uninterpreted_options_field = original_options->GetDescriptor()-> @@ -5567,14 +6328,17 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( const int num_uninterpreted_options = original_options->GetReflection()-> FieldSize(*original_options, original_uninterpreted_options_field); for (int i = 0; i < num_uninterpreted_options; ++i) { + src_path.push_back(i); uninterpreted_option_ = down_cast<const UninterpretedOption*>( &original_options->GetReflection()->GetRepeatedMessage( *original_options, original_uninterpreted_options_field, i)); - if (!InterpretSingleOption(options)) { + if (!InterpretSingleOption(options, src_path, + options_to_interpret->element_path)) { // Error already added by InterpretSingleOption(). failed = true; break; } + src_path.pop_back(); } // Reset these, so we don't have any dangling pointers. uninterpreted_option_ = NULL; @@ -5607,7 +6371,7 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( } bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( - Message* options) { + Message* options, std::vector<int>& src_path, std::vector<int>& opts_path) { // First do some basic validation. if (uninterpreted_option_->name_size() == 0) { // This should never happen unless the parser has gone seriously awry or @@ -5650,9 +6414,11 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // name in |debug_msg_name|, for use in error messages. const Descriptor* descriptor = options_descriptor; const FieldDescriptor* field = NULL; - vector<const FieldDescriptor*> intermediate_fields; + std::vector<const FieldDescriptor*> intermediate_fields; string debug_msg_name = ""; + std::vector<int> dest_path = opts_path; + for (int i = 0; i < uninterpreted_option_->name_size(); ++i) { const string& name_part = uninterpreted_option_->name(i).name_part(); if (debug_msg_name.size() > 0) { @@ -5715,19 +6481,24 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( "\" is not a field or extension of message \"" + descriptor->name() + "\"."); } - } else if (i < uninterpreted_option_->name_size() - 1) { - if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { - return AddNameError("Option \"" + debug_msg_name + - "\" is an atomic type, not a message."); - } else if (field->is_repeated()) { - return AddNameError("Option field \"" + debug_msg_name + - "\" is a repeated message. Repeated message " - "options must be initialized using an " - "aggregate value."); - } else { - // Drill down into the submessage. - intermediate_fields.push_back(field); - descriptor = field->message_type(); + } else { + // accumulate field numbers to form path to interpreted option + dest_path.push_back(field->number()); + + if (i < uninterpreted_option_->name_size() - 1) { + if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { + return AddNameError("Option \"" + debug_msg_name + + "\" is an atomic type, not a message."); + } else if (field->is_repeated()) { + return AddNameError("Option field \"" + debug_msg_name + + "\" is a repeated message. Repeated message " + "options must be initialized using an " + "aggregate value."); + } else { + // Drill down into the submessage. + intermediate_fields.push_back(field); + descriptor = field->message_type(); + } } } } @@ -5748,20 +6519,19 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( return false; // ExamineIfOptionIsSet() already added the error. } - // First set the value on the UnknownFieldSet corresponding to the // innermost message. - google::protobuf::scoped_ptr<UnknownFieldSet> unknown_fields(new UnknownFieldSet()); + std::unique_ptr<UnknownFieldSet> unknown_fields(new UnknownFieldSet()); if (!SetOptionValue(field, unknown_fields.get())) { return false; // SetOptionValue() already added the error. } // Now wrap the UnknownFieldSet with UnknownFieldSets corresponding to all // the intermediate messages. - for (vector<const FieldDescriptor*>::reverse_iterator iter = + for (std::vector<const FieldDescriptor*>::reverse_iterator iter = intermediate_fields.rbegin(); iter != intermediate_fields.rend(); ++iter) { - google::protobuf::scoped_ptr<UnknownFieldSet> parent_unknown_fields( + std::unique_ptr<UnknownFieldSet> parent_unknown_fields( new UnknownFieldSet()); switch ((*iter)->type()) { case FieldDescriptor::TYPE_MESSAGE: { @@ -5794,9 +6564,110 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( options->GetReflection()->MutableUnknownFields(options)->MergeFrom( *unknown_fields); + // record the element path of the interpreted option + if (field->is_repeated()) { + int index = repeated_option_counts_[dest_path]++; + dest_path.push_back(index); + } + interpreted_paths_[src_path] = dest_path; + return true; } +void DescriptorBuilder::OptionInterpreter::UpdateSourceCodeInfo( + SourceCodeInfo* info) { + + if (interpreted_paths_.empty()) { + // nothing to do! + return; + } + + // We find locations that match keys in interpreted_paths_ and + // 1) replace the path with the corresponding value in interpreted_paths_ + // 2) remove any subsequent sub-locations (sub-location is one whose path + // has the parent path as a prefix) + // + // To avoid quadratic behavior of removing interior rows as we go, + // we keep a copy. But we don't actually copy anything until we've + // found the first match (so if the source code info has no locations + // that need to be changed, there is zero copy overhead). + + RepeatedPtrField<SourceCodeInfo_Location>* locs = info->mutable_location(); + RepeatedPtrField<SourceCodeInfo_Location> new_locs; + bool copying = false; + + std::vector<int> pathv; + bool matched = false; + + for (RepeatedPtrField<SourceCodeInfo_Location>::iterator loc = locs->begin(); + loc != locs->end(); loc++) { + + if (matched) { + // see if this location is in the range to remove + bool loc_matches = true; + if (loc->path_size() < pathv.size()) { + loc_matches = false; + } else { + for (int j = 0; j < pathv.size(); j++) { + if (loc->path(j) != pathv[j]) { + loc_matches = false; + break; + } + } + } + + if (loc_matches) { + // don't copy this row since it is a sub-location that we're removing + continue; + } + + matched = false; + } + + pathv.clear(); + for (int j = 0; j < loc->path_size(); j++) { + pathv.push_back(loc->path(j)); + } + + std::map<std::vector<int>, std::vector<int>>::iterator entry = + interpreted_paths_.find(pathv); + + if (entry == interpreted_paths_.end()) { + // not a match + if (copying) { + new_locs.Add()->CopyFrom(*loc); + } + continue; + } + + matched = true; + + if (!copying) { + // initialize the copy we are building + copying = true; + new_locs.Reserve(locs->size()); + for (RepeatedPtrField<SourceCodeInfo_Location>::iterator it = + locs->begin(); it != loc; it++) { + new_locs.Add()->CopyFrom(*it); + } + } + + // add replacement and update its path + SourceCodeInfo_Location* replacement = new_locs.Add(); + replacement->CopyFrom(*loc); + replacement->clear_path(); + for (std::vector<int>::iterator rit = entry->second.begin(); + rit != entry->second.end(); rit++) { + replacement->add_path(*rit); + } + } + + // if we made a changed copy, put it in place + if (copying) { + *locs = new_locs; + } +} + void DescriptorBuilder::OptionInterpreter::AddWithoutInterpreting( const UninterpretedOption& uninterpreted_option, Message* options) { const FieldDescriptor* field = @@ -5808,8 +6679,9 @@ void DescriptorBuilder::OptionInterpreter::AddWithoutInterpreting( } bool DescriptorBuilder::OptionInterpreter::ExamineIfOptionIsSet( - vector<const FieldDescriptor*>::const_iterator intermediate_fields_iter, - vector<const FieldDescriptor*>::const_iterator intermediate_fields_end, + std::vector<const FieldDescriptor*>::const_iterator + intermediate_fields_iter, + std::vector<const FieldDescriptor*>::const_iterator intermediate_fields_end, const FieldDescriptor* innermost_field, const string& debug_msg_name, const UnknownFieldSet& unknown_fields) { // We do linear searches of the UnknownFieldSet and its sub-groups. This @@ -6148,7 +7020,7 @@ bool DescriptorBuilder::OptionInterpreter::SetAggregateOption( } const Descriptor* type = option_field->message_type(); - google::protobuf::scoped_ptr<Message> dynamic(dynamic_factory_.GetPrototype(type)->New()); + std::unique_ptr<Message> dynamic(dynamic_factory_.GetPrototype(type)->New()); GOOGLE_CHECK(dynamic.get() != NULL) << "Could not create an instance of " << option_field->DebugString(); @@ -6266,10 +7138,11 @@ void DescriptorBuilder::LogUnusedDependency(const FileDescriptorProto& proto, annotation_extensions.insert("google.protobuf.FieldOptions"); annotation_extensions.insert("google.protobuf.EnumOptions"); annotation_extensions.insert("google.protobuf.EnumValueOptions"); + annotation_extensions.insert("google.protobuf.EnumValueOptions"); annotation_extensions.insert("google.protobuf.ServiceOptions"); annotation_extensions.insert("google.protobuf.MethodOptions"); annotation_extensions.insert("google.protobuf.StreamOptions"); - for (set<const FileDescriptor*>::const_iterator + for (std::set<const FileDescriptor*>::const_iterator it = unused_dependency_.begin(); it != unused_dependency_.end(); ++it) { // Do not log warnings for proto files which extend annotations. @@ -6291,5 +7164,158 @@ void DescriptorBuilder::LogUnusedDependency(const FileDescriptorProto& proto, } } +Symbol DescriptorPool::CrossLinkOnDemandHelper(const string& name, + bool expecting_enum) const { + string lookup_name = name; + if (!lookup_name.empty() && lookup_name[0] == '.') { + lookup_name = lookup_name.substr(1); + } + Symbol result = tables_->FindByNameHelper(this, lookup_name); + return result; +} + +// Handle the lazy import building for a message field whose type wasn't built +// at cross link time. If that was the case, we saved the name of the type to +// be looked up when the accessor for the type was called. Set type_, +// enum_type_, message_type_, and default_value_enum_ appropriately. +void FieldDescriptor::InternalTypeOnceInit() const { + GOOGLE_CHECK(file()->finished_building_ == true); + if (type_name_) { + Symbol result = file()->pool()->CrossLinkOnDemandHelper( + *type_name_, type_ == FieldDescriptor::TYPE_ENUM); + if (result.type == Symbol::MESSAGE) { + type_ = FieldDescriptor::TYPE_MESSAGE; + message_type_ = result.descriptor; + } else if (result.type == Symbol::ENUM) { + type_ = FieldDescriptor::TYPE_ENUM; + enum_type_ = result.enum_descriptor; + } + } + if (enum_type_ && !default_value_enum_) { + if (default_value_enum_name_) { + // Have to build the full name now instead of at CrossLink time, + // because enum_type_ may not be known at the time. + string name = enum_type_->full_name(); + // Enum values reside in the same scope as the enum type. + string::size_type last_dot = name.find_last_of('.'); + if (last_dot != string::npos) { + name = name.substr(0, last_dot) + "." + *default_value_enum_name_; + } else { + name = *default_value_enum_name_; + } + Symbol result = file()->pool()->CrossLinkOnDemandHelper(name, true); + if (result.type == Symbol::ENUM_VALUE) { + default_value_enum_ = result.enum_value_descriptor; + } + } + if (!default_value_enum_) { + // We use the first defined value as the default + // if a default is not explicitly defined. + GOOGLE_CHECK(enum_type_->value_count()); + default_value_enum_ = enum_type_->value(0); + } + } +} + +void FieldDescriptor::TypeOnceInit(const FieldDescriptor* to_init) { + to_init->InternalTypeOnceInit(); +} + +// message_type(), enum_type(), default_value_enum(), and type() +// all share the same GoogleOnceDynamic init path to do lazy +// import building and cross linking of a field of a message. +const Descriptor* FieldDescriptor::message_type() const { + if (type_once_) { + type_once_->Init(&FieldDescriptor::TypeOnceInit, this); + } + return message_type_; +} + +const EnumDescriptor* FieldDescriptor::enum_type() const { + if (type_once_) { + type_once_->Init(&FieldDescriptor::TypeOnceInit, this); + } + return enum_type_; +} + +const EnumValueDescriptor* FieldDescriptor::default_value_enum() const { + if (type_once_) { + type_once_->Init(&FieldDescriptor::TypeOnceInit, this); + } + return default_value_enum_; +} + +void FileDescriptor::InternalDependenciesOnceInit() const { + GOOGLE_CHECK(finished_building_ == true); + for (int i = 0; i < dependency_count(); i++) { + if (dependencies_names_[i]) { + dependencies_[i] = pool_->FindFileByName(*dependencies_names_[i]); + } + } +} + +void FileDescriptor::DependenciesOnceInit(const FileDescriptor* to_init) { + to_init->InternalDependenciesOnceInit(); +} + +const FileDescriptor* FileDescriptor::dependency(int index) const { + if (dependencies_once_) { + // Do once init for all indicies, as it's unlikely only a single index would + // be called, and saves on GoogleOnceDynamic allocations. + dependencies_once_->Init(&FileDescriptor::DependenciesOnceInit, this); + } + return dependencies_[index]; +} + +const Descriptor* MethodDescriptor::input_type() const { + return input_type_.Get(); +} + +const Descriptor* MethodDescriptor::output_type() const { + return output_type_.Get(); +} + + +namespace internal { +void LazyDescriptor::Set(const Descriptor* descriptor) { + GOOGLE_CHECK(!name_); + GOOGLE_CHECK(!once_); + GOOGLE_CHECK(!file_); + descriptor_ = descriptor; +} + +void LazyDescriptor::SetLazy(const string& name, const FileDescriptor* file) { + // verify Init() has been called and Set hasn't been called yet. + GOOGLE_CHECK(!descriptor_); + GOOGLE_CHECK(!file_); + GOOGLE_CHECK(!name_); + GOOGLE_CHECK(!once_); + GOOGLE_CHECK(file && file->pool_); + GOOGLE_CHECK(file->pool_->lazily_build_dependencies_); + GOOGLE_CHECK(!file->finished_building_); + file_ = file; + name_ = file->pool_->tables_->AllocateString(name); + once_ = file->pool_->tables_->AllocateOnceDynamic(); +} + +void LazyDescriptor::Once() { + if (once_) { + once_->Init(&LazyDescriptor::OnceStatic, this); + } +} + +void LazyDescriptor::OnceStatic(LazyDescriptor* lazy) { lazy->OnceInternal(); } + +void LazyDescriptor::OnceInternal() { + GOOGLE_CHECK(file_->finished_building_); + if (!descriptor_ && name_) { + Symbol result = file_->pool_->CrossLinkOnDemandHelper(*name_, false); + if (!result.IsNull() && result.type == Symbol::MESSAGE) { + descriptor_ = result.descriptor; + } + } +} +} // namespace internal + } // namespace protobuf } // namespace google |