diff options
author | Feng Xiao <xfxyjwf@gmail.com> | 2015-12-11 17:09:20 -0800 |
---|---|---|
committer | Feng Xiao <xfxyjwf@gmail.com> | 2015-12-11 17:10:28 -0800 |
commit | e841bac4fcf47f809e089a70d5f84ac37b3883df (patch) | |
tree | d25dc5fc814db182c04c5f276ff1a609c5965a5a /python/google/protobuf/pyext | |
parent | 99a6a95c751a28a3cc33dd2384959179f83f682c (diff) |
Down-integrate from internal code base.
Diffstat (limited to 'python/google/protobuf/pyext')
-rw-r--r-- | python/google/protobuf/pyext/descriptor.cc | 29 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_database.cc | 145 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_database.h | 75 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_pool.cc | 111 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_pool.h | 10 | ||||
-rw-r--r-- | python/google/protobuf/pyext/extension_dict.cc | 47 | ||||
-rw-r--r-- | python/google/protobuf/pyext/extension_dict.h | 5 | ||||
-rw-r--r-- | python/google/protobuf/pyext/map_container.cc | 912 | ||||
-rw-r--r-- | python/google/protobuf/pyext/map_container.h (renamed from python/google/protobuf/pyext/message_map_container.h) | 73 | ||||
-rw-r--r-- | python/google/protobuf/pyext/message.cc | 299 | ||||
-rw-r--r-- | python/google/protobuf/pyext/message.h | 1 | ||||
-rw-r--r-- | python/google/protobuf/pyext/message_map_container.cc | 569 | ||||
-rw-r--r-- | python/google/protobuf/pyext/scalar_map_container.cc | 542 | ||||
-rw-r--r-- | python/google/protobuf/pyext/scalar_map_container.h | 119 |
14 files changed, 1498 insertions, 1439 deletions
diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 61a3d237..a875a7be 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -203,6 +203,14 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { PyObject* message_class(cdescriptor_pool::GetMessageClass( pool, message_type)); if (message_class == NULL) { + // The Options message was not found in the current DescriptorPool. + // In this case, there cannot be extensions to these options, and we can + // try to use the basic pool instead. + PyErr_Clear(); + message_class = cdescriptor_pool::GetMessageClass( + GetDefaultDescriptorPool(), message_type); + } + if (message_class == NULL) { PyErr_Format(PyExc_TypeError, "Could not retrieve class for Options: %s", message_type->full_name().c_str()); return NULL; @@ -211,6 +219,12 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { if (value == NULL) { return NULL; } + if (!PyObject_TypeCheck(value.get(), &CMessage_Type)) { + PyErr_Format(PyExc_TypeError, "Invalid class for %s: %s", + message_type->full_name().c_str(), + Py_TYPE(value.get())->tp_name); + return NULL; + } CMessage* cmsg = reinterpret_cast<CMessage*>(value.get()); const Reflection* reflection = options.GetReflection(); @@ -327,7 +341,8 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, PyDescriptorPool* pool = GetDescriptorPool_FromPool( GetFileDescriptor(descriptor)->pool()); if (pool == NULL) { - Py_DECREF(py_descriptor); + // Don't DECREF, the object is not fully initialized. + PyObject_Del(py_descriptor); return NULL; } Py_INCREF(pool); @@ -1213,6 +1228,13 @@ static void Dealloc(PyFileDescriptor* self) { descriptor::Dealloc(&self->base); } +static PyObject* GetPool(PyFileDescriptor *self, void *closure) { + PyObject* pool = reinterpret_cast<PyObject*>( + GetDescriptorPool_FromPool(_GetDescriptor(self)->pool())); + Py_XINCREF(pool); + return pool; +} + static PyObject* GetName(PyFileDescriptor *self, void *closure) { return PyString_FromCppString(_GetDescriptor(self)->name()); } @@ -1292,6 +1314,7 @@ static PyObject* CopyToProto(PyFileDescriptor *self, PyObject *target) { } static PyGetSetDef Getters[] = { + { "pool", (getter)GetPool, NULL, "pool"}, { "name", (getter)GetName, NULL, "name"}, { "package", (getter)GetPackage, NULL, "package"}, { "serialized_pb", (getter)GetSerializedPb}, @@ -1354,8 +1377,8 @@ PyTypeObject PyFileDescriptor_Type = { 0, // tp_descr_set 0, // tp_dictoffset 0, // tp_init - PyType_GenericAlloc, // tp_alloc - PyType_GenericNew, // tp_new + 0, // tp_alloc + 0, // tp_new PyObject_Del, // tp_free }; diff --git a/python/google/protobuf/pyext/descriptor_database.cc b/python/google/protobuf/pyext/descriptor_database.cc new file mode 100644 index 00000000..514722b4 --- /dev/null +++ b/python/google/protobuf/pyext/descriptor_database.cc @@ -0,0 +1,145 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file defines a C++ DescriptorDatabase, which wraps a Python Database +// and delegate all its operations to Python methods. + +#include <google/protobuf/pyext/descriptor_database.h> + +#include <google/protobuf/stubs/logging.h> +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/descriptor.pb.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +namespace google { +namespace protobuf { +namespace python { + +PyDescriptorDatabase::PyDescriptorDatabase(PyObject* py_database) + : py_database_(py_database) { + Py_INCREF(py_database_); +} + +PyDescriptorDatabase::~PyDescriptorDatabase() { Py_DECREF(py_database_); } + +// Convert a Python object to a FileDescriptorProto pointer. +// Handles all kinds of Python errors, which are simply logged. +static bool GetFileDescriptorProto(PyObject* py_descriptor, + FileDescriptorProto* output) { + if (py_descriptor == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + // Expected error: item was simply not found. + PyErr_Clear(); + } else { + GOOGLE_LOG(ERROR) << "DescriptorDatabase method raised an error"; + PyErr_Print(); + } + return false; + } + const Descriptor* filedescriptor_descriptor = + FileDescriptorProto::default_instance().GetDescriptor(); + CMessage* message = reinterpret_cast<CMessage*>(py_descriptor); + if (PyObject_TypeCheck(py_descriptor, &CMessage_Type) && + message->message->GetDescriptor() == filedescriptor_descriptor) { + // Fast path: Just use the pointer. + FileDescriptorProto* file_proto = + static_cast<FileDescriptorProto*>(message->message); + *output = *file_proto; + return true; + } else { + // Slow path: serialize the message. This allows to use databases which + // use a different implementation of FileDescriptorProto. + ScopedPyObjectPtr serialized_pb( + PyObject_CallMethod(py_descriptor, "SerializeToString", NULL)); + if (serialized_pb == NULL) { + GOOGLE_LOG(ERROR) + << "DescriptorDatabase method did not return a FileDescriptorProto"; + PyErr_Print(); + return false; + } + char* str; + Py_ssize_t len; + if (PyBytes_AsStringAndSize(serialized_pb.get(), &str, &len) < 0) { + GOOGLE_LOG(ERROR) + << "DescriptorDatabase method did not return a FileDescriptorProto"; + PyErr_Print(); + return false; + } + FileDescriptorProto file_proto; + if (!file_proto.ParseFromArray(str, len)) { + GOOGLE_LOG(ERROR) + << "DescriptorDatabase method did not return a FileDescriptorProto"; + return false; + } + *output = file_proto; + return true; + } +} + +// Find a file by file name. +bool PyDescriptorDatabase::FindFileByName(const string& filename, + FileDescriptorProto* output) { + ScopedPyObjectPtr py_descriptor(PyObject_CallMethod( + py_database_, "FindFileByName", "s#", filename.c_str(), filename.size())); + return GetFileDescriptorProto(py_descriptor.get(), output); +} + +// Find the file that declares the given fully-qualified symbol name. +bool PyDescriptorDatabase::FindFileContainingSymbol( + const string& symbol_name, FileDescriptorProto* output) { + ScopedPyObjectPtr py_descriptor( + PyObject_CallMethod(py_database_, "FindFileContainingSymbol", "s#", + symbol_name.c_str(), symbol_name.size())); + return GetFileDescriptorProto(py_descriptor.get(), output); +} + +// Find the file which defines an extension extending the given message type +// with the given field number. +// Python DescriptorDatabases are not required to implement this method. +bool PyDescriptorDatabase::FindFileContainingExtension( + const string& containing_type, int field_number, + FileDescriptorProto* output) { + ScopedPyObjectPtr py_method( + PyObject_GetAttrString(py_database_, "FindFileContainingExtension")); + if (py_method == NULL) { + // This method is not implemented, returns without error. + PyErr_Clear(); + return false; + } + ScopedPyObjectPtr py_descriptor( + PyObject_CallFunction(py_method.get(), "s#i", containing_type.c_str(), + containing_type.size(), field_number)); + return GetFileDescriptorProto(py_descriptor.get(), output); +} + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/descriptor_database.h b/python/google/protobuf/pyext/descriptor_database.h new file mode 100644 index 00000000..fc71c4bc --- /dev/null +++ b/python/google/protobuf/pyext/descriptor_database.h @@ -0,0 +1,75 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__ + +#include <Python.h> + +#include <google/protobuf/descriptor_database.h> + +namespace google { +namespace protobuf { +namespace python { + +class PyDescriptorDatabase : public DescriptorDatabase { + public: + explicit PyDescriptorDatabase(PyObject* py_database); + ~PyDescriptorDatabase(); + + // Implement the abstract interface. All these functions fill the output + // with a copy of FileDescriptorProto. + + // Find a file by file name. + bool FindFileByName(const string& filename, + FileDescriptorProto* output); + + // Find the file that declares the given fully-qualified symbol name. + bool FindFileContainingSymbol(const string& symbol_name, + FileDescriptorProto* output); + + // Find the file which defines an extension extending the given message type + // with the given field number. + // Containing_type must be a fully-qualified type name. + // Python objects are not required to implement this method. + bool FindFileContainingExtension(const string& containing_type, + int field_number, + FileDescriptorProto* output); + + private: + // The python object that implements the database. The reference is owned. + PyObject* py_database_; +}; + +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__ diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index 0f7487fa..0bc76bc9 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -34,8 +34,9 @@ #include <google/protobuf/descriptor.pb.h> #include <google/protobuf/dynamic_message.h> -#include <google/protobuf/pyext/descriptor_pool.h> #include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/descriptor_database.h> +#include <google/protobuf/pyext/descriptor_pool.h> #include <google/protobuf/pyext/message.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> @@ -60,38 +61,93 @@ static hash_map<const DescriptorPool*, PyDescriptorPool*> descriptor_pool_map; namespace cdescriptor_pool { -static PyDescriptorPool* NewDescriptorPool() { - PyDescriptorPool* cdescriptor_pool = PyObject_New( +// Create a Python DescriptorPool object, but does not fill the "pool" +// attribute. +static PyDescriptorPool* _CreateDescriptorPool() { + PyDescriptorPool* cpool = PyObject_New( PyDescriptorPool, &PyDescriptorPool_Type); - if (cdescriptor_pool == NULL) { + if (cpool == NULL) { return NULL; } - // Build a DescriptorPool for messages only declared in Python libraries. - // generated_pool() contains all messages linked in C++ libraries, and is used - // as underlay. - cdescriptor_pool->pool = new DescriptorPool(DescriptorPool::generated_pool()); + cpool->underlay = NULL; + cpool->database = NULL; DynamicMessageFactory* message_factory = new DynamicMessageFactory(); // This option might be the default some day. message_factory->SetDelegateToGeneratedFactory(true); - cdescriptor_pool->message_factory = message_factory; + cpool->message_factory = message_factory; // TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same // storage. - cdescriptor_pool->classes_by_descriptor = + cpool->classes_by_descriptor = new PyDescriptorPool::ClassesByMessageMap(); - cdescriptor_pool->descriptor_options = + cpool->descriptor_options = new hash_map<const void*, PyObject *>(); + return cpool; +} + +// Create a Python DescriptorPool, using the given pool as an underlay: +// new messages will be added to a custom pool, not to the underlay. +// +// Ownership of the underlay is not transferred, its pointer should +// stay alive. +static PyDescriptorPool* PyDescriptorPool_NewWithUnderlay( + const DescriptorPool* underlay) { + PyDescriptorPool* cpool = _CreateDescriptorPool(); + if (cpool == NULL) { + return NULL; + } + cpool->pool = new DescriptorPool(underlay); + cpool->underlay = underlay; + if (!descriptor_pool_map.insert( - std::make_pair(cdescriptor_pool->pool, cdescriptor_pool)).second) { + std::make_pair(cpool->pool, cpool)).second) { // Should never happen -- would indicate an internal error / bug. PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered"); return NULL; } - return cdescriptor_pool; + return cpool; +} + +static PyDescriptorPool* PyDescriptorPool_NewWithDatabase( + DescriptorDatabase* database) { + PyDescriptorPool* cpool = _CreateDescriptorPool(); + if (cpool == NULL) { + return NULL; + } + if (database != NULL) { + cpool->pool = new DescriptorPool(database); + cpool->database = database; + } else { + cpool->pool = new DescriptorPool(); + } + + if (!descriptor_pool_map.insert(std::make_pair(cpool->pool, cpool)).second) { + // Should never happen -- would indicate an internal error / bug. + PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered"); + return NULL; + } + + return cpool; +} + +// The public DescriptorPool constructor. +static PyObject* New(PyTypeObject* type, + PyObject* args, PyObject* kwargs) { + static char* kwlist[] = {"descriptor_db", 0}; + PyObject* py_database = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", kwlist, &py_database)) { + return NULL; + } + DescriptorDatabase* database = NULL; + if (py_database && py_database != Py_None) { + database = new PyDescriptorDatabase(py_database); + } + return reinterpret_cast<PyObject*>( + PyDescriptorPool_NewWithDatabase(database)); } static void Dealloc(PyDescriptorPool* self) { @@ -108,8 +164,9 @@ static void Dealloc(PyDescriptorPool* self) { Py_DECREF(it->second); } delete self->descriptor_options; - delete self->pool; delete self->message_factory; + delete self->database; + delete self->pool; Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); } @@ -354,6 +411,14 @@ PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) { char* message_type; Py_ssize_t message_len; + if (self->database != NULL) { + PyErr_SetString( + PyExc_ValueError, + "Cannot call Add on a DescriptorPool that uses a DescriptorDatabase. " + "Add your file to the underlying database."); + return NULL; + } + if (PyBytes_AsStringAndSize(serialized_pb, &message_type, &message_len) < 0) { return NULL; } @@ -366,8 +431,10 @@ PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) { // If the file was already part of a C++ library, all its descriptors are in // the underlying pool. No need to do anything else. - const FileDescriptor* generated_file = - DescriptorPool::generated_pool()->FindFileByName(file_proto.name()); + const FileDescriptor* generated_file = NULL; + if (self->underlay) { + generated_file = self->underlay->FindFileByName(file_proto.name()); + } if (generated_file != NULL) { return PyFileDescriptor_FromDescriptorWithSerializedPb( generated_file, serialized_pb); @@ -470,7 +537,7 @@ PyTypeObject PyDescriptorPool_Type = { 0, // tp_dictoffset 0, // tp_init 0, // tp_alloc - 0, // tp_new + cdescriptor_pool::New, // tp_new PyObject_Del, // tp_free }; @@ -482,7 +549,11 @@ bool InitDescriptorPool() { if (PyType_Ready(&PyDescriptorPool_Type) < 0) return false; - python_generated_pool = cdescriptor_pool::NewDescriptorPool(); + // The Pool of messages declared in Python libraries. + // generated_pool() contains all messages already linked in C++ libraries, and + // is used as underlay. + python_generated_pool = cdescriptor_pool::PyDescriptorPool_NewWithUnderlay( + DescriptorPool::generated_pool()); if (python_generated_pool == NULL) { return false; } @@ -494,6 +565,10 @@ bool InitDescriptorPool() { return true; } +// The default DescriptorPool used everywhere in this module. +// Today it's the python_generated_pool. +// TODO(amauryfa): Remove all usages of this function: the pool should be +// derived from the context. PyDescriptorPool* GetDefaultDescriptorPool() { return python_generated_pool; } diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h index eda73d38..16bc910c 100644 --- a/python/google/protobuf/pyext/descriptor_pool.h +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -55,8 +55,17 @@ namespace python { typedef struct PyDescriptorPool { PyObject_HEAD + // The C++ pool containing Descriptors. DescriptorPool* pool; + // The C++ pool acting as an underlay. Can be NULL. + // This pointer is not owned and must stay alive. + const DescriptorPool* underlay; + + // The C++ descriptor database used to fetch unknown protos. Can be NULL. + // This pointer is owned. + const DescriptorDatabase* database; + // DynamicMessageFactory used to create C++ instances of messages. // This object cache the descriptors that were used, so the DescriptorPool // needs to get rid of it before it can delete itself. @@ -138,6 +147,7 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg); // Retrieve the global descriptor pool owned by the _message module. // This is the one used by pb2.py generated modules. // Returns a *borrowed* reference. +// "Default" pool used to register messages from _pb2.py modules. PyDescriptorPool* GetDefaultDescriptorPool(); // Retrieve the python descriptor pool owning a C++ descriptor pool. diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index b361b342..555bd293 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -94,13 +94,13 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { if (descriptor == NULL) { return NULL; } - if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) { + if (!CheckFieldBelongsToMessage(descriptor, self->message)) { return NULL; } if (descriptor->label() != FieldDescriptor::LABEL_REPEATED && descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { - return cmessage::InternalGetScalar(self->parent->message, descriptor); + return cmessage::InternalGetScalar(self->message, descriptor); } PyObject* value = PyDict_GetItem(self->values, key); @@ -109,6 +109,14 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { return value; } + if (self->parent == NULL) { + // We are in "detached" state. Don't allow further modifications. + // TODO(amauryfa): Support adding non-scalars to a detached extension dict. + // This probably requires to store the type of the main message. + PyErr_SetObject(PyExc_KeyError, key); + return NULL; + } + if (descriptor->label() != FieldDescriptor::LABEL_REPEATED && descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { PyObject* sub_message = cmessage::InternalGetSubMessage( @@ -154,7 +162,7 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { if (descriptor == NULL) { return -1; } - if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) { + if (!CheckFieldBelongsToMessage(descriptor, self->message)) { return -1; } @@ -164,9 +172,11 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { "type"); return -1; } - cmessage::AssureWritable(self->parent); - if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) { - return -1; + if (self->parent) { + cmessage::AssureWritable(self->parent); + if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) { + return -1; + } } // TODO(tibell): We shouldn't write scalars to the cache. PyDict_SetItem(self->values, key, value); @@ -180,15 +190,17 @@ PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) { return NULL; } PyObject* value = PyDict_GetItem(self->values, extension); - if (value != NULL) { - if (ReleaseExtension(self, value, descriptor) < 0) { + if (self->parent) { + if (value != NULL) { + if (ReleaseExtension(self, value, descriptor) < 0) { + return NULL; + } + } + if (ScopedPyObjectPtr(cmessage::ClearFieldByDescriptor( + self->parent, descriptor)) == NULL) { return NULL; } } - if (ScopedPyObjectPtr(cmessage::ClearFieldByDescriptor( - self->parent, descriptor)) == NULL) { - return NULL; - } if (PyDict_DelItem(self->values, extension) < 0) { PyErr_Clear(); } @@ -201,8 +213,15 @@ PyObject* HasExtension(ExtensionDict* self, PyObject* extension) { if (descriptor == NULL) { return NULL; } - PyObject* result = cmessage::HasFieldByDescriptor(self->parent, descriptor); - return result; + if (self->parent) { + return cmessage::HasFieldByDescriptor(self->parent, descriptor); + } else { + int exists = PyDict_Contains(self->values, extension); + if (exists < 0) { + return NULL; + } + return PyBool_FromLong(exists); + } } PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* name) { diff --git a/python/google/protobuf/pyext/extension_dict.h b/python/google/protobuf/pyext/extension_dict.h index 7a66cb23..d92cf956 100644 --- a/python/google/protobuf/pyext/extension_dict.h +++ b/python/google/protobuf/pyext/extension_dict.h @@ -117,11 +117,6 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value); PyObject* ClearExtension(ExtensionDict* self, PyObject* extension); -// Checks if the dict has an extension. -// -// Returns a new python boolean reference. -PyObject* HasExtension(ExtensionDict* self, PyObject* extension); - // Gets an extension from the dict given the extension name as opposed to // descriptor. // diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc new file mode 100644 index 00000000..c39f7b83 --- /dev/null +++ b/python/google/protobuf/pyext/map_container.cc @@ -0,0 +1,912 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: haberman@google.com (Josh Haberman) + +#include <google/protobuf/pyext/map_container.h> + +#include <google/protobuf/stubs/logging.h> +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/stubs/scoped_ptr.h> +#include <google/protobuf/map_field.h> +#include <google/protobuf/map.h> +#include <google/protobuf/message.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +#if PY_MAJOR_VERSION >= 3 + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t +#endif + +namespace google { +namespace protobuf { +namespace python { + +// Functions that need access to map reflection functionality. +// They need to be contained in this class because it is friended. +class MapReflectionFriend { + public: + // Methods that are in common between the map types. + static PyObject* Contains(PyObject* _self, PyObject* key); + static Py_ssize_t Length(PyObject* _self); + static PyObject* GetIterator(PyObject *_self); + static PyObject* IterNext(PyObject* _self); + + // Methods that differ between the map types. + static PyObject* ScalarMapGetItem(PyObject* _self, PyObject* key); + static PyObject* MessageMapGetItem(PyObject* _self, PyObject* key); + static int ScalarMapSetItem(PyObject* _self, PyObject* key, PyObject* v); + static int MessageMapSetItem(PyObject* _self, PyObject* key, PyObject* v); +}; + +struct MapIterator { + PyObject_HEAD; + + scoped_ptr< ::google::protobuf::MapIterator> iter; + + // A pointer back to the container, so we can notice changes to the version. + // We own a ref on this. + MapContainer* container; + + // We need to keep a ref on the Message* too, because + // MapIterator::~MapIterator() accesses it. Normally this would be ok because + // the ref on container (above) would guarantee outlive semantics. However in + // the case of ClearField(), InitializeAndCopyToParentContainer() resets the + // message pointer (and the owner) to a different message, a copy of the + // original. But our iterator still points to the original, which could now + // get deleted before us. + // + // To prevent this, we ensure that the Message will always stay alive as long + // as this iterator does. This is solely for the benefit of the MapIterator + // destructor -- we should never actually access the iterator in this state + // except to delete it. + shared_ptr<Message> owner; + + // The version of the map when we took the iterator to it. + // + // We store this so that if the map is modified during iteration we can throw + // an error. + uint64 version; + + // True if the container is empty. We signal this separately to avoid calling + // any of the iteration methods, which are non-const. + bool empty; +}; + +Message* MapContainer::GetMutableMessage() { + cmessage::AssureWritable(parent); + return const_cast<Message*>(message); +} + +// Consumes a reference on the Python string object. +static bool PyStringToSTL(PyObject* py_string, string* stl_string) { + char *value; + Py_ssize_t value_len; + + if (!py_string) { + return false; + } + if (PyBytes_AsStringAndSize(py_string, &value, &value_len) < 0) { + Py_DECREF(py_string); + return false; + } else { + stl_string->assign(value, value_len); + Py_DECREF(py_string); + return true; + } +} + +static bool PythonToMapKey(PyObject* obj, + const FieldDescriptor* field_descriptor, + MapKey* key) { + switch (field_descriptor->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: { + GOOGLE_CHECK_GET_INT32(obj, value, false); + key->SetInt32Value(value); + break; + } + case FieldDescriptor::CPPTYPE_INT64: { + GOOGLE_CHECK_GET_INT64(obj, value, false); + key->SetInt64Value(value); + break; + } + case FieldDescriptor::CPPTYPE_UINT32: { + GOOGLE_CHECK_GET_UINT32(obj, value, false); + key->SetUInt32Value(value); + break; + } + case FieldDescriptor::CPPTYPE_UINT64: { + GOOGLE_CHECK_GET_UINT64(obj, value, false); + key->SetUInt64Value(value); + break; + } + case FieldDescriptor::CPPTYPE_BOOL: { + GOOGLE_CHECK_GET_BOOL(obj, value, false); + key->SetBoolValue(value); + break; + } + case FieldDescriptor::CPPTYPE_STRING: { + string str; + if (!PyStringToSTL(CheckString(obj, field_descriptor), &str)) { + return false; + } + key->SetStringValue(str); + break; + } + default: + PyErr_Format( + PyExc_SystemError, "Type %d cannot be a map key", + field_descriptor->cpp_type()); + return false; + } + return true; +} + +static PyObject* MapKeyToPython(const FieldDescriptor* field_descriptor, + const MapKey& key) { + switch (field_descriptor->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return PyInt_FromLong(key.GetInt32Value()); + case FieldDescriptor::CPPTYPE_INT64: + return PyLong_FromLongLong(key.GetInt64Value()); + case FieldDescriptor::CPPTYPE_UINT32: + return PyInt_FromSize_t(key.GetUInt32Value()); + case FieldDescriptor::CPPTYPE_UINT64: + return PyLong_FromUnsignedLongLong(key.GetUInt64Value()); + case FieldDescriptor::CPPTYPE_BOOL: + return PyBool_FromLong(key.GetBoolValue()); + case FieldDescriptor::CPPTYPE_STRING: + return ToStringObject(field_descriptor, key.GetStringValue()); + default: + PyErr_Format( + PyExc_SystemError, "Couldn't convert type %d to value", + field_descriptor->cpp_type()); + return NULL; + } +} + +// This is only used for ScalarMap, so we don't need to handle the +// CPPTYPE_MESSAGE case. +PyObject* MapValueRefToPython(const FieldDescriptor* field_descriptor, + MapValueRef* value) { + switch (field_descriptor->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return PyInt_FromLong(value->GetInt32Value()); + case FieldDescriptor::CPPTYPE_INT64: + return PyLong_FromLongLong(value->GetInt64Value()); + case FieldDescriptor::CPPTYPE_UINT32: + return PyInt_FromSize_t(value->GetUInt32Value()); + case FieldDescriptor::CPPTYPE_UINT64: + return PyLong_FromUnsignedLongLong(value->GetUInt64Value()); + case FieldDescriptor::CPPTYPE_FLOAT: + return PyFloat_FromDouble(value->GetFloatValue()); + case FieldDescriptor::CPPTYPE_DOUBLE: + return PyFloat_FromDouble(value->GetDoubleValue()); + case FieldDescriptor::CPPTYPE_BOOL: + return PyBool_FromLong(value->GetBoolValue()); + case FieldDescriptor::CPPTYPE_STRING: + return ToStringObject(field_descriptor, value->GetStringValue()); + case FieldDescriptor::CPPTYPE_ENUM: + return PyInt_FromLong(value->GetEnumValue()); + default: + PyErr_Format( + PyExc_SystemError, "Couldn't convert type %d to value", + field_descriptor->cpp_type()); + return NULL; + } +} + +// This is only used for ScalarMap, so we don't need to handle the +// CPPTYPE_MESSAGE case. +static bool PythonToMapValueRef(PyObject* obj, + const FieldDescriptor* field_descriptor, + bool allow_unknown_enum_values, + MapValueRef* value_ref) { + switch (field_descriptor->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: { + GOOGLE_CHECK_GET_INT32(obj, value, false); + value_ref->SetInt32Value(value); + return true; + } + case FieldDescriptor::CPPTYPE_INT64: { + GOOGLE_CHECK_GET_INT64(obj, value, false); + value_ref->SetInt64Value(value); + return true; + } + case FieldDescriptor::CPPTYPE_UINT32: { + GOOGLE_CHECK_GET_UINT32(obj, value, false); + value_ref->SetUInt32Value(value); + return true; + } + case FieldDescriptor::CPPTYPE_UINT64: { + GOOGLE_CHECK_GET_UINT64(obj, value, false); + value_ref->SetUInt64Value(value); + return true; + } + case FieldDescriptor::CPPTYPE_FLOAT: { + GOOGLE_CHECK_GET_FLOAT(obj, value, false); + value_ref->SetFloatValue(value); + return true; + } + case FieldDescriptor::CPPTYPE_DOUBLE: { + GOOGLE_CHECK_GET_DOUBLE(obj, value, false); + value_ref->SetDoubleValue(value); + return true; + } + case FieldDescriptor::CPPTYPE_BOOL: { + GOOGLE_CHECK_GET_BOOL(obj, value, false); + value_ref->SetBoolValue(value); + return true;; + } + case FieldDescriptor::CPPTYPE_STRING: { + string str; + if (!PyStringToSTL(CheckString(obj, field_descriptor), &str)) { + return false; + } + value_ref->SetStringValue(str); + return true; + } + case FieldDescriptor::CPPTYPE_ENUM: { + GOOGLE_CHECK_GET_INT32(obj, value, false); + if (allow_unknown_enum_values) { + value_ref->SetEnumValue(value); + return true; + } else { + const EnumDescriptor* enum_descriptor = field_descriptor->enum_type(); + const EnumValueDescriptor* enum_value = + enum_descriptor->FindValueByNumber(value); + if (enum_value != NULL) { + value_ref->SetEnumValue(value); + return true; + } else { + PyErr_Format(PyExc_ValueError, "Unknown enum value: %d", value); + return false; + } + } + break; + } + default: + PyErr_Format( + PyExc_SystemError, "Setting value to a field of unknown type %d", + field_descriptor->cpp_type()); + return false; + } +} + +// Map methods common to ScalarMap and MessageMap ////////////////////////////// + +static MapContainer* GetMap(PyObject* obj) { + return reinterpret_cast<MapContainer*>(obj); +} + +Py_ssize_t MapReflectionFriend::Length(PyObject* _self) { + MapContainer* self = GetMap(_self); + const google::protobuf::Message* message = self->message; + return message->GetReflection()->MapSize(*message, + self->parent_field_descriptor); +} + +PyObject* Clear(PyObject* _self) { + MapContainer* self = GetMap(_self); + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + + reflection->ClearField(message, self->parent_field_descriptor); + + Py_RETURN_NONE; +} + +PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) { + MapContainer* self = GetMap(_self); + + const Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return NULL; + } + + if (reflection->ContainsMapKey(*message, self->parent_field_descriptor, + map_key)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + +// Initializes the underlying Message object of "to" so it becomes a new parent +// repeated scalar, and copies all the values from "from" to it. A child scalar +// container can be released by passing it as both from and to (e.g. making it +// the recipient of the new parent message and copying the values from itself). +static int InitializeAndCopyToParentContainer(MapContainer* from, + MapContainer* to) { + // For now we require from == to, re-evaluate if we want to support deep copy + // as in repeated_scalar_container.cc. + GOOGLE_DCHECK(from == to); + Message* new_message = from->message->New(); + + if (MapReflectionFriend::Length(reinterpret_cast<PyObject*>(from)) > 0) { + // A somewhat roundabout way of copying just one field from old_message to + // new_message. This is the best we can do with what Reflection gives us. + Message* mutable_old = from->GetMutableMessage(); + vector<const FieldDescriptor*> fields; + fields.push_back(from->parent_field_descriptor); + + // Move the map field into the new message. + mutable_old->GetReflection()->SwapFields(mutable_old, new_message, fields); + + // If/when we support from != to, this will be required also to copy the + // map field back into the existing message: + // mutable_old->MergeFrom(*new_message); + } + + // If from == to this could delete old_message. + to->owner.reset(new_message); + + to->parent = NULL; + to->parent_field_descriptor = from->parent_field_descriptor; + to->message = new_message; + + // Invalidate iterators, since they point to the old copy of the field. + to->version++; + + return 0; +} + +int MapContainer::Release() { + return InitializeAndCopyToParentContainer(this, this); +} + + +// ScalarMap /////////////////////////////////////////////////////////////////// + +PyObject *NewScalarMapContainer( + CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) { + if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { + return NULL; + } + + ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0)); + if (obj.get() == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate new container."); + } + + MapContainer* self = GetMap(obj.get()); + + self->message = parent->message; + self->parent = parent; + self->parent_field_descriptor = parent_field_descriptor; + self->owner = parent->owner; + self->version = 0; + + self->key_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("key"); + self->value_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("value"); + + if (self->key_field_descriptor == NULL || + self->value_field_descriptor == NULL) { + return PyErr_Format(PyExc_KeyError, + "Map entry descriptor did not have key/value fields"); + } + + return obj.release(); +} + +PyObject* MapReflectionFriend::ScalarMapGetItem(PyObject* _self, + PyObject* key) { + MapContainer* self = GetMap(_self); + + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + MapValueRef value; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return NULL; + } + + if (reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor, + map_key, &value)) { + self->version++; + } + + return MapValueRefToPython(self->value_field_descriptor, &value); +} + +int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key, + PyObject* v) { + MapContainer* self = GetMap(_self); + + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + MapValueRef value; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return -1; + } + + self->version++; + + if (v) { + // Set item to v. + reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor, + map_key, &value); + + return PythonToMapValueRef(v, self->value_field_descriptor, + reflection->SupportsUnknownEnumValues(), &value) + ? 0 + : -1; + } else { + // Delete key from map. + if (reflection->DeleteMapValue(message, self->parent_field_descriptor, + map_key)) { + return 0; + } else { + PyErr_Format(PyExc_KeyError, "Key not present in map"); + return -1; + } + } +} + +static PyObject* ScalarMapGet(PyObject* self, PyObject* args) { + PyObject* key; + PyObject* default_value = NULL; + if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { + return NULL; + } + + ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key)); + if (is_present.get() == NULL) { + return NULL; + } + + if (PyObject_IsTrue(is_present.get())) { + return MapReflectionFriend::ScalarMapGetItem(self, key); + } else { + if (default_value != NULL) { + Py_INCREF(default_value); + return default_value; + } else { + Py_RETURN_NONE; + } + } +} + +static void ScalarMapDealloc(PyObject* _self) { + MapContainer* self = GetMap(_self); + self->owner.reset(); + Py_TYPE(_self)->tp_free(_self); +} + +static PyMappingMethods ScalarMapMappingMethods = { + MapReflectionFriend::Length, // mp_length + MapReflectionFriend::ScalarMapGetItem, // mp_subscript + MapReflectionFriend::ScalarMapSetItem, // mp_ass_subscript +}; + +static PyMethodDef ScalarMapMethods[] = { + { "__contains__", MapReflectionFriend::Contains, METH_O, + "Tests whether a key is a member of the map." }, + { "clear", (PyCFunction)Clear, METH_NOARGS, + "Removes all elements from the map." }, + { "get", ScalarMapGet, METH_VARARGS, + "Gets the value for the given key if present, or otherwise a default" }, + /* + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field." }, + */ + {NULL, NULL}, +}; + +PyTypeObject ScalarMapContainer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".ScalarMapContainer", // tp_name + sizeof(MapContainer), // tp_basicsize + 0, // tp_itemsize + ScalarMapDealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + &ScalarMapMappingMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A scalar map container", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + MapReflectionFriend::GetIterator, // tp_iter + 0, // tp_iternext + ScalarMapMethods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + + +// MessageMap ////////////////////////////////////////////////////////////////// + +static MessageMapContainer* GetMessageMap(PyObject* obj) { + return reinterpret_cast<MessageMapContainer*>(obj); +} + +static PyObject* GetCMessage(MessageMapContainer* self, Message* message) { + // Get or create the CMessage object corresponding to this message. + ScopedPyObjectPtr key(PyLong_FromVoidPtr(message)); + PyObject* ret = PyDict_GetItem(self->message_dict, key.get()); + + if (ret == NULL) { + CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init, + message->GetDescriptor()); + ret = reinterpret_cast<PyObject*>(cmsg); + + if (cmsg == NULL) { + return NULL; + } + cmsg->owner = self->owner; + cmsg->message = message; + cmsg->parent = self->parent; + + if (PyDict_SetItem(self->message_dict, key.get(), ret) < 0) { + Py_DECREF(ret); + return NULL; + } + } else { + Py_INCREF(ret); + } + + return ret; +} + +PyObject* NewMessageMapContainer( + CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor, + PyObject* concrete_class) { + if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { + return NULL; + } + + PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0); + if (obj == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate new container."); + } + + MessageMapContainer* self = GetMessageMap(obj); + + self->message = parent->message; + self->parent = parent; + self->parent_field_descriptor = parent_field_descriptor; + self->owner = parent->owner; + self->version = 0; + + self->key_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("key"); + self->value_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("value"); + + self->message_dict = PyDict_New(); + if (self->message_dict == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate message dict."); + } + + Py_INCREF(concrete_class); + self->subclass_init = concrete_class; + + if (self->key_field_descriptor == NULL || + self->value_field_descriptor == NULL) { + Py_DECREF(obj); + return PyErr_Format(PyExc_KeyError, + "Map entry descriptor did not have key/value fields"); + } + + return obj; +} + +int MapReflectionFriend::MessageMapSetItem(PyObject* _self, PyObject* key, + PyObject* v) { + if (v) { + PyErr_Format(PyExc_ValueError, + "Direct assignment of submessage not allowed"); + return -1; + } + + // Now we know that this is a delete, not a set. + + MessageMapContainer* self = GetMessageMap(_self); + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + MapValueRef value; + + self->version++; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return -1; + } + + // Delete key from map. + if (reflection->DeleteMapValue(message, self->parent_field_descriptor, + map_key)) { + return 0; + } else { + PyErr_Format(PyExc_KeyError, "Key not present in map"); + return -1; + } +} + +PyObject* MapReflectionFriend::MessageMapGetItem(PyObject* _self, + PyObject* key) { + MessageMapContainer* self = GetMessageMap(_self); + + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + MapValueRef value; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return NULL; + } + + if (reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor, + map_key, &value)) { + self->version++; + } + + return GetCMessage(self, value.MutableMessageValue()); +} + +PyObject* MessageMapGet(PyObject* self, PyObject* args) { + PyObject* key; + PyObject* default_value = NULL; + if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { + return NULL; + } + + ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key)); + if (is_present.get() == NULL) { + return NULL; + } + + if (PyObject_IsTrue(is_present.get())) { + return MapReflectionFriend::MessageMapGetItem(self, key); + } else { + if (default_value != NULL) { + Py_INCREF(default_value); + return default_value; + } else { + Py_RETURN_NONE; + } + } +} + +static void MessageMapDealloc(PyObject* _self) { + MessageMapContainer* self = GetMessageMap(_self); + self->owner.reset(); + Py_DECREF(self->message_dict); + Py_TYPE(_self)->tp_free(_self); +} + +static PyMappingMethods MessageMapMappingMethods = { + MapReflectionFriend::Length, // mp_length + MapReflectionFriend::MessageMapGetItem, // mp_subscript + MapReflectionFriend::MessageMapSetItem, // mp_ass_subscript +}; + +static PyMethodDef MessageMapMethods[] = { + { "__contains__", (PyCFunction)MapReflectionFriend::Contains, METH_O, + "Tests whether the map contains this element."}, + { "clear", (PyCFunction)Clear, METH_NOARGS, + "Removes all elements from the map."}, + { "get", MessageMapGet, METH_VARARGS, + "Gets the value for the given key if present, or otherwise a default" }, + { "get_or_create", MapReflectionFriend::MessageMapGetItem, METH_O, + "Alias for getitem, useful to make explicit that the map is mutated." }, + /* + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field." }, + */ + {NULL, NULL}, +}; + +PyTypeObject MessageMapContainer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".MessageMapContainer", // tp_name + sizeof(MessageMapContainer), // tp_basicsize + 0, // tp_itemsize + MessageMapDealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + &MessageMapMappingMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A map container for message", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + MapReflectionFriend::GetIterator, // tp_iter + 0, // tp_iternext + MessageMapMethods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +// MapIterator ///////////////////////////////////////////////////////////////// + +static MapIterator* GetIter(PyObject* obj) { + return reinterpret_cast<MapIterator*>(obj); +} + +PyObject* MapReflectionFriend::GetIterator(PyObject *_self) { + MapContainer* self = GetMap(_self); + + ScopedPyObjectPtr obj(PyType_GenericAlloc(&MapIterator_Type, 0)); + if (obj == NULL) { + return PyErr_Format(PyExc_KeyError, "Could not allocate iterator"); + } + + MapIterator* iter = GetIter(obj.get()); + + Py_INCREF(self); + iter->container = self; + iter->version = self->version; + iter->owner = self->owner; + + if (MapReflectionFriend::Length(_self) > 0) { + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + + iter->iter.reset(new ::google::protobuf::MapIterator( + reflection->MapBegin(message, self->parent_field_descriptor))); + } + + return obj.release(); +} + +PyObject* MapReflectionFriend::IterNext(PyObject* _self) { + MapIterator* self = GetIter(_self); + + // This won't catch mutations to the map performed by MergeFrom(); no easy way + // to address that. + if (self->version != self->container->version) { + return PyErr_Format(PyExc_RuntimeError, + "Map modified during iteration."); + } + + if (self->iter.get() == NULL) { + return NULL; + } + + Message* message = self->container->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + + if (*self->iter == + reflection->MapEnd(message, self->container->parent_field_descriptor)) { + return NULL; + } + + PyObject* ret = MapKeyToPython(self->container->key_field_descriptor, + self->iter->GetKey()); + + ++(*self->iter); + + return ret; +} + +static void DeallocMapIterator(PyObject* _self) { + MapIterator* self = GetIter(_self); + self->iter.reset(); + self->owner.reset(); + Py_XDECREF(self->container); + Py_TYPE(_self)->tp_free(_self); +} + +PyTypeObject MapIterator_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".MapIterator", // tp_name + sizeof(MapIterator), // tp_basicsize + 0, // tp_itemsize + DeallocMapIterator, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A scalar map iterator", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + PyObject_SelfIter, // tp_iter + MapReflectionFriend::IterNext, // tp_iternext + 0, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/message_map_container.h b/python/google/protobuf/pyext/map_container.h index 4f6cb26a..2de61187 100644 --- a/python/google/protobuf/pyext/message_map_container.h +++ b/python/google/protobuf/pyext/map_container.h @@ -28,8 +28,8 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__ -#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__ +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__ #include <Python.h> @@ -39,6 +39,7 @@ #endif #include <google/protobuf/descriptor.h> +#include <google/protobuf/message.h> namespace google { namespace protobuf { @@ -55,18 +56,23 @@ namespace python { struct CMessage; -struct MessageMapContainer { +// This struct is used directly for ScalarMap, and is the base class of +// MessageMapContainer, which is used for MessageMap. +struct MapContainer { PyObject_HEAD; // This is the top-level C++ Message object that owns the whole - // proto tree. Every Python MessageMapContainer holds a + // proto tree. Every Python MapContainer holds a // reference to it in order to keep it alive as long as there's a // Python object that references any part of the tree. shared_ptr<Message> owner; // Pointer to the C++ Message that contains this container. The - // MessageMapContainer does not own this pointer. - Message* message; + // MapContainer does not own this pointer. + const Message* message; + + // Use to get a mutable message when necessary. + Message* GetMutableMessage(); // Weak reference to a parent CMessage object (i.e. may be NULL.) // @@ -82,45 +88,46 @@ struct MessageMapContainer { const FieldDescriptor* key_field_descriptor; const FieldDescriptor* value_field_descriptor; + // We bump this whenever we perform a mutation, to invalidate existing + // iterators. + uint64 version; + + // Releases the messages in the container to a new message. + // + // Returns 0 on success, -1 on failure. + int Release(); + + // Set the owner field of self and any children of self. + void SetOwner(const shared_ptr<Message>& new_owner) { + owner = new_owner; + } +}; + +struct MessageMapContainer : public MapContainer { // A callable that is used to create new child messages. PyObject* subclass_init; // A dict mapping Message* -> CMessage. PyObject* message_dict; - - // We bump this whenever we perform a mutation, to invalidate existing - // iterators. - uint64 version; }; -#if PY_MAJOR_VERSION >= 3 - extern PyObject *MessageMapContainer_Type; - extern PyType_Spec MessageMapContainer_Type_spec; -#else - extern PyTypeObject MessageMapContainer_Type; -#endif -extern PyTypeObject MessageMapIterator_Type; +extern PyTypeObject ScalarMapContainer_Type; +extern PyTypeObject MessageMapContainer_Type; +extern PyTypeObject MapIterator_Type; // Both map types use the same iterator. -namespace message_map_container { - -// Builds a MessageMapContainer object, from a parent message and a +// Builds a MapContainer object, from a parent message and a // field descriptor. -extern PyObject* NewContainer(CMessage* parent, - const FieldDescriptor* parent_field_descriptor, - PyObject* concrete_class); - -// Releases the messages in the container to a new message. -// -// Returns 0 on success, -1 on failure. -int Release(MessageMapContainer* self); +extern PyObject* NewScalarMapContainer( + CMessage* parent, const FieldDescriptor* parent_field_descriptor); -// Set the owner field of self and any children of self. -void SetOwner(MessageMapContainer* self, - const shared_ptr<Message>& new_owner); +// Builds a MessageMap object, from a parent message and a +// field descriptor. +extern PyObject* NewMessageMapContainer( + CMessage* parent, const FieldDescriptor* parent_field_descriptor, + PyObject* concrete_class); -} // namespace message_map_container } // namespace python } // namespace protobuf } // namespace google -#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__ +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__ diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 72f51ec1..863cde01 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -33,6 +33,7 @@ #include <google/protobuf/pyext/message.h> +#include <map> #include <memory> #ifndef _SHARED_PTR_H #include <google/protobuf/stubs/shared_ptr.h> @@ -61,8 +62,7 @@ #include <google/protobuf/pyext/extension_dict.h> #include <google/protobuf/pyext/repeated_composite_container.h> #include <google/protobuf/pyext/repeated_scalar_container.h> -#include <google/protobuf/pyext/message_map_container.h> -#include <google/protobuf/pyext/scalar_map_container.h> +#include <google/protobuf/pyext/map_container.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> #include <google/protobuf/stubs/strutil.h> @@ -96,6 +96,7 @@ static PyObject* k_extensions_by_number; PyObject* EnumTypeWrapper_class; static PyObject* PythonMessage_class; static PyObject* kEmptyWeakref; +static PyObject* WKT_classes = NULL; // Defines the Metaclass of all Message classes. // It allows us to cache some C++ pointers in the class object itself, they are @@ -274,8 +275,32 @@ static PyObject* New(PyTypeObject* type, // Build the arguments to the base metaclass. // We change the __bases__ classes. - ScopedPyObjectPtr new_args(Py_BuildValue( - "s(OO)O", name, &CMessage_Type, PythonMessage_class, dict)); + ScopedPyObjectPtr new_args; + const Descriptor* message_descriptor = + PyMessageDescriptor_AsDescriptor(py_descriptor); + if (message_descriptor == NULL) { + return NULL; + } + + if (WKT_classes == NULL) { + ScopedPyObjectPtr well_known_types(PyImport_ImportModule( + "google.protobuf.internal.well_known_types")); + GOOGLE_DCHECK(well_known_types != NULL); + + WKT_classes = PyObject_GetAttrString(well_known_types.get(), "WKTBASES"); + GOOGLE_DCHECK(WKT_classes != NULL); + } + + PyObject* well_known_class = PyDict_GetItemString( + WKT_classes, message_descriptor->full_name().c_str()); + if (well_known_class == NULL) { + new_args.reset(Py_BuildValue("s(OO)O", name, &CMessage_Type, + PythonMessage_class, dict)); + } else { + new_args.reset(Py_BuildValue("s(OOO)O", name, &CMessage_Type, + PythonMessage_class, well_known_class, dict)); + } + if (new_args == NULL) { return NULL; } @@ -448,21 +473,9 @@ static int VisitCompositeField(const FieldDescriptor* descriptor, if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { if (descriptor->is_map()) { - const Descriptor* entry_type = descriptor->message_type(); - const FieldDescriptor* value_type = - entry_type->FindFieldByName("value"); - if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - MessageMapContainer* container = - reinterpret_cast<MessageMapContainer*>(child); - if (visitor.VisitMessageMapContainer(container) == -1) { - return -1; - } - } else { - ScalarMapContainer* container = - reinterpret_cast<ScalarMapContainer*>(child); - if (visitor.VisitScalarMapContainer(container) == -1) { - return -1; - } + MapContainer* container = reinterpret_cast<MapContainer*>(child); + if (visitor.VisitMapContainer(container) == -1) { + return -1; } } else { RepeatedCompositeContainer* container = @@ -579,12 +592,14 @@ bool CheckAndGetInteger( if (PyObject_RichCompareBool(min, arg, Py_LE) != 1 || PyObject_RichCompareBool(max, arg, Py_GE) != 1) { #endif - PyObject *s = PyObject_Str(arg); - if (s) { - PyErr_Format(PyExc_ValueError, - "Value out of range: %s", - PyString_AsString(s)); - Py_DECREF(s); + if (!PyErr_Occurred()) { + PyObject *s = PyObject_Str(arg); + if (s) { + PyErr_Format(PyExc_ValueError, + "Value out of range: %s", + PyString_AsString(s)); + Py_DECREF(s); + } } return false; } @@ -642,38 +657,51 @@ bool CheckAndGetBool(PyObject* arg, bool* value) { return true; } -bool CheckAndSetString( - PyObject* arg, Message* message, - const FieldDescriptor* descriptor, - const Reflection* reflection, - bool append, - int index) { +// Checks whether the given object (which must be "bytes" or "unicode") contains +// valid UTF-8. +bool IsValidUTF8(PyObject* obj) { + if (PyBytes_Check(obj)) { + PyObject* unicode = PyUnicode_FromEncodedObject(obj, "utf-8", NULL); + + // Clear the error indicator; we report our own error when desired. + PyErr_Clear(); + + if (unicode) { + Py_DECREF(unicode); + return true; + } else { + return false; + } + } else { + // Unicode object, known to be valid UTF-8. + return true; + } +} + +bool AllowInvalidUTF8(const FieldDescriptor* field) { return false; } + +PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor) { GOOGLE_DCHECK(descriptor->type() == FieldDescriptor::TYPE_STRING || descriptor->type() == FieldDescriptor::TYPE_BYTES); if (descriptor->type() == FieldDescriptor::TYPE_STRING) { if (!PyBytes_Check(arg) && !PyUnicode_Check(arg)) { FormatTypeError(arg, "bytes, unicode"); - return false; + return NULL; } - if (PyBytes_Check(arg)) { - PyObject* unicode = PyUnicode_FromEncodedObject(arg, "utf-8", NULL); - if (unicode == NULL) { - PyObject* repr = PyObject_Repr(arg); - PyErr_Format(PyExc_ValueError, - "%s has type str, but isn't valid UTF-8 " - "encoding. Non-UTF-8 strings must be converted to " - "unicode objects before being added.", - PyString_AsString(repr)); - Py_DECREF(repr); - return false; - } else { - Py_DECREF(unicode); - } + if (!IsValidUTF8(arg) && !AllowInvalidUTF8(descriptor)) { + PyObject* repr = PyObject_Repr(arg); + PyErr_Format(PyExc_ValueError, + "%s has type str, but isn't valid UTF-8 " + "encoding. Non-UTF-8 strings must be converted to " + "unicode objects before being added.", + PyString_AsString(repr)); + Py_DECREF(repr); + return NULL; } } else if (!PyBytes_Check(arg)) { FormatTypeError(arg, "bytes"); - return false; + return NULL; } PyObject* encoded_string = NULL; @@ -691,14 +719,24 @@ bool CheckAndSetString( Py_INCREF(encoded_string); } - if (encoded_string == NULL) { + return encoded_string; +} + +bool CheckAndSetString( + PyObject* arg, Message* message, + const FieldDescriptor* descriptor, + const Reflection* reflection, + bool append, + int index) { + ScopedPyObjectPtr encoded_string(CheckString(arg, descriptor)); + + if (encoded_string.get() == NULL) { return false; } char* value; Py_ssize_t value_len; - if (PyBytes_AsStringAndSize(encoded_string, &value, &value_len) < 0) { - Py_DECREF(encoded_string); + if (PyBytes_AsStringAndSize(encoded_string.get(), &value, &value_len) < 0) { return false; } @@ -710,7 +748,6 @@ bool CheckAndSetString( } else { reflection->SetRepeatedString(message, descriptor, index, value_string); } - Py_DECREF(encoded_string); return true; } @@ -823,12 +860,7 @@ struct FixupMessageReference : public ChildVisitor { return 0; } - int VisitScalarMapContainer(ScalarMapContainer* container) { - container->message = message_; - return 0; - } - - int VisitMessageMapContainer(MessageMapContainer* container) { + int VisitMapContainer(MapContainer* container) { container->message = message_; return 0; } @@ -870,9 +902,8 @@ int AssureWritable(CMessage* self) { // When a CMessage is made writable its Message pointer is updated // to point to a new mutable Message. When that happens we need to // update any references to the old, read-only CMessage. There are - // five places such references occur: RepeatedScalarContainer, - // RepeatedCompositeContainer, ScalarMapContainer, MessageMapContainer, - // and ExtensionDict. + // four places such references occur: RepeatedScalarContainer, + // RepeatedCompositeContainer, MapContainer, and ExtensionDict. if (self->extensions != NULL) self->extensions->message = self->message; if (ForEachCompositeField(self, FixupMessageReference(self->message)) == -1) @@ -1054,7 +1085,8 @@ int InitAttributes(CMessage* self, PyObject* kwargs) { } const FieldDescriptor* descriptor = GetFieldDescriptor(self, name); if (descriptor == NULL) { - PyErr_Format(PyExc_ValueError, "Protocol message has no \"%s\" field.", + PyErr_Format(PyExc_ValueError, "Protocol message %s has no \"%s\" field.", + self->message->GetDescriptor()->name().c_str(), PyString_AsString(name)); return -1; } @@ -1203,18 +1235,6 @@ CMessage* NewEmptyMessage(PyObject* type, const Descriptor *descriptor) { self->composite_fields = NULL; - // If there are extension_ranges, the message is "extendable". Allocate a - // dictionary to store the extension fields. - if (descriptor->extension_range_count() > 0) { - // TODO(amauryfa): Delay the construction of this dict until extensions are - // really used on the object. - ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self); - if (extension_dict == NULL) { - return NULL; - } - self->extensions = extension_dict; - } - return self; } @@ -1285,12 +1305,7 @@ struct ClearWeakReferences : public ChildVisitor { return 0; } - int VisitScalarMapContainer(ScalarMapContainer* container) { - container->parent = NULL; - return 0; - } - - int VisitMessageMapContainer(MessageMapContainer* container) { + int VisitMapContainer(MapContainer* container) { container->parent = NULL; return 0; } @@ -1305,6 +1320,9 @@ struct ClearWeakReferences : public ChildVisitor { static void Dealloc(CMessage* self) { // Null out all weak references from children to this message. GOOGLE_CHECK_EQ(0, ForEachCompositeField(self, ClearWeakReferences())); + if (self->extensions) { + self->extensions->parent = NULL; + } Py_CLEAR(self->extensions); Py_CLEAR(self->composite_fields); @@ -1466,20 +1484,27 @@ PyObject* HasField(CMessage* self, PyObject* arg) { Py_RETURN_FALSE; } -PyObject* ClearExtension(CMessage* self, PyObject* arg) { +PyObject* ClearExtension(CMessage* self, PyObject* extension) { if (self->extensions != NULL) { - return extension_dict::ClearExtension(self->extensions, arg); + return extension_dict::ClearExtension(self->extensions, extension); + } else { + const FieldDescriptor* descriptor = GetExtensionDescriptor(extension); + if (descriptor == NULL) { + return NULL; + } + if (ScopedPyObjectPtr(ClearFieldByDescriptor(self, descriptor)) == NULL) { + return NULL; + } } - PyErr_SetString(PyExc_TypeError, "Message is not extendable"); - return NULL; + Py_RETURN_NONE; } -PyObject* HasExtension(CMessage* self, PyObject* arg) { - if (self->extensions != NULL) { - return extension_dict::HasExtension(self->extensions, arg); +PyObject* HasExtension(CMessage* self, PyObject* extension) { + const FieldDescriptor* descriptor = GetExtensionDescriptor(extension); + if (descriptor == NULL) { + return NULL; } - PyErr_SetString(PyExc_TypeError, "Message is not extendable"); - return NULL; + return HasFieldByDescriptor(self, descriptor); } // --------------------------------------------------------------------- @@ -1529,13 +1554,8 @@ struct SetOwnerVisitor : public ChildVisitor { return 0; } - int VisitScalarMapContainer(ScalarMapContainer* container) { - scalar_map_container::SetOwner(container, new_owner_); - return 0; - } - - int VisitMessageMapContainer(MessageMapContainer* container) { - message_map_container::SetOwner(container, new_owner_); + int VisitMapContainer(MapContainer* container) { + container->SetOwner(new_owner_); return 0; } @@ -1608,14 +1628,8 @@ struct ReleaseChild : public ChildVisitor { reinterpret_cast<RepeatedScalarContainer*>(container)); } - int VisitScalarMapContainer(ScalarMapContainer* container) { - return scalar_map_container::Release( - reinterpret_cast<ScalarMapContainer*>(container)); - } - - int VisitMessageMapContainer(MessageMapContainer* container) { - return message_map_container::Release( - reinterpret_cast<MessageMapContainer*>(container)); + int VisitMapContainer(MapContainer* container) { + return reinterpret_cast<MapContainer*>(container)->Release(); } int VisitCMessage(CMessage* cmessage, @@ -1707,17 +1721,7 @@ PyObject* Clear(CMessage* self) { AssureWritable(self); if (ForEachCompositeField(self, ReleaseChild(self)) == -1) return NULL; - - // The old ExtensionDict still aliases this CMessage, but all its - // fields have been released. - if (self->extensions != NULL) { - Py_CLEAR(self->extensions); - ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self); - if (extension_dict == NULL) { - return NULL; - } - self->extensions = extension_dict; - } + Py_CLEAR(self->extensions); if (self->composite_fields) { PyDict_Clear(self->composite_fields); } @@ -1997,7 +2001,6 @@ static PyObject* RegisterExtension(PyObject* cls, if (descriptor->is_extension() && descriptor->containing_type()->options().message_set_wire_format() && descriptor->type() == FieldDescriptor::TYPE_MESSAGE && - descriptor->message_type() == descriptor->extension_scope() && descriptor->label() == FieldDescriptor::LABEL_OPTIONAL) { ScopedPyObjectPtr message_name(PyString_FromStringAndSize( descriptor->message_type()->full_name().c_str(), @@ -2042,6 +2045,8 @@ static PyObject* WhichOneof(CMessage* self, PyObject* arg) { } } +static PyObject* GetExtensionDict(CMessage* self, void *closure); + static PyObject* ListFields(CMessage* self) { vector<const FieldDescriptor*> fields; self->message->GetReflection()->ListFields(*self->message, &fields); @@ -2079,12 +2084,13 @@ static PyObject* ListFields(CMessage* self) { PyErr_Clear(); continue; } - PyObject* extensions = reinterpret_cast<PyObject*>(self->extensions); + ScopedPyObjectPtr extensions(GetExtensionDict(self, NULL)); if (extensions == NULL) { return NULL; } // 'extension' reference later stolen by PyTuple_SET_ITEM. - PyObject* extension = PyObject_GetItem(extensions, extension_field.get()); + PyObject* extension = PyObject_GetItem( + extensions.get(), extension_field.get()); if (extension == NULL) { return NULL; } @@ -2493,9 +2499,31 @@ PyObject* _CheckCalledFromGeneratedFile(PyObject* unused, Py_RETURN_NONE; } -static PyMemberDef Members[] = { - {"Extensions", T_OBJECT_EX, offsetof(CMessage, extensions), 0, - "Extension dict"}, +static PyObject* GetExtensionDict(CMessage* self, void *closure) { + if (self->extensions) { + Py_INCREF(self->extensions); + return reinterpret_cast<PyObject*>(self->extensions); + } + + // If there are extension_ranges, the message is "extendable". Allocate a + // dictionary to store the extension fields. + const Descriptor* descriptor = GetMessageDescriptor(Py_TYPE(self)); + if (descriptor->extension_range_count() > 0) { + ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self); + if (extension_dict == NULL) { + return NULL; + } + self->extensions = extension_dict; + Py_INCREF(self->extensions); + return reinterpret_cast<PyObject*>(self->extensions); + } + + PyErr_SetNone(PyExc_AttributeError); + return NULL; +} + +static PyGetSetDef Getters[] = { + {"Extensions", (getter)GetExtensionDict, NULL, "Extension dict"}, {NULL} }; @@ -2592,10 +2620,10 @@ PyObject* GetAttr(CMessage* self, PyObject* name) { if (value_class == NULL) { return NULL; } - py_container = message_map_container::NewContainer(self, field_descriptor, - value_class); + py_container = + NewMessageMapContainer(self, field_descriptor, value_class); } else { - py_container = scalar_map_container::NewContainer(self, field_descriptor); + py_container = NewScalarMapContainer(self, field_descriptor); } if (py_container == NULL) { return NULL; @@ -2672,7 +2700,10 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) { } } - PyErr_Format(PyExc_AttributeError, "Assignment not allowed"); + PyErr_Format(PyExc_AttributeError, + "Assignment not allowed " + "(no field \"%s\"in protocol message object).", + PyString_AsString(name)); return -1; } @@ -2707,8 +2738,8 @@ PyTypeObject CMessage_Type = { 0, // tp_iter 0, // tp_iternext cmessage::Methods, // tp_methods - cmessage::Members, // tp_members - 0, // tp_getset + 0, // tp_members + cmessage::Getters, // tp_getset 0, // tp_base 0, // tp_dict 0, // tp_descr_get @@ -2910,12 +2941,12 @@ bool InitProto2MessageModule(PyObject *m) { reinterpret_cast<PyObject*>(&ScalarMapContainer_Type)); #endif - if (PyType_Ready(&ScalarMapIterator_Type) < 0) { + if (PyType_Ready(&MapIterator_Type) < 0) { return false; } - PyModule_AddObject(m, "ScalarMapIterator", - reinterpret_cast<PyObject*>(&ScalarMapIterator_Type)); + PyModule_AddObject(m, "MapIterator", + reinterpret_cast<PyObject*>(&MapIterator_Type)); #if PY_MAJOR_VERSION >= 3 @@ -2934,13 +2965,6 @@ bool InitProto2MessageModule(PyObject *m) { PyModule_AddObject(m, "MessageMapContainer", reinterpret_cast<PyObject*>(&MessageMapContainer_Type)); #endif - - if (PyType_Ready(&MessageMapIterator_Type) < 0) { - return false; - } - - PyModule_AddObject(m, "MessageMapIterator", - reinterpret_cast<PyObject*>(&MessageMapIterator_Type)); } if (PyType_Ready(&ExtensionDict_Type) < 0) { @@ -2957,6 +2981,9 @@ bool InitProto2MessageModule(PyObject *m) { PyModule_AddObject(m, "default_pool", reinterpret_cast<PyObject*>(GetDefaultDescriptorPool())); + PyModule_AddObject(m, "DescriptorPool", reinterpret_cast<PyObject*>( + &PyDescriptorPool_Type)); + // This implementation provides full Descriptor types, we advertise it so that // descriptor.py can use them in replacement of the Python classes. PyModule_AddIntConstant(m, "_USE_C_DESCRIPTORS", 1); diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index 94de4551..cc0012e9 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -307,6 +307,7 @@ bool CheckAndGetInteger( bool CheckAndGetDouble(PyObject* arg, double* value); bool CheckAndGetFloat(PyObject* arg, float* value); bool CheckAndGetBool(PyObject* arg, bool* value); +PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor); bool CheckAndSetString( PyObject* arg, Message* message, const FieldDescriptor* descriptor, diff --git a/python/google/protobuf/pyext/message_map_container.cc b/python/google/protobuf/pyext/message_map_container.cc deleted file mode 100644 index 8902fa00..00000000 --- a/python/google/protobuf/pyext/message_map_container.cc +++ /dev/null @@ -1,569 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: haberman@google.com (Josh Haberman) - -#include <google/protobuf/pyext/message_map_container.h> - -#include <google/protobuf/stubs/logging.h> -#include <google/protobuf/stubs/common.h> -#include <google/protobuf/message.h> -#include <google/protobuf/pyext/message.h> -#include <google/protobuf/pyext/scoped_pyobject_ptr.h> - -namespace google { -namespace protobuf { -namespace python { - -struct MessageMapIterator { - PyObject_HEAD; - - // This dict contains the full contents of what we want to iterate over. - // There's no way to avoid building this, because the list representation - // (which is canonical) can contain duplicate keys. So at the very least we - // need a set that lets us skip duplicate keys. And at the point that we're - // doing that, we might as well just build the actual dict we're iterating - // over and use dict's built-in iterator. - PyObject* dict; - - // An iterator on dict. - PyObject* iter; - - // A pointer back to the container, so we can notice changes to the version. - MessageMapContainer* container; - - // The version of the map when we took the iterator to it. - // - // We store this so that if the map is modified during iteration we can throw - // an error. - uint64 version; -}; - -static MessageMapIterator* GetIter(PyObject* obj) { - return reinterpret_cast<MessageMapIterator*>(obj); -} - -namespace message_map_container { - -static MessageMapContainer* GetMap(PyObject* obj) { - return reinterpret_cast<MessageMapContainer*>(obj); -} - -// The private constructor of MessageMapContainer objects. -PyObject* NewContainer(CMessage* parent, - const google::protobuf::FieldDescriptor* parent_field_descriptor, - PyObject* concrete_class) { - if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { - return NULL; - } - -#if PY_MAJOR_VERSION >= 3 - PyObject* obj = PyType_GenericAlloc( - reinterpret_cast<PyTypeObject *>(MessageMapContainer_Type), 0); -#else - PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0); -#endif - if (obj == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate new container."); - } - - MessageMapContainer* self = GetMap(obj); - - self->message = parent->message; - self->parent = parent; - self->parent_field_descriptor = parent_field_descriptor; - self->owner = parent->owner; - self->version = 0; - - self->key_field_descriptor = - parent_field_descriptor->message_type()->FindFieldByName("key"); - self->value_field_descriptor = - parent_field_descriptor->message_type()->FindFieldByName("value"); - - self->message_dict = PyDict_New(); - if (self->message_dict == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate message dict."); - } - - Py_INCREF(concrete_class); - self->subclass_init = concrete_class; - - if (self->key_field_descriptor == NULL || - self->value_field_descriptor == NULL) { - Py_DECREF(obj); - return PyErr_Format(PyExc_KeyError, - "Map entry descriptor did not have key/value fields"); - } - - return obj; -} - -// Initializes the underlying Message object of "to" so it becomes a new parent -// repeated scalar, and copies all the values from "from" to it. A child scalar -// container can be released by passing it as both from and to (e.g. making it -// the recipient of the new parent message and copying the values from itself). -static int InitializeAndCopyToParentContainer( - MessageMapContainer* from, - MessageMapContainer* to) { - // For now we require from == to, re-evaluate if we want to support deep copy - // as in repeated_composite_container.cc. - GOOGLE_DCHECK(from == to); - Message* old_message = from->message; - Message* new_message = old_message->New(); - to->parent = NULL; - to->parent_field_descriptor = from->parent_field_descriptor; - to->message = new_message; - to->owner.reset(new_message); - - vector<const FieldDescriptor*> fields; - fields.push_back(from->parent_field_descriptor); - old_message->GetReflection()->SwapFields(old_message, new_message, fields); - return 0; -} - -static PyObject* GetCMessage(MessageMapContainer* self, Message* entry) { - // Get or create the CMessage object corresponding to this message. - Message* message = entry->GetReflection()->MutableMessage( - entry, self->value_field_descriptor); - ScopedPyObjectPtr key(PyLong_FromVoidPtr(message)); - PyObject* ret = PyDict_GetItem(self->message_dict, key.get()); - - if (ret == NULL) { - CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init, - message->GetDescriptor()); - ret = reinterpret_cast<PyObject*>(cmsg); - - if (cmsg == NULL) { - return NULL; - } - cmsg->owner = self->owner; - cmsg->message = message; - cmsg->parent = self->parent; - - if (PyDict_SetItem(self->message_dict, key.get(), ret) < 0) { - Py_DECREF(ret); - return NULL; - } - } else { - Py_INCREF(ret); - } - - return ret; -} - -int Release(MessageMapContainer* self) { - InitializeAndCopyToParentContainer(self, self); - return 0; -} - -void SetOwner(MessageMapContainer* self, - const shared_ptr<Message>& new_owner) { - self->owner = new_owner; -} - -Py_ssize_t Length(PyObject* _self) { - MessageMapContainer* self = GetMap(_self); - google::protobuf::Message* message = self->message; - return message->GetReflection()->FieldSize(*message, - self->parent_field_descriptor); -} - -int MapKeyMatches(MessageMapContainer* self, const Message* entry, - PyObject* key) { - // TODO(haberman): do we need more strict type checking? - ScopedPyObjectPtr entry_key( - cmessage::InternalGetScalar(entry, self->key_field_descriptor)); - int ret = PyObject_RichCompareBool(key, entry_key.get(), Py_EQ); - return ret; -} - -int SetItem(PyObject *_self, PyObject *key, PyObject *v) { - if (v) { - PyErr_Format(PyExc_ValueError, - "Direct assignment of submessage not allowed"); - return -1; - } - - // Now we know that this is a delete, not a set. - - MessageMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. We need to search from the end because the underlying - // representation can have duplicates if a user calls MergeFrom(); the last - // one needs to win. - // - // TODO(haberman): add lookup API to Reflection API. - bool found = false; - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return -1; - if (matches) { - found = true; - if (i != (int)size - 1) { - reflection->SwapElements(message, self->parent_field_descriptor, i, - size - 1); - } - reflection->RemoveLast(message, self->parent_field_descriptor); - - // Can't exit now, the repeated field representation of maps allows - // duplicate keys, and we have to be sure to remove all of them. - } - } - - if (!found) { - PyErr_Format(PyExc_KeyError, "Key not present in map"); - return -1; - } - - self->version++; - - return 0; -} - -PyObject* GetIterator(PyObject *_self) { - MessageMapContainer* self = GetMap(_self); - - ScopedPyObjectPtr obj(PyType_GenericAlloc(&MessageMapIterator_Type, 0)); - if (obj == NULL) { - return PyErr_Format(PyExc_KeyError, "Could not allocate iterator"); - } - - MessageMapIterator* iter = GetIter(obj.get()); - - Py_INCREF(self); - iter->container = self; - iter->version = self->version; - iter->dict = PyDict_New(); - if (iter->dict == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate dict for iterator."); - } - - // Build the entire map into a dict right now. Start from the beginning so - // that later entries win in the case of duplicates. - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. We need to search from the end because the underlying - // representation can have duplicates if a user calls MergeFrom(); the last - // one needs to win. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - ScopedPyObjectPtr key( - cmessage::InternalGetScalar(entry, self->key_field_descriptor)); - if (PyDict_SetItem(iter->dict, key.get(), GetCMessage(self, entry)) < 0) { - return PyErr_Format(PyExc_RuntimeError, - "SetItem failed in iterator construction."); - } - } - - iter->iter = PyObject_GetIter(iter->dict); - - return obj.release(); -} - -PyObject* GetItem(PyObject* _self, PyObject* key) { - MessageMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. We need to search from the end because the underlying - // representation can have duplicates if a user calls MergeFrom(); the last - // one needs to win. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return NULL; - if (matches) { - return GetCMessage(self, entry); - } - } - - // Key is not already present; insert a new entry. - Message* entry = - reflection->AddMessage(message, self->parent_field_descriptor); - - self->version++; - - if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor, - key) < 0) { - reflection->RemoveLast(message, self->parent_field_descriptor); - return NULL; - } - - return GetCMessage(self, entry); -} - -PyObject* Contains(PyObject* _self, PyObject* key) { - MessageMapContainer* self = GetMap(_self); - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. - // - // TODO(haberman): add lookup API to Reflection API. - int size = - reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = 0; i < size; i++) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return NULL; - if (matches) { - Py_RETURN_TRUE; - } - } - - Py_RETURN_FALSE; -} - -PyObject* Clear(PyObject* _self) { - MessageMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - self->version++; - reflection->ClearField(message, self->parent_field_descriptor); - - Py_RETURN_NONE; -} - -PyObject* Get(PyObject* self, PyObject* args) { - PyObject* key; - PyObject* default_value = NULL; - if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { - return NULL; - } - - ScopedPyObjectPtr is_present(Contains(self, key)); - if (is_present.get() == NULL) { - return NULL; - } - - if (PyObject_IsTrue(is_present.get())) { - return GetItem(self, key); - } else { - if (default_value != NULL) { - Py_INCREF(default_value); - return default_value; - } else { - Py_RETURN_NONE; - } - } -} - -static void Dealloc(PyObject* _self) { - MessageMapContainer* self = GetMap(_self); - self->owner.reset(); - Py_DECREF(self->message_dict); - Py_TYPE(_self)->tp_free(_self); -} - -static PyMethodDef Methods[] = { - { "__contains__", (PyCFunction)Contains, METH_O, - "Tests whether the map contains this element."}, - { "clear", (PyCFunction)Clear, METH_NOARGS, - "Removes all elements from the map."}, - { "get", Get, METH_VARARGS, - "Gets the value for the given key if present, or otherwise a default" }, - { "get_or_create", GetItem, METH_O, - "Alias for getitem, useful to make explicit that the map is mutated." }, - /* - { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, - "Makes a deep copy of the class." }, - { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, - "Outputs picklable representation of the repeated field." }, - */ - {NULL, NULL}, -}; - -} // namespace message_map_container - -namespace message_map_iterator { - -static void Dealloc(PyObject* _self) { - MessageMapIterator* self = GetIter(_self); - Py_DECREF(self->dict); - Py_DECREF(self->iter); - Py_DECREF(self->container); - Py_TYPE(_self)->tp_free(_self); -} - -PyObject* IterNext(PyObject* _self) { - MessageMapIterator* self = GetIter(_self); - - // This won't catch mutations to the map performed by MergeFrom(); no easy way - // to address that. - if (self->version != self->container->version) { - return PyErr_Format(PyExc_RuntimeError, - "Map modified during iteration."); - } - - return PyIter_Next(self->iter); -} - -} // namespace message_map_iterator - -#if PY_MAJOR_VERSION >= 3 - static PyType_Slot MessageMapContainer_Type_slots[] = { - {Py_tp_dealloc, (void *)message_map_container::Dealloc}, - {Py_mp_length, (void *)message_map_container::Length}, - {Py_mp_subscript, (void *)message_map_container::GetItem}, - {Py_mp_ass_subscript, (void *)message_map_container::SetItem}, - {Py_tp_methods, (void *)message_map_container::Methods}, - {Py_tp_iter, (void *)message_map_container::GetIterator}, - {0, 0} - }; - - PyType_Spec MessageMapContainer_Type_spec = { - FULL_MODULE_NAME ".MessageMapContainer", - sizeof(MessageMapContainer), - 0, - Py_TPFLAGS_DEFAULT, - MessageMapContainer_Type_slots - }; - - PyObject *MessageMapContainer_Type; - -#else - static PyMappingMethods MpMethods = { - message_map_container::Length, // mp_length - message_map_container::GetItem, // mp_subscript - message_map_container::SetItem, // mp_ass_subscript - }; - - PyTypeObject MessageMapContainer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".MessageMapContainer", // tp_name - sizeof(MessageMapContainer), // tp_basicsize - 0, // tp_itemsize - message_map_container::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - &MpMethods, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A map container for message", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - message_map_container::GetIterator, // tp_iter - 0, // tp_iternext - message_map_container::Methods, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - }; -#endif - -PyTypeObject MessageMapIterator_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".MessageMapIterator", // tp_name - sizeof(MessageMapIterator), // tp_basicsize - 0, // tp_itemsize - message_map_iterator::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A scalar map iterator", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - PyObject_SelfIter, // tp_iter - message_map_iterator::IterNext, // tp_iternext - 0, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init -}; - -} // namespace python -} // namespace protobuf -} // namespace google diff --git a/python/google/protobuf/pyext/scalar_map_container.cc b/python/google/protobuf/pyext/scalar_map_container.cc deleted file mode 100644 index 0b0d5a3d..00000000 --- a/python/google/protobuf/pyext/scalar_map_container.cc +++ /dev/null @@ -1,542 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: haberman@google.com (Josh Haberman) - -#include <google/protobuf/pyext/scalar_map_container.h> - -#include <google/protobuf/stubs/logging.h> -#include <google/protobuf/stubs/common.h> -#include <google/protobuf/message.h> -#include <google/protobuf/pyext/message.h> -#include <google/protobuf/pyext/scoped_pyobject_ptr.h> - -namespace google { -namespace protobuf { -namespace python { - -struct ScalarMapIterator { - PyObject_HEAD; - - // This dict contains the full contents of what we want to iterate over. - // There's no way to avoid building this, because the list representation - // (which is canonical) can contain duplicate keys. So at the very least we - // need a set that lets us skip duplicate keys. And at the point that we're - // doing that, we might as well just build the actual dict we're iterating - // over and use dict's built-in iterator. - PyObject* dict; - - // An iterator on dict. - PyObject* iter; - - // A pointer back to the container, so we can notice changes to the version. - ScalarMapContainer* container; - - // The version of the map when we took the iterator to it. - // - // We store this so that if the map is modified during iteration we can throw - // an error. - uint64 version; -}; - -static ScalarMapIterator* GetIter(PyObject* obj) { - return reinterpret_cast<ScalarMapIterator*>(obj); -} - -namespace scalar_map_container { - -static ScalarMapContainer* GetMap(PyObject* obj) { - return reinterpret_cast<ScalarMapContainer*>(obj); -} - -// The private constructor of ScalarMapContainer objects. -PyObject *NewContainer( - CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) { - if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { - return NULL; - } - -#if PY_MAJOR_VERSION >= 3 - ScopedPyObjectPtr obj(PyType_GenericAlloc( - reinterpret_cast<PyTypeObject *>(ScalarMapContainer_Type), 0)); -#else - ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0)); -#endif - if (obj.get() == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate new container."); - } - - ScalarMapContainer* self = GetMap(obj.get()); - - self->message = parent->message; - self->parent = parent; - self->parent_field_descriptor = parent_field_descriptor; - self->owner = parent->owner; - self->version = 0; - - self->key_field_descriptor = - parent_field_descriptor->message_type()->FindFieldByName("key"); - self->value_field_descriptor = - parent_field_descriptor->message_type()->FindFieldByName("value"); - - if (self->key_field_descriptor == NULL || - self->value_field_descriptor == NULL) { - return PyErr_Format(PyExc_KeyError, - "Map entry descriptor did not have key/value fields"); - } - - return obj.release(); -} - -// Initializes the underlying Message object of "to" so it becomes a new parent -// repeated scalar, and copies all the values from "from" to it. A child scalar -// container can be released by passing it as both from and to (e.g. making it -// the recipient of the new parent message and copying the values from itself). -static int InitializeAndCopyToParentContainer( - ScalarMapContainer* from, - ScalarMapContainer* to) { - // For now we require from == to, re-evaluate if we want to support deep copy - // as in repeated_scalar_container.cc. - GOOGLE_DCHECK(from == to); - Message* old_message = from->message; - Message* new_message = old_message->New(); - to->parent = NULL; - to->parent_field_descriptor = from->parent_field_descriptor; - to->message = new_message; - to->owner.reset(new_message); - - vector<const FieldDescriptor*> fields; - fields.push_back(from->parent_field_descriptor); - old_message->GetReflection()->SwapFields(old_message, new_message, fields); - return 0; -} - -int Release(ScalarMapContainer* self) { - return InitializeAndCopyToParentContainer(self, self); -} - -void SetOwner(ScalarMapContainer* self, - const shared_ptr<Message>& new_owner) { - self->owner = new_owner; -} - -Py_ssize_t Length(PyObject* _self) { - ScalarMapContainer* self = GetMap(_self); - google::protobuf::Message* message = self->message; - return message->GetReflection()->FieldSize(*message, - self->parent_field_descriptor); -} - -int MapKeyMatches(ScalarMapContainer* self, const Message* entry, - PyObject* key) { - // TODO(haberman): do we need more strict type checking? - ScopedPyObjectPtr entry_key( - cmessage::InternalGetScalar(entry, self->key_field_descriptor)); - int ret = PyObject_RichCompareBool(key, entry_key.get(), Py_EQ); - return ret; -} - -PyObject* GetItem(PyObject* _self, PyObject* key) { - ScalarMapContainer* self = GetMap(_self); - - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = size - 1; i >= 0; i--) { - const Message& entry = reflection->GetRepeatedMessage( - *message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, &entry, key); - if (matches < 0) return NULL; - if (matches) { - return cmessage::InternalGetScalar(&entry, self->value_field_descriptor); - } - } - - // Need to add a new entry. - Message* entry = - reflection->AddMessage(message, self->parent_field_descriptor); - PyObject* ret = NULL; - - if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor, - key) >= 0) { - ret = cmessage::InternalGetScalar(entry, self->value_field_descriptor); - } - - self->version++; - - // If there was a type error above, it set the Python exception. - return ret; -} - -int SetItem(PyObject *_self, PyObject *key, PyObject *v) { - ScalarMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - self->version++; - - if (v) { - // Set item. - // - // Right now the Reflection API doesn't support map lookup, so we implement - // it via linear search. - // - // TODO(haberman): add lookup API to Reflection API. - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return -1; - if (matches) { - return cmessage::InternalSetNonOneofScalar( - entry, self->value_field_descriptor, v); - } - } - - // Key is not already present; insert a new entry. - Message* entry = - reflection->AddMessage(message, self->parent_field_descriptor); - - if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor, - key) < 0 || - cmessage::InternalSetNonOneofScalar(entry, self->value_field_descriptor, - v) < 0) { - reflection->RemoveLast(message, self->parent_field_descriptor); - return -1; - } - - return 0; - } else { - bool found = false; - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return -1; - if (matches) { - found = true; - if (i != (int)size - 1) { - reflection->SwapElements(message, self->parent_field_descriptor, i, - size - 1); - } - reflection->RemoveLast(message, self->parent_field_descriptor); - - // Can't exit now, the repeated field representation of maps allows - // duplicate keys, and we have to be sure to remove all of them. - } - } - - if (found) { - return 0; - } else { - PyErr_Format(PyExc_KeyError, "Key not present in map"); - return -1; - } - } -} - -PyObject* GetIterator(PyObject *_self) { - ScalarMapContainer* self = GetMap(_self); - - ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapIterator_Type, 0)); - if (obj == NULL) { - return PyErr_Format(PyExc_KeyError, "Could not allocate iterator"); - } - - ScalarMapIterator* iter = GetIter(obj.get()); - - Py_INCREF(self); - iter->container = self; - iter->version = self->version; - iter->dict = PyDict_New(); - if (iter->dict == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate dict for iterator."); - } - - // Build the entire map into a dict right now. Start from the beginning so - // that later entries win in the case of duplicates. - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. We need to search from the end because the underlying - // representation can have duplicates if a user calls MergeFrom(); the last - // one needs to win. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - for (size_t i = 0; i < size; i++) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - ScopedPyObjectPtr key( - cmessage::InternalGetScalar(entry, self->key_field_descriptor)); - ScopedPyObjectPtr val( - cmessage::InternalGetScalar(entry, self->value_field_descriptor)); - if (PyDict_SetItem(iter->dict, key.get(), val.get()) < 0) { - return PyErr_Format(PyExc_RuntimeError, - "SetItem failed in iterator construction."); - } - } - - - iter->iter = PyObject_GetIter(iter->dict); - - - return obj.release(); -} - -PyObject* Clear(PyObject* _self) { - ScalarMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - reflection->ClearField(message, self->parent_field_descriptor); - - Py_RETURN_NONE; -} - -PyObject* Contains(PyObject* _self, PyObject* key) { - ScalarMapContainer* self = GetMap(_self); - - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = size - 1; i >= 0; i--) { - const Message& entry = reflection->GetRepeatedMessage( - *message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, &entry, key); - if (matches < 0) return NULL; - if (matches) { - Py_RETURN_TRUE; - } - } - - Py_RETURN_FALSE; -} - -PyObject* Get(PyObject* self, PyObject* args) { - PyObject* key; - PyObject* default_value = NULL; - if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { - return NULL; - } - - ScopedPyObjectPtr is_present(Contains(self, key)); - if (is_present.get() == NULL) { - return NULL; - } - - if (PyObject_IsTrue(is_present.get())) { - return GetItem(self, key); - } else { - if (default_value != NULL) { - Py_INCREF(default_value); - return default_value; - } else { - Py_RETURN_NONE; - } - } -} - -static void Dealloc(PyObject* _self) { - ScalarMapContainer* self = GetMap(_self); - self->owner.reset(); - Py_TYPE(_self)->tp_free(_self); -} - -static PyMethodDef Methods[] = { - { "__contains__", Contains, METH_O, - "Tests whether a key is a member of the map." }, - { "clear", (PyCFunction)Clear, METH_NOARGS, - "Removes all elements from the map." }, - { "get", Get, METH_VARARGS, - "Gets the value for the given key if present, or otherwise a default" }, - /* - { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, - "Makes a deep copy of the class." }, - { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, - "Outputs picklable representation of the repeated field." }, - */ - {NULL, NULL}, -}; - -} // namespace scalar_map_container - -namespace scalar_map_iterator { - -static void Dealloc(PyObject* _self) { - ScalarMapIterator* self = GetIter(_self); - Py_DECREF(self->dict); - Py_DECREF(self->iter); - Py_DECREF(self->container); - Py_TYPE(_self)->tp_free(_self); -} - -PyObject* IterNext(PyObject* _self) { - ScalarMapIterator* self = GetIter(_self); - - // This won't catch mutations to the map performed by MergeFrom(); no easy way - // to address that. - if (self->version != self->container->version) { - return PyErr_Format(PyExc_RuntimeError, - "Map modified during iteration."); - } - - return PyIter_Next(self->iter); -} - -} // namespace scalar_map_iterator - - -#if PY_MAJOR_VERSION >= 3 - static PyType_Slot ScalarMapContainer_Type_slots[] = { - {Py_tp_dealloc, (void *)scalar_map_container::Dealloc}, - {Py_mp_length, (void *)scalar_map_container::Length}, - {Py_mp_subscript, (void *)scalar_map_container::GetItem}, - {Py_mp_ass_subscript, (void *)scalar_map_container::SetItem}, - {Py_tp_methods, (void *)scalar_map_container::Methods}, - {Py_tp_iter, (void *)scalar_map_container::GetIterator}, - {0, 0}, - }; - - PyType_Spec ScalarMapContainer_Type_spec = { - FULL_MODULE_NAME ".ScalarMapContainer", - sizeof(ScalarMapContainer), - 0, - Py_TPFLAGS_DEFAULT, - ScalarMapContainer_Type_slots - }; - PyObject *ScalarMapContainer_Type; -#else - static PyMappingMethods MpMethods = { - scalar_map_container::Length, // mp_length - scalar_map_container::GetItem, // mp_subscript - scalar_map_container::SetItem, // mp_ass_subscript - }; - - PyTypeObject ScalarMapContainer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".ScalarMapContainer", // tp_name - sizeof(ScalarMapContainer), // tp_basicsize - 0, // tp_itemsize - scalar_map_container::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - &MpMethods, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A scalar map container", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - scalar_map_container::GetIterator, // tp_iter - 0, // tp_iternext - scalar_map_container::Methods, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - }; -#endif - -PyTypeObject ScalarMapIterator_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".ScalarMapIterator", // tp_name - sizeof(ScalarMapIterator), // tp_basicsize - 0, // tp_itemsize - scalar_map_iterator::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A scalar map iterator", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - PyObject_SelfIter, // tp_iter - scalar_map_iterator::IterNext, // tp_iternext - 0, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init -}; - -} // namespace python -} // namespace protobuf -} // namespace google diff --git a/python/google/protobuf/pyext/scalar_map_container.h b/python/google/protobuf/pyext/scalar_map_container.h deleted file mode 100644 index 4d663b88..00000000 --- a/python/google/protobuf/pyext/scalar_map_container.h +++ /dev/null @@ -1,119 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__ -#define GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__ - -#include <Python.h> - -#include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif - -#include <google/protobuf/descriptor.h> - -namespace google { -namespace protobuf { - -class Message; - -#ifdef _SHARED_PTR_H -using std::shared_ptr; -#else -using internal::shared_ptr; -#endif - -namespace python { - -struct CMessage; - -struct ScalarMapContainer { - PyObject_HEAD; - - // This is the top-level C++ Message object that owns the whole - // proto tree. Every Python ScalarMapContainer holds a - // reference to it in order to keep it alive as long as there's a - // Python object that references any part of the tree. - shared_ptr<Message> owner; - - // Pointer to the C++ Message that contains this container. The - // ScalarMapContainer does not own this pointer. - Message* message; - - // Weak reference to a parent CMessage object (i.e. may be NULL.) - // - // Used to make sure all ancestors are also mutable when first - // modifying the container. - CMessage* parent; - - // Pointer to the parent's descriptor that describes this - // field. Used together with the parent's message when making a - // default message instance mutable. - // The pointer is owned by the global DescriptorPool. - const FieldDescriptor* parent_field_descriptor; - const FieldDescriptor* key_field_descriptor; - const FieldDescriptor* value_field_descriptor; - - // We bump this whenever we perform a mutation, to invalidate existing - // iterators. - uint64 version; -}; - -#if PY_MAJOR_VERSION >= 3 - extern PyObject *ScalarMapContainer_Type; - extern PyType_Spec ScalarMapContainer_Type_spec; -#else - extern PyTypeObject ScalarMapContainer_Type; -#endif -extern PyTypeObject ScalarMapIterator_Type; - -namespace scalar_map_container { - -// Builds a ScalarMapContainer object, from a parent message and a -// field descriptor. -extern PyObject *NewContainer( - CMessage* parent, const FieldDescriptor* parent_field_descriptor); - -// Releases the messages in the container to a new message. -// -// Returns 0 on success, -1 on failure. -int Release(ScalarMapContainer* self); - -// Set the owner field of self and any children of self. -void SetOwner(ScalarMapContainer* self, - const shared_ptr<Message>& new_owner); - -} // namespace scalar_map_container -} // namespace python -} // namespace protobuf - -} // namespace google -#endif // GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__ |