aboutsummaryrefslogtreecommitdiffhomepage
path: root/python/google/protobuf/pyext
diff options
context:
space:
mode:
authorGravatar Feng Xiao <xfxyjwf@gmail.com>2014-11-10 17:34:54 -0800
committerGravatar Feng Xiao <xfxyjwf@gmail.com>2014-11-10 17:34:54 -0800
commit6ef984af4b0c63c1c33127a12dcfc8e6359f0c9e (patch)
treed17c61ff9f3ae28224fbddac6d26bfc59e2cf755 /python/google/protobuf/pyext
parentbaca1a8a1aa180c42de6278d3b8286c4496c6a10 (diff)
Down-integrate from internal code base.
Diffstat (limited to 'python/google/protobuf/pyext')
-rw-r--r--python/google/protobuf/pyext/cpp_message.py6
-rw-r--r--python/google/protobuf/pyext/descriptor.cc222
-rw-r--r--python/google/protobuf/pyext/descriptor.h70
-rw-r--r--python/google/protobuf/pyext/extension_dict.cc110
-rw-r--r--python/google/protobuf/pyext/extension_dict.h22
-rw-r--r--python/google/protobuf/pyext/message.cc495
-rw-r--r--python/google/protobuf/pyext/message.h45
-rw-r--r--python/google/protobuf/pyext/proto2_api_test.proto2
-rw-r--r--python/google/protobuf/pyext/python.proto2
-rw-r--r--python/google/protobuf/pyext/repeated_composite_container.cc135
-rw-r--r--python/google/protobuf/pyext/repeated_composite_container.h13
-rw-r--r--python/google/protobuf/pyext/repeated_scalar_container.cc102
-rw-r--r--python/google/protobuf/pyext/repeated_scalar_container.h12
-rw-r--r--python/google/protobuf/pyext/scoped_pyobject_ptr.h2
14 files changed, 692 insertions, 546 deletions
diff --git a/python/google/protobuf/pyext/cpp_message.py b/python/google/protobuf/pyext/cpp_message.py
index dcf34a02..037bb72c 100644
--- a/python/google/protobuf/pyext/cpp_message.py
+++ b/python/google/protobuf/pyext/cpp_message.py
@@ -53,9 +53,5 @@ def NewMessage(bases, message_descriptor, dictionary):
def InitMessage(message_descriptor, cls):
- """Constructs a new message instance (called before instance's __init__)."""
-
- def SubInit(self, **kwargs):
- super(cls, self).__init__(message_descriptor, **kwargs)
- cls.__init__ = SubInit
+ """Finalizes the creation of a message class."""
cls.AddDescriptors(message_descriptor)
diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc
index 3f7be73c..55bb0b72 100644
--- a/python/google/protobuf/pyext/descriptor.cc
+++ b/python/google/protobuf/pyext/descriptor.cc
@@ -35,6 +35,7 @@
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/pyext/descriptor.h>
+#include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#define C(str) const_cast<char*>(str)
@@ -46,7 +47,7 @@
#error "Python 3.0 - 3.2 are not supported."
#else
#define PyString_AsString(ob) \
- (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob))
+ (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob))
#endif
#endif
@@ -65,10 +66,80 @@ namespace python {
static google::protobuf::DescriptorPool* g_descriptor_pool = NULL;
+namespace cmessage_descriptor {
+
+static void Dealloc(CMessageDescriptor* self) {
+ Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
+}
+
+static PyObject* GetFullName(CMessageDescriptor* self, void *closure) {
+ return PyString_FromStringAndSize(
+ self->descriptor->full_name().c_str(),
+ self->descriptor->full_name().size());
+}
+
+static PyObject* GetName(CMessageDescriptor *self, void *closure) {
+ return PyString_FromStringAndSize(
+ self->descriptor->name().c_str(),
+ self->descriptor->name().size());
+}
+
+static PyGetSetDef Getters[] = {
+ { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
+ { C("name"), (getter)GetName, NULL, "Unqualified name", NULL},
+ {NULL}
+};
+
+} // namespace cmessage_descriptor
+
+PyTypeObject CMessageDescriptor_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ C("google.protobuf.internal."
+ "_net_proto2___python."
+ "CMessageDescriptor"), // tp_name
+ sizeof(CMessageDescriptor), // tp_basicsize
+ 0, // tp_itemsize
+ (destructor)cmessage_descriptor::Dealloc, // tp_dealloc
+ 0, // tp_print
+ 0, // tp_getattr
+ 0, // tp_setattr
+ 0, // tp_compare
+ 0, // tp_repr
+ 0, // tp_as_number
+ 0, // tp_as_sequence
+ 0, // tp_as_mapping
+ 0, // tp_hash
+ 0, // tp_call
+ 0, // tp_str
+ 0, // tp_getattro
+ 0, // tp_setattro
+ 0, // tp_as_buffer
+ Py_TPFLAGS_DEFAULT, // tp_flags
+ C("A Message Descriptor"), // tp_doc
+ 0, // tp_traverse
+ 0, // tp_clear
+ 0, // tp_richcompare
+ 0, // tp_weaklistoffset
+ 0, // tp_iter
+ 0, // tp_iternext
+ 0, // tp_methods
+ 0, // tp_members
+ cmessage_descriptor::Getters, // tp_getset
+ 0, // tp_base
+ 0, // tp_dict
+ 0, // tp_descr_get
+ 0, // tp_descr_set
+ 0, // tp_dictoffset
+ 0, // tp_init
+ PyType_GenericAlloc, // tp_alloc
+ PyType_GenericNew, // tp_new
+ PyObject_Del, // tp_free
+};
+
+
namespace cfield_descriptor {
static void Dealloc(CFieldDescriptor* self) {
- Py_CLEAR(self->descriptor_field);
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
@@ -98,7 +169,7 @@ static PyObject* GetID(CFieldDescriptor *self, void *closure) {
static PyGetSetDef Getters[] = {
{ C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
- { C("name"), (getter)GetName, NULL, "last name", NULL},
+ { C("name"), (getter)GetName, NULL, "Unqualified name", NULL},
{ C("cpp_type"), (getter)GetCppType, NULL, "C++ Type", NULL},
{ C("label"), (getter)GetLabel, NULL, "Label", NULL},
{ C("id"), (getter)GetID, NULL, "ID", NULL},
@@ -151,13 +222,56 @@ PyTypeObject CFieldDescriptor_Type = {
PyObject_Del, // tp_free
};
+
namespace cdescriptor_pool {
-static void Dealloc(CDescriptorPool* self) {
+PyDescriptorPool* NewDescriptorPool() {
+ PyDescriptorPool* cdescriptor_pool = PyObject_New(
+ PyDescriptorPool, &PyDescriptorPool_Type);
+ if (cdescriptor_pool == NULL) {
+ return NULL;
+ }
+
+ // Build a DescriptorPool for messages only declared in Python libraries.
+ // generated_pool() contains all messages linked in C++ libraries, and is used
+ // as underlay.
+ cdescriptor_pool->pool = new google::protobuf::DescriptorPool(
+ google::protobuf::DescriptorPool::generated_pool());
+
+ // TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same
+ // storage.
+ cdescriptor_pool->classes_by_descriptor =
+ new PyDescriptorPool::ClassesByMessageMap();
+
+ return cdescriptor_pool;
+}
+
+static void Dealloc(PyDescriptorPool* self) {
+ for (auto it : (*self->classes_by_descriptor)) {
+ Py_DECREF(it.second);
+ }
+ delete self->classes_by_descriptor;
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
-static PyObject* NewCDescriptor(
+const google::protobuf::Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
+ const string& name) {
+ return self->pool->FindMessageTypeByName(name);
+}
+
+static PyObject* NewCMessageDescriptor(
+ const google::protobuf::Descriptor* message_descriptor) {
+ CMessageDescriptor* cmessage_descriptor = PyObject_New(
+ CMessageDescriptor, &CMessageDescriptor_Type);
+ if (cmessage_descriptor == NULL) {
+ return NULL;
+ }
+ cmessage_descriptor->descriptor = message_descriptor;
+
+ return reinterpret_cast<PyObject*>(cmessage_descriptor);
+}
+
+static PyObject* NewCFieldDescriptor(
const google::protobuf::FieldDescriptor* field_descriptor) {
CFieldDescriptor* cfield_descriptor = PyObject_New(
CFieldDescriptor, &CFieldDescriptor_Type);
@@ -165,12 +279,61 @@ static PyObject* NewCDescriptor(
return NULL;
}
cfield_descriptor->descriptor = field_descriptor;
- cfield_descriptor->descriptor_field = NULL;
return reinterpret_cast<PyObject*>(cfield_descriptor);
}
-PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name) {
+// Add a message class to our database.
+const google::protobuf::Descriptor* RegisterMessageClass(
+ PyDescriptorPool* self, PyObject *message_class, PyObject* descriptor) {
+ ScopedPyObjectPtr full_message_name(
+ PyObject_GetAttrString(descriptor, "full_name"));
+ const char* full_name = PyString_AsString(full_message_name);
+ if (full_name == NULL) {
+ return NULL;
+ }
+ const Descriptor *message_descriptor =
+ self->pool->FindMessageTypeByName(full_name);
+ if (!message_descriptor) {
+ PyErr_Format(PyExc_TypeError, "Could not find C++ descriptor for '%s'",
+ full_name);
+ return NULL;
+ }
+ Py_INCREF(message_class);
+ auto ret = self->classes_by_descriptor->insert(
+ make_pair(message_descriptor, message_class));
+ if (!ret.second) {
+ // Update case: DECREF the previous value.
+ Py_DECREF(ret.first->second);
+ ret.first->second = message_class;
+ }
+
+ // Also add the C++ descriptor to the Python descriptor class.
+ ScopedPyObjectPtr cdescriptor(NewCMessageDescriptor(message_descriptor));
+ if (cdescriptor == NULL) {
+ return NULL;
+ }
+ if (PyObject_SetAttrString(
+ descriptor, "_cdescriptor", cdescriptor) < 0) {
+ return NULL;
+ }
+ return message_descriptor;
+}
+
+// Retrieve the message class added to our database.
+PyObject *GetMessageClass(PyDescriptorPool* self,
+ const Descriptor *message_descriptor) {
+ auto ret = self->classes_by_descriptor->find(message_descriptor);
+ if (ret == self->classes_by_descriptor->end()) {
+ PyErr_Format(PyExc_TypeError, "No message class registered for '%s'",
+ message_descriptor->full_name().c_str());
+ return NULL;
+ } else {
+ return ret->second;
+ }
+}
+
+PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name) {
const char* full_field_name = PyString_AsString(name);
if (full_field_name == NULL) {
return NULL;
@@ -186,10 +349,10 @@ PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name) {
return NULL;
}
- return NewCDescriptor(field_descriptor);
+ return NewCFieldDescriptor(field_descriptor);
}
-PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg) {
+PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
const char* full_field_name = PyString_AsString(arg);
if (full_field_name == NULL) {
return NULL;
@@ -203,7 +366,7 @@ PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg) {
return NULL;
}
- return NewCDescriptor(field_descriptor);
+ return NewCFieldDescriptor(field_descriptor);
}
static PyMethodDef Methods[] = {
@@ -220,12 +383,12 @@ static PyMethodDef Methods[] = {
} // namespace cdescriptor_pool
-PyTypeObject CDescriptorPool_Type = {
+PyTypeObject PyDescriptorPool_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
C("google.protobuf.internal."
"_net_proto2___python."
"CFieldDescriptor"), // tp_name
- sizeof(CDescriptorPool), // tp_basicsize
+ sizeof(PyDescriptorPool), // tp_basicsize
0, // tp_itemsize
(destructor)cdescriptor_pool::Dealloc, // tp_dealloc
0, // tp_print
@@ -259,29 +422,11 @@ PyTypeObject CDescriptorPool_Type = {
0, // tp_descr_set
0, // tp_dictoffset
0, // tp_init
- PyType_GenericAlloc, // tp_alloc
- PyType_GenericNew, // tp_new
+ 0, // tp_alloc
+ 0, // tp_new
PyObject_Del, // tp_free
};
-google::protobuf::DescriptorPool* GetDescriptorPool() {
- if (g_descriptor_pool == NULL) {
- g_descriptor_pool = new google::protobuf::DescriptorPool(
- google::protobuf::DescriptorPool::generated_pool());
- }
- return g_descriptor_pool;
-}
-
-PyObject* Python_NewCDescriptorPool(PyObject* ignored, PyObject* args) {
- CDescriptorPool* cdescriptor_pool = PyObject_New(
- CDescriptorPool, &CDescriptorPool_Type);
- if (cdescriptor_pool == NULL) {
- return NULL;
- }
- cdescriptor_pool->pool = GetDescriptorPool();
- return reinterpret_cast<PyObject*>(cdescriptor_pool);
-}
-
// Collects errors that occur during proto file building to allow them to be
// propagated in the python exception instead of only living in ERROR logs.
@@ -321,6 +466,8 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* arg) {
return NULL;
}
+ // If the file was already part of a C++ library, all its descriptors are in
+ // the underlying pool. No need to do anything else.
if (google::protobuf::DescriptorPool::generated_pool()->FindFileByName(
file_proto.name()) != NULL) {
Py_RETURN_NONE;
@@ -328,8 +475,8 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* arg) {
BuildFileErrorCollector error_collector;
const google::protobuf::FileDescriptor* descriptor =
- GetDescriptorPool()->BuildFileCollectingErrors(file_proto,
- &error_collector);
+ GetDescriptorPool()->pool->BuildFileCollectingErrors(file_proto,
+ &error_collector);
if (descriptor == NULL) {
PyErr_Format(PyExc_TypeError,
"Couldn't build proto file into descriptor pool!\n%s",
@@ -341,12 +488,13 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* arg) {
}
bool InitDescriptor() {
- CFieldDescriptor_Type.tp_new = PyType_GenericNew;
+ if (PyType_Ready(&CMessageDescriptor_Type) < 0)
+ return false;
if (PyType_Ready(&CFieldDescriptor_Type) < 0)
return false;
- CDescriptorPool_Type.tp_new = PyType_GenericNew;
- if (PyType_Ready(&CDescriptorPool_Type) < 0)
+ PyDescriptorPool_Type.tp_new = PyType_GenericNew;
+ if (PyType_Ready(&PyDescriptorPool_Type) < 0)
return false;
return true;
diff --git a/python/google/protobuf/pyext/descriptor.h b/python/google/protobuf/pyext/descriptor.h
index ae7a1b9c..9e5957b5 100644
--- a/python/google/protobuf/pyext/descriptor.h
+++ b/python/google/protobuf/pyext/descriptor.h
@@ -36,6 +36,8 @@
#include <Python.h>
#include <structmember.h>
+#include <google/protobuf/stubs/hash.h>
+
#include <google/protobuf/descriptor.h>
#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
@@ -48,46 +50,90 @@ namespace google {
namespace protobuf {
namespace python {
+typedef struct CMessageDescriptor {
+ PyObject_HEAD
+
+ // The proto2 descriptor that this object represents.
+ const google::protobuf::Descriptor* descriptor;
+} CMessageDescriptor;
+
+
typedef struct CFieldDescriptor {
PyObject_HEAD
// The proto2 descriptor that this object represents.
const google::protobuf::FieldDescriptor* descriptor;
-
- // Reference to the original field object in the Python DESCRIPTOR.
- PyObject* descriptor_field;
} CFieldDescriptor;
-typedef struct {
+
+// Wraps operations to the global DescriptorPool which contains information
+// about all messages and fields.
+//
+// There is normally one pool per process. We make it a Python object only
+// because it contains many Python references.
+// TODO(amauryfa): See whether such objects can appear in reference cycles, and
+// consider adding support for the cyclic GC.
+//
+// "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool
+// namespace.
+typedef struct PyDescriptorPool {
PyObject_HEAD
- const google::protobuf::DescriptorPool* pool;
-} CDescriptorPool;
+ google::protobuf::DescriptorPool* pool;
+ // Make our own mapping to retrieve Python classes from C++ descriptors.
+ //
+ // Descriptor pointers stored here are owned by the DescriptorPool above.
+ // Python references to classes are owned by this PyDescriptorPool.
+ typedef hash_map<const Descriptor *, PyObject *> ClassesByMessageMap;
+ ClassesByMessageMap *classes_by_descriptor;
+} PyDescriptorPool;
+
+
+extern PyTypeObject CMessageDescriptor_Type;
extern PyTypeObject CFieldDescriptor_Type;
-extern PyTypeObject CDescriptorPool_Type;
+extern PyTypeObject PyDescriptorPool_Type;
+
namespace cdescriptor_pool {
+// Builds a new DescriptorPool. Normally called only once per process.
+PyDescriptorPool* NewDescriptorPool();
+
+// Looks up a message by name.
+// Returns a message Descriptor, or NULL if not found.
+const google::protobuf::Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
+ const string& name);
+
+// Registers a new Python class for the given message descriptor.
+// Returns the message Descriptor.
+// On error, returns NULL with a Python exception set.
+const google::protobuf::Descriptor* RegisterMessageClass(
+ PyDescriptorPool* self, PyObject *message_class, PyObject *descriptor);
+
+// Retrieves the Python class registered with the given message descriptor.
+//
+// Returns a *borrowed* reference if found, otherwise returns NULL with an
+// exception set.
+PyObject *GetMessageClass(PyDescriptorPool* self,
+ const Descriptor *message_descriptor);
+
// Looks up a field by name. Returns a CDescriptor corresponding to
// the field on success, or NULL on failure.
//
// Returns a new reference.
-PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name);
+PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name);
// Looks up an extension by name. Returns a CDescriptor corresponding
// to the field on success, or NULL on failure.
//
// Returns a new reference.
-PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg);
-
+PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg);
} // namespace cdescriptor_pool
-PyObject* Python_NewCDescriptorPool(PyObject* ignored, PyObject* args);
PyObject* Python_BuildFile(PyObject* ignored, PyObject* args);
bool InitDescriptor();
-google::protobuf::DescriptorPool* GetDescriptorPool();
} // namespace python
} // namespace protobuf
diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc
index 3861c794..d83b57d5 100644
--- a/python/google/protobuf/pyext/extension_dict.cc
+++ b/python/google/protobuf/pyext/extension_dict.cc
@@ -62,22 +62,6 @@ static google::protobuf::Message* GetMessage(ExtensionDict* self) {
}
}
-CFieldDescriptor* InternalGetCDescriptorFromExtension(PyObject* extension) {
- PyObject* cdescriptor = PyObject_GetAttrString(extension, "_cdescriptor");
- if (cdescriptor == NULL) {
- PyErr_SetString(PyExc_KeyError, "Unregistered extension.");
- return NULL;
- }
- if (!PyObject_TypeCheck(cdescriptor, &CFieldDescriptor_Type)) {
- PyErr_SetString(PyExc_TypeError, "Not a CFieldDescriptor");
- Py_DECREF(cdescriptor);
- return NULL;
- }
- CFieldDescriptor* descriptor =
- reinterpret_cast<CFieldDescriptor*>(cdescriptor);
- return descriptor;
-}
-
PyObject* len(ExtensionDict* self) {
#if PY_MAJOR_VERSION >= 3
return PyLong_FromLong(PyDict_Size(self->values));
@@ -118,16 +102,15 @@ int ReleaseExtension(ExtensionDict* self,
}
PyObject* subscript(ExtensionDict* self, PyObject* key) {
- CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
- key);
- if (cdescriptor == NULL) {
+ const google::protobuf::FieldDescriptor* descriptor =
+ cmessage::GetExtensionDescriptor(key);
+ if (descriptor == NULL) {
return NULL;
}
- ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
- const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor;
- if (descriptor == NULL) {
+ if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) {
return NULL;
}
+
if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) {
return cmessage::InternalGetScalar(self->parent, descriptor);
@@ -142,7 +125,7 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
PyObject* sub_message = cmessage::InternalGetSubMessage(
- self->parent, cdescriptor);
+ self->parent, descriptor);
if (sub_message == NULL) {
return NULL;
}
@@ -152,33 +135,21 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
- // COPIED
- PyObject* py_container = PyObject_CallObject(
- reinterpret_cast<PyObject*>(&RepeatedCompositeContainer_Type),
- NULL);
+ PyObject *message_class = cdescriptor_pool::GetMessageClass(
+ GetDescriptorPool(), descriptor->message_type());
+ if (message_class == NULL) {
+ return NULL;
+ }
+ PyObject* py_container = repeated_composite_container::NewContainer(
+ self->parent, descriptor, message_class);
if (py_container == NULL) {
return NULL;
}
- RepeatedCompositeContainer* container =
- reinterpret_cast<RepeatedCompositeContainer*>(py_container);
- PyObject* field = cdescriptor->descriptor_field;
- PyObject* message_type = PyObject_GetAttrString(field, "message_type");
- PyObject* concrete_class = PyObject_GetAttrString(message_type,
- "_concrete_class");
- container->owner = self->owner;
- container->parent = self->parent;
- container->message = self->parent->message;
- container->parent_field = cdescriptor;
- container->subclass_init = concrete_class;
- Py_DECREF(message_type);
PyDict_SetItem(self->values, key, py_container);
return py_container;
} else {
- // COPIED
- ScopedPyObjectPtr init_args(PyTuple_Pack(2, self->parent, cdescriptor));
- PyObject* py_container = PyObject_CallObject(
- reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type),
- init_args);
+ PyObject* py_container = repeated_scalar_container::NewContainer(
+ self->parent, descriptor);
if (py_container == NULL) {
return NULL;
}
@@ -191,13 +162,15 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
}
int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
- CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
- key);
- if (cdescriptor == NULL) {
+ const google::protobuf::FieldDescriptor* descriptor =
+ cmessage::GetExtensionDescriptor(key);
+ if (descriptor == NULL) {
+ return -1;
+ }
+ if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) {
return -1;
}
- ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
- const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor;
+
if (descriptor->label() != FieldDescriptor::LABEL_OPTIONAL ||
descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
PyErr_SetString(PyExc_TypeError, "Extension is repeated and/or composite "
@@ -214,20 +187,18 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
}
PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
- CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
- extension);
- if (cdescriptor == NULL) {
+ const google::protobuf::FieldDescriptor* descriptor =
+ cmessage::GetExtensionDescriptor(extension);
+ if (descriptor == NULL) {
return NULL;
}
- ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
PyObject* value = PyDict_GetItem(self->values, extension);
if (value != NULL) {
- if (ReleaseExtension(self, value, cdescriptor->descriptor) < 0) {
+ if (ReleaseExtension(self, value, descriptor) < 0) {
return NULL;
}
}
- if (cmessage::ClearFieldByDescriptor(self->parent,
- cdescriptor->descriptor) == NULL) {
+ if (cmessage::ClearFieldByDescriptor(self->parent, descriptor) == NULL) {
return NULL;
}
if (PyDict_DelItem(self->values, extension) < 0) {
@@ -237,14 +208,12 @@ PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
}
PyObject* HasExtension(ExtensionDict* self, PyObject* extension) {
- CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
- extension);
- if (cdescriptor == NULL) {
+ const google::protobuf::FieldDescriptor* descriptor =
+ cmessage::GetExtensionDescriptor(extension);
+ if (descriptor == NULL) {
return NULL;
}
- ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
- PyObject* result = cmessage::HasFieldByDescriptor(
- self->parent, cdescriptor->descriptor);
+ PyObject* result = cmessage::HasFieldByDescriptor(self->parent, descriptor);
return result;
}
@@ -263,11 +232,18 @@ PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* name) {
}
}
-int init(ExtensionDict* self, PyObject* args, PyObject* kwargs) {
- self->parent = NULL;
- self->message = NULL;
+ExtensionDict* NewExtensionDict(CMessage *parent) {
+ ExtensionDict* self = reinterpret_cast<ExtensionDict*>(
+ PyType_GenericAlloc(&ExtensionDict_Type, 0));
+ if (self == NULL) {
+ return NULL;
+ }
+
+ self->parent = parent; // Store a borrowed reference.
+ self->message = parent->message;
+ self->owner = parent->owner;
self->values = PyDict_New();
- return 0;
+ return self;
}
void dealloc(ExtensionDict* self) {
@@ -330,7 +306,7 @@ PyTypeObject ExtensionDict_Type = {
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
- (initproc)extension_dict::init, // tp_init
+ 0, // tp_init
};
} // namespace python
diff --git a/python/google/protobuf/pyext/extension_dict.h b/python/google/protobuf/pyext/extension_dict.h
index 13c874a4..47625e23 100644
--- a/python/google/protobuf/pyext/extension_dict.h
+++ b/python/google/protobuf/pyext/extension_dict.h
@@ -53,13 +53,26 @@ using internal::shared_ptr;
namespace python {
struct CMessage;
-struct CFieldDescriptor;
typedef struct ExtensionDict {
PyObject_HEAD;
+
+ // This is the top-level C++ Message object that owns the whole
+ // proto tree. Every Python container class holds a
+ // reference to it in order to keep it alive as long as there's a
+ // Python object that references any part of the tree.
shared_ptr<Message> owner;
+
+ // Weak reference to parent message. Used to make sure
+ // the parent is writable when an extension field is modified.
CMessage* parent;
+
+ // Pointer to the C++ Message that this ExtensionDict extends.
+ // Not owned by us.
Message* message;
+
+ // A dict of child messages, indexed by Extension descriptors.
+ // Similar to CMessage::composite_fields.
PyObject* values;
} ExtensionDict;
@@ -67,11 +80,8 @@ extern PyTypeObject ExtensionDict_Type;
namespace extension_dict {
-// Gets the _cdescriptor reference to a CFieldDescriptor object given a
-// python descriptor object.
-//
-// Returns a new reference.
-CFieldDescriptor* InternalGetCDescriptorFromExtension(PyObject* extension);
+// Builds an Extensions dict for a specific message.
+ExtensionDict* NewExtensionDict(CMessage *parent);
// Gets the number of extension values in this ExtensionDict as a python object.
//
diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
index 9fb7083f..cd956e0e 100644
--- a/python/google/protobuf/pyext/message.cc
+++ b/python/google/protobuf/pyext/message.cc
@@ -71,7 +71,7 @@
#error "Python 3.0 - 3.2 are not supported."
#else
#define PyString_AsString(ob) \
- (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob))
+ (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob))
#endif
#endif
@@ -81,7 +81,9 @@ namespace python {
// Forward declarations
namespace cmessage {
-static PyObject* GetDescriptor(CMessage* self, PyObject* name);
+static const google::protobuf::FieldDescriptor* GetFieldDescriptor(
+ CMessage* self, PyObject* name);
+static const google::protobuf::Descriptor* GetMessageDescriptor(PyTypeObject* cls);
static string GetMessageName(CMessage* self);
int InternalReleaseFieldByDescriptor(
const google::protobuf::FieldDescriptor* field_descriptor,
@@ -147,12 +149,15 @@ int ForEachCompositeField(CMessage* self, Visitor visitor) {
PyObject* key;
PyObject* field;
+ // Never use self->message in this function, it may be already freed.
+ const google::protobuf::Descriptor* message_descriptor =
+ cmessage::GetMessageDescriptor(Py_TYPE(self));
+
// Visit normal fields.
while (PyDict_Next(self->composite_fields, &pos, &key, &field)) {
- PyObject* cdescriptor = cmessage::GetDescriptor(self, key);
- if (cdescriptor != NULL) {
- const google::protobuf::FieldDescriptor* descriptor =
- reinterpret_cast<CFieldDescriptor*>(cdescriptor)->descriptor;
+ const google::protobuf::FieldDescriptor* descriptor =
+ message_descriptor->FindFieldByName(PyString_AsString(key));
+ if (descriptor != NULL) {
if (VisitCompositeField(descriptor, field, visitor) == -1)
return -1;
}
@@ -161,11 +166,11 @@ int ForEachCompositeField(CMessage* self, Visitor visitor) {
// Visit extension fields.
if (self->extensions != NULL) {
while (PyDict_Next(self->extensions->values, &pos, &key, &field)) {
- CFieldDescriptor* cdescriptor =
- extension_dict::InternalGetCDescriptorFromExtension(key);
- if (cdescriptor == NULL)
+ const google::protobuf::FieldDescriptor* descriptor =
+ cmessage::GetExtensionDescriptor(key);
+ if (descriptor == NULL)
return -1;
- if (VisitCompositeField(cdescriptor->descriptor, field, visitor) == -1)
+ if (VisitCompositeField(descriptor, field, visitor) == -1)
return -1;
}
}
@@ -191,18 +196,19 @@ PyObject* PickleError_class;
// Constant PyString values used for GetAttr/GetItem.
static PyObject* kDESCRIPTOR;
-static PyObject* k__descriptors;
+static PyObject* k_cdescriptor;
static PyObject* kfull_name;
static PyObject* kname;
-static PyObject* kmessage_type;
-static PyObject* kis_extendable;
static PyObject* kextensions_by_name;
static PyObject* k_extensions_by_name;
static PyObject* k_extensions_by_number;
-static PyObject* k_concrete_class;
static PyObject* kfields_by_name;
-static CDescriptorPool* descriptor_pool;
+static PyDescriptorPool* descriptor_pool;
+
+PyDescriptorPool* GetDescriptorPool() {
+ return descriptor_pool;
+}
/* Is 64bit */
void FormatTypeError(PyObject* arg, char* expected_types) {
@@ -313,12 +319,12 @@ bool CheckAndSetString(
}
if (PyBytes_Check(arg)) {
- PyObject* unicode = PyUnicode_FromEncodedObject(arg, "ascii", NULL);
+ PyObject* unicode = PyUnicode_FromEncodedObject(arg, "utf-8", NULL);
if (unicode == NULL) {
PyObject* repr = PyObject_Repr(arg);
PyErr_Format(PyExc_ValueError,
- "%s has type str, but isn't in 7-bit ASCII "
- "encoding. Non-ASCII strings must be converted to "
+ "%s has type str, but isn't valid UTF-8 "
+ "encoding. Non-UTF-8 strings must be converted to "
"unicode objects before being added.",
PyString_AsString(repr));
Py_DECREF(repr);
@@ -335,12 +341,9 @@ bool CheckAndSetString(
PyObject* encoded_string = NULL;
if (descriptor->type() == google::protobuf::FieldDescriptor::TYPE_STRING) {
if (PyBytes_Check(arg)) {
-#if PY_MAJOR_VERSION < 3
- encoded_string = PyString_AsEncodedObject(arg, "utf-8", NULL);
-#else
+ // The bytes were already validated as correctly encoded UTF-8 above.
encoded_string = arg; // Already encoded.
Py_INCREF(encoded_string);
-#endif
} else {
encoded_string = PyUnicode_AsEncodedObject(arg, "utf-8", NULL);
}
@@ -391,6 +394,17 @@ PyObject* ToStringObject(
return result;
}
+bool CheckFieldBelongsToMessage(const google::protobuf::FieldDescriptor* field_descriptor,
+ const google::protobuf::Message* message) {
+ if (message->GetDescriptor() == field_descriptor->containing_type()) {
+ return true;
+ }
+ PyErr_Format(PyExc_KeyError, "Field '%s' does not belong to message '%s'",
+ field_descriptor->full_name().c_str(),
+ message->GetDescriptor()->full_name().c_str());
+ return false;
+}
+
google::protobuf::DynamicMessageFactory* global_message_factory;
namespace cmessage {
@@ -489,7 +503,7 @@ int AssureWritable(CMessage* self) {
google::protobuf::Message* parent_message = self->parent->message;
google::protobuf::Message* mutable_message = GetMutableMessage(
self->parent,
- self->parent_field->descriptor);
+ self->parent_field_descriptor);
if (mutable_message == NULL) {
return -1;
}
@@ -512,26 +526,61 @@ int AssureWritable(CMessage* self) {
// --- Globals:
-static PyObject* GetDescriptor(CMessage* self, PyObject* name) {
- PyObject* descriptors =
- PyDict_GetItem(Py_TYPE(self)->tp_dict, k__descriptors);
- if (descriptors == NULL) {
- PyErr_SetString(PyExc_TypeError, "No __descriptors");
+// Retrieve the C++ Descriptor of a message class.
+// On error, returns NULL with an exception set.
+static const google::protobuf::Descriptor* GetMessageDescriptor(PyTypeObject* cls) {
+ ScopedPyObjectPtr descriptor(PyObject_GetAttr(
+ reinterpret_cast<PyObject*>(cls), kDESCRIPTOR));
+ if (descriptor == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Message class has no DESCRIPTOR");
+ return NULL;
+ }
+ ScopedPyObjectPtr cdescriptor(PyObject_GetAttr(descriptor, k_cdescriptor));
+ if (cdescriptor == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Unregistered message.");
+ return NULL;
+ }
+ if (!PyObject_TypeCheck(cdescriptor, &CMessageDescriptor_Type)) {
+ PyErr_SetString(PyExc_TypeError, "Not a CMessageDescriptor");
return NULL;
}
+ return reinterpret_cast<CMessageDescriptor*>(cdescriptor.get())->descriptor;
+}
- return PyDict_GetItem(descriptors, name);
+// Retrieve a C++ FieldDescriptor for a message attribute.
+// The C++ message must be valid.
+// TODO(amauryfa): This function should stay internal, because exception
+// handling is not consistent.
+static const google::protobuf::FieldDescriptor* GetFieldDescriptor(
+ CMessage* self, PyObject* name) {
+ const google::protobuf::Descriptor *message_descriptor = self->message->GetDescriptor();
+ const char* field_name = PyString_AsString(name);
+ if (field_name == NULL) {
+ return NULL;
+ }
+ const google::protobuf::FieldDescriptor *field_descriptor =
+ message_descriptor->FindFieldByName(field_name);
+ if (field_descriptor == NULL) {
+ // Note: No exception is set!
+ return NULL;
+ }
+ return field_descriptor;
}
-static const google::protobuf::Message* CreateMessage(const char* message_type) {
- string message_name(message_type);
- const google::protobuf::Descriptor* descriptor =
- GetDescriptorPool()->FindMessageTypeByName(message_name);
- if (descriptor == NULL) {
- PyErr_SetString(PyExc_TypeError, message_type);
+// Retrieve a C++ FieldDescriptor for an extension handle.
+const google::protobuf::FieldDescriptor* GetExtensionDescriptor(PyObject* extension) {
+ ScopedPyObjectPtr cdescriptor(
+ PyObject_GetAttrString(extension, "_cdescriptor"));
+ if (cdescriptor == NULL) {
+ PyErr_SetString(PyExc_KeyError, "Unregistered extension.");
+ return NULL;
+ }
+ if (!PyObject_TypeCheck(cdescriptor, &CFieldDescriptor_Type)) {
+ PyErr_SetString(PyExc_TypeError, "Not a CFieldDescriptor");
+ Py_DECREF(cdescriptor);
return NULL;
}
- return global_message_factory->GetPrototype(descriptor);
+ return reinterpret_cast<CFieldDescriptor*>(cdescriptor.get())->descriptor;
}
// If cmessage_list is not NULL, this function releases values into the
@@ -627,39 +676,8 @@ int InternalDeleteRepeatedField(
return 0;
}
-int InitAttributes(CMessage* self, PyObject* arg, PyObject* kwargs) {
- ScopedPyObjectPtr descriptor;
- if (arg == NULL) {
- descriptor.reset(
- PyObject_GetAttr(reinterpret_cast<PyObject*>(self), kDESCRIPTOR));
- if (descriptor == NULL) {
- return NULL;
- }
- } else {
- descriptor.reset(arg);
- descriptor.inc();
- }
- ScopedPyObjectPtr is_extendable(PyObject_GetAttr(descriptor, kis_extendable));
- if (is_extendable == NULL) {
- return NULL;
- }
- int retcode = PyObject_IsTrue(is_extendable);
- if (retcode == -1) {
- return NULL;
- }
- if (retcode) {
- PyObject* py_extension_dict = PyObject_CallObject(
- reinterpret_cast<PyObject*>(&ExtensionDict_Type), NULL);
- if (py_extension_dict == NULL) {
- return NULL;
- }
- ExtensionDict* extension_dict = reinterpret_cast<ExtensionDict*>(
- py_extension_dict);
- extension_dict->parent = self;
- extension_dict->message = self->message;
- self->extensions = extension_dict;
- }
-
+// Initializes fields of a message. Used in constructors.
+int InitAttributes(CMessage* self, PyObject* kwargs) {
if (kwargs == NULL) {
return 0;
}
@@ -672,14 +690,12 @@ int InitAttributes(CMessage* self, PyObject* arg, PyObject* kwargs) {
PyErr_SetString(PyExc_ValueError, "Field name must be a string");
return -1;
}
- PyObject* py_cdescriptor = GetDescriptor(self, name);
- if (py_cdescriptor == NULL) {
+ const google::protobuf::FieldDescriptor* descriptor = GetFieldDescriptor(self, name);
+ if (descriptor == NULL) {
PyErr_Format(PyExc_ValueError, "Protocol message has no \"%s\" field.",
PyString_AsString(name));
return -1;
}
- const google::protobuf::FieldDescriptor* descriptor =
- reinterpret_cast<CFieldDescriptor*>(py_cdescriptor)->descriptor;
if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
ScopedPyObjectPtr container(GetAttr(self, name));
if (container == NULL) {
@@ -719,15 +735,19 @@ int InitAttributes(CMessage* self, PyObject* arg, PyObject* kwargs) {
return 0;
}
-static PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
- CMessage* self = reinterpret_cast<CMessage*>(type->tp_alloc(type, 0));
+// Allocates an incomplete Python Message: the caller must fill self->message,
+// self->owner and eventually self->parent.
+CMessage* NewEmptyMessage(PyObject* type,
+ const google::protobuf::Descriptor *descriptor) {
+ CMessage* self = reinterpret_cast<CMessage*>(
+ PyType_GenericAlloc(reinterpret_cast<PyTypeObject*>(type), 0));
if (self == NULL) {
return NULL;
}
self->message = NULL;
self->parent = NULL;
- self->parent_field = NULL;
+ self->parent_field_descriptor = NULL;
self->read_only = false;
self->extensions = NULL;
@@ -735,43 +755,58 @@ static PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
if (self->composite_fields == NULL) {
return NULL;
}
- return reinterpret_cast<PyObject*>(self);
-}
-
-PyObject* NewEmpty(PyObject* type) {
- return New(reinterpret_cast<PyTypeObject*>(type), NULL, NULL);
-}
-static int Init(CMessage* self, PyObject* args, PyObject* kwargs) {
- if (kwargs == NULL) {
- // TODO(anuraag): Set error
- return -1;
+ // If there are extension_ranges, the message is "extendable". Allocate a
+ // dictionary to store the extension fields.
+ if (descriptor->extension_range_count() > 0) {
+ // TODO(amauryfa): Delay the construction of this dict until extensions are
+ // really used on the object.
+ ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self);
+ if (extension_dict == NULL) {
+ return NULL;
+ }
+ self->extensions = extension_dict;
}
- PyObject* descriptor = PyTuple_GetItem(args, 0);
- if (descriptor == NULL || PyTuple_Size(args) != 1) {
- PyErr_SetString(PyExc_ValueError, "args must contain one arg: descriptor");
- return -1;
- }
+ return self;
+}
- ScopedPyObjectPtr py_message_type(PyObject_GetAttr(descriptor, kfull_name));
- if (py_message_type == NULL) {
- return -1;
+// The __new__ method of Message classes.
+// Creates a new C++ message and takes ownership.
+static PyObject* New(PyTypeObject* type,
+ PyObject* unused_args, PyObject* unused_kwargs) {
+ // Retrieve the message descriptor and the default instance (=prototype).
+ const google::protobuf::Descriptor* message_descriptor = GetMessageDescriptor(type);
+ if (message_descriptor == NULL) {
+ return NULL;
}
-
- const char* message_type = PyString_AsString(py_message_type.get());
- const google::protobuf::Message* message = CreateMessage(message_type);
- if (message == NULL) {
- return -1;
+ const google::protobuf::Message* default_message =
+ global_message_factory->GetPrototype(message_descriptor);
+ if (default_message == NULL) {
+ PyErr_SetString(PyExc_TypeError, message_descriptor->full_name().c_str());
+ return NULL;
}
- self->message = message->New();
+ CMessage* self = NewEmptyMessage(reinterpret_cast<PyObject*>(type),
+ message_descriptor);
+ if (self == NULL) {
+ return NULL;
+ }
+ self->message = default_message->New();
self->owner.reset(self->message);
- if (InitAttributes(self, descriptor, kwargs) < 0) {
+ return reinterpret_cast<PyObject*>(self);
+}
+
+// The __init__ method of Message classes.
+// It initializes fields from keywords passed to the constructor.
+static int Init(CMessage* self, PyObject* args, PyObject* kwargs) {
+ if (PyTuple_Size(args) != 0) {
+ PyErr_SetString(PyExc_TypeError, "No positional arguments allowed");
return -1;
}
- return 0;
+
+ return InitAttributes(self, kwargs);
}
// ---------------------------------------------------------------------
@@ -853,9 +888,7 @@ PyObject* IsInitialized(CMessage* self, PyObject* args) {
PyObject* HasFieldByDescriptor(
CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor) {
google::protobuf::Message* message = self->message;
- if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
- PyErr_SetString(PyExc_KeyError,
- "Field does not belong to message!");
+ if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
return NULL;
}
if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
@@ -1048,7 +1081,7 @@ int ReleaseSubMessage(google::protobuf::Message* message,
child_cmessage->message = released_message.get();
child_cmessage->owner.swap(released_message);
child_cmessage->parent = NULL;
- child_cmessage->parent_field = NULL;
+ child_cmessage->parent_field_descriptor = NULL;
child_cmessage->read_only = false;
return ForEachCompositeField(child_cmessage,
SetOwnerVisitor(child_cmessage->owner));
@@ -1090,10 +1123,8 @@ int InternalReleaseFieldByDescriptor(
int InternalReleaseField(CMessage* self, PyObject* composite_field,
PyObject* name) {
- PyObject* cdescriptor = GetDescriptor(self, name);
- if (cdescriptor != NULL) {
- const google::protobuf::FieldDescriptor* descriptor =
- reinterpret_cast<CFieldDescriptor*>(cdescriptor)->descriptor;
+ const google::protobuf::FieldDescriptor* descriptor = GetFieldDescriptor(self, name);
+ if (descriptor != NULL) {
return InternalReleaseFieldByDescriptor(
descriptor, composite_field, self->message);
}
@@ -1104,9 +1135,7 @@ int InternalReleaseField(CMessage* self, PyObject* composite_field,
PyObject* ClearFieldByDescriptor(
CMessage* self,
const google::protobuf::FieldDescriptor* descriptor) {
- if (!FIELD_BELONGS_TO_MESSAGE(descriptor, self->message)) {
- PyErr_SetString(PyExc_KeyError,
- "Field does not belong to message!");
+ if (!CheckFieldBelongsToMessage(descriptor, self->message)) {
return NULL;
}
AssureWritable(self);
@@ -1177,15 +1206,10 @@ PyObject* Clear(CMessage* self) {
// fields have been released.
if (self->extensions != NULL) {
Py_CLEAR(self->extensions);
- PyObject* py_extension_dict = PyObject_CallObject(
- reinterpret_cast<PyObject*>(&ExtensionDict_Type), NULL);
- if (py_extension_dict == NULL) {
+ ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self);
+ if (extension_dict == NULL) {
return NULL;
}
- ExtensionDict* extension_dict = reinterpret_cast<ExtensionDict*>(
- py_extension_dict);
- extension_dict->parent = self;
- extension_dict->message = self->message;
self->extensions = extension_dict;
}
PyDict_Clear(self->composite_fields);
@@ -1196,8 +1220,8 @@ PyObject* Clear(CMessage* self) {
// ---------------------------------------------------------------------
static string GetMessageName(CMessage* self) {
- if (self->parent_field != NULL) {
- return self->parent_field->descriptor->full_name();
+ if (self->parent_field_descriptor != NULL) {
+ return self->parent_field_descriptor->full_name();
} else {
return self->message->GetDescriptor()->full_name();
}
@@ -1219,7 +1243,7 @@ static PyObject* SerializeToString(CMessage* self, PyObject* args) {
return NULL;
}
PyErr_Format(EncodeError_class, "Message %s is missing required fields: %s",
- GetMessageName(self).c_str(), PyString_AsString(joined.get()));
+ GetMessageName(self).c_str(), PyString_AsString(joined));
return NULL;
}
int size = self->message->ByteSize();
@@ -1361,7 +1385,7 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
AssureWritable(self);
google::protobuf::io::CodedInputStream input(
reinterpret_cast<const uint8*>(data), data_length);
- input.SetExtensionRegistry(GetDescriptorPool(), global_message_factory);
+ input.SetExtensionRegistry(descriptor_pool->pool, global_message_factory);
bool success = self->message->MergePartialFromCodedStream(&input);
if (success) {
return PyInt_FromLong(input.CurrentPosition());
@@ -1421,15 +1445,11 @@ static PyObject* RegisterExtension(PyObject* cls,
return NULL;
}
- CFieldDescriptor* cdescriptor =
- extension_dict::InternalGetCDescriptorFromExtension(extension_handle);
- ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
- if (cdescriptor == NULL) {
+ const google::protobuf::FieldDescriptor* descriptor =
+ GetExtensionDescriptor(extension_handle);
+ if (descriptor == NULL) {
return NULL;
}
- Py_INCREF(extension_handle);
- cdescriptor->descriptor_field = extension_handle;
- const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor;
// Check if it's a message set
if (descriptor->is_extension() &&
descriptor->containing_type()->options().message_set_wire_format() &&
@@ -1608,8 +1628,14 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) {
}
if (opid == Py_EQ || opid == Py_NE) {
ScopedPyObjectPtr self_fields(ListFields(self));
+ if (!self_fields) {
+ return NULL;
+ }
ScopedPyObjectPtr other_fields(ListFields(
reinterpret_cast<CMessage*>(other)));
+ if (!other_fields) {
+ return NULL;
+ }
return PyObject_RichCompare(self_fields, other_fields, opid);
} else {
Py_INCREF(Py_NotImplemented);
@@ -1623,9 +1649,7 @@ PyObject* InternalGetScalar(
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
- if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
- PyErr_SetString(
- PyExc_KeyError, "Field does not belong to message!");
+ if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
return NULL;
}
@@ -1701,43 +1725,31 @@ PyObject* InternalGetScalar(
return result;
}
-PyObject* InternalGetSubMessage(CMessage* self,
- CFieldDescriptor* cfield_descriptor) {
- PyObject* field = cfield_descriptor->descriptor_field;
- ScopedPyObjectPtr message_type(PyObject_GetAttr(field, kmessage_type));
- if (message_type == NULL) {
- return NULL;
- }
- ScopedPyObjectPtr concrete_class(
- PyObject_GetAttr(message_type, k_concrete_class));
- if (concrete_class == NULL) {
+PyObject* InternalGetSubMessage(
+ CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor) {
+ const google::protobuf::Reflection* reflection = self->message->GetReflection();
+ const google::protobuf::Message& sub_message = reflection->GetMessage(
+ *self->message, field_descriptor, global_message_factory);
+
+ PyObject *message_class = cdescriptor_pool::GetMessageClass(
+ descriptor_pool, field_descriptor->message_type());
+ if (message_class == NULL) {
return NULL;
}
- PyObject* py_cmsg = cmessage::NewEmpty(concrete_class);
- if (py_cmsg == NULL) {
+
+ CMessage* cmsg = cmessage::NewEmptyMessage(message_class,
+ sub_message.GetDescriptor());
+ if (cmsg == NULL) {
return NULL;
}
- if (!PyObject_TypeCheck(py_cmsg, &CMessage_Type)) {
- PyErr_SetString(PyExc_TypeError, "Not a CMessage!");
- }
- CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
- const google::protobuf::FieldDescriptor* field_descriptor =
- cfield_descriptor->descriptor;
- const google::protobuf::Reflection* reflection = self->message->GetReflection();
- const google::protobuf::Message& sub_message = reflection->GetMessage(
- *self->message, field_descriptor, global_message_factory);
cmsg->owner = self->owner;
cmsg->parent = self;
- cmsg->parent_field = cfield_descriptor;
+ cmsg->parent_field_descriptor = field_descriptor;
cmsg->read_only = !reflection->HasField(*self->message, field_descriptor);
cmsg->message = const_cast<google::protobuf::Message*>(&sub_message);
- if (InitAttributes(cmsg, NULL, NULL) < 0) {
- Py_DECREF(py_cmsg);
- return NULL;
- }
- return py_cmsg;
+ return reinterpret_cast<PyObject*>(cmsg);
}
int InternalSetScalar(
@@ -1747,9 +1759,7 @@ int InternalSetScalar(
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
- if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
- PyErr_SetString(
- PyExc_KeyError, "Field does not belong to message!");
+ if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
return -1;
}
@@ -1838,25 +1848,35 @@ PyObject* FromString(PyTypeObject* cls, PyObject* serialized) {
return NULL;
}
- if (InitAttributes(cmsg, NULL, NULL) < 0) {
- Py_DECREF(py_cmsg);
- return NULL;
- }
return py_cmsg;
}
+
+// Finalize the creation of the Message class.
+// Called from its metaclass: GeneratedProtocolMessageType.__init__().
static PyObject* AddDescriptors(PyTypeObject* cls,
PyObject* descriptor) {
- if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
- k_extensions_by_name, PyDict_New()) < 0) {
- return NULL;
- }
- if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
- k_extensions_by_number, PyDict_New()) < 0) {
+ const google::protobuf::Descriptor* message_descriptor =
+ cdescriptor_pool::RegisterMessageClass(
+ descriptor_pool, reinterpret_cast<PyObject*>(cls), descriptor);
+ if (message_descriptor == NULL) {
return NULL;
}
- ScopedPyObjectPtr field_descriptors(PyDict_New());
+ // If there are extension_ranges, the message is "extendable", and extension
+ // classes will register themselves in this class.
+ if (message_descriptor->extension_range_count() > 0) {
+ ScopedPyObjectPtr by_name(PyDict_New());
+ if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
+ k_extensions_by_name, by_name) < 0) {
+ return NULL;
+ }
+ ScopedPyObjectPtr by_number(PyDict_New());
+ if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
+ k_extensions_by_number, by_number) < 0) {
+ return NULL;
+ }
+ }
ScopedPyObjectPtr fields(PyObject_GetAttrString(descriptor, "fields"));
if (fields == NULL) {
@@ -1878,19 +1898,14 @@ static PyObject* AddDescriptors(PyTypeObject* cls,
return NULL;
}
- PyObject* field_descriptor =
- cdescriptor_pool::FindFieldByName(descriptor_pool, full_field_name);
+ ScopedPyObjectPtr field_descriptor(
+ cdescriptor_pool::FindFieldByName(descriptor_pool, full_field_name));
if (field_descriptor == NULL) {
PyErr_SetString(PyExc_TypeError, "Couldn't find field");
return NULL;
}
- Py_INCREF(field);
CFieldDescriptor* cfield_descriptor = reinterpret_cast<CFieldDescriptor*>(
- field_descriptor);
- cfield_descriptor->descriptor_field = field;
- if (PyDict_SetItem(field_descriptors, field_name, field_descriptor) < 0) {
- return NULL;
- }
+ field_descriptor.get());
// The FieldDescriptor's name field might either be of type bytes or
// of type unicode, depending on whether the FieldDescriptor was
@@ -1919,8 +1934,6 @@ static PyObject* AddDescriptors(PyTypeObject* cls,
}
}
- PyDict_SetItem(cls->tp_dict, k__descriptors, field_descriptors);
-
// Enum Values
ScopedPyObjectPtr enum_types(PyObject_GetAttrString(descriptor,
"enum_types"));
@@ -1994,15 +2007,11 @@ static PyObject* AddDescriptors(PyTypeObject* cls,
extension_name, extension_field) == -1) {
return NULL;
}
- ScopedPyObjectPtr py_cfield_descriptor(
- PyObject_GetAttrString(extension_field, "_cdescriptor"));
- if (py_cfield_descriptor == NULL) {
+ const google::protobuf::FieldDescriptor* field_descriptor =
+ GetExtensionDescriptor(extension_field);
+ if (field_descriptor == NULL) {
return NULL;
}
- CFieldDescriptor* cfield_descriptor =
- reinterpret_cast<CFieldDescriptor*>(py_cfield_descriptor.get());
- Py_INCREF(extension_field);
- cfield_descriptor->descriptor_field = extension_field;
ScopedPyObjectPtr field_name_upcased(
PyObject_CallMethod(extension_name, "upper", NULL));
@@ -2015,13 +2024,12 @@ static PyObject* AddDescriptors(PyTypeObject* cls,
return NULL;
}
ScopedPyObjectPtr number(PyInt_FromLong(
- cfield_descriptor->descriptor->number()));
+ field_descriptor->number()));
if (number == NULL) {
return NULL;
}
if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls),
- field_number_name, PyInt_FromLong(
- cfield_descriptor->descriptor->number())) == -1) {
+ field_number_name, number) == -1) {
return NULL;
}
}
@@ -2039,10 +2047,6 @@ PyObject* DeepCopy(CMessage* self, PyObject* arg) {
Py_DECREF(clone);
return NULL;
}
- if (InitAttributes(reinterpret_cast<CMessage*>(clone), NULL, NULL) < 0) {
- Py_DECREF(clone);
- return NULL;
- }
if (MergeFrom(reinterpret_cast<CMessage*>(clone),
reinterpret_cast<PyObject*>(self)) == NULL) {
Py_DECREF(clone);
@@ -2202,48 +2206,30 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
return value;
}
- PyObject* descriptor = GetDescriptor(self, name);
- if (descriptor != NULL) {
- CFieldDescriptor* cdescriptor =
- reinterpret_cast<CFieldDescriptor*>(descriptor);
- const google::protobuf::FieldDescriptor* field_descriptor = cdescriptor->descriptor;
+ const google::protobuf::FieldDescriptor* field_descriptor = GetFieldDescriptor(
+ self, name);
+ if (field_descriptor != NULL) {
if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
if (field_descriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
- PyObject* py_container = PyObject_CallObject(
- reinterpret_cast<PyObject*>(&RepeatedCompositeContainer_Type),
- NULL);
- if (py_container == NULL) {
+ PyObject *message_class = cdescriptor_pool::GetMessageClass(
+ descriptor_pool, field_descriptor->message_type());
+ if (message_class == NULL) {
return NULL;
}
- RepeatedCompositeContainer* container =
- reinterpret_cast<RepeatedCompositeContainer*>(py_container);
- PyObject* field = cdescriptor->descriptor_field;
- PyObject* message_type = PyObject_GetAttr(field, kmessage_type);
- if (message_type == NULL) {
- return NULL;
- }
- PyObject* concrete_class =
- PyObject_GetAttr(message_type, k_concrete_class);
- if (concrete_class == NULL) {
+ PyObject* py_container = repeated_composite_container::NewContainer(
+ self, field_descriptor, message_class);
+ if (py_container == NULL) {
return NULL;
}
- container->parent = self;
- container->parent_field = cdescriptor;
- container->message = self->message;
- container->owner = self->owner;
- container->subclass_init = concrete_class;
- Py_DECREF(message_type);
if (PyDict_SetItem(self->composite_fields, name, py_container) < 0) {
Py_DECREF(py_container);
return NULL;
}
return py_container;
} else {
- ScopedPyObjectPtr init_args(PyTuple_Pack(2, self, cdescriptor));
- PyObject* py_container = PyObject_CallObject(
- reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type),
- init_args);
+ PyObject* py_container = repeated_scalar_container::NewContainer(
+ self, field_descriptor);
if (py_container == NULL) {
return NULL;
}
@@ -2256,7 +2242,7 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
} else {
if (field_descriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
- PyObject* sub_message = InternalGetSubMessage(self, cdescriptor);
+ PyObject* sub_message = InternalGetSubMessage(self, field_descriptor);
if (PyDict_SetItem(self->composite_fields, name, sub_message) < 0) {
Py_DECREF(sub_message);
return NULL;
@@ -2278,12 +2264,10 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) {
return -1;
}
- PyObject* descriptor = GetDescriptor(self, name);
- if (descriptor != NULL) {
+ const google::protobuf::FieldDescriptor* field_descriptor =
+ GetFieldDescriptor(self, name);
+ if (field_descriptor != NULL) {
AssureWritable(self);
- CFieldDescriptor* cdescriptor =
- reinterpret_cast<CFieldDescriptor*>(descriptor);
- const google::protobuf::FieldDescriptor* field_descriptor = cdescriptor->descriptor;
if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
PyErr_Format(PyExc_AttributeError, "Assignment not allowed to repeated "
"field \"%s\" in protocol message object.",
@@ -2401,22 +2385,18 @@ void InitGlobals() {
kuint64max_py = PyLong_FromUnsignedLongLong(kuint64max);
kDESCRIPTOR = PyString_FromString("DESCRIPTOR");
- k__descriptors = PyString_FromString("__descriptors");
+ k_cdescriptor = PyString_FromString("_cdescriptor");
kfull_name = PyString_FromString("full_name");
- kis_extendable = PyString_FromString("is_extendable");
kextensions_by_name = PyString_FromString("extensions_by_name");
k_extensions_by_name = PyString_FromString("_extensions_by_name");
k_extensions_by_number = PyString_FromString("_extensions_by_number");
- k_concrete_class = PyString_FromString("_concrete_class");
- kmessage_type = PyString_FromString("message_type");
kname = PyString_FromString("name");
kfields_by_name = PyString_FromString("fields_by_name");
- global_message_factory = new DynamicMessageFactory(GetDescriptorPool());
- global_message_factory->SetDelegateToGeneratedFactory(true);
+ descriptor_pool = cdescriptor_pool::NewDescriptorPool();
- descriptor_pool = reinterpret_cast<google::protobuf::python::CDescriptorPool*>(
- Python_NewCDescriptorPool(NULL, NULL));
+ global_message_factory = new DynamicMessageFactory(descriptor_pool->pool);
+ global_message_factory->SetDelegateToGeneratedFactory(true);
}
bool InitProto2MessageModule(PyObject *m) {
@@ -2427,19 +2407,32 @@ bool InitProto2MessageModule(PyObject *m) {
return false;
}
- // All three of these are actually set elsewhere, directly onto the child
- // protocol buffer message class, but set them here as well to document that
- // subclasses need to set these.
+ // DESCRIPTOR is set on each protocol buffer message class elsewhere, but set
+ // it here as well to document that subclasses need to set it.
PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict, kDESCRIPTOR, Py_None);
- PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict,
- k_extensions_by_name, Py_None);
- PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict,
- k_extensions_by_number, Py_None);
+ // Subclasses with message extensions will override _extensions_by_name and
+ // _extensions_by_number with fresh mutable dictionaries in AddDescriptors.
+ // All other classes can share this same immutable mapping.
+ ScopedPyObjectPtr empty_dict(PyDict_New());
+ if (empty_dict == NULL) {
+ return false;
+ }
+ ScopedPyObjectPtr immutable_dict(PyDictProxy_New(empty_dict));
+ if (immutable_dict == NULL) {
+ return false;
+ }
+ if (PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict,
+ k_extensions_by_name, immutable_dict) < 0) {
+ return false;
+ }
+ if (PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict,
+ k_extensions_by_number, immutable_dict) < 0) {
+ return false;
+ }
PyModule_AddObject(m, "Message", reinterpret_cast<PyObject*>(
&google::protobuf::python::CMessage_Type));
- google::protobuf::python::RepeatedScalarContainer_Type.tp_new = PyType_GenericNew;
google::protobuf::python::RepeatedScalarContainer_Type.tp_hash =
PyObject_HashNotImplemented;
if (PyType_Ready(&google::protobuf::python::RepeatedScalarContainer_Type) < 0) {
@@ -2450,7 +2443,6 @@ bool InitProto2MessageModule(PyObject *m) {
reinterpret_cast<PyObject*>(
&google::protobuf::python::RepeatedScalarContainer_Type));
- google::protobuf::python::RepeatedCompositeContainer_Type.tp_new = PyType_GenericNew;
google::protobuf::python::RepeatedCompositeContainer_Type.tp_hash =
PyObject_HashNotImplemented;
if (PyType_Ready(&google::protobuf::python::RepeatedCompositeContainer_Type) < 0) {
@@ -2462,7 +2454,6 @@ bool InitProto2MessageModule(PyObject *m) {
reinterpret_cast<PyObject*>(
&google::protobuf::python::RepeatedCompositeContainer_Type));
- google::protobuf::python::ExtensionDict_Type.tp_new = PyType_GenericNew;
google::protobuf::python::ExtensionDict_Type.tp_hash = PyObject_HashNotImplemented;
if (PyType_Ready(&google::protobuf::python::ExtensionDict_Type) < 0) {
return false;
diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h
index 9f4978f4..73e5a7d3 100644
--- a/python/google/protobuf/pyext/message.h
+++ b/python/google/protobuf/pyext/message.h
@@ -49,12 +49,13 @@ namespace protobuf {
class Message;
class Reflection;
class FieldDescriptor;
+class Descriptor;
using internal::shared_ptr;
namespace python {
-struct CFieldDescriptor;
+struct PyDescriptorPool;
struct ExtensionDict;
typedef struct CMessage {
@@ -79,13 +80,11 @@ typedef struct CMessage {
// to use this pointer will result in a crash.
struct CMessage* parent;
- // Weak reference to the parent's descriptor that describes this submessage.
+ // Pointer to the parent's descriptor that describes this submessage.
// Used together with the parent's message when making a default message
// instance mutable.
- // TODO(anuraag): With a bit of work on the Python/C++ layer, it should be
- // possible to make this a direct pointer to a C++ FieldDescriptor, this would
- // be easier if this implementation replaces upstream.
- CFieldDescriptor* parent_field;
+ // The pointer is owned by the global DescriptorPool.
+ const google::protobuf::FieldDescriptor* parent_field_descriptor;
// Pointer to the C++ Message object for this CMessage. The
// CMessage does not own this pointer.
@@ -113,8 +112,11 @@ extern PyTypeObject CMessage_Type;
namespace cmessage {
-// Create a new empty message that can be populated by the parent.
-PyObject* NewEmpty(PyObject* type);
+// Internal function to create a new empty Message Python object, but with empty
+// pointers to the C++ objects.
+// The caller must fill self->message, self->owner and eventually self->parent.
+CMessage* NewEmptyMessage(PyObject* type,
+ const google::protobuf::Descriptor* descriptor);
// Release a submessage from its proto tree, making it a new top-level messgae.
// A new message will be created if this is a read-only default instance.
@@ -124,12 +126,16 @@ int ReleaseSubMessage(google::protobuf::Message* message,
const google::protobuf::FieldDescriptor* field_descriptor,
CMessage* child_cmessage);
+// Retrieves the C++ descriptor of a Python Extension descriptor.
+// On error, return NULL with an exception set.
+const google::protobuf::FieldDescriptor* GetExtensionDescriptor(PyObject* extension);
+
// Initializes a new CMessage instance for a submessage. Only called once per
// submessage as the result is cached in composite_fields.
//
// Corresponds to reflection api method GetMessage.
-PyObject* InternalGetSubMessage(CMessage* self,
- CFieldDescriptor* cfield_descriptor);
+PyObject* InternalGetSubMessage(
+ CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor);
// Deletes a range of C++ submessages in a repeated field (following a
// removal in a RepeatedCompositeContainer).
@@ -190,10 +196,8 @@ PyObject* HasFieldByDescriptor(
// Corresponds to reflection api method HasField.
PyObject* HasField(CMessage* self, PyObject* arg);
-// Initializes constants/enum values on a message. This is called by
-// RepeatedCompositeContainer and ExtensionDict after calling the constructor.
-// TODO(anuraag): Make it always called from within the constructor since it can
-int InitAttributes(CMessage* self, PyObject* descriptor, PyObject* kwargs);
+// Initializes values of fields on a newly constructed message.
+int InitAttributes(CMessage* self, PyObject* kwargs);
PyObject* MergeFrom(CMessage* self, PyObject* arg);
@@ -218,12 +222,14 @@ int AssureWritable(CMessage* self);
} // namespace cmessage
+
+// Retrieve the global descriptor pool owned by the _message module.
+PyDescriptorPool* GetDescriptorPool();
+
+
/* Is 64bit */
#define IS_64BIT (SIZEOF_LONG == 8)
-#define FIELD_BELONGS_TO_MESSAGE(field_descriptor, message) \
- ((message)->GetDescriptor() == (field_descriptor)->containing_type())
-
#define FIELD_IS_REPEATED(field_descriptor) \
((field_descriptor)->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
@@ -296,6 +302,11 @@ bool CheckAndSetString(
PyObject* ToStringObject(
const google::protobuf::FieldDescriptor* descriptor, string value);
+// Check if the passed field descriptor belongs to the given message.
+// If not, return false and set a Python exception (a KeyError)
+bool CheckFieldBelongsToMessage(const google::protobuf::FieldDescriptor* field_descriptor,
+ const google::protobuf::Message* message);
+
extern PyObject* PickleError_class;
} // namespace python
diff --git a/python/google/protobuf/pyext/proto2_api_test.proto b/python/google/protobuf/pyext/proto2_api_test.proto
index 72c31b1c..18aecfb7 100644
--- a/python/google/protobuf/pyext/proto2_api_test.proto
+++ b/python/google/protobuf/pyext/proto2_api_test.proto
@@ -28,6 +28,8 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+syntax = "proto2";
+
import "google/protobuf/internal/cpp/proto1_api_test.proto";
package google.protobuf.python.internal;
diff --git a/python/google/protobuf/pyext/python.proto b/python/google/protobuf/pyext/python.proto
index d47d402c..cce645d7 100644
--- a/python/google/protobuf/pyext/python.proto
+++ b/python/google/protobuf/pyext/python.proto
@@ -33,6 +33,7 @@
// These message definitions are used to exercises known corner cases
// in the C++ implementation of the Python API.
+syntax = "proto2";
package google.protobuf.python.internal;
@@ -63,4 +64,5 @@ message TestAllExtensions {
extend TestAllExtensions {
optional TestAllTypes.NestedMessage optional_nested_message_extension = 1;
+ repeated TestAllTypes.NestedMessage repeated_nested_message_extension = 2;
}
diff --git a/python/google/protobuf/pyext/repeated_composite_container.cc b/python/google/protobuf/pyext/repeated_composite_container.cc
index 5c05b3d8..36fe86ae 100644
--- a/python/google/protobuf/pyext/repeated_composite_container.cc
+++ b/python/google/protobuf/pyext/repeated_composite_container.cc
@@ -65,14 +65,14 @@ namespace repeated_composite_container {
#define GOOGLE_CHECK_ATTACHED(self) \
do { \
GOOGLE_CHECK_NOTNULL((self)->message); \
- GOOGLE_CHECK_NOTNULL((self)->parent_field); \
+ GOOGLE_CHECK_NOTNULL((self)->parent_field_descriptor); \
} while (0);
#define GOOGLE_CHECK_RELEASED(self) \
do { \
GOOGLE_CHECK((self)->owner.get() == NULL); \
GOOGLE_CHECK((self)->message == NULL); \
- GOOGLE_CHECK((self)->parent_field == NULL); \
+ GOOGLE_CHECK((self)->parent_field_descriptor == NULL); \
GOOGLE_CHECK((self)->parent == NULL); \
} while (0);
@@ -122,7 +122,7 @@ static int InternalQuickSort(RepeatedCompositeContainer* self,
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
- const google::protobuf::FieldDescriptor* descriptor = self->parent_field->descriptor;
+ const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor;
Py_ssize_t left;
Py_ssize_t right;
@@ -202,7 +202,7 @@ static Py_ssize_t Length(RepeatedCompositeContainer* self) {
google::protobuf::Message* message = self->message;
if (message != NULL) {
return message->GetReflection()->FieldSize(*message,
- self->parent_field->descriptor);
+ self->parent_field_descriptor);
} else {
// The container has been released (i.e. by a call to Clear() or
// ClearField() on the parent) and thus there's no message.
@@ -225,19 +225,19 @@ static int UpdateChildMessages(RepeatedCompositeContainer* self) {
const google::protobuf::Reflection* reflection = message->GetReflection();
for (Py_ssize_t i = child_length; i < message_length; ++i) {
const Message& sub_message = reflection->GetRepeatedMessage(
- *(self->message), self->parent_field->descriptor, i);
- ScopedPyObjectPtr py_cmsg(cmessage::NewEmpty(self->subclass_init));
- if (py_cmsg == NULL) {
+ *(self->message), self->parent_field_descriptor, i);
+ CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
+ sub_message.GetDescriptor());
+ ScopedPyObjectPtr py_cmsg(reinterpret_cast<PyObject*>(cmsg));
+ if (cmsg == NULL) {
return -1;
}
- CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg.get());
cmsg->owner = self->owner;
cmsg->message = const_cast<google::protobuf::Message*>(&sub_message);
cmsg->parent = self->parent;
- if (cmessage::InitAttributes(cmsg, NULL, NULL) < 0) {
+ if (PyList_Append(self->child_messages, py_cmsg) < 0) {
return -1;
}
- PyList_Append(self->child_messages, py_cmsg);
}
return 0;
}
@@ -258,23 +258,25 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self,
google::protobuf::Message* message = self->message;
google::protobuf::Message* sub_message =
message->GetReflection()->AddMessage(message,
- self->parent_field->descriptor);
- PyObject* py_cmsg = cmessage::NewEmpty(self->subclass_init);
- if (py_cmsg == NULL) {
+ self->parent_field_descriptor);
+ CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
+ sub_message->GetDescriptor());
+ if (cmsg == NULL)
return NULL;
- }
- CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
cmsg->owner = self->owner;
cmsg->message = sub_message;
cmsg->parent = self->parent;
- // cmessage::InitAttributes must be called after cmsg->message has
- // been set.
- if (cmessage::InitAttributes(cmsg, NULL, kwargs) < 0) {
+ if (cmessage::InitAttributes(cmsg, kwargs) < 0) {
+ Py_DECREF(cmsg);
+ return NULL;
+ }
+
+ PyObject* py_cmsg = reinterpret_cast<PyObject*>(cmsg);
+ if (PyList_Append(self->child_messages, py_cmsg) < 0) {
Py_DECREF(py_cmsg);
return NULL;
}
- PyList_Append(self->child_messages, py_cmsg);
return py_cmsg;
}
@@ -283,20 +285,16 @@ static PyObject* AddToReleased(RepeatedCompositeContainer* self,
PyObject* kwargs) {
GOOGLE_CHECK_RELEASED(self);
- // Create the CMessage
- PyObject* py_cmsg = PyObject_CallObject(self->subclass_init, NULL);
+ // Create a new Message detached from the rest.
+ PyObject* py_cmsg = PyEval_CallObjectWithKeywords(
+ self->subclass_init, NULL, kwargs);
if (py_cmsg == NULL)
return NULL;
- CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
- if (cmessage::InitAttributes(cmsg, NULL, kwargs) < 0) {
+
+ if (PyList_Append(self->child_messages, py_cmsg) < 0) {
Py_DECREF(py_cmsg);
return NULL;
}
-
- // The Message got created by the call to subclass_init above and
- // it set self->owner to the newly allocated message.
-
- PyList_Append(self->child_messages, py_cmsg);
return py_cmsg;
}
@@ -354,35 +352,9 @@ PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* slice) {
if (UpdateChildMessages(self) < 0) {
return NULL;
}
- Py_ssize_t from;
- Py_ssize_t to;
- Py_ssize_t step;
- Py_ssize_t length = Length(self);
- Py_ssize_t slicelength;
- if (PySlice_Check(slice)) {
-#if PY_MAJOR_VERSION >= 3
- if (PySlice_GetIndicesEx(slice,
-#else
- if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice),
-#endif
- length, &from, &to, &step, &slicelength) == -1) {
- return NULL;
- }
- return PyList_GetSlice(self->child_messages, from, to);
- } else if (PyInt_Check(slice) || PyLong_Check(slice)) {
- from = to = PyLong_AsLong(slice);
- if (from < 0) {
- from = to = length + from;
- }
- PyObject* result = PyList_GetItem(self->child_messages, from);
- if (result == NULL) {
- return NULL;
- }
- Py_INCREF(result);
- return result;
- }
- PyErr_SetString(PyExc_TypeError, "index must be an integer or slice");
- return NULL;
+ // Just forward the call to the subscript-handling function of the
+ // list containing the child messages.
+ return PyObject_GetItem(self->child_messages, slice);
}
int AssignSubscript(RepeatedCompositeContainer* self,
@@ -399,7 +371,7 @@ int AssignSubscript(RepeatedCompositeContainer* self,
// Delete from the underlying Message, if any.
if (self->message != NULL) {
if (cmessage::InternalDeleteRepeatedField(self->message,
- self->parent_field->descriptor,
+ self->parent_field_descriptor,
slice,
self->child_messages) < 0) {
return -1;
@@ -512,7 +484,7 @@ static PyObject* SortAttached(RepeatedCompositeContainer* self,
if (reverse) {
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
- const google::protobuf::FieldDescriptor* descriptor = self->parent_field->descriptor;
+ const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor;
// Reverse the Message array.
for (int i = 0; i < length / 2; ++i)
@@ -554,8 +526,9 @@ static PyObject* Sort(RepeatedCompositeContainer* self,
}
}
- if (UpdateChildMessages(self) < 0)
+ if (UpdateChildMessages(self) < 0) {
return NULL;
+ }
if (self->message == NULL) {
return SortReleased(self, args, kwds);
} else {
@@ -617,7 +590,7 @@ void ReleaseLastTo(const FieldDescriptor* field,
shared_ptr<Message> released_message(
ReleaseLast(field, cmessage->message->GetDescriptor(), message));
cmessage->parent = NULL;
- cmessage->parent_field = NULL;
+ cmessage->parent_field_descriptor = NULL;
cmessage->message = released_message.get();
cmessage->read_only = false;
cmessage::SetOwner(cmessage, released_message);
@@ -633,7 +606,7 @@ int Release(RepeatedCompositeContainer* self) {
}
Message* message = self->message;
- const FieldDescriptor* field = self->parent_field->descriptor;
+ const FieldDescriptor* field = self->parent_field_descriptor;
// The reflection API only lets us release the last message in a
// repeated field. Therefore we iterate through the children
@@ -648,7 +621,7 @@ int Release(RepeatedCompositeContainer* self) {
// Detach from containing message.
self->parent = NULL;
- self->parent_field = NULL;
+ self->parent_field_descriptor = NULL;
self->message = NULL;
self->owner.reset();
@@ -670,22 +643,40 @@ int SetOwner(RepeatedCompositeContainer* self,
return 0;
}
-static int Init(RepeatedCompositeContainer* self,
- PyObject* args,
- PyObject* kwargs) {
- self->message = NULL;
- self->parent = NULL;
- self->parent_field = NULL;
- self->subclass_init = NULL;
+// The private constructor of RepeatedCompositeContainer objects.
+PyObject *NewContainer(
+ CMessage* parent,
+ const google::protobuf::FieldDescriptor* parent_field_descriptor,
+ PyObject *concrete_class) {
+ if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+ return NULL;
+ }
+
+ RepeatedCompositeContainer* self =
+ reinterpret_cast<RepeatedCompositeContainer*>(
+ PyType_GenericAlloc(&RepeatedCompositeContainer_Type, 0));
+ if (self == NULL) {
+ return NULL;
+ }
+
+ self->message = parent->message;
+ self->parent = parent;
+ self->parent_field_descriptor = parent_field_descriptor;
+ self->owner = parent->owner;
+ Py_INCREF(concrete_class);
+ self->subclass_init = concrete_class;
self->child_messages = PyList_New(0);
- return 0;
+
+ return reinterpret_cast<PyObject*>(self);
}
static void Dealloc(RepeatedCompositeContainer* self) {
Py_CLEAR(self->child_messages);
+ Py_CLEAR(self->subclass_init);
// TODO(tibell): Do we need to call delete on these objects to make
// sure their destructors are called?
self->owner.reset();
+
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
@@ -755,7 +746,7 @@ PyTypeObject RepeatedCompositeContainer_Type = {
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
- (initproc)repeated_composite_container::Init, // tp_init
+ 0, // tp_init
};
} // namespace python
diff --git a/python/google/protobuf/pyext/repeated_composite_container.h b/python/google/protobuf/pyext/repeated_composite_container.h
index 898ef5a7..a76a5d6a 100644
--- a/python/google/protobuf/pyext/repeated_composite_container.h
+++ b/python/google/protobuf/pyext/repeated_composite_container.h
@@ -55,7 +55,6 @@ using internal::shared_ptr;
namespace python {
struct CMessage;
-struct CFieldDescriptor;
// A RepeatedCompositeContainer can be in one of two states: attached
// or released.
@@ -66,7 +65,7 @@ struct CFieldDescriptor;
// 'child_messages' are owner by the 'owner'.
//
// When in the released state 'message', 'owner', 'parent', and
-// 'parent_field' are NULL.
+// 'parent_field_descriptor' are NULL.
typedef struct RepeatedCompositeContainer {
PyObject_HEAD;
@@ -82,7 +81,8 @@ typedef struct RepeatedCompositeContainer {
CMessage* parent;
// A descriptor used to modify the underlying 'message'.
- CFieldDescriptor* parent_field;
+ // The pointer is owned by the global DescriptorPool.
+ const google::protobuf::FieldDescriptor* parent_field_descriptor;
// Pointer to the C++ Message that contains this container. The
// RepeatedCompositeContainer does not own this pointer.
@@ -102,6 +102,13 @@ extern PyTypeObject RepeatedCompositeContainer_Type;
namespace repeated_composite_container {
+// Builds a RepeatedCompositeContainer object, from a parent message and a
+// field descriptor.
+PyObject *NewContainer(
+ CMessage* parent,
+ const google::protobuf::FieldDescriptor* parent_field_descriptor,
+ PyObject *concrete_class);
+
// Returns the number of items in this repeated composite container.
static Py_ssize_t Length(RepeatedCompositeContainer* self);
diff --git a/python/google/protobuf/pyext/repeated_scalar_container.cc b/python/google/protobuf/pyext/repeated_scalar_container.cc
index e627d37d..49d23fd6 100644
--- a/python/google/protobuf/pyext/repeated_scalar_container.cc
+++ b/python/google/protobuf/pyext/repeated_scalar_container.cc
@@ -52,7 +52,7 @@
#error "Python 3.0 - 3.2 are not supported."
#else
#define PyString_AsString(ob) \
- (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob))
+ (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob))
#endif
#endif
@@ -67,7 +67,7 @@ namespace repeated_scalar_container {
static int InternalAssignRepeatedField(
RepeatedScalarContainer* self, PyObject* list) {
self->message->GetReflection()->ClearField(self->message,
- self->parent_field->descriptor);
+ self->parent_field_descriptor);
for (Py_ssize_t i = 0; i < PyList_GET_SIZE(list); ++i) {
PyObject* value = PyList_GET_ITEM(list, i);
if (Append(self, value) == NULL) {
@@ -80,7 +80,7 @@ static int InternalAssignRepeatedField(
static Py_ssize_t Len(RepeatedScalarContainer* self) {
google::protobuf::Message* message = self->message;
return message->GetReflection()->FieldSize(*message,
- self->parent_field->descriptor);
+ self->parent_field_descriptor);
}
static int AssignItem(RepeatedScalarContainer* self,
@@ -89,12 +89,7 @@ static int AssignItem(RepeatedScalarContainer* self,
cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
- self->parent_field->descriptor;
- if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
- PyErr_SetString(
- PyExc_KeyError, "Field does not belong to message!");
- return -1;
- }
+ self->parent_field_descriptor;
const google::protobuf::Reflection* reflection = message->GetReflection();
int field_size = reflection->FieldSize(*message, field_descriptor);
@@ -175,7 +170,7 @@ static int AssignItem(RepeatedScalarContainer* self,
ScopedPyObjectPtr s(PyObject_Str(arg));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
- PyString_AsString(s.get()));
+ PyString_AsString(s));
}
return -1;
}
@@ -193,7 +188,7 @@ static int AssignItem(RepeatedScalarContainer* self,
static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
- self->parent_field->descriptor;
+ self->parent_field_descriptor;
const google::protobuf::Reflection* reflection = message->GetReflection();
int field_size = reflection->FieldSize(*message, field_descriptor);
@@ -358,13 +353,7 @@ PyObject* Append(RepeatedScalarContainer* self, PyObject* item) {
cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
- self->parent_field->descriptor;
-
- if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
- PyErr_SetString(
- PyExc_KeyError, "Field does not belong to message!");
- return NULL;
- }
+ self->parent_field_descriptor;
const google::protobuf::Reflection* reflection = message->GetReflection();
switch (field_descriptor->cpp_type()) {
@@ -422,7 +411,7 @@ PyObject* Append(RepeatedScalarContainer* self, PyObject* item) {
ScopedPyObjectPtr s(PyObject_Str(item));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
- PyString_AsString(s.get()));
+ PyString_AsString(s));
}
return NULL;
}
@@ -451,7 +440,7 @@ static int AssSubscript(RepeatedScalarContainer* self,
cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
- self->parent_field->descriptor;
+ self->parent_field_descriptor;
#if PY_MAJOR_VERSION < 3
if (PyInt_Check(slice)) {
@@ -638,47 +627,25 @@ static PyObject* Sort(RepeatedScalarContainer* self,
Py_RETURN_NONE;
}
-static int Init(RepeatedScalarContainer* self,
- PyObject* args,
- PyObject* kwargs) {
- PyObject* py_parent;
- PyObject* py_parent_field;
- if (!PyArg_UnpackTuple(args, "__init__()", 2, 2, &py_parent,
- &py_parent_field)) {
- return -1;
- }
-
- if (!PyObject_TypeCheck(py_parent, &CMessage_Type)) {
- PyErr_Format(PyExc_TypeError,
- "expect %s, but got %s",
- CMessage_Type.tp_name,
- Py_TYPE(py_parent)->tp_name);
- return -1;
+// The private constructor of RepeatedScalarContainer objects.
+PyObject *NewContainer(
+ CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) {
+ if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+ return NULL;
}
- if (!PyObject_TypeCheck(py_parent_field, &CFieldDescriptor_Type)) {
- PyErr_Format(PyExc_TypeError,
- "expect %s, but got %s",
- CFieldDescriptor_Type.tp_name,
- Py_TYPE(py_parent_field)->tp_name);
- return -1;
+ RepeatedScalarContainer* self = reinterpret_cast<RepeatedScalarContainer*>(
+ PyType_GenericAlloc(&RepeatedScalarContainer_Type, 0));
+ if (self == NULL) {
+ return NULL;
}
- CMessage* cmessage = reinterpret_cast<CMessage*>(py_parent);
- CFieldDescriptor* cdescriptor = reinterpret_cast<CFieldDescriptor*>(
- py_parent_field);
-
- if (!FIELD_BELONGS_TO_MESSAGE(cdescriptor->descriptor, cmessage->message)) {
- PyErr_SetString(
- PyExc_KeyError, "Field does not belong to message!");
- return -1;
- }
+ self->message = parent->message;
+ self->parent = parent;
+ self->parent_field_descriptor = parent_field_descriptor;
+ self->owner = parent->owner;
- self->message = cmessage->message;
- self->parent = cmessage;
- self->parent_field = cdescriptor;
- self->owner = cmessage->owner;
- return 0;
+ return reinterpret_cast<PyObject*>(self);
}
// Initializes the underlying Message object of "to" so it becomes a new parent
@@ -699,10 +666,7 @@ static int InitializeAndCopyToParentContainer(
google::protobuf::Message* new_message = global_message_factory->GetPrototype(
from->message->GetDescriptor())->New();
to->parent = NULL;
- // TODO(anuraag): Document why it's OK to hang on to parent_field,
- // even though it's a weak reference. It ought to be enough to
- // hold on to the FieldDescriptor only.
- to->parent_field = from->parent_field;
+ to->parent_field_descriptor = from->parent_field_descriptor;
to->message = new_message;
to->owner.reset(new_message);
if (InternalAssignRepeatedField(to, values) < 0) {
@@ -716,23 +680,17 @@ int Release(RepeatedScalarContainer* self) {
}
PyObject* DeepCopy(RepeatedScalarContainer* self, PyObject* arg) {
- ScopedPyObjectPtr init_args(
- PyTuple_Pack(2, self->parent, self->parent_field));
- PyObject* clone = PyObject_CallObject(
- reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type), init_args);
+ RepeatedScalarContainer* clone = reinterpret_cast<RepeatedScalarContainer*>(
+ PyType_GenericAlloc(&RepeatedScalarContainer_Type, 0));
if (clone == NULL) {
return NULL;
}
- if (!PyObject_TypeCheck(clone, &RepeatedScalarContainer_Type)) {
- Py_DECREF(clone);
- return NULL;
- }
- if (InitializeAndCopyToParentContainer(
- self, reinterpret_cast<RepeatedScalarContainer*>(clone)) < 0) {
+
+ if (InitializeAndCopyToParentContainer(self, clone) < 0) {
Py_DECREF(clone);
return NULL;
}
- return clone;
+ return reinterpret_cast<PyObject*>(clone);
}
static void Dealloc(RepeatedScalarContainer* self) {
@@ -817,7 +775,7 @@ PyTypeObject RepeatedScalarContainer_Type = {
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
- (initproc)repeated_scalar_container::Init, // tp_init
+ 0, // tp_init
};
} // namespace python
diff --git a/python/google/protobuf/pyext/repeated_scalar_container.h b/python/google/protobuf/pyext/repeated_scalar_container.h
index 69d15d5c..513bfe48 100644
--- a/python/google/protobuf/pyext/repeated_scalar_container.h
+++ b/python/google/protobuf/pyext/repeated_scalar_container.h
@@ -41,6 +41,7 @@
#include <google/protobuf/stubs/shared_ptr.h>
#endif
+#include <google/protobuf/descriptor.h>
namespace google {
namespace protobuf {
@@ -51,7 +52,6 @@ using internal::shared_ptr;
namespace python {
-struct CFieldDescriptor;
struct CMessage;
typedef struct RepeatedScalarContainer {
@@ -73,16 +73,22 @@ typedef struct RepeatedScalarContainer {
// modifying the container.
CMessage* parent;
- // Weak reference to the parent's descriptor that describes this
+ // Pointer to the parent's descriptor that describes this
// field. Used together with the parent's message when making a
// default message instance mutable.
- CFieldDescriptor* parent_field;
+ // The pointer is owned by the global DescriptorPool.
+ const google::protobuf::FieldDescriptor* parent_field_descriptor;
} RepeatedScalarContainer;
extern PyTypeObject RepeatedScalarContainer_Type;
namespace repeated_scalar_container {
+// Builds a RepeatedScalarContainer object, from a parent message and a
+// field descriptor.
+extern PyObject *NewContainer(
+ CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor);
+
// Appends the scalar 'item' to the end of the container 'self'.
//
// Returns None if successful; returns NULL and sets an exception if
diff --git a/python/google/protobuf/pyext/scoped_pyobject_ptr.h b/python/google/protobuf/pyext/scoped_pyobject_ptr.h
index 9f337c3c..fefebb6a 100644
--- a/python/google/protobuf/pyext/scoped_pyobject_ptr.h
+++ b/python/google/protobuf/pyext/scoped_pyobject_ptr.h
@@ -33,6 +33,8 @@
#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__
#define GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__
+#include <google/protobuf/stubs/common.h>
+
#include <Python.h>
namespace google {