aboutsummaryrefslogtreecommitdiffhomepage
path: root/python/google/protobuf/pyext
diff options
context:
space:
mode:
authorGravatar Feng Xiao <xfxyjwf@gmail.com>2015-12-11 17:09:20 -0800
committerGravatar Feng Xiao <xfxyjwf@gmail.com>2015-12-11 17:10:28 -0800
commite841bac4fcf47f809e089a70d5f84ac37b3883df (patch)
treed25dc5fc814db182c04c5f276ff1a609c5965a5a /python/google/protobuf/pyext
parent99a6a95c751a28a3cc33dd2384959179f83f682c (diff)
Down-integrate from internal code base.
Diffstat (limited to 'python/google/protobuf/pyext')
-rw-r--r--python/google/protobuf/pyext/descriptor.cc29
-rw-r--r--python/google/protobuf/pyext/descriptor_database.cc145
-rw-r--r--python/google/protobuf/pyext/descriptor_database.h75
-rw-r--r--python/google/protobuf/pyext/descriptor_pool.cc111
-rw-r--r--python/google/protobuf/pyext/descriptor_pool.h10
-rw-r--r--python/google/protobuf/pyext/extension_dict.cc47
-rw-r--r--python/google/protobuf/pyext/extension_dict.h5
-rw-r--r--python/google/protobuf/pyext/map_container.cc912
-rw-r--r--python/google/protobuf/pyext/map_container.h (renamed from python/google/protobuf/pyext/message_map_container.h)73
-rw-r--r--python/google/protobuf/pyext/message.cc299
-rw-r--r--python/google/protobuf/pyext/message.h1
-rw-r--r--python/google/protobuf/pyext/message_map_container.cc569
-rw-r--r--python/google/protobuf/pyext/scalar_map_container.cc542
-rw-r--r--python/google/protobuf/pyext/scalar_map_container.h119
14 files changed, 1498 insertions, 1439 deletions
diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc
index 61a3d237..a875a7be 100644
--- a/python/google/protobuf/pyext/descriptor.cc
+++ b/python/google/protobuf/pyext/descriptor.cc
@@ -203,6 +203,14 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) {
PyObject* message_class(cdescriptor_pool::GetMessageClass(
pool, message_type));
if (message_class == NULL) {
+ // The Options message was not found in the current DescriptorPool.
+ // In this case, there cannot be extensions to these options, and we can
+ // try to use the basic pool instead.
+ PyErr_Clear();
+ message_class = cdescriptor_pool::GetMessageClass(
+ GetDefaultDescriptorPool(), message_type);
+ }
+ if (message_class == NULL) {
PyErr_Format(PyExc_TypeError, "Could not retrieve class for Options: %s",
message_type->full_name().c_str());
return NULL;
@@ -211,6 +219,12 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) {
if (value == NULL) {
return NULL;
}
+ if (!PyObject_TypeCheck(value.get(), &CMessage_Type)) {
+ PyErr_Format(PyExc_TypeError, "Invalid class for %s: %s",
+ message_type->full_name().c_str(),
+ Py_TYPE(value.get())->tp_name);
+ return NULL;
+ }
CMessage* cmsg = reinterpret_cast<CMessage*>(value.get());
const Reflection* reflection = options.GetReflection();
@@ -327,7 +341,8 @@ PyObject* NewInternedDescriptor(PyTypeObject* type,
PyDescriptorPool* pool = GetDescriptorPool_FromPool(
GetFileDescriptor(descriptor)->pool());
if (pool == NULL) {
- Py_DECREF(py_descriptor);
+ // Don't DECREF, the object is not fully initialized.
+ PyObject_Del(py_descriptor);
return NULL;
}
Py_INCREF(pool);
@@ -1213,6 +1228,13 @@ static void Dealloc(PyFileDescriptor* self) {
descriptor::Dealloc(&self->base);
}
+static PyObject* GetPool(PyFileDescriptor *self, void *closure) {
+ PyObject* pool = reinterpret_cast<PyObject*>(
+ GetDescriptorPool_FromPool(_GetDescriptor(self)->pool()));
+ Py_XINCREF(pool);
+ return pool;
+}
+
static PyObject* GetName(PyFileDescriptor *self, void *closure) {
return PyString_FromCppString(_GetDescriptor(self)->name());
}
@@ -1292,6 +1314,7 @@ static PyObject* CopyToProto(PyFileDescriptor *self, PyObject *target) {
}
static PyGetSetDef Getters[] = {
+ { "pool", (getter)GetPool, NULL, "pool"},
{ "name", (getter)GetName, NULL, "name"},
{ "package", (getter)GetPackage, NULL, "package"},
{ "serialized_pb", (getter)GetSerializedPb},
@@ -1354,8 +1377,8 @@ PyTypeObject PyFileDescriptor_Type = {
0, // tp_descr_set
0, // tp_dictoffset
0, // tp_init
- PyType_GenericAlloc, // tp_alloc
- PyType_GenericNew, // tp_new
+ 0, // tp_alloc
+ 0, // tp_new
PyObject_Del, // tp_free
};
diff --git a/python/google/protobuf/pyext/descriptor_database.cc b/python/google/protobuf/pyext/descriptor_database.cc
new file mode 100644
index 00000000..514722b4
--- /dev/null
+++ b/python/google/protobuf/pyext/descriptor_database.cc
@@ -0,0 +1,145 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// This file defines a C++ DescriptorDatabase, which wraps a Python Database
+// and delegate all its operations to Python methods.
+
+#include <google/protobuf/pyext/descriptor_database.h>
+
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/descriptor.pb.h>
+#include <google/protobuf/pyext/message.h>
+#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+PyDescriptorDatabase::PyDescriptorDatabase(PyObject* py_database)
+ : py_database_(py_database) {
+ Py_INCREF(py_database_);
+}
+
+PyDescriptorDatabase::~PyDescriptorDatabase() { Py_DECREF(py_database_); }
+
+// Convert a Python object to a FileDescriptorProto pointer.
+// Handles all kinds of Python errors, which are simply logged.
+static bool GetFileDescriptorProto(PyObject* py_descriptor,
+ FileDescriptorProto* output) {
+ if (py_descriptor == NULL) {
+ if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+ // Expected error: item was simply not found.
+ PyErr_Clear();
+ } else {
+ GOOGLE_LOG(ERROR) << "DescriptorDatabase method raised an error";
+ PyErr_Print();
+ }
+ return false;
+ }
+ const Descriptor* filedescriptor_descriptor =
+ FileDescriptorProto::default_instance().GetDescriptor();
+ CMessage* message = reinterpret_cast<CMessage*>(py_descriptor);
+ if (PyObject_TypeCheck(py_descriptor, &CMessage_Type) &&
+ message->message->GetDescriptor() == filedescriptor_descriptor) {
+ // Fast path: Just use the pointer.
+ FileDescriptorProto* file_proto =
+ static_cast<FileDescriptorProto*>(message->message);
+ *output = *file_proto;
+ return true;
+ } else {
+ // Slow path: serialize the message. This allows to use databases which
+ // use a different implementation of FileDescriptorProto.
+ ScopedPyObjectPtr serialized_pb(
+ PyObject_CallMethod(py_descriptor, "SerializeToString", NULL));
+ if (serialized_pb == NULL) {
+ GOOGLE_LOG(ERROR)
+ << "DescriptorDatabase method did not return a FileDescriptorProto";
+ PyErr_Print();
+ return false;
+ }
+ char* str;
+ Py_ssize_t len;
+ if (PyBytes_AsStringAndSize(serialized_pb.get(), &str, &len) < 0) {
+ GOOGLE_LOG(ERROR)
+ << "DescriptorDatabase method did not return a FileDescriptorProto";
+ PyErr_Print();
+ return false;
+ }
+ FileDescriptorProto file_proto;
+ if (!file_proto.ParseFromArray(str, len)) {
+ GOOGLE_LOG(ERROR)
+ << "DescriptorDatabase method did not return a FileDescriptorProto";
+ return false;
+ }
+ *output = file_proto;
+ return true;
+ }
+}
+
+// Find a file by file name.
+bool PyDescriptorDatabase::FindFileByName(const string& filename,
+ FileDescriptorProto* output) {
+ ScopedPyObjectPtr py_descriptor(PyObject_CallMethod(
+ py_database_, "FindFileByName", "s#", filename.c_str(), filename.size()));
+ return GetFileDescriptorProto(py_descriptor.get(), output);
+}
+
+// Find the file that declares the given fully-qualified symbol name.
+bool PyDescriptorDatabase::FindFileContainingSymbol(
+ const string& symbol_name, FileDescriptorProto* output) {
+ ScopedPyObjectPtr py_descriptor(
+ PyObject_CallMethod(py_database_, "FindFileContainingSymbol", "s#",
+ symbol_name.c_str(), symbol_name.size()));
+ return GetFileDescriptorProto(py_descriptor.get(), output);
+}
+
+// Find the file which defines an extension extending the given message type
+// with the given field number.
+// Python DescriptorDatabases are not required to implement this method.
+bool PyDescriptorDatabase::FindFileContainingExtension(
+ const string& containing_type, int field_number,
+ FileDescriptorProto* output) {
+ ScopedPyObjectPtr py_method(
+ PyObject_GetAttrString(py_database_, "FindFileContainingExtension"));
+ if (py_method == NULL) {
+ // This method is not implemented, returns without error.
+ PyErr_Clear();
+ return false;
+ }
+ ScopedPyObjectPtr py_descriptor(
+ PyObject_CallFunction(py_method.get(), "s#i", containing_type.c_str(),
+ containing_type.size(), field_number));
+ return GetFileDescriptorProto(py_descriptor.get(), output);
+}
+
+} // namespace python
+} // namespace protobuf
+} // namespace google
diff --git a/python/google/protobuf/pyext/descriptor_database.h b/python/google/protobuf/pyext/descriptor_database.h
new file mode 100644
index 00000000..fc71c4bc
--- /dev/null
+++ b/python/google/protobuf/pyext/descriptor_database.h
@@ -0,0 +1,75 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__
+
+#include <Python.h>
+
+#include <google/protobuf/descriptor_database.h>
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+class PyDescriptorDatabase : public DescriptorDatabase {
+ public:
+ explicit PyDescriptorDatabase(PyObject* py_database);
+ ~PyDescriptorDatabase();
+
+ // Implement the abstract interface. All these functions fill the output
+ // with a copy of FileDescriptorProto.
+
+ // Find a file by file name.
+ bool FindFileByName(const string& filename,
+ FileDescriptorProto* output);
+
+ // Find the file that declares the given fully-qualified symbol name.
+ bool FindFileContainingSymbol(const string& symbol_name,
+ FileDescriptorProto* output);
+
+ // Find the file which defines an extension extending the given message type
+ // with the given field number.
+ // Containing_type must be a fully-qualified type name.
+ // Python objects are not required to implement this method.
+ bool FindFileContainingExtension(const string& containing_type,
+ int field_number,
+ FileDescriptorProto* output);
+
+ private:
+ // The python object that implements the database. The reference is owned.
+ PyObject* py_database_;
+};
+
+} // namespace python
+} // namespace protobuf
+
+} // namespace google
+#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__
diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc
index 0f7487fa..0bc76bc9 100644
--- a/python/google/protobuf/pyext/descriptor_pool.cc
+++ b/python/google/protobuf/pyext/descriptor_pool.cc
@@ -34,8 +34,9 @@
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/dynamic_message.h>
-#include <google/protobuf/pyext/descriptor_pool.h>
#include <google/protobuf/pyext/descriptor.h>
+#include <google/protobuf/pyext/descriptor_database.h>
+#include <google/protobuf/pyext/descriptor_pool.h>
#include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
@@ -60,38 +61,93 @@ static hash_map<const DescriptorPool*, PyDescriptorPool*> descriptor_pool_map;
namespace cdescriptor_pool {
-static PyDescriptorPool* NewDescriptorPool() {
- PyDescriptorPool* cdescriptor_pool = PyObject_New(
+// Create a Python DescriptorPool object, but does not fill the "pool"
+// attribute.
+static PyDescriptorPool* _CreateDescriptorPool() {
+ PyDescriptorPool* cpool = PyObject_New(
PyDescriptorPool, &PyDescriptorPool_Type);
- if (cdescriptor_pool == NULL) {
+ if (cpool == NULL) {
return NULL;
}
- // Build a DescriptorPool for messages only declared in Python libraries.
- // generated_pool() contains all messages linked in C++ libraries, and is used
- // as underlay.
- cdescriptor_pool->pool = new DescriptorPool(DescriptorPool::generated_pool());
+ cpool->underlay = NULL;
+ cpool->database = NULL;
DynamicMessageFactory* message_factory = new DynamicMessageFactory();
// This option might be the default some day.
message_factory->SetDelegateToGeneratedFactory(true);
- cdescriptor_pool->message_factory = message_factory;
+ cpool->message_factory = message_factory;
// TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same
// storage.
- cdescriptor_pool->classes_by_descriptor =
+ cpool->classes_by_descriptor =
new PyDescriptorPool::ClassesByMessageMap();
- cdescriptor_pool->descriptor_options =
+ cpool->descriptor_options =
new hash_map<const void*, PyObject *>();
+ return cpool;
+}
+
+// Create a Python DescriptorPool, using the given pool as an underlay:
+// new messages will be added to a custom pool, not to the underlay.
+//
+// Ownership of the underlay is not transferred, its pointer should
+// stay alive.
+static PyDescriptorPool* PyDescriptorPool_NewWithUnderlay(
+ const DescriptorPool* underlay) {
+ PyDescriptorPool* cpool = _CreateDescriptorPool();
+ if (cpool == NULL) {
+ return NULL;
+ }
+ cpool->pool = new DescriptorPool(underlay);
+ cpool->underlay = underlay;
+
if (!descriptor_pool_map.insert(
- std::make_pair(cdescriptor_pool->pool, cdescriptor_pool)).second) {
+ std::make_pair(cpool->pool, cpool)).second) {
// Should never happen -- would indicate an internal error / bug.
PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered");
return NULL;
}
- return cdescriptor_pool;
+ return cpool;
+}
+
+static PyDescriptorPool* PyDescriptorPool_NewWithDatabase(
+ DescriptorDatabase* database) {
+ PyDescriptorPool* cpool = _CreateDescriptorPool();
+ if (cpool == NULL) {
+ return NULL;
+ }
+ if (database != NULL) {
+ cpool->pool = new DescriptorPool(database);
+ cpool->database = database;
+ } else {
+ cpool->pool = new DescriptorPool();
+ }
+
+ if (!descriptor_pool_map.insert(std::make_pair(cpool->pool, cpool)).second) {
+ // Should never happen -- would indicate an internal error / bug.
+ PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered");
+ return NULL;
+ }
+
+ return cpool;
+}
+
+// The public DescriptorPool constructor.
+static PyObject* New(PyTypeObject* type,
+ PyObject* args, PyObject* kwargs) {
+ static char* kwlist[] = {"descriptor_db", 0};
+ PyObject* py_database = NULL;
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", kwlist, &py_database)) {
+ return NULL;
+ }
+ DescriptorDatabase* database = NULL;
+ if (py_database && py_database != Py_None) {
+ database = new PyDescriptorDatabase(py_database);
+ }
+ return reinterpret_cast<PyObject*>(
+ PyDescriptorPool_NewWithDatabase(database));
}
static void Dealloc(PyDescriptorPool* self) {
@@ -108,8 +164,9 @@ static void Dealloc(PyDescriptorPool* self) {
Py_DECREF(it->second);
}
delete self->descriptor_options;
- delete self->pool;
delete self->message_factory;
+ delete self->database;
+ delete self->pool;
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
@@ -354,6 +411,14 @@ PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) {
char* message_type;
Py_ssize_t message_len;
+ if (self->database != NULL) {
+ PyErr_SetString(
+ PyExc_ValueError,
+ "Cannot call Add on a DescriptorPool that uses a DescriptorDatabase. "
+ "Add your file to the underlying database.");
+ return NULL;
+ }
+
if (PyBytes_AsStringAndSize(serialized_pb, &message_type, &message_len) < 0) {
return NULL;
}
@@ -366,8 +431,10 @@ PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) {
// If the file was already part of a C++ library, all its descriptors are in
// the underlying pool. No need to do anything else.
- const FileDescriptor* generated_file =
- DescriptorPool::generated_pool()->FindFileByName(file_proto.name());
+ const FileDescriptor* generated_file = NULL;
+ if (self->underlay) {
+ generated_file = self->underlay->FindFileByName(file_proto.name());
+ }
if (generated_file != NULL) {
return PyFileDescriptor_FromDescriptorWithSerializedPb(
generated_file, serialized_pb);
@@ -470,7 +537,7 @@ PyTypeObject PyDescriptorPool_Type = {
0, // tp_dictoffset
0, // tp_init
0, // tp_alloc
- 0, // tp_new
+ cdescriptor_pool::New, // tp_new
PyObject_Del, // tp_free
};
@@ -482,7 +549,11 @@ bool InitDescriptorPool() {
if (PyType_Ready(&PyDescriptorPool_Type) < 0)
return false;
- python_generated_pool = cdescriptor_pool::NewDescriptorPool();
+ // The Pool of messages declared in Python libraries.
+ // generated_pool() contains all messages already linked in C++ libraries, and
+ // is used as underlay.
+ python_generated_pool = cdescriptor_pool::PyDescriptorPool_NewWithUnderlay(
+ DescriptorPool::generated_pool());
if (python_generated_pool == NULL) {
return false;
}
@@ -494,6 +565,10 @@ bool InitDescriptorPool() {
return true;
}
+// The default DescriptorPool used everywhere in this module.
+// Today it's the python_generated_pool.
+// TODO(amauryfa): Remove all usages of this function: the pool should be
+// derived from the context.
PyDescriptorPool* GetDefaultDescriptorPool() {
return python_generated_pool;
}
diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h
index eda73d38..16bc910c 100644
--- a/python/google/protobuf/pyext/descriptor_pool.h
+++ b/python/google/protobuf/pyext/descriptor_pool.h
@@ -55,8 +55,17 @@ namespace python {
typedef struct PyDescriptorPool {
PyObject_HEAD
+ // The C++ pool containing Descriptors.
DescriptorPool* pool;
+ // The C++ pool acting as an underlay. Can be NULL.
+ // This pointer is not owned and must stay alive.
+ const DescriptorPool* underlay;
+
+ // The C++ descriptor database used to fetch unknown protos. Can be NULL.
+ // This pointer is owned.
+ const DescriptorDatabase* database;
+
// DynamicMessageFactory used to create C++ instances of messages.
// This object cache the descriptors that were used, so the DescriptorPool
// needs to get rid of it before it can delete itself.
@@ -138,6 +147,7 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg);
// Retrieve the global descriptor pool owned by the _message module.
// This is the one used by pb2.py generated modules.
// Returns a *borrowed* reference.
+// "Default" pool used to register messages from _pb2.py modules.
PyDescriptorPool* GetDefaultDescriptorPool();
// Retrieve the python descriptor pool owning a C++ descriptor pool.
diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc
index b361b342..555bd293 100644
--- a/python/google/protobuf/pyext/extension_dict.cc
+++ b/python/google/protobuf/pyext/extension_dict.cc
@@ -94,13 +94,13 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
if (descriptor == NULL) {
return NULL;
}
- if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) {
+ if (!CheckFieldBelongsToMessage(descriptor, self->message)) {
return NULL;
}
if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) {
- return cmessage::InternalGetScalar(self->parent->message, descriptor);
+ return cmessage::InternalGetScalar(self->message, descriptor);
}
PyObject* value = PyDict_GetItem(self->values, key);
@@ -109,6 +109,14 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
return value;
}
+ if (self->parent == NULL) {
+ // We are in "detached" state. Don't allow further modifications.
+ // TODO(amauryfa): Support adding non-scalars to a detached extension dict.
+ // This probably requires to store the type of the main message.
+ PyErr_SetObject(PyExc_KeyError, key);
+ return NULL;
+ }
+
if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
PyObject* sub_message = cmessage::InternalGetSubMessage(
@@ -154,7 +162,7 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
if (descriptor == NULL) {
return -1;
}
- if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) {
+ if (!CheckFieldBelongsToMessage(descriptor, self->message)) {
return -1;
}
@@ -164,9 +172,11 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
"type");
return -1;
}
- cmessage::AssureWritable(self->parent);
- if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) {
- return -1;
+ if (self->parent) {
+ cmessage::AssureWritable(self->parent);
+ if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) {
+ return -1;
+ }
}
// TODO(tibell): We shouldn't write scalars to the cache.
PyDict_SetItem(self->values, key, value);
@@ -180,15 +190,17 @@ PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
return NULL;
}
PyObject* value = PyDict_GetItem(self->values, extension);
- if (value != NULL) {
- if (ReleaseExtension(self, value, descriptor) < 0) {
+ if (self->parent) {
+ if (value != NULL) {
+ if (ReleaseExtension(self, value, descriptor) < 0) {
+ return NULL;
+ }
+ }
+ if (ScopedPyObjectPtr(cmessage::ClearFieldByDescriptor(
+ self->parent, descriptor)) == NULL) {
return NULL;
}
}
- if (ScopedPyObjectPtr(cmessage::ClearFieldByDescriptor(
- self->parent, descriptor)) == NULL) {
- return NULL;
- }
if (PyDict_DelItem(self->values, extension) < 0) {
PyErr_Clear();
}
@@ -201,8 +213,15 @@ PyObject* HasExtension(ExtensionDict* self, PyObject* extension) {
if (descriptor == NULL) {
return NULL;
}
- PyObject* result = cmessage::HasFieldByDescriptor(self->parent, descriptor);
- return result;
+ if (self->parent) {
+ return cmessage::HasFieldByDescriptor(self->parent, descriptor);
+ } else {
+ int exists = PyDict_Contains(self->values, extension);
+ if (exists < 0) {
+ return NULL;
+ }
+ return PyBool_FromLong(exists);
+ }
}
PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* name) {
diff --git a/python/google/protobuf/pyext/extension_dict.h b/python/google/protobuf/pyext/extension_dict.h
index 7a66cb23..d92cf956 100644
--- a/python/google/protobuf/pyext/extension_dict.h
+++ b/python/google/protobuf/pyext/extension_dict.h
@@ -117,11 +117,6 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value);
PyObject* ClearExtension(ExtensionDict* self,
PyObject* extension);
-// Checks if the dict has an extension.
-//
-// Returns a new python boolean reference.
-PyObject* HasExtension(ExtensionDict* self, PyObject* extension);
-
// Gets an extension from the dict given the extension name as opposed to
// descriptor.
//
diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc
new file mode 100644
index 00000000..c39f7b83
--- /dev/null
+++ b/python/google/protobuf/pyext/map_container.cc
@@ -0,0 +1,912 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: haberman@google.com (Josh Haberman)
+
+#include <google/protobuf/pyext/map_container.h>
+
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/stubs/scoped_ptr.h>
+#include <google/protobuf/map_field.h>
+#include <google/protobuf/map.h>
+#include <google/protobuf/message.h>
+#include <google/protobuf/pyext/message.h>
+#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
+
+#if PY_MAJOR_VERSION >= 3
+ #define PyInt_FromLong PyLong_FromLong
+ #define PyInt_FromSize_t PyLong_FromSize_t
+#endif
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+// Functions that need access to map reflection functionality.
+// They need to be contained in this class because it is friended.
+class MapReflectionFriend {
+ public:
+ // Methods that are in common between the map types.
+ static PyObject* Contains(PyObject* _self, PyObject* key);
+ static Py_ssize_t Length(PyObject* _self);
+ static PyObject* GetIterator(PyObject *_self);
+ static PyObject* IterNext(PyObject* _self);
+
+ // Methods that differ between the map types.
+ static PyObject* ScalarMapGetItem(PyObject* _self, PyObject* key);
+ static PyObject* MessageMapGetItem(PyObject* _self, PyObject* key);
+ static int ScalarMapSetItem(PyObject* _self, PyObject* key, PyObject* v);
+ static int MessageMapSetItem(PyObject* _self, PyObject* key, PyObject* v);
+};
+
+struct MapIterator {
+ PyObject_HEAD;
+
+ scoped_ptr< ::google::protobuf::MapIterator> iter;
+
+ // A pointer back to the container, so we can notice changes to the version.
+ // We own a ref on this.
+ MapContainer* container;
+
+ // We need to keep a ref on the Message* too, because
+ // MapIterator::~MapIterator() accesses it. Normally this would be ok because
+ // the ref on container (above) would guarantee outlive semantics. However in
+ // the case of ClearField(), InitializeAndCopyToParentContainer() resets the
+ // message pointer (and the owner) to a different message, a copy of the
+ // original. But our iterator still points to the original, which could now
+ // get deleted before us.
+ //
+ // To prevent this, we ensure that the Message will always stay alive as long
+ // as this iterator does. This is solely for the benefit of the MapIterator
+ // destructor -- we should never actually access the iterator in this state
+ // except to delete it.
+ shared_ptr<Message> owner;
+
+ // The version of the map when we took the iterator to it.
+ //
+ // We store this so that if the map is modified during iteration we can throw
+ // an error.
+ uint64 version;
+
+ // True if the container is empty. We signal this separately to avoid calling
+ // any of the iteration methods, which are non-const.
+ bool empty;
+};
+
+Message* MapContainer::GetMutableMessage() {
+ cmessage::AssureWritable(parent);
+ return const_cast<Message*>(message);
+}
+
+// Consumes a reference on the Python string object.
+static bool PyStringToSTL(PyObject* py_string, string* stl_string) {
+ char *value;
+ Py_ssize_t value_len;
+
+ if (!py_string) {
+ return false;
+ }
+ if (PyBytes_AsStringAndSize(py_string, &value, &value_len) < 0) {
+ Py_DECREF(py_string);
+ return false;
+ } else {
+ stl_string->assign(value, value_len);
+ Py_DECREF(py_string);
+ return true;
+ }
+}
+
+static bool PythonToMapKey(PyObject* obj,
+ const FieldDescriptor* field_descriptor,
+ MapKey* key) {
+ switch (field_descriptor->cpp_type()) {
+ case FieldDescriptor::CPPTYPE_INT32: {
+ GOOGLE_CHECK_GET_INT32(obj, value, false);
+ key->SetInt32Value(value);
+ break;
+ }
+ case FieldDescriptor::CPPTYPE_INT64: {
+ GOOGLE_CHECK_GET_INT64(obj, value, false);
+ key->SetInt64Value(value);
+ break;
+ }
+ case FieldDescriptor::CPPTYPE_UINT32: {
+ GOOGLE_CHECK_GET_UINT32(obj, value, false);
+ key->SetUInt32Value(value);
+ break;
+ }
+ case FieldDescriptor::CPPTYPE_UINT64: {
+ GOOGLE_CHECK_GET_UINT64(obj, value, false);
+ key->SetUInt64Value(value);
+ break;
+ }
+ case FieldDescriptor::CPPTYPE_BOOL: {
+ GOOGLE_CHECK_GET_BOOL(obj, value, false);
+ key->SetBoolValue(value);
+ break;
+ }
+ case FieldDescriptor::CPPTYPE_STRING: {
+ string str;
+ if (!PyStringToSTL(CheckString(obj, field_descriptor), &str)) {
+ return false;
+ }
+ key->SetStringValue(str);
+ break;
+ }
+ default:
+ PyErr_Format(
+ PyExc_SystemError, "Type %d cannot be a map key",
+ field_descriptor->cpp_type());
+ return false;
+ }
+ return true;
+}
+
+static PyObject* MapKeyToPython(const FieldDescriptor* field_descriptor,
+ const MapKey& key) {
+ switch (field_descriptor->cpp_type()) {
+ case FieldDescriptor::CPPTYPE_INT32:
+ return PyInt_FromLong(key.GetInt32Value());
+ case FieldDescriptor::CPPTYPE_INT64:
+ return PyLong_FromLongLong(key.GetInt64Value());
+ case FieldDescriptor::CPPTYPE_UINT32:
+ return PyInt_FromSize_t(key.GetUInt32Value());
+ case FieldDescriptor::CPPTYPE_UINT64:
+ return PyLong_FromUnsignedLongLong(key.GetUInt64Value());
+ case FieldDescriptor::CPPTYPE_BOOL:
+ return PyBool_FromLong(key.GetBoolValue());
+ case FieldDescriptor::CPPTYPE_STRING:
+ return ToStringObject(field_descriptor, key.GetStringValue());
+ default:
+ PyErr_Format(
+ PyExc_SystemError, "Couldn't convert type %d to value",
+ field_descriptor->cpp_type());
+ return NULL;
+ }
+}
+
+// This is only used for ScalarMap, so we don't need to handle the
+// CPPTYPE_MESSAGE case.
+PyObject* MapValueRefToPython(const FieldDescriptor* field_descriptor,
+ MapValueRef* value) {
+ switch (field_descriptor->cpp_type()) {
+ case FieldDescriptor::CPPTYPE_INT32:
+ return PyInt_FromLong(value->GetInt32Value());
+ case FieldDescriptor::CPPTYPE_INT64:
+ return PyLong_FromLongLong(value->GetInt64Value());
+ case FieldDescriptor::CPPTYPE_UINT32:
+ return PyInt_FromSize_t(value->GetUInt32Value());
+ case FieldDescriptor::CPPTYPE_UINT64:
+ return PyLong_FromUnsignedLongLong(value->GetUInt64Value());
+ case FieldDescriptor::CPPTYPE_FLOAT:
+ return PyFloat_FromDouble(value->GetFloatValue());
+ case FieldDescriptor::CPPTYPE_DOUBLE:
+ return PyFloat_FromDouble(value->GetDoubleValue());
+ case FieldDescriptor::CPPTYPE_BOOL:
+ return PyBool_FromLong(value->GetBoolValue());
+ case FieldDescriptor::CPPTYPE_STRING:
+ return ToStringObject(field_descriptor, value->GetStringValue());
+ case FieldDescriptor::CPPTYPE_ENUM:
+ return PyInt_FromLong(value->GetEnumValue());
+ default:
+ PyErr_Format(
+ PyExc_SystemError, "Couldn't convert type %d to value",
+ field_descriptor->cpp_type());
+ return NULL;
+ }
+}
+
+// This is only used for ScalarMap, so we don't need to handle the
+// CPPTYPE_MESSAGE case.
+static bool PythonToMapValueRef(PyObject* obj,
+ const FieldDescriptor* field_descriptor,
+ bool allow_unknown_enum_values,
+ MapValueRef* value_ref) {
+ switch (field_descriptor->cpp_type()) {
+ case FieldDescriptor::CPPTYPE_INT32: {
+ GOOGLE_CHECK_GET_INT32(obj, value, false);
+ value_ref->SetInt32Value(value);
+ return true;
+ }
+ case FieldDescriptor::CPPTYPE_INT64: {
+ GOOGLE_CHECK_GET_INT64(obj, value, false);
+ value_ref->SetInt64Value(value);
+ return true;
+ }
+ case FieldDescriptor::CPPTYPE_UINT32: {
+ GOOGLE_CHECK_GET_UINT32(obj, value, false);
+ value_ref->SetUInt32Value(value);
+ return true;
+ }
+ case FieldDescriptor::CPPTYPE_UINT64: {
+ GOOGLE_CHECK_GET_UINT64(obj, value, false);
+ value_ref->SetUInt64Value(value);
+ return true;
+ }
+ case FieldDescriptor::CPPTYPE_FLOAT: {
+ GOOGLE_CHECK_GET_FLOAT(obj, value, false);
+ value_ref->SetFloatValue(value);
+ return true;
+ }
+ case FieldDescriptor::CPPTYPE_DOUBLE: {
+ GOOGLE_CHECK_GET_DOUBLE(obj, value, false);
+ value_ref->SetDoubleValue(value);
+ return true;
+ }
+ case FieldDescriptor::CPPTYPE_BOOL: {
+ GOOGLE_CHECK_GET_BOOL(obj, value, false);
+ value_ref->SetBoolValue(value);
+ return true;;
+ }
+ case FieldDescriptor::CPPTYPE_STRING: {
+ string str;
+ if (!PyStringToSTL(CheckString(obj, field_descriptor), &str)) {
+ return false;
+ }
+ value_ref->SetStringValue(str);
+ return true;
+ }
+ case FieldDescriptor::CPPTYPE_ENUM: {
+ GOOGLE_CHECK_GET_INT32(obj, value, false);
+ if (allow_unknown_enum_values) {
+ value_ref->SetEnumValue(value);
+ return true;
+ } else {
+ const EnumDescriptor* enum_descriptor = field_descriptor->enum_type();
+ const EnumValueDescriptor* enum_value =
+ enum_descriptor->FindValueByNumber(value);
+ if (enum_value != NULL) {
+ value_ref->SetEnumValue(value);
+ return true;
+ } else {
+ PyErr_Format(PyExc_ValueError, "Unknown enum value: %d", value);
+ return false;
+ }
+ }
+ break;
+ }
+ default:
+ PyErr_Format(
+ PyExc_SystemError, "Setting value to a field of unknown type %d",
+ field_descriptor->cpp_type());
+ return false;
+ }
+}
+
+// Map methods common to ScalarMap and MessageMap //////////////////////////////
+
+static MapContainer* GetMap(PyObject* obj) {
+ return reinterpret_cast<MapContainer*>(obj);
+}
+
+Py_ssize_t MapReflectionFriend::Length(PyObject* _self) {
+ MapContainer* self = GetMap(_self);
+ const google::protobuf::Message* message = self->message;
+ return message->GetReflection()->MapSize(*message,
+ self->parent_field_descriptor);
+}
+
+PyObject* Clear(PyObject* _self) {
+ MapContainer* self = GetMap(_self);
+ Message* message = self->GetMutableMessage();
+ const Reflection* reflection = message->GetReflection();
+
+ reflection->ClearField(message, self->parent_field_descriptor);
+
+ Py_RETURN_NONE;
+}
+
+PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
+ MapContainer* self = GetMap(_self);
+
+ const Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+ MapKey map_key;
+
+ if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+ return NULL;
+ }
+
+ if (reflection->ContainsMapKey(*message, self->parent_field_descriptor,
+ map_key)) {
+ Py_RETURN_TRUE;
+ } else {
+ Py_RETURN_FALSE;
+ }
+}
+
+// Initializes the underlying Message object of "to" so it becomes a new parent
+// repeated scalar, and copies all the values from "from" to it. A child scalar
+// container can be released by passing it as both from and to (e.g. making it
+// the recipient of the new parent message and copying the values from itself).
+static int InitializeAndCopyToParentContainer(MapContainer* from,
+ MapContainer* to) {
+ // For now we require from == to, re-evaluate if we want to support deep copy
+ // as in repeated_scalar_container.cc.
+ GOOGLE_DCHECK(from == to);
+ Message* new_message = from->message->New();
+
+ if (MapReflectionFriend::Length(reinterpret_cast<PyObject*>(from)) > 0) {
+ // A somewhat roundabout way of copying just one field from old_message to
+ // new_message. This is the best we can do with what Reflection gives us.
+ Message* mutable_old = from->GetMutableMessage();
+ vector<const FieldDescriptor*> fields;
+ fields.push_back(from->parent_field_descriptor);
+
+ // Move the map field into the new message.
+ mutable_old->GetReflection()->SwapFields(mutable_old, new_message, fields);
+
+ // If/when we support from != to, this will be required also to copy the
+ // map field back into the existing message:
+ // mutable_old->MergeFrom(*new_message);
+ }
+
+ // If from == to this could delete old_message.
+ to->owner.reset(new_message);
+
+ to->parent = NULL;
+ to->parent_field_descriptor = from->parent_field_descriptor;
+ to->message = new_message;
+
+ // Invalidate iterators, since they point to the old copy of the field.
+ to->version++;
+
+ return 0;
+}
+
+int MapContainer::Release() {
+ return InitializeAndCopyToParentContainer(this, this);
+}
+
+
+// ScalarMap ///////////////////////////////////////////////////////////////////
+
+PyObject *NewScalarMapContainer(
+ CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) {
+ if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+ return NULL;
+ }
+
+ ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0));
+ if (obj.get() == NULL) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Could not allocate new container.");
+ }
+
+ MapContainer* self = GetMap(obj.get());
+
+ self->message = parent->message;
+ self->parent = parent;
+ self->parent_field_descriptor = parent_field_descriptor;
+ self->owner = parent->owner;
+ self->version = 0;
+
+ self->key_field_descriptor =
+ parent_field_descriptor->message_type()->FindFieldByName("key");
+ self->value_field_descriptor =
+ parent_field_descriptor->message_type()->FindFieldByName("value");
+
+ if (self->key_field_descriptor == NULL ||
+ self->value_field_descriptor == NULL) {
+ return PyErr_Format(PyExc_KeyError,
+ "Map entry descriptor did not have key/value fields");
+ }
+
+ return obj.release();
+}
+
+PyObject* MapReflectionFriend::ScalarMapGetItem(PyObject* _self,
+ PyObject* key) {
+ MapContainer* self = GetMap(_self);
+
+ Message* message = self->GetMutableMessage();
+ const Reflection* reflection = message->GetReflection();
+ MapKey map_key;
+ MapValueRef value;
+
+ if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+ return NULL;
+ }
+
+ if (reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor,
+ map_key, &value)) {
+ self->version++;
+ }
+
+ return MapValueRefToPython(self->value_field_descriptor, &value);
+}
+
+int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key,
+ PyObject* v) {
+ MapContainer* self = GetMap(_self);
+
+ Message* message = self->GetMutableMessage();
+ const Reflection* reflection = message->GetReflection();
+ MapKey map_key;
+ MapValueRef value;
+
+ if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+ return -1;
+ }
+
+ self->version++;
+
+ if (v) {
+ // Set item to v.
+ reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor,
+ map_key, &value);
+
+ return PythonToMapValueRef(v, self->value_field_descriptor,
+ reflection->SupportsUnknownEnumValues(), &value)
+ ? 0
+ : -1;
+ } else {
+ // Delete key from map.
+ if (reflection->DeleteMapValue(message, self->parent_field_descriptor,
+ map_key)) {
+ return 0;
+ } else {
+ PyErr_Format(PyExc_KeyError, "Key not present in map");
+ return -1;
+ }
+ }
+}
+
+static PyObject* ScalarMapGet(PyObject* self, PyObject* args) {
+ PyObject* key;
+ PyObject* default_value = NULL;
+ if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) {
+ return NULL;
+ }
+
+ ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key));
+ if (is_present.get() == NULL) {
+ return NULL;
+ }
+
+ if (PyObject_IsTrue(is_present.get())) {
+ return MapReflectionFriend::ScalarMapGetItem(self, key);
+ } else {
+ if (default_value != NULL) {
+ Py_INCREF(default_value);
+ return default_value;
+ } else {
+ Py_RETURN_NONE;
+ }
+ }
+}
+
+static void ScalarMapDealloc(PyObject* _self) {
+ MapContainer* self = GetMap(_self);
+ self->owner.reset();
+ Py_TYPE(_self)->tp_free(_self);
+}
+
+static PyMappingMethods ScalarMapMappingMethods = {
+ MapReflectionFriend::Length, // mp_length
+ MapReflectionFriend::ScalarMapGetItem, // mp_subscript
+ MapReflectionFriend::ScalarMapSetItem, // mp_ass_subscript
+};
+
+static PyMethodDef ScalarMapMethods[] = {
+ { "__contains__", MapReflectionFriend::Contains, METH_O,
+ "Tests whether a key is a member of the map." },
+ { "clear", (PyCFunction)Clear, METH_NOARGS,
+ "Removes all elements from the map." },
+ { "get", ScalarMapGet, METH_VARARGS,
+ "Gets the value for the given key if present, or otherwise a default" },
+ /*
+ { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+ "Makes a deep copy of the class." },
+ { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+ "Outputs picklable representation of the repeated field." },
+ */
+ {NULL, NULL},
+};
+
+PyTypeObject ScalarMapContainer_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ FULL_MODULE_NAME ".ScalarMapContainer", // tp_name
+ sizeof(MapContainer), // tp_basicsize
+ 0, // tp_itemsize
+ ScalarMapDealloc, // tp_dealloc
+ 0, // tp_print
+ 0, // tp_getattr
+ 0, // tp_setattr
+ 0, // tp_compare
+ 0, // tp_repr
+ 0, // tp_as_number
+ 0, // tp_as_sequence
+ &ScalarMapMappingMethods, // tp_as_mapping
+ 0, // tp_hash
+ 0, // tp_call
+ 0, // tp_str
+ 0, // tp_getattro
+ 0, // tp_setattro
+ 0, // tp_as_buffer
+ Py_TPFLAGS_DEFAULT, // tp_flags
+ "A scalar map container", // tp_doc
+ 0, // tp_traverse
+ 0, // tp_clear
+ 0, // tp_richcompare
+ 0, // tp_weaklistoffset
+ MapReflectionFriend::GetIterator, // tp_iter
+ 0, // tp_iternext
+ ScalarMapMethods, // tp_methods
+ 0, // tp_members
+ 0, // tp_getset
+ 0, // tp_base
+ 0, // tp_dict
+ 0, // tp_descr_get
+ 0, // tp_descr_set
+ 0, // tp_dictoffset
+ 0, // tp_init
+};
+
+
+// MessageMap //////////////////////////////////////////////////////////////////
+
+static MessageMapContainer* GetMessageMap(PyObject* obj) {
+ return reinterpret_cast<MessageMapContainer*>(obj);
+}
+
+static PyObject* GetCMessage(MessageMapContainer* self, Message* message) {
+ // Get or create the CMessage object corresponding to this message.
+ ScopedPyObjectPtr key(PyLong_FromVoidPtr(message));
+ PyObject* ret = PyDict_GetItem(self->message_dict, key.get());
+
+ if (ret == NULL) {
+ CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
+ message->GetDescriptor());
+ ret = reinterpret_cast<PyObject*>(cmsg);
+
+ if (cmsg == NULL) {
+ return NULL;
+ }
+ cmsg->owner = self->owner;
+ cmsg->message = message;
+ cmsg->parent = self->parent;
+
+ if (PyDict_SetItem(self->message_dict, key.get(), ret) < 0) {
+ Py_DECREF(ret);
+ return NULL;
+ }
+ } else {
+ Py_INCREF(ret);
+ }
+
+ return ret;
+}
+
+PyObject* NewMessageMapContainer(
+ CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor,
+ PyObject* concrete_class) {
+ if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+ return NULL;
+ }
+
+ PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0);
+ if (obj == NULL) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Could not allocate new container.");
+ }
+
+ MessageMapContainer* self = GetMessageMap(obj);
+
+ self->message = parent->message;
+ self->parent = parent;
+ self->parent_field_descriptor = parent_field_descriptor;
+ self->owner = parent->owner;
+ self->version = 0;
+
+ self->key_field_descriptor =
+ parent_field_descriptor->message_type()->FindFieldByName("key");
+ self->value_field_descriptor =
+ parent_field_descriptor->message_type()->FindFieldByName("value");
+
+ self->message_dict = PyDict_New();
+ if (self->message_dict == NULL) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Could not allocate message dict.");
+ }
+
+ Py_INCREF(concrete_class);
+ self->subclass_init = concrete_class;
+
+ if (self->key_field_descriptor == NULL ||
+ self->value_field_descriptor == NULL) {
+ Py_DECREF(obj);
+ return PyErr_Format(PyExc_KeyError,
+ "Map entry descriptor did not have key/value fields");
+ }
+
+ return obj;
+}
+
+int MapReflectionFriend::MessageMapSetItem(PyObject* _self, PyObject* key,
+ PyObject* v) {
+ if (v) {
+ PyErr_Format(PyExc_ValueError,
+ "Direct assignment of submessage not allowed");
+ return -1;
+ }
+
+ // Now we know that this is a delete, not a set.
+
+ MessageMapContainer* self = GetMessageMap(_self);
+ Message* message = self->GetMutableMessage();
+ const Reflection* reflection = message->GetReflection();
+ MapKey map_key;
+ MapValueRef value;
+
+ self->version++;
+
+ if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+ return -1;
+ }
+
+ // Delete key from map.
+ if (reflection->DeleteMapValue(message, self->parent_field_descriptor,
+ map_key)) {
+ return 0;
+ } else {
+ PyErr_Format(PyExc_KeyError, "Key not present in map");
+ return -1;
+ }
+}
+
+PyObject* MapReflectionFriend::MessageMapGetItem(PyObject* _self,
+ PyObject* key) {
+ MessageMapContainer* self = GetMessageMap(_self);
+
+ Message* message = self->GetMutableMessage();
+ const Reflection* reflection = message->GetReflection();
+ MapKey map_key;
+ MapValueRef value;
+
+ if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) {
+ return NULL;
+ }
+
+ if (reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor,
+ map_key, &value)) {
+ self->version++;
+ }
+
+ return GetCMessage(self, value.MutableMessageValue());
+}
+
+PyObject* MessageMapGet(PyObject* self, PyObject* args) {
+ PyObject* key;
+ PyObject* default_value = NULL;
+ if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) {
+ return NULL;
+ }
+
+ ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key));
+ if (is_present.get() == NULL) {
+ return NULL;
+ }
+
+ if (PyObject_IsTrue(is_present.get())) {
+ return MapReflectionFriend::MessageMapGetItem(self, key);
+ } else {
+ if (default_value != NULL) {
+ Py_INCREF(default_value);
+ return default_value;
+ } else {
+ Py_RETURN_NONE;
+ }
+ }
+}
+
+static void MessageMapDealloc(PyObject* _self) {
+ MessageMapContainer* self = GetMessageMap(_self);
+ self->owner.reset();
+ Py_DECREF(self->message_dict);
+ Py_TYPE(_self)->tp_free(_self);
+}
+
+static PyMappingMethods MessageMapMappingMethods = {
+ MapReflectionFriend::Length, // mp_length
+ MapReflectionFriend::MessageMapGetItem, // mp_subscript
+ MapReflectionFriend::MessageMapSetItem, // mp_ass_subscript
+};
+
+static PyMethodDef MessageMapMethods[] = {
+ { "__contains__", (PyCFunction)MapReflectionFriend::Contains, METH_O,
+ "Tests whether the map contains this element."},
+ { "clear", (PyCFunction)Clear, METH_NOARGS,
+ "Removes all elements from the map."},
+ { "get", MessageMapGet, METH_VARARGS,
+ "Gets the value for the given key if present, or otherwise a default" },
+ { "get_or_create", MapReflectionFriend::MessageMapGetItem, METH_O,
+ "Alias for getitem, useful to make explicit that the map is mutated." },
+ /*
+ { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+ "Makes a deep copy of the class." },
+ { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+ "Outputs picklable representation of the repeated field." },
+ */
+ {NULL, NULL},
+};
+
+PyTypeObject MessageMapContainer_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ FULL_MODULE_NAME ".MessageMapContainer", // tp_name
+ sizeof(MessageMapContainer), // tp_basicsize
+ 0, // tp_itemsize
+ MessageMapDealloc, // tp_dealloc
+ 0, // tp_print
+ 0, // tp_getattr
+ 0, // tp_setattr
+ 0, // tp_compare
+ 0, // tp_repr
+ 0, // tp_as_number
+ 0, // tp_as_sequence
+ &MessageMapMappingMethods, // tp_as_mapping
+ 0, // tp_hash
+ 0, // tp_call
+ 0, // tp_str
+ 0, // tp_getattro
+ 0, // tp_setattro
+ 0, // tp_as_buffer
+ Py_TPFLAGS_DEFAULT, // tp_flags
+ "A map container for message", // tp_doc
+ 0, // tp_traverse
+ 0, // tp_clear
+ 0, // tp_richcompare
+ 0, // tp_weaklistoffset
+ MapReflectionFriend::GetIterator, // tp_iter
+ 0, // tp_iternext
+ MessageMapMethods, // tp_methods
+ 0, // tp_members
+ 0, // tp_getset
+ 0, // tp_base
+ 0, // tp_dict
+ 0, // tp_descr_get
+ 0, // tp_descr_set
+ 0, // tp_dictoffset
+ 0, // tp_init
+};
+
+// MapIterator /////////////////////////////////////////////////////////////////
+
+static MapIterator* GetIter(PyObject* obj) {
+ return reinterpret_cast<MapIterator*>(obj);
+}
+
+PyObject* MapReflectionFriend::GetIterator(PyObject *_self) {
+ MapContainer* self = GetMap(_self);
+
+ ScopedPyObjectPtr obj(PyType_GenericAlloc(&MapIterator_Type, 0));
+ if (obj == NULL) {
+ return PyErr_Format(PyExc_KeyError, "Could not allocate iterator");
+ }
+
+ MapIterator* iter = GetIter(obj.get());
+
+ Py_INCREF(self);
+ iter->container = self;
+ iter->version = self->version;
+ iter->owner = self->owner;
+
+ if (MapReflectionFriend::Length(_self) > 0) {
+ Message* message = self->GetMutableMessage();
+ const Reflection* reflection = message->GetReflection();
+
+ iter->iter.reset(new ::google::protobuf::MapIterator(
+ reflection->MapBegin(message, self->parent_field_descriptor)));
+ }
+
+ return obj.release();
+}
+
+PyObject* MapReflectionFriend::IterNext(PyObject* _self) {
+ MapIterator* self = GetIter(_self);
+
+ // This won't catch mutations to the map performed by MergeFrom(); no easy way
+ // to address that.
+ if (self->version != self->container->version) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Map modified during iteration.");
+ }
+
+ if (self->iter.get() == NULL) {
+ return NULL;
+ }
+
+ Message* message = self->container->GetMutableMessage();
+ const Reflection* reflection = message->GetReflection();
+
+ if (*self->iter ==
+ reflection->MapEnd(message, self->container->parent_field_descriptor)) {
+ return NULL;
+ }
+
+ PyObject* ret = MapKeyToPython(self->container->key_field_descriptor,
+ self->iter->GetKey());
+
+ ++(*self->iter);
+
+ return ret;
+}
+
+static void DeallocMapIterator(PyObject* _self) {
+ MapIterator* self = GetIter(_self);
+ self->iter.reset();
+ self->owner.reset();
+ Py_XDECREF(self->container);
+ Py_TYPE(_self)->tp_free(_self);
+}
+
+PyTypeObject MapIterator_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ FULL_MODULE_NAME ".MapIterator", // tp_name
+ sizeof(MapIterator), // tp_basicsize
+ 0, // tp_itemsize
+ DeallocMapIterator, // tp_dealloc
+ 0, // tp_print
+ 0, // tp_getattr
+ 0, // tp_setattr
+ 0, // tp_compare
+ 0, // tp_repr
+ 0, // tp_as_number
+ 0, // tp_as_sequence
+ 0, // tp_as_mapping
+ 0, // tp_hash
+ 0, // tp_call
+ 0, // tp_str
+ 0, // tp_getattro
+ 0, // tp_setattro
+ 0, // tp_as_buffer
+ Py_TPFLAGS_DEFAULT, // tp_flags
+ "A scalar map iterator", // tp_doc
+ 0, // tp_traverse
+ 0, // tp_clear
+ 0, // tp_richcompare
+ 0, // tp_weaklistoffset
+ PyObject_SelfIter, // tp_iter
+ MapReflectionFriend::IterNext, // tp_iternext
+ 0, // tp_methods
+ 0, // tp_members
+ 0, // tp_getset
+ 0, // tp_base
+ 0, // tp_dict
+ 0, // tp_descr_get
+ 0, // tp_descr_set
+ 0, // tp_dictoffset
+ 0, // tp_init
+};
+
+} // namespace python
+} // namespace protobuf
+} // namespace google
diff --git a/python/google/protobuf/pyext/message_map_container.h b/python/google/protobuf/pyext/map_container.h
index 4f6cb26a..2de61187 100644
--- a/python/google/protobuf/pyext/message_map_container.h
+++ b/python/google/protobuf/pyext/map_container.h
@@ -28,8 +28,8 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__
-#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__
#include <Python.h>
@@ -39,6 +39,7 @@
#endif
#include <google/protobuf/descriptor.h>
+#include <google/protobuf/message.h>
namespace google {
namespace protobuf {
@@ -55,18 +56,23 @@ namespace python {
struct CMessage;
-struct MessageMapContainer {
+// This struct is used directly for ScalarMap, and is the base class of
+// MessageMapContainer, which is used for MessageMap.
+struct MapContainer {
PyObject_HEAD;
// This is the top-level C++ Message object that owns the whole
- // proto tree. Every Python MessageMapContainer holds a
+ // proto tree. Every Python MapContainer holds a
// reference to it in order to keep it alive as long as there's a
// Python object that references any part of the tree.
shared_ptr<Message> owner;
// Pointer to the C++ Message that contains this container. The
- // MessageMapContainer does not own this pointer.
- Message* message;
+ // MapContainer does not own this pointer.
+ const Message* message;
+
+ // Use to get a mutable message when necessary.
+ Message* GetMutableMessage();
// Weak reference to a parent CMessage object (i.e. may be NULL.)
//
@@ -82,45 +88,46 @@ struct MessageMapContainer {
const FieldDescriptor* key_field_descriptor;
const FieldDescriptor* value_field_descriptor;
+ // We bump this whenever we perform a mutation, to invalidate existing
+ // iterators.
+ uint64 version;
+
+ // Releases the messages in the container to a new message.
+ //
+ // Returns 0 on success, -1 on failure.
+ int Release();
+
+ // Set the owner field of self and any children of self.
+ void SetOwner(const shared_ptr<Message>& new_owner) {
+ owner = new_owner;
+ }
+};
+
+struct MessageMapContainer : public MapContainer {
// A callable that is used to create new child messages.
PyObject* subclass_init;
// A dict mapping Message* -> CMessage.
PyObject* message_dict;
-
- // We bump this whenever we perform a mutation, to invalidate existing
- // iterators.
- uint64 version;
};
-#if PY_MAJOR_VERSION >= 3
- extern PyObject *MessageMapContainer_Type;
- extern PyType_Spec MessageMapContainer_Type_spec;
-#else
- extern PyTypeObject MessageMapContainer_Type;
-#endif
-extern PyTypeObject MessageMapIterator_Type;
+extern PyTypeObject ScalarMapContainer_Type;
+extern PyTypeObject MessageMapContainer_Type;
+extern PyTypeObject MapIterator_Type; // Both map types use the same iterator.
-namespace message_map_container {
-
-// Builds a MessageMapContainer object, from a parent message and a
+// Builds a MapContainer object, from a parent message and a
// field descriptor.
-extern PyObject* NewContainer(CMessage* parent,
- const FieldDescriptor* parent_field_descriptor,
- PyObject* concrete_class);
-
-// Releases the messages in the container to a new message.
-//
-// Returns 0 on success, -1 on failure.
-int Release(MessageMapContainer* self);
+extern PyObject* NewScalarMapContainer(
+ CMessage* parent, const FieldDescriptor* parent_field_descriptor);
-// Set the owner field of self and any children of self.
-void SetOwner(MessageMapContainer* self,
- const shared_ptr<Message>& new_owner);
+// Builds a MessageMap object, from a parent message and a
+// field descriptor.
+extern PyObject* NewMessageMapContainer(
+ CMessage* parent, const FieldDescriptor* parent_field_descriptor,
+ PyObject* concrete_class);
-} // namespace message_map_container
} // namespace python
} // namespace protobuf
} // namespace google
-#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__
+#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__
diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
index 72f51ec1..863cde01 100644
--- a/python/google/protobuf/pyext/message.cc
+++ b/python/google/protobuf/pyext/message.cc
@@ -33,6 +33,7 @@
#include <google/protobuf/pyext/message.h>
+#include <map>
#include <memory>
#ifndef _SHARED_PTR_H
#include <google/protobuf/stubs/shared_ptr.h>
@@ -61,8 +62,7 @@
#include <google/protobuf/pyext/extension_dict.h>
#include <google/protobuf/pyext/repeated_composite_container.h>
#include <google/protobuf/pyext/repeated_scalar_container.h>
-#include <google/protobuf/pyext/message_map_container.h>
-#include <google/protobuf/pyext/scalar_map_container.h>
+#include <google/protobuf/pyext/map_container.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#include <google/protobuf/stubs/strutil.h>
@@ -96,6 +96,7 @@ static PyObject* k_extensions_by_number;
PyObject* EnumTypeWrapper_class;
static PyObject* PythonMessage_class;
static PyObject* kEmptyWeakref;
+static PyObject* WKT_classes = NULL;
// Defines the Metaclass of all Message classes.
// It allows us to cache some C++ pointers in the class object itself, they are
@@ -274,8 +275,32 @@ static PyObject* New(PyTypeObject* type,
// Build the arguments to the base metaclass.
// We change the __bases__ classes.
- ScopedPyObjectPtr new_args(Py_BuildValue(
- "s(OO)O", name, &CMessage_Type, PythonMessage_class, dict));
+ ScopedPyObjectPtr new_args;
+ const Descriptor* message_descriptor =
+ PyMessageDescriptor_AsDescriptor(py_descriptor);
+ if (message_descriptor == NULL) {
+ return NULL;
+ }
+
+ if (WKT_classes == NULL) {
+ ScopedPyObjectPtr well_known_types(PyImport_ImportModule(
+ "google.protobuf.internal.well_known_types"));
+ GOOGLE_DCHECK(well_known_types != NULL);
+
+ WKT_classes = PyObject_GetAttrString(well_known_types.get(), "WKTBASES");
+ GOOGLE_DCHECK(WKT_classes != NULL);
+ }
+
+ PyObject* well_known_class = PyDict_GetItemString(
+ WKT_classes, message_descriptor->full_name().c_str());
+ if (well_known_class == NULL) {
+ new_args.reset(Py_BuildValue("s(OO)O", name, &CMessage_Type,
+ PythonMessage_class, dict));
+ } else {
+ new_args.reset(Py_BuildValue("s(OOO)O", name, &CMessage_Type,
+ PythonMessage_class, well_known_class, dict));
+ }
+
if (new_args == NULL) {
return NULL;
}
@@ -448,21 +473,9 @@ static int VisitCompositeField(const FieldDescriptor* descriptor,
if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
if (descriptor->is_map()) {
- const Descriptor* entry_type = descriptor->message_type();
- const FieldDescriptor* value_type =
- entry_type->FindFieldByName("value");
- if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
- MessageMapContainer* container =
- reinterpret_cast<MessageMapContainer*>(child);
- if (visitor.VisitMessageMapContainer(container) == -1) {
- return -1;
- }
- } else {
- ScalarMapContainer* container =
- reinterpret_cast<ScalarMapContainer*>(child);
- if (visitor.VisitScalarMapContainer(container) == -1) {
- return -1;
- }
+ MapContainer* container = reinterpret_cast<MapContainer*>(child);
+ if (visitor.VisitMapContainer(container) == -1) {
+ return -1;
}
} else {
RepeatedCompositeContainer* container =
@@ -579,12 +592,14 @@ bool CheckAndGetInteger(
if (PyObject_RichCompareBool(min, arg, Py_LE) != 1 ||
PyObject_RichCompareBool(max, arg, Py_GE) != 1) {
#endif
- PyObject *s = PyObject_Str(arg);
- if (s) {
- PyErr_Format(PyExc_ValueError,
- "Value out of range: %s",
- PyString_AsString(s));
- Py_DECREF(s);
+ if (!PyErr_Occurred()) {
+ PyObject *s = PyObject_Str(arg);
+ if (s) {
+ PyErr_Format(PyExc_ValueError,
+ "Value out of range: %s",
+ PyString_AsString(s));
+ Py_DECREF(s);
+ }
}
return false;
}
@@ -642,38 +657,51 @@ bool CheckAndGetBool(PyObject* arg, bool* value) {
return true;
}
-bool CheckAndSetString(
- PyObject* arg, Message* message,
- const FieldDescriptor* descriptor,
- const Reflection* reflection,
- bool append,
- int index) {
+// Checks whether the given object (which must be "bytes" or "unicode") contains
+// valid UTF-8.
+bool IsValidUTF8(PyObject* obj) {
+ if (PyBytes_Check(obj)) {
+ PyObject* unicode = PyUnicode_FromEncodedObject(obj, "utf-8", NULL);
+
+ // Clear the error indicator; we report our own error when desired.
+ PyErr_Clear();
+
+ if (unicode) {
+ Py_DECREF(unicode);
+ return true;
+ } else {
+ return false;
+ }
+ } else {
+ // Unicode object, known to be valid UTF-8.
+ return true;
+ }
+}
+
+bool AllowInvalidUTF8(const FieldDescriptor* field) { return false; }
+
+PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor) {
GOOGLE_DCHECK(descriptor->type() == FieldDescriptor::TYPE_STRING ||
descriptor->type() == FieldDescriptor::TYPE_BYTES);
if (descriptor->type() == FieldDescriptor::TYPE_STRING) {
if (!PyBytes_Check(arg) && !PyUnicode_Check(arg)) {
FormatTypeError(arg, "bytes, unicode");
- return false;
+ return NULL;
}
- if (PyBytes_Check(arg)) {
- PyObject* unicode = PyUnicode_FromEncodedObject(arg, "utf-8", NULL);
- if (unicode == NULL) {
- PyObject* repr = PyObject_Repr(arg);
- PyErr_Format(PyExc_ValueError,
- "%s has type str, but isn't valid UTF-8 "
- "encoding. Non-UTF-8 strings must be converted to "
- "unicode objects before being added.",
- PyString_AsString(repr));
- Py_DECREF(repr);
- return false;
- } else {
- Py_DECREF(unicode);
- }
+ if (!IsValidUTF8(arg) && !AllowInvalidUTF8(descriptor)) {
+ PyObject* repr = PyObject_Repr(arg);
+ PyErr_Format(PyExc_ValueError,
+ "%s has type str, but isn't valid UTF-8 "
+ "encoding. Non-UTF-8 strings must be converted to "
+ "unicode objects before being added.",
+ PyString_AsString(repr));
+ Py_DECREF(repr);
+ return NULL;
}
} else if (!PyBytes_Check(arg)) {
FormatTypeError(arg, "bytes");
- return false;
+ return NULL;
}
PyObject* encoded_string = NULL;
@@ -691,14 +719,24 @@ bool CheckAndSetString(
Py_INCREF(encoded_string);
}
- if (encoded_string == NULL) {
+ return encoded_string;
+}
+
+bool CheckAndSetString(
+ PyObject* arg, Message* message,
+ const FieldDescriptor* descriptor,
+ const Reflection* reflection,
+ bool append,
+ int index) {
+ ScopedPyObjectPtr encoded_string(CheckString(arg, descriptor));
+
+ if (encoded_string.get() == NULL) {
return false;
}
char* value;
Py_ssize_t value_len;
- if (PyBytes_AsStringAndSize(encoded_string, &value, &value_len) < 0) {
- Py_DECREF(encoded_string);
+ if (PyBytes_AsStringAndSize(encoded_string.get(), &value, &value_len) < 0) {
return false;
}
@@ -710,7 +748,6 @@ bool CheckAndSetString(
} else {
reflection->SetRepeatedString(message, descriptor, index, value_string);
}
- Py_DECREF(encoded_string);
return true;
}
@@ -823,12 +860,7 @@ struct FixupMessageReference : public ChildVisitor {
return 0;
}
- int VisitScalarMapContainer(ScalarMapContainer* container) {
- container->message = message_;
- return 0;
- }
-
- int VisitMessageMapContainer(MessageMapContainer* container) {
+ int VisitMapContainer(MapContainer* container) {
container->message = message_;
return 0;
}
@@ -870,9 +902,8 @@ int AssureWritable(CMessage* self) {
// When a CMessage is made writable its Message pointer is updated
// to point to a new mutable Message. When that happens we need to
// update any references to the old, read-only CMessage. There are
- // five places such references occur: RepeatedScalarContainer,
- // RepeatedCompositeContainer, ScalarMapContainer, MessageMapContainer,
- // and ExtensionDict.
+ // four places such references occur: RepeatedScalarContainer,
+ // RepeatedCompositeContainer, MapContainer, and ExtensionDict.
if (self->extensions != NULL)
self->extensions->message = self->message;
if (ForEachCompositeField(self, FixupMessageReference(self->message)) == -1)
@@ -1054,7 +1085,8 @@ int InitAttributes(CMessage* self, PyObject* kwargs) {
}
const FieldDescriptor* descriptor = GetFieldDescriptor(self, name);
if (descriptor == NULL) {
- PyErr_Format(PyExc_ValueError, "Protocol message has no \"%s\" field.",
+ PyErr_Format(PyExc_ValueError, "Protocol message %s has no \"%s\" field.",
+ self->message->GetDescriptor()->name().c_str(),
PyString_AsString(name));
return -1;
}
@@ -1203,18 +1235,6 @@ CMessage* NewEmptyMessage(PyObject* type, const Descriptor *descriptor) {
self->composite_fields = NULL;
- // If there are extension_ranges, the message is "extendable". Allocate a
- // dictionary to store the extension fields.
- if (descriptor->extension_range_count() > 0) {
- // TODO(amauryfa): Delay the construction of this dict until extensions are
- // really used on the object.
- ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self);
- if (extension_dict == NULL) {
- return NULL;
- }
- self->extensions = extension_dict;
- }
-
return self;
}
@@ -1285,12 +1305,7 @@ struct ClearWeakReferences : public ChildVisitor {
return 0;
}
- int VisitScalarMapContainer(ScalarMapContainer* container) {
- container->parent = NULL;
- return 0;
- }
-
- int VisitMessageMapContainer(MessageMapContainer* container) {
+ int VisitMapContainer(MapContainer* container) {
container->parent = NULL;
return 0;
}
@@ -1305,6 +1320,9 @@ struct ClearWeakReferences : public ChildVisitor {
static void Dealloc(CMessage* self) {
// Null out all weak references from children to this message.
GOOGLE_CHECK_EQ(0, ForEachCompositeField(self, ClearWeakReferences()));
+ if (self->extensions) {
+ self->extensions->parent = NULL;
+ }
Py_CLEAR(self->extensions);
Py_CLEAR(self->composite_fields);
@@ -1466,20 +1484,27 @@ PyObject* HasField(CMessage* self, PyObject* arg) {
Py_RETURN_FALSE;
}
-PyObject* ClearExtension(CMessage* self, PyObject* arg) {
+PyObject* ClearExtension(CMessage* self, PyObject* extension) {
if (self->extensions != NULL) {
- return extension_dict::ClearExtension(self->extensions, arg);
+ return extension_dict::ClearExtension(self->extensions, extension);
+ } else {
+ const FieldDescriptor* descriptor = GetExtensionDescriptor(extension);
+ if (descriptor == NULL) {
+ return NULL;
+ }
+ if (ScopedPyObjectPtr(ClearFieldByDescriptor(self, descriptor)) == NULL) {
+ return NULL;
+ }
}
- PyErr_SetString(PyExc_TypeError, "Message is not extendable");
- return NULL;
+ Py_RETURN_NONE;
}
-PyObject* HasExtension(CMessage* self, PyObject* arg) {
- if (self->extensions != NULL) {
- return extension_dict::HasExtension(self->extensions, arg);
+PyObject* HasExtension(CMessage* self, PyObject* extension) {
+ const FieldDescriptor* descriptor = GetExtensionDescriptor(extension);
+ if (descriptor == NULL) {
+ return NULL;
}
- PyErr_SetString(PyExc_TypeError, "Message is not extendable");
- return NULL;
+ return HasFieldByDescriptor(self, descriptor);
}
// ---------------------------------------------------------------------
@@ -1529,13 +1554,8 @@ struct SetOwnerVisitor : public ChildVisitor {
return 0;
}
- int VisitScalarMapContainer(ScalarMapContainer* container) {
- scalar_map_container::SetOwner(container, new_owner_);
- return 0;
- }
-
- int VisitMessageMapContainer(MessageMapContainer* container) {
- message_map_container::SetOwner(container, new_owner_);
+ int VisitMapContainer(MapContainer* container) {
+ container->SetOwner(new_owner_);
return 0;
}
@@ -1608,14 +1628,8 @@ struct ReleaseChild : public ChildVisitor {
reinterpret_cast<RepeatedScalarContainer*>(container));
}
- int VisitScalarMapContainer(ScalarMapContainer* container) {
- return scalar_map_container::Release(
- reinterpret_cast<ScalarMapContainer*>(container));
- }
-
- int VisitMessageMapContainer(MessageMapContainer* container) {
- return message_map_container::Release(
- reinterpret_cast<MessageMapContainer*>(container));
+ int VisitMapContainer(MapContainer* container) {
+ return reinterpret_cast<MapContainer*>(container)->Release();
}
int VisitCMessage(CMessage* cmessage,
@@ -1707,17 +1721,7 @@ PyObject* Clear(CMessage* self) {
AssureWritable(self);
if (ForEachCompositeField(self, ReleaseChild(self)) == -1)
return NULL;
-
- // The old ExtensionDict still aliases this CMessage, but all its
- // fields have been released.
- if (self->extensions != NULL) {
- Py_CLEAR(self->extensions);
- ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self);
- if (extension_dict == NULL) {
- return NULL;
- }
- self->extensions = extension_dict;
- }
+ Py_CLEAR(self->extensions);
if (self->composite_fields) {
PyDict_Clear(self->composite_fields);
}
@@ -1997,7 +2001,6 @@ static PyObject* RegisterExtension(PyObject* cls,
if (descriptor->is_extension() &&
descriptor->containing_type()->options().message_set_wire_format() &&
descriptor->type() == FieldDescriptor::TYPE_MESSAGE &&
- descriptor->message_type() == descriptor->extension_scope() &&
descriptor->label() == FieldDescriptor::LABEL_OPTIONAL) {
ScopedPyObjectPtr message_name(PyString_FromStringAndSize(
descriptor->message_type()->full_name().c_str(),
@@ -2042,6 +2045,8 @@ static PyObject* WhichOneof(CMessage* self, PyObject* arg) {
}
}
+static PyObject* GetExtensionDict(CMessage* self, void *closure);
+
static PyObject* ListFields(CMessage* self) {
vector<const FieldDescriptor*> fields;
self->message->GetReflection()->ListFields(*self->message, &fields);
@@ -2079,12 +2084,13 @@ static PyObject* ListFields(CMessage* self) {
PyErr_Clear();
continue;
}
- PyObject* extensions = reinterpret_cast<PyObject*>(self->extensions);
+ ScopedPyObjectPtr extensions(GetExtensionDict(self, NULL));
if (extensions == NULL) {
return NULL;
}
// 'extension' reference later stolen by PyTuple_SET_ITEM.
- PyObject* extension = PyObject_GetItem(extensions, extension_field.get());
+ PyObject* extension = PyObject_GetItem(
+ extensions.get(), extension_field.get());
if (extension == NULL) {
return NULL;
}
@@ -2493,9 +2499,31 @@ PyObject* _CheckCalledFromGeneratedFile(PyObject* unused,
Py_RETURN_NONE;
}
-static PyMemberDef Members[] = {
- {"Extensions", T_OBJECT_EX, offsetof(CMessage, extensions), 0,
- "Extension dict"},
+static PyObject* GetExtensionDict(CMessage* self, void *closure) {
+ if (self->extensions) {
+ Py_INCREF(self->extensions);
+ return reinterpret_cast<PyObject*>(self->extensions);
+ }
+
+ // If there are extension_ranges, the message is "extendable". Allocate a
+ // dictionary to store the extension fields.
+ const Descriptor* descriptor = GetMessageDescriptor(Py_TYPE(self));
+ if (descriptor->extension_range_count() > 0) {
+ ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self);
+ if (extension_dict == NULL) {
+ return NULL;
+ }
+ self->extensions = extension_dict;
+ Py_INCREF(self->extensions);
+ return reinterpret_cast<PyObject*>(self->extensions);
+ }
+
+ PyErr_SetNone(PyExc_AttributeError);
+ return NULL;
+}
+
+static PyGetSetDef Getters[] = {
+ {"Extensions", (getter)GetExtensionDict, NULL, "Extension dict"},
{NULL}
};
@@ -2592,10 +2620,10 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
if (value_class == NULL) {
return NULL;
}
- py_container = message_map_container::NewContainer(self, field_descriptor,
- value_class);
+ py_container =
+ NewMessageMapContainer(self, field_descriptor, value_class);
} else {
- py_container = scalar_map_container::NewContainer(self, field_descriptor);
+ py_container = NewScalarMapContainer(self, field_descriptor);
}
if (py_container == NULL) {
return NULL;
@@ -2672,7 +2700,10 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) {
}
}
- PyErr_Format(PyExc_AttributeError, "Assignment not allowed");
+ PyErr_Format(PyExc_AttributeError,
+ "Assignment not allowed "
+ "(no field \"%s\"in protocol message object).",
+ PyString_AsString(name));
return -1;
}
@@ -2707,8 +2738,8 @@ PyTypeObject CMessage_Type = {
0, // tp_iter
0, // tp_iternext
cmessage::Methods, // tp_methods
- cmessage::Members, // tp_members
- 0, // tp_getset
+ 0, // tp_members
+ cmessage::Getters, // tp_getset
0, // tp_base
0, // tp_dict
0, // tp_descr_get
@@ -2910,12 +2941,12 @@ bool InitProto2MessageModule(PyObject *m) {
reinterpret_cast<PyObject*>(&ScalarMapContainer_Type));
#endif
- if (PyType_Ready(&ScalarMapIterator_Type) < 0) {
+ if (PyType_Ready(&MapIterator_Type) < 0) {
return false;
}
- PyModule_AddObject(m, "ScalarMapIterator",
- reinterpret_cast<PyObject*>(&ScalarMapIterator_Type));
+ PyModule_AddObject(m, "MapIterator",
+ reinterpret_cast<PyObject*>(&MapIterator_Type));
#if PY_MAJOR_VERSION >= 3
@@ -2934,13 +2965,6 @@ bool InitProto2MessageModule(PyObject *m) {
PyModule_AddObject(m, "MessageMapContainer",
reinterpret_cast<PyObject*>(&MessageMapContainer_Type));
#endif
-
- if (PyType_Ready(&MessageMapIterator_Type) < 0) {
- return false;
- }
-
- PyModule_AddObject(m, "MessageMapIterator",
- reinterpret_cast<PyObject*>(&MessageMapIterator_Type));
}
if (PyType_Ready(&ExtensionDict_Type) < 0) {
@@ -2957,6 +2981,9 @@ bool InitProto2MessageModule(PyObject *m) {
PyModule_AddObject(m, "default_pool",
reinterpret_cast<PyObject*>(GetDefaultDescriptorPool()));
+ PyModule_AddObject(m, "DescriptorPool", reinterpret_cast<PyObject*>(
+ &PyDescriptorPool_Type));
+
// This implementation provides full Descriptor types, we advertise it so that
// descriptor.py can use them in replacement of the Python classes.
PyModule_AddIntConstant(m, "_USE_C_DESCRIPTORS", 1);
diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h
index 94de4551..cc0012e9 100644
--- a/python/google/protobuf/pyext/message.h
+++ b/python/google/protobuf/pyext/message.h
@@ -307,6 +307,7 @@ bool CheckAndGetInteger(
bool CheckAndGetDouble(PyObject* arg, double* value);
bool CheckAndGetFloat(PyObject* arg, float* value);
bool CheckAndGetBool(PyObject* arg, bool* value);
+PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor);
bool CheckAndSetString(
PyObject* arg, Message* message,
const FieldDescriptor* descriptor,
diff --git a/python/google/protobuf/pyext/message_map_container.cc b/python/google/protobuf/pyext/message_map_container.cc
deleted file mode 100644
index 8902fa00..00000000
--- a/python/google/protobuf/pyext/message_map_container.cc
+++ /dev/null
@@ -1,569 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2008 Google Inc. All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-// Author: haberman@google.com (Josh Haberman)
-
-#include <google/protobuf/pyext/message_map_container.h>
-
-#include <google/protobuf/stubs/logging.h>
-#include <google/protobuf/stubs/common.h>
-#include <google/protobuf/message.h>
-#include <google/protobuf/pyext/message.h>
-#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
-
-namespace google {
-namespace protobuf {
-namespace python {
-
-struct MessageMapIterator {
- PyObject_HEAD;
-
- // This dict contains the full contents of what we want to iterate over.
- // There's no way to avoid building this, because the list representation
- // (which is canonical) can contain duplicate keys. So at the very least we
- // need a set that lets us skip duplicate keys. And at the point that we're
- // doing that, we might as well just build the actual dict we're iterating
- // over and use dict's built-in iterator.
- PyObject* dict;
-
- // An iterator on dict.
- PyObject* iter;
-
- // A pointer back to the container, so we can notice changes to the version.
- MessageMapContainer* container;
-
- // The version of the map when we took the iterator to it.
- //
- // We store this so that if the map is modified during iteration we can throw
- // an error.
- uint64 version;
-};
-
-static MessageMapIterator* GetIter(PyObject* obj) {
- return reinterpret_cast<MessageMapIterator*>(obj);
-}
-
-namespace message_map_container {
-
-static MessageMapContainer* GetMap(PyObject* obj) {
- return reinterpret_cast<MessageMapContainer*>(obj);
-}
-
-// The private constructor of MessageMapContainer objects.
-PyObject* NewContainer(CMessage* parent,
- const google::protobuf::FieldDescriptor* parent_field_descriptor,
- PyObject* concrete_class) {
- if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
- return NULL;
- }
-
-#if PY_MAJOR_VERSION >= 3
- PyObject* obj = PyType_GenericAlloc(
- reinterpret_cast<PyTypeObject *>(MessageMapContainer_Type), 0);
-#else
- PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0);
-#endif
- if (obj == NULL) {
- return PyErr_Format(PyExc_RuntimeError,
- "Could not allocate new container.");
- }
-
- MessageMapContainer* self = GetMap(obj);
-
- self->message = parent->message;
- self->parent = parent;
- self->parent_field_descriptor = parent_field_descriptor;
- self->owner = parent->owner;
- self->version = 0;
-
- self->key_field_descriptor =
- parent_field_descriptor->message_type()->FindFieldByName("key");
- self->value_field_descriptor =
- parent_field_descriptor->message_type()->FindFieldByName("value");
-
- self->message_dict = PyDict_New();
- if (self->message_dict == NULL) {
- return PyErr_Format(PyExc_RuntimeError,
- "Could not allocate message dict.");
- }
-
- Py_INCREF(concrete_class);
- self->subclass_init = concrete_class;
-
- if (self->key_field_descriptor == NULL ||
- self->value_field_descriptor == NULL) {
- Py_DECREF(obj);
- return PyErr_Format(PyExc_KeyError,
- "Map entry descriptor did not have key/value fields");
- }
-
- return obj;
-}
-
-// Initializes the underlying Message object of "to" so it becomes a new parent
-// repeated scalar, and copies all the values from "from" to it. A child scalar
-// container can be released by passing it as both from and to (e.g. making it
-// the recipient of the new parent message and copying the values from itself).
-static int InitializeAndCopyToParentContainer(
- MessageMapContainer* from,
- MessageMapContainer* to) {
- // For now we require from == to, re-evaluate if we want to support deep copy
- // as in repeated_composite_container.cc.
- GOOGLE_DCHECK(from == to);
- Message* old_message = from->message;
- Message* new_message = old_message->New();
- to->parent = NULL;
- to->parent_field_descriptor = from->parent_field_descriptor;
- to->message = new_message;
- to->owner.reset(new_message);
-
- vector<const FieldDescriptor*> fields;
- fields.push_back(from->parent_field_descriptor);
- old_message->GetReflection()->SwapFields(old_message, new_message, fields);
- return 0;
-}
-
-static PyObject* GetCMessage(MessageMapContainer* self, Message* entry) {
- // Get or create the CMessage object corresponding to this message.
- Message* message = entry->GetReflection()->MutableMessage(
- entry, self->value_field_descriptor);
- ScopedPyObjectPtr key(PyLong_FromVoidPtr(message));
- PyObject* ret = PyDict_GetItem(self->message_dict, key.get());
-
- if (ret == NULL) {
- CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
- message->GetDescriptor());
- ret = reinterpret_cast<PyObject*>(cmsg);
-
- if (cmsg == NULL) {
- return NULL;
- }
- cmsg->owner = self->owner;
- cmsg->message = message;
- cmsg->parent = self->parent;
-
- if (PyDict_SetItem(self->message_dict, key.get(), ret) < 0) {
- Py_DECREF(ret);
- return NULL;
- }
- } else {
- Py_INCREF(ret);
- }
-
- return ret;
-}
-
-int Release(MessageMapContainer* self) {
- InitializeAndCopyToParentContainer(self, self);
- return 0;
-}
-
-void SetOwner(MessageMapContainer* self,
- const shared_ptr<Message>& new_owner) {
- self->owner = new_owner;
-}
-
-Py_ssize_t Length(PyObject* _self) {
- MessageMapContainer* self = GetMap(_self);
- google::protobuf::Message* message = self->message;
- return message->GetReflection()->FieldSize(*message,
- self->parent_field_descriptor);
-}
-
-int MapKeyMatches(MessageMapContainer* self, const Message* entry,
- PyObject* key) {
- // TODO(haberman): do we need more strict type checking?
- ScopedPyObjectPtr entry_key(
- cmessage::InternalGetScalar(entry, self->key_field_descriptor));
- int ret = PyObject_RichCompareBool(key, entry_key.get(), Py_EQ);
- return ret;
-}
-
-int SetItem(PyObject *_self, PyObject *key, PyObject *v) {
- if (v) {
- PyErr_Format(PyExc_ValueError,
- "Direct assignment of submessage not allowed");
- return -1;
- }
-
- // Now we know that this is a delete, not a set.
-
- MessageMapContainer* self = GetMap(_self);
- cmessage::AssureWritable(self->parent);
-
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
- size_t size =
- reflection->FieldSize(*message, self->parent_field_descriptor);
-
- // Right now the Reflection API doesn't support map lookup, so we implement it
- // via linear search. We need to search from the end because the underlying
- // representation can have duplicates if a user calls MergeFrom(); the last
- // one needs to win.
- //
- // TODO(haberman): add lookup API to Reflection API.
- bool found = false;
- for (int i = size - 1; i >= 0; i--) {
- Message* entry = reflection->MutableRepeatedMessage(
- message, self->parent_field_descriptor, i);
- int matches = MapKeyMatches(self, entry, key);
- if (matches < 0) return -1;
- if (matches) {
- found = true;
- if (i != (int)size - 1) {
- reflection->SwapElements(message, self->parent_field_descriptor, i,
- size - 1);
- }
- reflection->RemoveLast(message, self->parent_field_descriptor);
-
- // Can't exit now, the repeated field representation of maps allows
- // duplicate keys, and we have to be sure to remove all of them.
- }
- }
-
- if (!found) {
- PyErr_Format(PyExc_KeyError, "Key not present in map");
- return -1;
- }
-
- self->version++;
-
- return 0;
-}
-
-PyObject* GetIterator(PyObject *_self) {
- MessageMapContainer* self = GetMap(_self);
-
- ScopedPyObjectPtr obj(PyType_GenericAlloc(&MessageMapIterator_Type, 0));
- if (obj == NULL) {
- return PyErr_Format(PyExc_KeyError, "Could not allocate iterator");
- }
-
- MessageMapIterator* iter = GetIter(obj.get());
-
- Py_INCREF(self);
- iter->container = self;
- iter->version = self->version;
- iter->dict = PyDict_New();
- if (iter->dict == NULL) {
- return PyErr_Format(PyExc_RuntimeError,
- "Could not allocate dict for iterator.");
- }
-
- // Build the entire map into a dict right now. Start from the beginning so
- // that later entries win in the case of duplicates.
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
-
- // Right now the Reflection API doesn't support map lookup, so we implement it
- // via linear search. We need to search from the end because the underlying
- // representation can have duplicates if a user calls MergeFrom(); the last
- // one needs to win.
- //
- // TODO(haberman): add lookup API to Reflection API.
- size_t size =
- reflection->FieldSize(*message, self->parent_field_descriptor);
- for (int i = size - 1; i >= 0; i--) {
- Message* entry = reflection->MutableRepeatedMessage(
- message, self->parent_field_descriptor, i);
- ScopedPyObjectPtr key(
- cmessage::InternalGetScalar(entry, self->key_field_descriptor));
- if (PyDict_SetItem(iter->dict, key.get(), GetCMessage(self, entry)) < 0) {
- return PyErr_Format(PyExc_RuntimeError,
- "SetItem failed in iterator construction.");
- }
- }
-
- iter->iter = PyObject_GetIter(iter->dict);
-
- return obj.release();
-}
-
-PyObject* GetItem(PyObject* _self, PyObject* key) {
- MessageMapContainer* self = GetMap(_self);
- cmessage::AssureWritable(self->parent);
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
-
- // Right now the Reflection API doesn't support map lookup, so we implement it
- // via linear search. We need to search from the end because the underlying
- // representation can have duplicates if a user calls MergeFrom(); the last
- // one needs to win.
- //
- // TODO(haberman): add lookup API to Reflection API.
- size_t size =
- reflection->FieldSize(*message, self->parent_field_descriptor);
- for (int i = size - 1; i >= 0; i--) {
- Message* entry = reflection->MutableRepeatedMessage(
- message, self->parent_field_descriptor, i);
- int matches = MapKeyMatches(self, entry, key);
- if (matches < 0) return NULL;
- if (matches) {
- return GetCMessage(self, entry);
- }
- }
-
- // Key is not already present; insert a new entry.
- Message* entry =
- reflection->AddMessage(message, self->parent_field_descriptor);
-
- self->version++;
-
- if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor,
- key) < 0) {
- reflection->RemoveLast(message, self->parent_field_descriptor);
- return NULL;
- }
-
- return GetCMessage(self, entry);
-}
-
-PyObject* Contains(PyObject* _self, PyObject* key) {
- MessageMapContainer* self = GetMap(_self);
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
-
- // Right now the Reflection API doesn't support map lookup, so we implement it
- // via linear search.
- //
- // TODO(haberman): add lookup API to Reflection API.
- int size =
- reflection->FieldSize(*message, self->parent_field_descriptor);
- for (int i = 0; i < size; i++) {
- Message* entry = reflection->MutableRepeatedMessage(
- message, self->parent_field_descriptor, i);
- int matches = MapKeyMatches(self, entry, key);
- if (matches < 0) return NULL;
- if (matches) {
- Py_RETURN_TRUE;
- }
- }
-
- Py_RETURN_FALSE;
-}
-
-PyObject* Clear(PyObject* _self) {
- MessageMapContainer* self = GetMap(_self);
- cmessage::AssureWritable(self->parent);
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
-
- self->version++;
- reflection->ClearField(message, self->parent_field_descriptor);
-
- Py_RETURN_NONE;
-}
-
-PyObject* Get(PyObject* self, PyObject* args) {
- PyObject* key;
- PyObject* default_value = NULL;
- if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) {
- return NULL;
- }
-
- ScopedPyObjectPtr is_present(Contains(self, key));
- if (is_present.get() == NULL) {
- return NULL;
- }
-
- if (PyObject_IsTrue(is_present.get())) {
- return GetItem(self, key);
- } else {
- if (default_value != NULL) {
- Py_INCREF(default_value);
- return default_value;
- } else {
- Py_RETURN_NONE;
- }
- }
-}
-
-static void Dealloc(PyObject* _self) {
- MessageMapContainer* self = GetMap(_self);
- self->owner.reset();
- Py_DECREF(self->message_dict);
- Py_TYPE(_self)->tp_free(_self);
-}
-
-static PyMethodDef Methods[] = {
- { "__contains__", (PyCFunction)Contains, METH_O,
- "Tests whether the map contains this element."},
- { "clear", (PyCFunction)Clear, METH_NOARGS,
- "Removes all elements from the map."},
- { "get", Get, METH_VARARGS,
- "Gets the value for the given key if present, or otherwise a default" },
- { "get_or_create", GetItem, METH_O,
- "Alias for getitem, useful to make explicit that the map is mutated." },
- /*
- { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
- "Makes a deep copy of the class." },
- { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
- "Outputs picklable representation of the repeated field." },
- */
- {NULL, NULL},
-};
-
-} // namespace message_map_container
-
-namespace message_map_iterator {
-
-static void Dealloc(PyObject* _self) {
- MessageMapIterator* self = GetIter(_self);
- Py_DECREF(self->dict);
- Py_DECREF(self->iter);
- Py_DECREF(self->container);
- Py_TYPE(_self)->tp_free(_self);
-}
-
-PyObject* IterNext(PyObject* _self) {
- MessageMapIterator* self = GetIter(_self);
-
- // This won't catch mutations to the map performed by MergeFrom(); no easy way
- // to address that.
- if (self->version != self->container->version) {
- return PyErr_Format(PyExc_RuntimeError,
- "Map modified during iteration.");
- }
-
- return PyIter_Next(self->iter);
-}
-
-} // namespace message_map_iterator
-
-#if PY_MAJOR_VERSION >= 3
- static PyType_Slot MessageMapContainer_Type_slots[] = {
- {Py_tp_dealloc, (void *)message_map_container::Dealloc},
- {Py_mp_length, (void *)message_map_container::Length},
- {Py_mp_subscript, (void *)message_map_container::GetItem},
- {Py_mp_ass_subscript, (void *)message_map_container::SetItem},
- {Py_tp_methods, (void *)message_map_container::Methods},
- {Py_tp_iter, (void *)message_map_container::GetIterator},
- {0, 0}
- };
-
- PyType_Spec MessageMapContainer_Type_spec = {
- FULL_MODULE_NAME ".MessageMapContainer",
- sizeof(MessageMapContainer),
- 0,
- Py_TPFLAGS_DEFAULT,
- MessageMapContainer_Type_slots
- };
-
- PyObject *MessageMapContainer_Type;
-
-#else
- static PyMappingMethods MpMethods = {
- message_map_container::Length, // mp_length
- message_map_container::GetItem, // mp_subscript
- message_map_container::SetItem, // mp_ass_subscript
- };
-
- PyTypeObject MessageMapContainer_Type = {
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
- FULL_MODULE_NAME ".MessageMapContainer", // tp_name
- sizeof(MessageMapContainer), // tp_basicsize
- 0, // tp_itemsize
- message_map_container::Dealloc, // tp_dealloc
- 0, // tp_print
- 0, // tp_getattr
- 0, // tp_setattr
- 0, // tp_compare
- 0, // tp_repr
- 0, // tp_as_number
- 0, // tp_as_sequence
- &MpMethods, // tp_as_mapping
- 0, // tp_hash
- 0, // tp_call
- 0, // tp_str
- 0, // tp_getattro
- 0, // tp_setattro
- 0, // tp_as_buffer
- Py_TPFLAGS_DEFAULT, // tp_flags
- "A map container for message", // tp_doc
- 0, // tp_traverse
- 0, // tp_clear
- 0, // tp_richcompare
- 0, // tp_weaklistoffset
- message_map_container::GetIterator, // tp_iter
- 0, // tp_iternext
- message_map_container::Methods, // tp_methods
- 0, // tp_members
- 0, // tp_getset
- 0, // tp_base
- 0, // tp_dict
- 0, // tp_descr_get
- 0, // tp_descr_set
- 0, // tp_dictoffset
- 0, // tp_init
- };
-#endif
-
-PyTypeObject MessageMapIterator_Type = {
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
- FULL_MODULE_NAME ".MessageMapIterator", // tp_name
- sizeof(MessageMapIterator), // tp_basicsize
- 0, // tp_itemsize
- message_map_iterator::Dealloc, // tp_dealloc
- 0, // tp_print
- 0, // tp_getattr
- 0, // tp_setattr
- 0, // tp_compare
- 0, // tp_repr
- 0, // tp_as_number
- 0, // tp_as_sequence
- 0, // tp_as_mapping
- 0, // tp_hash
- 0, // tp_call
- 0, // tp_str
- 0, // tp_getattro
- 0, // tp_setattro
- 0, // tp_as_buffer
- Py_TPFLAGS_DEFAULT, // tp_flags
- "A scalar map iterator", // tp_doc
- 0, // tp_traverse
- 0, // tp_clear
- 0, // tp_richcompare
- 0, // tp_weaklistoffset
- PyObject_SelfIter, // tp_iter
- message_map_iterator::IterNext, // tp_iternext
- 0, // tp_methods
- 0, // tp_members
- 0, // tp_getset
- 0, // tp_base
- 0, // tp_dict
- 0, // tp_descr_get
- 0, // tp_descr_set
- 0, // tp_dictoffset
- 0, // tp_init
-};
-
-} // namespace python
-} // namespace protobuf
-} // namespace google
diff --git a/python/google/protobuf/pyext/scalar_map_container.cc b/python/google/protobuf/pyext/scalar_map_container.cc
deleted file mode 100644
index 0b0d5a3d..00000000
--- a/python/google/protobuf/pyext/scalar_map_container.cc
+++ /dev/null
@@ -1,542 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2008 Google Inc. All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-// Author: haberman@google.com (Josh Haberman)
-
-#include <google/protobuf/pyext/scalar_map_container.h>
-
-#include <google/protobuf/stubs/logging.h>
-#include <google/protobuf/stubs/common.h>
-#include <google/protobuf/message.h>
-#include <google/protobuf/pyext/message.h>
-#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
-
-namespace google {
-namespace protobuf {
-namespace python {
-
-struct ScalarMapIterator {
- PyObject_HEAD;
-
- // This dict contains the full contents of what we want to iterate over.
- // There's no way to avoid building this, because the list representation
- // (which is canonical) can contain duplicate keys. So at the very least we
- // need a set that lets us skip duplicate keys. And at the point that we're
- // doing that, we might as well just build the actual dict we're iterating
- // over and use dict's built-in iterator.
- PyObject* dict;
-
- // An iterator on dict.
- PyObject* iter;
-
- // A pointer back to the container, so we can notice changes to the version.
- ScalarMapContainer* container;
-
- // The version of the map when we took the iterator to it.
- //
- // We store this so that if the map is modified during iteration we can throw
- // an error.
- uint64 version;
-};
-
-static ScalarMapIterator* GetIter(PyObject* obj) {
- return reinterpret_cast<ScalarMapIterator*>(obj);
-}
-
-namespace scalar_map_container {
-
-static ScalarMapContainer* GetMap(PyObject* obj) {
- return reinterpret_cast<ScalarMapContainer*>(obj);
-}
-
-// The private constructor of ScalarMapContainer objects.
-PyObject *NewContainer(
- CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) {
- if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
- return NULL;
- }
-
-#if PY_MAJOR_VERSION >= 3
- ScopedPyObjectPtr obj(PyType_GenericAlloc(
- reinterpret_cast<PyTypeObject *>(ScalarMapContainer_Type), 0));
-#else
- ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0));
-#endif
- if (obj.get() == NULL) {
- return PyErr_Format(PyExc_RuntimeError,
- "Could not allocate new container.");
- }
-
- ScalarMapContainer* self = GetMap(obj.get());
-
- self->message = parent->message;
- self->parent = parent;
- self->parent_field_descriptor = parent_field_descriptor;
- self->owner = parent->owner;
- self->version = 0;
-
- self->key_field_descriptor =
- parent_field_descriptor->message_type()->FindFieldByName("key");
- self->value_field_descriptor =
- parent_field_descriptor->message_type()->FindFieldByName("value");
-
- if (self->key_field_descriptor == NULL ||
- self->value_field_descriptor == NULL) {
- return PyErr_Format(PyExc_KeyError,
- "Map entry descriptor did not have key/value fields");
- }
-
- return obj.release();
-}
-
-// Initializes the underlying Message object of "to" so it becomes a new parent
-// repeated scalar, and copies all the values from "from" to it. A child scalar
-// container can be released by passing it as both from and to (e.g. making it
-// the recipient of the new parent message and copying the values from itself).
-static int InitializeAndCopyToParentContainer(
- ScalarMapContainer* from,
- ScalarMapContainer* to) {
- // For now we require from == to, re-evaluate if we want to support deep copy
- // as in repeated_scalar_container.cc.
- GOOGLE_DCHECK(from == to);
- Message* old_message = from->message;
- Message* new_message = old_message->New();
- to->parent = NULL;
- to->parent_field_descriptor = from->parent_field_descriptor;
- to->message = new_message;
- to->owner.reset(new_message);
-
- vector<const FieldDescriptor*> fields;
- fields.push_back(from->parent_field_descriptor);
- old_message->GetReflection()->SwapFields(old_message, new_message, fields);
- return 0;
-}
-
-int Release(ScalarMapContainer* self) {
- return InitializeAndCopyToParentContainer(self, self);
-}
-
-void SetOwner(ScalarMapContainer* self,
- const shared_ptr<Message>& new_owner) {
- self->owner = new_owner;
-}
-
-Py_ssize_t Length(PyObject* _self) {
- ScalarMapContainer* self = GetMap(_self);
- google::protobuf::Message* message = self->message;
- return message->GetReflection()->FieldSize(*message,
- self->parent_field_descriptor);
-}
-
-int MapKeyMatches(ScalarMapContainer* self, const Message* entry,
- PyObject* key) {
- // TODO(haberman): do we need more strict type checking?
- ScopedPyObjectPtr entry_key(
- cmessage::InternalGetScalar(entry, self->key_field_descriptor));
- int ret = PyObject_RichCompareBool(key, entry_key.get(), Py_EQ);
- return ret;
-}
-
-PyObject* GetItem(PyObject* _self, PyObject* key) {
- ScalarMapContainer* self = GetMap(_self);
-
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
-
- // Right now the Reflection API doesn't support map lookup, so we implement it
- // via linear search.
- //
- // TODO(haberman): add lookup API to Reflection API.
- size_t size = reflection->FieldSize(*message, self->parent_field_descriptor);
- for (int i = size - 1; i >= 0; i--) {
- const Message& entry = reflection->GetRepeatedMessage(
- *message, self->parent_field_descriptor, i);
- int matches = MapKeyMatches(self, &entry, key);
- if (matches < 0) return NULL;
- if (matches) {
- return cmessage::InternalGetScalar(&entry, self->value_field_descriptor);
- }
- }
-
- // Need to add a new entry.
- Message* entry =
- reflection->AddMessage(message, self->parent_field_descriptor);
- PyObject* ret = NULL;
-
- if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor,
- key) >= 0) {
- ret = cmessage::InternalGetScalar(entry, self->value_field_descriptor);
- }
-
- self->version++;
-
- // If there was a type error above, it set the Python exception.
- return ret;
-}
-
-int SetItem(PyObject *_self, PyObject *key, PyObject *v) {
- ScalarMapContainer* self = GetMap(_self);
- cmessage::AssureWritable(self->parent);
-
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
- size_t size =
- reflection->FieldSize(*message, self->parent_field_descriptor);
- self->version++;
-
- if (v) {
- // Set item.
- //
- // Right now the Reflection API doesn't support map lookup, so we implement
- // it via linear search.
- //
- // TODO(haberman): add lookup API to Reflection API.
- for (int i = size - 1; i >= 0; i--) {
- Message* entry = reflection->MutableRepeatedMessage(
- message, self->parent_field_descriptor, i);
- int matches = MapKeyMatches(self, entry, key);
- if (matches < 0) return -1;
- if (matches) {
- return cmessage::InternalSetNonOneofScalar(
- entry, self->value_field_descriptor, v);
- }
- }
-
- // Key is not already present; insert a new entry.
- Message* entry =
- reflection->AddMessage(message, self->parent_field_descriptor);
-
- if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor,
- key) < 0 ||
- cmessage::InternalSetNonOneofScalar(entry, self->value_field_descriptor,
- v) < 0) {
- reflection->RemoveLast(message, self->parent_field_descriptor);
- return -1;
- }
-
- return 0;
- } else {
- bool found = false;
- for (int i = size - 1; i >= 0; i--) {
- Message* entry = reflection->MutableRepeatedMessage(
- message, self->parent_field_descriptor, i);
- int matches = MapKeyMatches(self, entry, key);
- if (matches < 0) return -1;
- if (matches) {
- found = true;
- if (i != (int)size - 1) {
- reflection->SwapElements(message, self->parent_field_descriptor, i,
- size - 1);
- }
- reflection->RemoveLast(message, self->parent_field_descriptor);
-
- // Can't exit now, the repeated field representation of maps allows
- // duplicate keys, and we have to be sure to remove all of them.
- }
- }
-
- if (found) {
- return 0;
- } else {
- PyErr_Format(PyExc_KeyError, "Key not present in map");
- return -1;
- }
- }
-}
-
-PyObject* GetIterator(PyObject *_self) {
- ScalarMapContainer* self = GetMap(_self);
-
- ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapIterator_Type, 0));
- if (obj == NULL) {
- return PyErr_Format(PyExc_KeyError, "Could not allocate iterator");
- }
-
- ScalarMapIterator* iter = GetIter(obj.get());
-
- Py_INCREF(self);
- iter->container = self;
- iter->version = self->version;
- iter->dict = PyDict_New();
- if (iter->dict == NULL) {
- return PyErr_Format(PyExc_RuntimeError,
- "Could not allocate dict for iterator.");
- }
-
- // Build the entire map into a dict right now. Start from the beginning so
- // that later entries win in the case of duplicates.
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
-
- // Right now the Reflection API doesn't support map lookup, so we implement it
- // via linear search. We need to search from the end because the underlying
- // representation can have duplicates if a user calls MergeFrom(); the last
- // one needs to win.
- //
- // TODO(haberman): add lookup API to Reflection API.
- size_t size =
- reflection->FieldSize(*message, self->parent_field_descriptor);
- for (size_t i = 0; i < size; i++) {
- Message* entry = reflection->MutableRepeatedMessage(
- message, self->parent_field_descriptor, i);
- ScopedPyObjectPtr key(
- cmessage::InternalGetScalar(entry, self->key_field_descriptor));
- ScopedPyObjectPtr val(
- cmessage::InternalGetScalar(entry, self->value_field_descriptor));
- if (PyDict_SetItem(iter->dict, key.get(), val.get()) < 0) {
- return PyErr_Format(PyExc_RuntimeError,
- "SetItem failed in iterator construction.");
- }
- }
-
-
- iter->iter = PyObject_GetIter(iter->dict);
-
-
- return obj.release();
-}
-
-PyObject* Clear(PyObject* _self) {
- ScalarMapContainer* self = GetMap(_self);
- cmessage::AssureWritable(self->parent);
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
-
- reflection->ClearField(message, self->parent_field_descriptor);
-
- Py_RETURN_NONE;
-}
-
-PyObject* Contains(PyObject* _self, PyObject* key) {
- ScalarMapContainer* self = GetMap(_self);
-
- Message* message = self->message;
- const Reflection* reflection = message->GetReflection();
-
- // Right now the Reflection API doesn't support map lookup, so we implement it
- // via linear search.
- //
- // TODO(haberman): add lookup API to Reflection API.
- size_t size = reflection->FieldSize(*message, self->parent_field_descriptor);
- for (int i = size - 1; i >= 0; i--) {
- const Message& entry = reflection->GetRepeatedMessage(
- *message, self->parent_field_descriptor, i);
- int matches = MapKeyMatches(self, &entry, key);
- if (matches < 0) return NULL;
- if (matches) {
- Py_RETURN_TRUE;
- }
- }
-
- Py_RETURN_FALSE;
-}
-
-PyObject* Get(PyObject* self, PyObject* args) {
- PyObject* key;
- PyObject* default_value = NULL;
- if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) {
- return NULL;
- }
-
- ScopedPyObjectPtr is_present(Contains(self, key));
- if (is_present.get() == NULL) {
- return NULL;
- }
-
- if (PyObject_IsTrue(is_present.get())) {
- return GetItem(self, key);
- } else {
- if (default_value != NULL) {
- Py_INCREF(default_value);
- return default_value;
- } else {
- Py_RETURN_NONE;
- }
- }
-}
-
-static void Dealloc(PyObject* _self) {
- ScalarMapContainer* self = GetMap(_self);
- self->owner.reset();
- Py_TYPE(_self)->tp_free(_self);
-}
-
-static PyMethodDef Methods[] = {
- { "__contains__", Contains, METH_O,
- "Tests whether a key is a member of the map." },
- { "clear", (PyCFunction)Clear, METH_NOARGS,
- "Removes all elements from the map." },
- { "get", Get, METH_VARARGS,
- "Gets the value for the given key if present, or otherwise a default" },
- /*
- { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
- "Makes a deep copy of the class." },
- { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
- "Outputs picklable representation of the repeated field." },
- */
- {NULL, NULL},
-};
-
-} // namespace scalar_map_container
-
-namespace scalar_map_iterator {
-
-static void Dealloc(PyObject* _self) {
- ScalarMapIterator* self = GetIter(_self);
- Py_DECREF(self->dict);
- Py_DECREF(self->iter);
- Py_DECREF(self->container);
- Py_TYPE(_self)->tp_free(_self);
-}
-
-PyObject* IterNext(PyObject* _self) {
- ScalarMapIterator* self = GetIter(_self);
-
- // This won't catch mutations to the map performed by MergeFrom(); no easy way
- // to address that.
- if (self->version != self->container->version) {
- return PyErr_Format(PyExc_RuntimeError,
- "Map modified during iteration.");
- }
-
- return PyIter_Next(self->iter);
-}
-
-} // namespace scalar_map_iterator
-
-
-#if PY_MAJOR_VERSION >= 3
- static PyType_Slot ScalarMapContainer_Type_slots[] = {
- {Py_tp_dealloc, (void *)scalar_map_container::Dealloc},
- {Py_mp_length, (void *)scalar_map_container::Length},
- {Py_mp_subscript, (void *)scalar_map_container::GetItem},
- {Py_mp_ass_subscript, (void *)scalar_map_container::SetItem},
- {Py_tp_methods, (void *)scalar_map_container::Methods},
- {Py_tp_iter, (void *)scalar_map_container::GetIterator},
- {0, 0},
- };
-
- PyType_Spec ScalarMapContainer_Type_spec = {
- FULL_MODULE_NAME ".ScalarMapContainer",
- sizeof(ScalarMapContainer),
- 0,
- Py_TPFLAGS_DEFAULT,
- ScalarMapContainer_Type_slots
- };
- PyObject *ScalarMapContainer_Type;
-#else
- static PyMappingMethods MpMethods = {
- scalar_map_container::Length, // mp_length
- scalar_map_container::GetItem, // mp_subscript
- scalar_map_container::SetItem, // mp_ass_subscript
- };
-
- PyTypeObject ScalarMapContainer_Type = {
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
- FULL_MODULE_NAME ".ScalarMapContainer", // tp_name
- sizeof(ScalarMapContainer), // tp_basicsize
- 0, // tp_itemsize
- scalar_map_container::Dealloc, // tp_dealloc
- 0, // tp_print
- 0, // tp_getattr
- 0, // tp_setattr
- 0, // tp_compare
- 0, // tp_repr
- 0, // tp_as_number
- 0, // tp_as_sequence
- &MpMethods, // tp_as_mapping
- 0, // tp_hash
- 0, // tp_call
- 0, // tp_str
- 0, // tp_getattro
- 0, // tp_setattro
- 0, // tp_as_buffer
- Py_TPFLAGS_DEFAULT, // tp_flags
- "A scalar map container", // tp_doc
- 0, // tp_traverse
- 0, // tp_clear
- 0, // tp_richcompare
- 0, // tp_weaklistoffset
- scalar_map_container::GetIterator, // tp_iter
- 0, // tp_iternext
- scalar_map_container::Methods, // tp_methods
- 0, // tp_members
- 0, // tp_getset
- 0, // tp_base
- 0, // tp_dict
- 0, // tp_descr_get
- 0, // tp_descr_set
- 0, // tp_dictoffset
- 0, // tp_init
- };
-#endif
-
-PyTypeObject ScalarMapIterator_Type = {
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
- FULL_MODULE_NAME ".ScalarMapIterator", // tp_name
- sizeof(ScalarMapIterator), // tp_basicsize
- 0, // tp_itemsize
- scalar_map_iterator::Dealloc, // tp_dealloc
- 0, // tp_print
- 0, // tp_getattr
- 0, // tp_setattr
- 0, // tp_compare
- 0, // tp_repr
- 0, // tp_as_number
- 0, // tp_as_sequence
- 0, // tp_as_mapping
- 0, // tp_hash
- 0, // tp_call
- 0, // tp_str
- 0, // tp_getattro
- 0, // tp_setattro
- 0, // tp_as_buffer
- Py_TPFLAGS_DEFAULT, // tp_flags
- "A scalar map iterator", // tp_doc
- 0, // tp_traverse
- 0, // tp_clear
- 0, // tp_richcompare
- 0, // tp_weaklistoffset
- PyObject_SelfIter, // tp_iter
- scalar_map_iterator::IterNext, // tp_iternext
- 0, // tp_methods
- 0, // tp_members
- 0, // tp_getset
- 0, // tp_base
- 0, // tp_dict
- 0, // tp_descr_get
- 0, // tp_descr_set
- 0, // tp_dictoffset
- 0, // tp_init
-};
-
-} // namespace python
-} // namespace protobuf
-} // namespace google
diff --git a/python/google/protobuf/pyext/scalar_map_container.h b/python/google/protobuf/pyext/scalar_map_container.h
deleted file mode 100644
index 4d663b88..00000000
--- a/python/google/protobuf/pyext/scalar_map_container.h
+++ /dev/null
@@ -1,119 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2008 Google Inc. All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__
-#define GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__
-
-#include <Python.h>
-
-#include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
-
-#include <google/protobuf/descriptor.h>
-
-namespace google {
-namespace protobuf {
-
-class Message;
-
-#ifdef _SHARED_PTR_H
-using std::shared_ptr;
-#else
-using internal::shared_ptr;
-#endif
-
-namespace python {
-
-struct CMessage;
-
-struct ScalarMapContainer {
- PyObject_HEAD;
-
- // This is the top-level C++ Message object that owns the whole
- // proto tree. Every Python ScalarMapContainer holds a
- // reference to it in order to keep it alive as long as there's a
- // Python object that references any part of the tree.
- shared_ptr<Message> owner;
-
- // Pointer to the C++ Message that contains this container. The
- // ScalarMapContainer does not own this pointer.
- Message* message;
-
- // Weak reference to a parent CMessage object (i.e. may be NULL.)
- //
- // Used to make sure all ancestors are also mutable when first
- // modifying the container.
- CMessage* parent;
-
- // Pointer to the parent's descriptor that describes this
- // field. Used together with the parent's message when making a
- // default message instance mutable.
- // The pointer is owned by the global DescriptorPool.
- const FieldDescriptor* parent_field_descriptor;
- const FieldDescriptor* key_field_descriptor;
- const FieldDescriptor* value_field_descriptor;
-
- // We bump this whenever we perform a mutation, to invalidate existing
- // iterators.
- uint64 version;
-};
-
-#if PY_MAJOR_VERSION >= 3
- extern PyObject *ScalarMapContainer_Type;
- extern PyType_Spec ScalarMapContainer_Type_spec;
-#else
- extern PyTypeObject ScalarMapContainer_Type;
-#endif
-extern PyTypeObject ScalarMapIterator_Type;
-
-namespace scalar_map_container {
-
-// Builds a ScalarMapContainer object, from a parent message and a
-// field descriptor.
-extern PyObject *NewContainer(
- CMessage* parent, const FieldDescriptor* parent_field_descriptor);
-
-// Releases the messages in the container to a new message.
-//
-// Returns 0 on success, -1 on failure.
-int Release(ScalarMapContainer* self);
-
-// Set the owner field of self and any children of self.
-void SetOwner(ScalarMapContainer* self,
- const shared_ptr<Message>& new_owner);
-
-} // namespace scalar_map_container
-} // namespace python
-} // namespace protobuf
-
-} // namespace google
-#endif // GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__