Integrate from internal code base.

8 files changed, 226 insertions, 36 deletions

diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.cc b/src/google/protobuf/util/internal/default_value_objectwriter.cc
index 21d7a2e4..1e8dab70 100644
--- a/src/google/protobuf/util/internal/default_value_objectwriter.cc
+++ b/src/google/protobuf/util/internal/default_value_objectwriter.cc
@@ -64,6 +64,7 @@ DefaultValueObjectWriter::DefaultValueObjectWriter(
       type_(type),
       current_(NULL),
       root_(NULL),
+      suppress_empty_list_(false),
       field_scrub_callback_(NULL),
       ow_(ow) {}
 
@@ -184,12 +185,10 @@ void DefaultValueObjectWriter::RegisterFieldScrubCallBack(
   field_scrub_callback_.reset(field_scrub_callback.release());
 }
 
-DefaultValueObjectWriter::Node::Node(const string& name,
-                                     const google::protobuf::Type* type,
-                                     NodeKind kind, const DataPiece& data,
-                                     bool is_placeholder,
-                                     const vector<string>& path,
-                                     FieldScrubCallBack* field_scrub_callback)
+DefaultValueObjectWriter::Node::Node(
+    const string& name, const google::protobuf::Type* type, NodeKind kind,
+    const DataPiece& data, bool is_placeholder, const vector<string>& path,
+    bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback)
     : name_(name),
       type_(type),
       kind_(kind),
@@ -197,6 +196,7 @@ DefaultValueObjectWriter::Node::Node(const string& name,
       data_(data),
       is_placeholder_(is_placeholder),
       path_(path),
+      suppress_empty_list_(suppress_empty_list),
       field_scrub_callback_(field_scrub_callback) {}
 
 DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild(
@@ -230,6 +230,9 @@ void DefaultValueObjectWriter::Node::WriteTo(ObjectWriter* ow) {
   // Write out lists. If we didn't have any list in response, write out empty
   // list.
   if (kind_ == LIST) {
+    // Suppress empty lists if requested.
+    if (suppress_empty_list_ && is_placeholder_) return;
+
     ow->StartList(name_);
     WriteChildren(ow);
     ow->EndList();
@@ -366,7 +369,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren(
         field.json_name(), field_type, kind,
         kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo)
                           : DataPiece::NullData(),
-        true, path, field_scrub_callback_));
+        true, path, suppress_empty_list_, field_scrub_callback_));
     new_children.push_back(child.release());
   }
   // Adds all leftover nodes in children_ to the beginning of new_child.
@@ -462,7 +465,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject(
   if (current_ == NULL) {
     vector<string> path;
     root_.reset(new Node(name.ToString(), &type_, OBJECT, DataPiece::NullData(),
-                         false, path, field_scrub_callback_.get()));
+                         false, path, suppress_empty_list_,
+                         field_scrub_callback_.get()));
     root_->PopulateChildren(typeinfo_);
     current_ = root_.get();
     return this;
@@ -478,7 +482,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject(
                               : NULL),
         OBJECT, DataPiece::NullData(), false,
         child == NULL ? current_->path() : child->path(),
-        field_scrub_callback_.get()));
+        suppress_empty_list_, field_scrub_callback_.get()));
     child = node.get();
     current_->AddChild(node.release());
   }
@@ -509,7 +513,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList(
   if (current_ == NULL) {
     vector<string> path;
     root_.reset(new Node(name.ToString(), &type_, LIST, DataPiece::NullData(),
-                         false, path, field_scrub_callback_.get()));
+                         false, path, suppress_empty_list_,
+                         field_scrub_callback_.get()));
     current_ = root_.get();
     return this;
   }
@@ -519,7 +524,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList(
     google::protobuf::scoped_ptr<Node> node(
         new Node(name.ToString(), NULL, LIST, DataPiece::NullData(), false,
                  child == NULL ? current_->path() : child->path(),
-                 field_scrub_callback_.get()));
+                 suppress_empty_list_, field_scrub_callback_.get()));
     child = node.get();
     current_->AddChild(node.release());
   }
@@ -577,7 +582,7 @@ void DefaultValueObjectWriter::RenderDataPiece(StringPiece name,
     google::protobuf::scoped_ptr<Node> node(
         new Node(name.ToString(), NULL, PRIMITIVE, data, false,
                  child == NULL ? current_->path() : child->path(),
-                 field_scrub_callback_.get()));
+                 suppress_empty_list_, field_scrub_callback_.get()));
     child = node.get();
     current_->AddChild(node.release());
   } else {
diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.h b/src/google/protobuf/util/internal/default_value_objectwriter.h
index 1d85bed8..5f3b25f3 100644
--- a/src/google/protobuf/util/internal/default_value_objectwriter.h
+++ b/src/google/protobuf/util/internal/default_value_objectwriter.h
@@ -122,6 +122,10 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
   // field_scrub_callback pointer is also transferred to this class
   void RegisterFieldScrubCallBack(FieldScrubCallBackPtr field_scrub_callback);
 
+  // If set to true, empty lists are suppressed from output when default values
+  // are written.
+  void set_suppress_empty_list(bool value) { suppress_empty_list_ = value; }
+
  private:
   enum NodeKind {
     PRIMITIVE = 0,
@@ -136,7 +140,7 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
    public:
     Node(const string& name, const google::protobuf::Type* type, NodeKind kind,
          const DataPiece& data, bool is_placeholder, const vector<string>& path,
-         FieldScrubCallBack* field_scrub_callback);
+         bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback);
     virtual ~Node() {
       for (int i = 0; i < children_.size(); ++i) {
         delete children_[i];
@@ -212,6 +216,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
     // Path of the field of this node
     std::vector<string> path_;
 
+    // Whether to suppress empty list output.
+    bool suppress_empty_list_;
+
     // Pointer to function for determining whether a field needs to be scrubbed
     // or not. This callback is owned by the creator of this node.
     FieldScrubCallBack* field_scrub_callback_;
@@ -257,6 +264,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
   // The stack to hold the path of Nodes from current_ to root_;
   std::stack<Node*> stack_;
 
+  // Whether to suppress output of empty lists.
+  bool suppress_empty_list_;
+
   // Unique Pointer to function for determining whether a field needs to be
   // scrubbed or not.
   FieldScrubCallBackPtr field_scrub_callback_;
diff --git a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc
index 8254c0fa..e1dd697a 100644
--- a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc
+++ b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc
@@ -149,6 +149,39 @@ TEST_P(DefaultValueObjectWriterTest, ShouldRetainUnknownField) {
 }
 
 
+class DefaultValueObjectWriterSuppressListTest
+    : public BaseDefaultValueObjectWriterTest {
+ protected:
+  DefaultValueObjectWriterSuppressListTest()
+      : BaseDefaultValueObjectWriterTest(DefaultValueTest::descriptor()) {
+    testing_->set_suppress_empty_list(true);
+  }
+  ~DefaultValueObjectWriterSuppressListTest() {}
+};
+
+INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest,
+                        DefaultValueObjectWriterSuppressListTest,
+                        ::testing::Values(
+                            testing::USE_TYPE_RESOLVER));
+
+TEST_P(DefaultValueObjectWriterSuppressListTest, Empty) {
+  // Set expectation. Emtpy lists should be suppressed.
+  expects_.StartObject("")
+      ->RenderDouble("doubleValue", 0.0)
+      ->RenderFloat("floatValue", 0.0)
+      ->RenderInt64("int64Value", 0)
+      ->RenderUint64("uint64Value", 0)
+      ->RenderInt32("int32Value", 0)
+      ->RenderUint32("uint32Value", 0)
+      ->RenderBool("boolValue", false)
+      ->RenderString("stringValue", "")
+      ->RenderBytes("bytesValue", "")
+      ->RenderString("enumValue", "ENUM_FIRST")
+      ->EndObject();
+
+  // Actual testing
+  testing_->StartObject("")->EndObject();
+}
 }  // namespace testing
 }  // namespace converter
 }  // namespace util
diff --git a/src/google/protobuf/util/internal/proto_writer.cc b/src/google/protobuf/util/internal/proto_writer.cc
index 7a1a6cbd..0c38aeb9 100644
--- a/src/google/protobuf/util/internal/proto_writer.cc
+++ b/src/google/protobuf/util/internal/proto_writer.cc
@@ -298,7 +298,9 @@ ProtoWriter::ProtoElement::ProtoElement(const TypeInfo* typeinfo,
       proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3),
       type_(type),
       size_index_(-1),
-      array_index_(-1) {
+      array_index_(-1),
+      // oneof_indices_ values are 1-indexed (0 means not present).
+      oneof_indices_(type.oneofs_size() + 1) {
   if (!proto3_) {
     required_fields_ = GetRequiredFields(type_);
   }
@@ -312,13 +314,15 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent,
       ow_(this->parent()->ow_),
       parent_field_(field),
       typeinfo_(this->parent()->typeinfo_),
-      proto3_(this->parent()->proto3_),
+      proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3),
       type_(type),
       size_index_(
           !is_list && field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE
               ? ow_->size_insert_.size()
               : -1),
-      array_index_(is_list ? 0 : -1) {
+      array_index_(is_list ? 0 : -1),
+      // oneof_indices_ values are 1-indexed (0 means not present).
+      oneof_indices_(type_.oneofs_size() + 1) {
   if (!is_list) {
     if (ow_->IsRepeated(*field)) {
       // Update array_index_ if it is an explicit list.
@@ -411,11 +415,11 @@ string ProtoWriter::ProtoElement::ToString() const {
 }
 
 bool ProtoWriter::ProtoElement::IsOneofIndexTaken(int32 index) {
-  return ContainsKey(oneof_indices_, index);
+  return oneof_indices_[index];
 }
 
 void ProtoWriter::ProtoElement::TakeOneofIndex(int32 index) {
-  InsertIfNotPresent(&oneof_indices_, index);
+  oneof_indices_[index] = true;
 }
 
 void ProtoWriter::InvalidName(StringPiece unknown_name, StringPiece message) {
@@ -573,10 +577,19 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField(
 
   // Pushing a ProtoElement and then pop it off at the end for 2 purposes:
   // error location reporting and required field accounting.
-  element_.reset(new ProtoElement(element_.release(), &field, type, false));
+  //
+  // For proto3, since there is no required field tracking, we only need to push
+  // ProtoElement for error cases.
+  if (!element_->proto3()) {
+    element_.reset(new ProtoElement(element_.release(), &field, type, false));
+  }
 
   if (field.kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN ||
       field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) {
+    // Push a ProtoElement for location reporting purposes.
+    if (element_->proto3()) {
+      element_.reset(new ProtoElement(element_.release(), &field, type, false));
+    }
     InvalidValue(field.type_url().empty()
                      ? google::protobuf::Field_Kind_Name(field.kind())
                      : field.type_url(),
@@ -657,11 +670,18 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField(
   }
 
   if (!status.ok()) {
+    // Push a ProtoElement for location reporting purposes.
+    if (element_->proto3()) {
+      element_.reset(new ProtoElement(element_.release(), &field, type, false));
+    }
     InvalidValue(google::protobuf::Field_Kind_Name(field.kind()),
                  status.error_message());
+    element_.reset(element()->pop());
+    return this;
   }
 
-  element_.reset(element()->pop());
+  if (!element_->proto3()) element_.reset(element()->pop());
+
   return this;
 }
 
diff --git a/src/google/protobuf/util/internal/proto_writer.h b/src/google/protobuf/util/internal/proto_writer.h
index 8b7c6c34..7f1108ab 100644
--- a/src/google/protobuf/util/internal/proto_writer.h
+++ b/src/google/protobuf/util/internal/proto_writer.h
@@ -32,8 +32,8 @@
 #define GOOGLE_PROTOBUF_UTIL_CONVERTER_PROTO_WRITER_H__
 
 #include <deque>
-#include <google/protobuf/stubs/hash.h>
 #include <string>
+#include <vector>
 
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/io/coded_stream.h>
@@ -45,6 +45,7 @@
 #include <google/protobuf/util/internal/structured_objectwriter.h>
 #include <google/protobuf/util/type_resolver.h>
 #include <google/protobuf/stubs/bytestream.h>
+#include <google/protobuf/stubs/hash.h>
 
 namespace google {
 namespace protobuf {
@@ -191,6 +192,8 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
     // generate an error.
     void TakeOneofIndex(int32 index);
 
+    bool proto3() { return proto3_; }
+
    private:
     // Used for access to variables of the enclosing instance of
     // ProtoWriter.
@@ -203,7 +206,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
     // TypeInfo to lookup types.
     const TypeInfo* typeinfo_;
 
-    // Whether the root type is a proto3 or not.
+    // Whether the type_ is proto3 or not.
     bool proto3_;
 
     // Additional variables if this element is a message:
@@ -221,7 +224,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
 
     // Set of oneof indices already seen for the type_. Used to validate
     // incoming messages so no more than one oneof is set.
-    hash_set<int32> oneof_indices_;
+    std::vector<bool> oneof_indices_;
 
     GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement);
   };
diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc
index 6d45a4f9..d7ac2dba 100644
--- a/src/google/protobuf/util/json_util.cc
+++ b/src/google/protobuf/util/json_util.cc
@@ -177,6 +177,66 @@ util::Status JsonToBinaryString(TypeResolver* resolver,
       resolver, type_url, &input_stream, &output_stream, options);
 }
 
+namespace {
+const char* kTypeUrlPrefix = "type.googleapis.com";
+TypeResolver* generated_type_resolver_ = NULL;
+GOOGLE_PROTOBUF_DECLARE_ONCE(generated_type_resolver_init_);
+
+string GetTypeUrl(const Message& message) {
+  return string(kTypeUrlPrefix) + "/" + message.GetDescriptor()->full_name();
+}
+
+void DeleteGeneratedTypeResolver() { delete generated_type_resolver_; }
+
+void InitGeneratedTypeResolver() {
+  generated_type_resolver_ = NewTypeResolverForDescriptorPool(
+      kTypeUrlPrefix, DescriptorPool::generated_pool());
+  ::google::protobuf::internal::OnShutdown(&DeleteGeneratedTypeResolver);
+}
+
+TypeResolver* GetGeneratedTypeResolver() {
+  ::google::protobuf::GoogleOnceInit(&generated_type_resolver_init_, &InitGeneratedTypeResolver);
+  return generated_type_resolver_;
+}
+}  // namespace
+
+util::Status MessageToJsonString(const Message& message, string* output,
+                                   const JsonOptions& options) {
+  const DescriptorPool* pool = message.GetDescriptor()->file()->pool();
+  TypeResolver* resolver =
+      pool == DescriptorPool::generated_pool()
+          ? GetGeneratedTypeResolver()
+          : NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool);
+  util::Status result =
+      BinaryToJsonString(resolver, GetTypeUrl(message),
+                         message.SerializeAsString(), output, options);
+  if (pool != DescriptorPool::generated_pool()) {
+    delete resolver;
+  }
+  return result;
+}
+
+util::Status JsonStringToMessage(const string& input, Message* message,
+                                   const JsonParseOptions& options) {
+  const DescriptorPool* pool = message->GetDescriptor()->file()->pool();
+  TypeResolver* resolver =
+      pool == DescriptorPool::generated_pool()
+          ? GetGeneratedTypeResolver()
+          : NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool);
+  string binary;
+  util::Status result = JsonToBinaryString(
+      resolver, GetTypeUrl(*message), input, &binary, options);
+  if (result.ok() && !message->ParseFromString(binary)) {
+    result =
+        util::Status(util::error::INVALID_ARGUMENT,
+                       "JSON transcoder produced invalid protobuf output.");
+  }
+  if (pool != DescriptorPool::generated_pool()) {
+    delete resolver;
+  }
+  return result;
+}
+
 }  // namespace util
 }  // namespace protobuf
 }  // namespace google
diff --git a/src/google/protobuf/util/json_util.h b/src/google/protobuf/util/json_util.h
index b4c2579b..170ae91b 100644
--- a/src/google/protobuf/util/json_util.h
+++ b/src/google/protobuf/util/json_util.h
@@ -33,6 +33,7 @@
 #ifndef GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__
 #define GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__
 
+#include <google/protobuf/message.h>
 #include <google/protobuf/util/type_resolver.h>
 #include <google/protobuf/stubs/bytestream.h>
 
@@ -69,6 +70,30 @@ struct JsonPrintOptions {
 // DEPRECATED. Use JsonPrintOptions instead.
 typedef JsonPrintOptions JsonOptions;
 
+// Converts from protobuf message to JSON. This is a simple wrapper of
+// BinaryToJsonString(). It will use the DescriptorPool of the passed-in
+// message to resolve Any types.
+util::Status MessageToJsonString(const Message& message,
+                                   string* output,
+                                   const JsonOptions& options);
+
+inline util::Status MessageToJsonString(const Message& message,
+                                          string* output) {
+  return MessageToJsonString(message, output, JsonOptions());
+}
+
+// Converts from JSON to protobuf message. This is a simple wrapper of
+// JsonStringToBinary(). It will use the DescriptorPool of the passed-in
+// message to resolve Any types.
+util::Status JsonStringToMessage(const string& input,
+                                   Message* message,
+                                   const JsonParseOptions& options);
+
+inline util::Status JsonStringToMessage(const string& input,
+                                          Message* message) {
+  return JsonStringToMessage(input, message, JsonParseOptions());
+}
+
 // Converts protobuf binary data to JSON.
 // The conversion will fail if:
 //   1. TypeResolver fails to resolve a type.
diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc
index c7d5c59e..dacac5e0 100644
--- a/src/google/protobuf/util/json_util_test.cc
+++ b/src/google/protobuf/util/json_util_test.cc
@@ -34,6 +34,8 @@
 #include <string>
 
 #include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/descriptor_database.h>
+#include <google/protobuf/dynamic_message.h>
 #include <google/protobuf/util/json_format_proto3.pb.h>
 #include <google/protobuf/util/type_resolver.h>
 #include <google/protobuf/util/type_resolver_util.h>
@@ -63,28 +65,21 @@ static string GetTypeUrl(const Descriptor* message) {
 class JsonUtilTest : public testing::Test {
  protected:
   JsonUtilTest() {
-    resolver_.reset(NewTypeResolverForDescriptorPool(
-        kTypeUrlPrefix, DescriptorPool::generated_pool()));
   }
 
   string ToJson(const Message& message, const JsonPrintOptions& options) {
     string result;
-    GOOGLE_CHECK_OK(BinaryToJsonString(resolver_.get(),
-                                GetTypeUrl(message.GetDescriptor()),
-                                message.SerializeAsString(), &result, options));
+    GOOGLE_CHECK_OK(MessageToJsonString(message, &result, options));
     return result;
   }
 
   bool FromJson(const string& json, Message* message,
                 const JsonParseOptions& options) {
-    string binary;
-    if (!JsonToBinaryString(resolver_.get(),
-                            GetTypeUrl(message->GetDescriptor()), json, &binary,
-                            options)
-             .ok()) {
-      return false;
-    }
-    return message->ParseFromString(binary);
+    return JsonStringToMessage(json, message, options).ok();
+  }
+
+  bool FromJson(const string& json, Message* message) {
+    return FromJson(json, message, JsonParseOptions());
   }
 
   google::protobuf::scoped_ptr<TypeResolver> resolver_;
@@ -189,6 +184,45 @@ TEST_F(JsonUtilTest, TestParseErrors) {
   EXPECT_FALSE(FromJson("{\"int32Value\":2147483648}", &m, options));
 }
 
+TEST_F(JsonUtilTest, TestDynamicMessage) {
+  // Some random message but good enough to test the wrapper functions.
+  string input =
+      "{\n"
+      "  \"int32Value\": 1024,\n"
+      "  \"repeatedInt32Value\": [1, 2],\n"
+      "  \"messageValue\": {\n"
+      "    \"value\": 2048\n"
+      "  },\n"
+      "  \"repeatedMessageValue\": [\n"
+      "    {\"value\": 40}, {\"value\": 96}\n"
+      "  ]\n"
+      "}\n";
+
+  // Create a new DescriptorPool with the same protos as the generated one.
+  DescriptorPoolDatabase database(*DescriptorPool::generated_pool());
+  DescriptorPool pool(&database);
+  // A dynamic version of the test proto.
+  DynamicMessageFactory factory;
+  google::protobuf::scoped_ptr<Message> message(factory.GetPrototype(
+      pool.FindMessageTypeByName("proto3.TestMessage"))->New());
+  EXPECT_TRUE(FromJson(input, message.get()));
+
+  // Convert to generated message for easy inspection.
+  TestMessage generated;
+  EXPECT_TRUE(generated.ParseFromString(message->SerializeAsString()));
+  EXPECT_EQ(1024, generated.int32_value());
+  ASSERT_EQ(2, generated.repeated_int32_value_size());
+  EXPECT_EQ(1, generated.repeated_int32_value(0));
+  EXPECT_EQ(2, generated.repeated_int32_value(1));
+  EXPECT_EQ(2048, generated.message_value().value());
+  ASSERT_EQ(2, generated.repeated_message_value_size());
+  EXPECT_EQ(40, generated.repeated_message_value(0).value());
+  EXPECT_EQ(96, generated.repeated_message_value(1).value());
+
+  JsonOptions options;
+  EXPECT_EQ(ToJson(generated, options), ToJson(*message, options));
+}
+
 typedef pair<char*, int> Segment;
 // A ZeroCopyOutputStream that writes to multiple buffers.
 class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream {