Merge from google internal

7 files changed, 179 insertions, 38 deletions

diff --git a/src/google/protobuf/util/field_mask_util.cc b/src/google/protobuf/util/field_mask_util.cc
index 982d6407..4d0d3a46 100644
--- a/src/google/protobuf/util/field_mask_util.cc
+++ b/src/google/protobuf/util/field_mask_util.cc
@@ -192,6 +192,13 @@ class FieldMaskTree {
   // children removed because the path matches all the node's children.
   void AddPath(const string& path);
 
+  // Remove a path from the tree.
+  // If the path is a sub-path of an existing field path in the tree, it means
+  // we need remove the existing fied path and add all sub-paths except
+  // specified path. If the path matches an existing node in the tree, this node
+  // will be moved.
+  void RemovePath(const string& path, const Descriptor* descriptor);
+
   // Calculate the intersection part of a field path with this tree and add
   // the intersection field path into out.
   void IntersectPath(const string& path, FieldMaskTree* out);
@@ -333,6 +340,59 @@ void FieldMaskTree::AddPath(const string& path) {
   }
 }
 
+void FieldMaskTree::RemovePath(const string& path,
+                               const Descriptor* descriptor) {
+  std::vector<string> parts = Split(path, ".");
+  if (parts.empty()) {
+    return;
+  }
+  std::vector<Node*> nodes(parts.size());
+  Node* node = &root_;
+  const Descriptor* current_descriptor = descriptor;
+  Node* new_branch_node = NULL;
+  for (int i = 0; i < parts.size(); ++i) {
+    nodes[i] = node;
+    const FieldDescriptor* field_descriptor =
+        current_descriptor->FindFieldByName(parts[i]);
+    if (field_descriptor == NULL ||
+        (field_descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE &&
+         i != parts.size() - 1)) {
+      // Invalid path.
+      if (new_branch_node != NULL) {
+        // If add any new nodes, cleanup.
+        new_branch_node->ClearChildren();
+      }
+      return;
+    }
+
+    if (node->children.empty()) {
+      if (new_branch_node == NULL) {
+        new_branch_node = node;
+      }
+      for (int i = 0; i < current_descriptor->field_count(); ++i) {
+        node->children[current_descriptor->field(i)->name()] = new Node();
+      }
+    }
+    if (ContainsKey(node->children, parts[i])) {
+      node = node->children.at(parts[i]);
+      if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+        current_descriptor = field_descriptor->message_type();
+      }
+    } else {
+      // Path does not exist.
+      return;
+    }
+  }
+  // Remove path.
+  for (int i = parts.size() - 1; i >= 0; i--) {
+    delete nodes[i]->children[parts[i]];
+    nodes[i]->children.erase(parts[i]);
+    if (!nodes[i]->children.empty()) {
+      break;
+    }
+  }
+}
+
 void FieldMaskTree::IntersectPath(const string& path, FieldMaskTree* out) {
   std::vector<string> parts = Split(path, ".");
   if (parts.empty()) {
@@ -560,6 +620,22 @@ void FieldMaskUtil::Intersect(const FieldMask& mask1, const FieldMask& mask2,
   intersection.MergeToFieldMask(out);
 }
 
+void FieldMaskUtil::InternalSubtract(const Descriptor* descriptor,
+                                     const FieldMask& mask1,
+                                     const FieldMask& mask2, FieldMask* out) {
+  if (mask1.paths().empty()) {
+    out->Clear();
+    return;
+  }
+  FieldMaskTree tree;
+  tree.MergeFromFieldMask(mask1);
+  for (int i = 0; i < mask2.paths_size(); ++i) {
+    tree.RemovePath(mask2.paths(i), descriptor);
+  }
+  out->Clear();
+  tree.MergeToFieldMask(out);
+}
+
 bool FieldMaskUtil::IsPathInFieldMask(StringPiece path, const FieldMask& mask) {
   for (int i = 0; i < mask.paths_size(); ++i) {
     const string& mask_path = mask.paths(i);
diff --git a/src/google/protobuf/util/field_mask_util.h b/src/google/protobuf/util/field_mask_util.h
index 71c68fec..01642c6f 100644
--- a/src/google/protobuf/util/field_mask_util.h
+++ b/src/google/protobuf/util/field_mask_util.h
@@ -113,6 +113,13 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil {
   static void Intersect(const FieldMask& mask1, const FieldMask& mask2,
                         FieldMask* out);
 
+  // Subtracts mask2 from mask1 base of type T.
+  template <typename T>
+  static void Subtract(const FieldMask& mask1, const FieldMask& mask2,
+                       FieldMask* out) {
+    InternalSubtract(T::descriptor(), mask1, mask2, out);
+  }
+
   // Returns true if path is covered by the given FieldMask. Note that path
   // "foo.bar" covers all paths like "foo.bar.baz", "foo.bar.quz.x", etc.
   static bool IsPathInFieldMask(StringPiece path, const FieldMask& mask);
@@ -167,6 +174,10 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil {
 
   static void InternalGetFieldMaskForAllFields(const Descriptor* descriptor,
                                                FieldMask* out);
+
+  static void InternalSubtract(const Descriptor* descriptor,
+                               const FieldMask& mask1, const FieldMask& mask2,
+                               FieldMask* out);
 };
 
 // Note that for compatibility with the defined behaviour for FieldMask in
diff --git a/src/google/protobuf/util/field_mask_util_test.cc b/src/google/protobuf/util/field_mask_util_test.cc
index 24943ed1..7939f733 100644
--- a/src/google/protobuf/util/field_mask_util_test.cc
+++ b/src/google/protobuf/util/field_mask_util_test.cc
@@ -348,6 +348,46 @@ TEST(FieldMaskUtilTest, TestIntersect) {
   EXPECT_EQ("foo.bar.baz", FieldMaskUtil::ToString(out));
 }
 
+TEST(FieldMaskUtilTest, TestSubtract) {
+  FieldMask mask1, mask2, out;
+  // Normal case.
+  FieldMaskUtil::FromString(
+      "optional_int32,optional_uint64,optional_nested_message,optional_foreign_"
+      "message,repeated_int32,repeated_foreign_message,repeated_nested_message."
+      "bb",
+      &mask1);
+
+  FieldMaskUtil::FromString(
+      "optional_int32,optional_nested_message.bb,optional_foreign_message.c,"
+      "repeated_int32,repeated_nested_message.bb,repeated_foreign_message.f,"
+      "repeated_foreign_message.d,repeated_nested_message.bb,repeated_uint32",
+      &mask2);
+
+  FieldMaskUtil::Subtract<TestAllTypes>(mask1, mask2, &out);
+  EXPECT_EQ(
+      "optional_foreign_message.d,optional_uint64,repeated_foreign_message.c",
+      FieldMaskUtil::ToString(out));
+
+  // mask1 is empty.
+  FieldMaskUtil::FromString("", &mask1);
+  FieldMaskUtil::Subtract<TestAllTypes>(mask1, mask2, &out);
+  EXPECT_EQ("", FieldMaskUtil::ToString(out));
+
+  // mask1 is "optional_nested_message" and mask2 is
+  // "optional_nested_message.nonexist_field".
+  FieldMaskUtil::FromString("optional_nested_message", &mask1);
+  FieldMaskUtil::FromString("optional_nested_message.nonexist_field", &mask2);
+  FieldMaskUtil::Subtract<TestAllTypes>(mask1, mask2, &out);
+  EXPECT_EQ("optional_nested_message", FieldMaskUtil::ToString(out));
+
+  // mask1 is "optional_nested_message" and mask2 is
+  // "optional_nested_message".
+  FieldMaskUtil::FromString("optional_nested_message", &mask1);
+  FieldMaskUtil::FromString("optional_nested_message", &mask2);
+  FieldMaskUtil::Subtract<TestAllTypes>(mask1, mask2, &out);
+  EXPECT_EQ("", FieldMaskUtil::ToString(out));
+}
+
 TEST(FieldMaskUtilTest, TestIspathInFieldMask) {
   FieldMask mask;
   FieldMaskUtil::FromString("foo.bar", &mask);
diff --git a/src/google/protobuf/util/internal/json_stream_parser.cc b/src/google/protobuf/util/internal/json_stream_parser.cc
index 047c14e1..2ada3583 100644
--- a/src/google/protobuf/util/internal/json_stream_parser.cc
+++ b/src/google/protobuf/util/internal/json_stream_parser.cc
@@ -498,6 +498,19 @@ util::Status JsonStreamParser::ParseNumber() {
   return result;
 }
 
+util::Status JsonStreamParser::ParseDoubleHelper(
+    const string& number, NumberResult* result) {
+  if (!safe_strtod(number, &result->double_val)) {
+    return ReportFailure("Unable to parse number.");
+  }
+  if (!loose_float_number_conversion_ &&
+      !MathLimits<double>::IsFinite(result->double_val)) {
+    return ReportFailure("Number exceeds the range of double.");
+  }
+  result->type = NumberResult::DOUBLE;
+  return util::Status();
+}
+
 util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) {
   const char* data = p_.data();
   int length = p_.length();
@@ -533,16 +546,11 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) {
 
   // Floating point number, parse as a double.
   if (floating) {
-    if (!safe_strtod(number, &result->double_val)) {
-      return ReportFailure("Unable to parse number.");
+    util::Status status = ParseDoubleHelper(number, result);
+    if (status.ok()) {
+      p_.remove_prefix(index);
     }
-    if (!loose_float_number_conversion_ &&
-        !MathLimits<double>::IsFinite(result->double_val)) {
-      return ReportFailure("Number exceeds the range of double.");
-    }
-    result->type = NumberResult::DOUBLE;
-    p_.remove_prefix(index);
-    return util::Status();
+    return status;
   }
 
   // Positive non-floating point number, parse as a uint64.
@@ -551,12 +559,18 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) {
     if (number.length() >= 2 && number[0] == '0') {
       return ReportFailure("Octal/hex numbers are not valid JSON values.");
     }
-    if (!safe_strtou64(number, &result->uint_val)) {
-      return ReportFailure("Unable to parse number.");
+    if (safe_strtou64(number, &result->uint_val)) {
+      result->type = NumberResult::UINT;
+      p_.remove_prefix(index);
+      return util::Status();
+    } else {
+      // If the value is too large, parse it as double.
+      util::Status status = ParseDoubleHelper(number, result);
+      if (status.ok()) {
+        p_.remove_prefix(index);
+      }
+      return status;
     }
-    result->type = NumberResult::UINT;
-    p_.remove_prefix(index);
-    return util::Status();
   }
 
   // Octal/Hex numbers are not valid JSON values.
@@ -564,12 +578,18 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) {
     return ReportFailure("Octal/hex numbers are not valid JSON values.");
   }
   // Negative non-floating point number, parse as an int64.
-  if (!safe_strto64(number, &result->int_val)) {
-    return ReportFailure("Unable to parse number.");
+  if (safe_strto64(number, &result->int_val)) {
+    result->type = NumberResult::INT;
+    p_.remove_prefix(index);
+    return util::Status();
+  } else {
+    // If the value is too large, parse it as double.
+    util::Status status = ParseDoubleHelper(number, result);
+    if (status.ok()) {
+      p_.remove_prefix(index);
+    }
+    return status;
   }
-  result->type = NumberResult::INT;
-  p_.remove_prefix(index);
-  return util::Status();
 }
 
 util::Status JsonStreamParser::HandleBeginObject() {
diff --git a/src/google/protobuf/util/internal/json_stream_parser.h b/src/google/protobuf/util/internal/json_stream_parser.h
index 6b9d46ee..31933b67 100644
--- a/src/google/protobuf/util/internal/json_stream_parser.h
+++ b/src/google/protobuf/util/internal/json_stream_parser.h
@@ -154,6 +154,9 @@ class LIBPROTOBUF_EXPORT JsonStreamParser {
   // component.
   util::Status ParseNumberHelper(NumberResult* result);
 
+  // Parse a number as double into a NumberResult.
+  util::Status ParseDoubleHelper(const string& number, NumberResult* result);
+
   // Handles a { during parsing of a value.
   util::Status HandleBeginObject();
 
diff --git a/src/google/protobuf/util/internal/json_stream_parser_test.cc b/src/google/protobuf/util/internal/json_stream_parser_test.cc
index ca71ff24..a11e9be0 100644
--- a/src/google/protobuf/util/internal/json_stream_parser_test.cc
+++ b/src/google/protobuf/util/internal/json_stream_parser_test.cc
@@ -694,20 +694,19 @@ TEST_F(JsonStreamParserTest, ExtraCharactersAfterObject) {
   }
 }
 
-// numbers too large
-TEST_F(JsonStreamParserTest, PositiveNumberTooBig) {
-  StringPiece str = "[18446744073709551616]";  // 2^64
+TEST_F(JsonStreamParserTest, PositiveNumberTooBigIsDouble) {
+  StringPiece str = "18446744073709551616";  // 2^64
   for (int i = 0; i <= str.length(); ++i) {
-    ow_.StartList("");
-    DoErrorTest(str, i, "Unable to parse number.");
+    ow_.RenderDouble("", 18446744073709552000.0);
+    DoTest(str, i);
   }
 }
 
-TEST_F(JsonStreamParserTest, NegativeNumberTooBig) {
-  StringPiece str = "[-18446744073709551616]";
+TEST_F(JsonStreamParserTest, NegativeNumberTooBigIsDouble) {
+  StringPiece str = "-18446744073709551616";
   for (int i = 0; i <= str.length(); ++i) {
-    ow_.StartList("");
-    DoErrorTest(str, i, "Unable to parse number.");
+    ow_.RenderDouble("", -18446744073709551616.0);
+    DoTest(str, i);
   }
 }
 
diff --git a/src/google/protobuf/util/message_differencer.cc b/src/google/protobuf/util/message_differencer.cc
index d62038ad..e964e041 100644
--- a/src/google/protobuf/util/message_differencer.cc
+++ b/src/google/protobuf/util/message_differencer.cc
@@ -1440,9 +1440,6 @@ bool MessageDifferencer::MatchRepeatedFieldIndices(
   match_list1->assign(count1, -1);
   match_list2->assign(count2, -1);
 
-  SpecificField specific_field;
-  specific_field.field = repeated_field;
-
   bool success = true;
   // Find potential match if this is a special repeated field.
   if (key_comparator != NULL || IsTreatedAsSet(repeated_field)) {
@@ -1469,20 +1466,15 @@ bool MessageDifferencer::MatchRepeatedFieldIndices(
         // Indicates any matched elements for this repeated field.
         bool match = false;
 
-        specific_field.index = i;
-        specific_field.new_index = i;
-
         for (int j = 0; j < count2; j++) {
           if (match_list2->at(j) != -1) continue;
-          specific_field.index = i;
-          specific_field.new_index = j;
 
           match = IsMatch(repeated_field, key_comparator,
                           &message1, &message2, parent_fields, i, j);
 
           if (match) {
-            match_list1->at(specific_field.index) = specific_field.new_index;
-            match_list2->at(specific_field.new_index) = specific_field.index;
+            match_list1->at(i) = j;
+            match_list2->at(j) = i;
             break;
           }
         }