Reserve unknown in Ruby (#3763)

* Reserve unknown in ruby

* Revert ruby tests. Wait for cpp impl for conformance test

* Add conformance test for preserving unknown

* Add unknown field conformance test to csharp failure list.

* Fix comments

* Fix comment

* Fix comments

* Fix typo

* Use stringsink_string directly

* Mark hd unused

* Remove unused encodeunknown_handlerfunc

5 files changed, 69 insertions, 5 deletions

diff --git a/conformance/conformance_php.php b/conformance/conformance_php.php
index 2e3f7831..4dc18f59 100755
--- a/conformance/conformance_php.php
+++ b/conformance/conformance_php.php
@@ -57,10 +57,10 @@ function doTest($request)
           return $response;
         }
       } elseif ($request->getMessageType() == "protobuf_test_messages.proto2.TestAllTypesProto2") {
-	$response->setSkipped("PHP doesn't support proto2");
-	return $response;
+        $response->setSkipped("PHP doesn't support proto2");
+        return $response;
       } else {
-	trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR);
+        trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR);
       }
     } elseif ($request->getPayload() == "json_payload") {
       try {
diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc
index f1b92056..c54d4ccd 100644
--- a/conformance/conformance_test.cc
+++ b/conformance/conformance_test.cc
@@ -290,6 +290,28 @@ void ConformanceTestSuite::RunValidInputTest(
       TextFormat::ParseFromString(equivalent_text_format, reference_message))
           << "Failed to parse data for test case: " << test_name
           << ", data: " << equivalent_text_format;
+  const string equivalent_wire_format = reference_message->SerializeAsString();
+  RunValidBinaryInputTest(test_name, level, input, input_format,
+                          equivalent_wire_format, requested_output, isProto3);
+}
+
+void ConformanceTestSuite::RunValidBinaryInputTest(
+    const string& test_name, ConformanceLevel level, const string& input,
+    WireFormat input_format, const string& equivalent_wire_format,
+    WireFormat requested_output, bool isProto3) {
+  auto newTestMessage = [&isProto3]() {
+    Message* newMessage;
+    if (isProto3) {
+      newMessage = new TestAllTypesProto3;
+    } else {
+      newMessage = new TestAllTypesProto2;
+    }
+    return newMessage;
+  };
+  Message* reference_message = newTestMessage();
+  GOOGLE_CHECK(
+      reference_message->ParseFromString(equivalent_wire_format))
+          << "Failed to parse wire data for test case: " << test_name;
 
   ConformanceRequest request;
   ConformanceResponse response;
@@ -493,6 +515,19 @@ void ConformanceTestSuite::RunValidProtobufTest(
   }
 }
 
+void ConformanceTestSuite::RunValidBinaryProtobufTest(
+    const string& test_name, ConformanceLevel level,
+    const string& input_protobuf, bool isProto3) {
+  string rname = ".Proto3";
+  if (!isProto3) {
+    rname = ".Proto2";
+  }
+  RunValidBinaryInputTest(
+      ConformanceLevelToString(level) + rname + ".ProtobufInput." + test_name +
+      ".ProtobufOutput", level, input_protobuf, conformance::PROTOBUF,
+      input_protobuf, conformance::PROTOBUF, isProto3);
+}
+
 void ConformanceTestSuite::RunValidProtobufTestWithMessage(
     const string& test_name, ConformanceLevel level, const Message *input,
     const string& equivalent_text_format, bool isProto3) {
@@ -811,6 +846,14 @@ void ConformanceTestSuite::TestOneofMessage (MessageType &message,
       "OneofZeroEnum", RECOMMENDED, &message, "oneof_enum: FOO", isProto3);
 }
 
+template <class MessageType>
+void ConformanceTestSuite::TestUnknownMessage(MessageType& message,
+                                              bool isProto3) {
+  message.ParseFromString("\xA8\x1F\x01");
+  RunValidBinaryProtobufTest("UnknownVarint", REQUIRED,
+                             message.SerializeAsString(), isProto3);
+}
+
 bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
                                     std::string* output) {
   runner_ = runner;
@@ -1847,6 +1890,14 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
       "StringFieldSingleQuoteBoth", RECOMMENDED,
       R"({'optionalString': 'Hello world!'})");
 
+  // Unknown fields.
+  {
+    TestAllTypesProto3 messageProto3;
+    TestAllTypesProto2 messageProto2;
+    TestUnknownMessage(messageProto3, true);
+    TestUnknownMessage(messageProto2, false);
+  }
+
   // Wrapper types.
   RunValidJsonTest(
       "OptionalBoolWrapper", REQUIRED,
diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h
index d1a822eb..4882412c 100644
--- a/conformance/conformance_test.h
+++ b/conformance/conformance_test.h
@@ -167,6 +167,13 @@ class ConformanceTestSuite {
                          const string& equivalent_text_format,
                          conformance::WireFormat requested_output,
                          bool isProto3);
+  void RunValidBinaryInputTest(const string& test_name,
+                               ConformanceLevel level,
+                               const string& input,
+                               conformance::WireFormat input_format,
+                               const string& equivalent_wire_format,
+                               conformance::WireFormat requested_output,
+                               bool isProto3);
   void RunValidJsonTest(const string& test_name,
                         ConformanceLevel level,
                         const string& input_json,
@@ -180,6 +187,10 @@ class ConformanceTestSuite {
                             const string& input_protobuf,
                             const string& equivalent_text_format,
                             bool isProto3);
+  void RunValidBinaryProtobufTest(const string& test_name,
+                                  ConformanceLevel level,
+                                  const string& input_protobuf,
+                                  bool isProto3);
   void RunValidProtobufTestWithMessage(
       const string& test_name, ConformanceLevel level,
       const Message *input,
@@ -212,6 +223,9 @@ class ConformanceTestSuite {
   template <class MessageType>
   void TestOneofMessage (MessageType &message,
                          bool isProto3);
+  template <class MessageType>
+  void TestUnknownMessage (MessageType &message,
+                           bool isProto3);
   void TestValidDataForType(
       google::protobuf::FieldDescriptor::Type,
       std::vector<std::pair<std::string, std::string>> values);
diff --git a/conformance/failure_list_csharp.txt b/conformance/failure_list_csharp.txt
index 2a20aa78..cd84e693 100644
--- a/conformance/failure_list_csharp.txt
+++ b/conformance/failure_list_csharp.txt
@@ -1,2 +1,3 @@
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
+Required.Proto3.ProtobufInput.UnknownVarint.ProtobufOutput
diff --git a/conformance/failure_list_ruby.txt b/conformance/failure_list_ruby.txt
index 45cfaca6..1bab1359 100644
--- a/conformance/failure_list_ruby.txt
+++ b/conformance/failure_list_ruby.txt
@@ -59,8 +59,6 @@ Required.Proto3.JsonInput.FieldMask.ProtobufOutput
 Required.Proto3.JsonInput.FloatFieldInfinity.JsonOutput
 Required.Proto3.JsonInput.FloatFieldNan.JsonOutput
 Required.Proto3.JsonInput.FloatFieldNegativeInfinity.JsonOutput
-Required.Proto3.JsonInput.FloatFieldTooLarge
-Required.Proto3.JsonInput.FloatFieldTooSmall
 Required.Proto3.JsonInput.OneofFieldDuplicate
 Required.Proto3.JsonInput.OptionalBoolWrapper.JsonOutput
 Required.Proto3.JsonInput.OptionalBoolWrapper.ProtobufOutput