aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--conformance/conformance_test.cc55
-rw-r--r--conformance/conformance_test.h10
-rw-r--r--conformance/conformance_test_runner.cc4
-rw-r--r--conformance/failure_list_cpp.txt58
-rw-r--r--conformance/failure_list_java.txt4
5 files changed, 51 insertions, 80 deletions
diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc
index fc0605bf..59a61e51 100644
--- a/conformance/conformance_test.cc
+++ b/conformance/conformance_test.cc
@@ -30,6 +30,7 @@
#include <stdarg.h>
#include <string>
+#include <fstream>
#include "conformance.pb.h"
#include "conformance_test.h"
@@ -575,24 +576,41 @@ void ConformanceTestSuite::TestPrematureEOFForType(FieldDescriptor::Type type) {
}
}
-void ConformanceTestSuite::SetFailureList(const vector<string>& failure_list) {
+void ConformanceTestSuite::SetFailureList(const string& filename,
+ const vector<string>& failure_list) {
+ failure_list_filename_ = filename;
expected_to_fail_.clear();
std::copy(failure_list.begin(), failure_list.end(),
std::inserter(expected_to_fail_, expected_to_fail_.end()));
}
bool ConformanceTestSuite::CheckSetEmpty(const set<string>& set_to_check,
- const char* msg) {
+ const std::string& write_to_file,
+ const std::string& msg) {
if (set_to_check.empty()) {
return true;
} else {
StringAppendF(&output_, "\n");
- StringAppendF(&output_, "%s:\n", msg);
+ StringAppendF(&output_, "%s\n\n", msg.c_str());
for (set<string>::const_iterator iter = set_to_check.begin();
iter != set_to_check.end(); ++iter) {
StringAppendF(&output_, " %s\n", iter->c_str());
}
StringAppendF(&output_, "\n");
+
+ if (!write_to_file.empty()) {
+ std::ofstream os(write_to_file);
+ if (os) {
+ for (set<string>::const_iterator iter = set_to_check.begin();
+ iter != set_to_check.end(); ++iter) {
+ os << *iter << "\n";
+ }
+ } else {
+ StringAppendF(&output_, "Failed to open file: %s\n",
+ write_to_file.c_str());
+ }
+ }
+
return false;
}
}
@@ -1965,27 +1983,34 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
)");
bool ok = true;
- if (!CheckSetEmpty(expected_to_fail_,
+ if (!CheckSetEmpty(expected_to_fail_, "nonexistent_tests.txt",
"These tests were listed in the failure list, but they "
- "don't exist. Remove them from the failure list")) {
+ "don't exist. Remove them from the failure list by "
+ "running:\n"
+ " ./update_failure_list.py " + failure_list_filename_ +
+ " --remove nonexistent_tests.txt")) {
ok = false;
}
- if (!CheckSetEmpty(unexpected_failing_tests_,
+ if (!CheckSetEmpty(unexpected_failing_tests_, "failing_tests.txt",
"These tests failed. If they can't be fixed right now, "
"you can add them to the failure list so the overall "
- "suite can succeed")) {
+ "suite can succeed. Add them to the failure list by "
+ "running:\n"
+ " ./update_failure_list.py " + failure_list_filename_ +
+ " --add failing_tests.txt")) {
+ ok = false;
+ }
+ if (!CheckSetEmpty(unexpected_succeeding_tests_, "succeeding_tests.txt",
+ "These tests succeeded, even though they were listed in "
+ "the failure list. Remove them from the failure list "
+ "by running:\n"
+ " ./update_failure_list.py " + failure_list_filename_ +
+ " --remove succeeding_tests.txt")) {
ok = false;
}
-
- // Sometimes the testee may be fixed before we update the failure list (e.g.,
- // the testee is from a different component). We warn about this case but
- // don't consider it an overall test failure.
- CheckSetEmpty(unexpected_succeeding_tests_,
- "These tests succeeded, even though they were listed in "
- "the failure list. Remove them from the failure list");
if (verbose_) {
- CheckSetEmpty(skipped_,
+ CheckSetEmpty(skipped_, "",
"These tests were skipped (probably because support for some "
"features is not implemented)");
}
diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h
index 75fc97bc..c9c5213c 100644
--- a/conformance/conformance_test.h
+++ b/conformance/conformance_test.h
@@ -98,7 +98,11 @@ class ConformanceTestSuite {
// Sets the list of tests that are expected to fail when RunSuite() is called.
// RunSuite() will fail unless the set of failing tests is exactly the same
// as this list.
- void SetFailureList(const std::vector<std::string>& failure_list);
+ //
+ // The filename here is *only* used to create/format useful error messages for
+ // how to update the failure list. We do NOT read this file at all.
+ void SetFailureList(const std::string& filename,
+ const std::vector<std::string>& failure_list);
// Run all the conformance tests against the given test runner.
// Test output will be stored in "output".
@@ -143,12 +147,14 @@ class ConformanceTestSuite {
void ExpectHardParseFailureForProto(const std::string& proto,
const std::string& test_name);
void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type);
- bool CheckSetEmpty(const set<string>& set_to_check, const char* msg);
+ bool CheckSetEmpty(const set<string>& set_to_check,
+ const std::string& write_to_file, const std::string& msg);
ConformanceTestRunner* runner_;
int successes_;
int expected_failures_;
bool verbose_;
std::string output_;
+ std::string failure_list_filename_;
// The set of test names that are expected to fail in this run, but haven't
// failed yet.
diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc
index 376a60b9..d6b1175c 100644
--- a/conformance/conformance_test_runner.cc
+++ b/conformance/conformance_test_runner.cc
@@ -280,11 +280,13 @@ int main(int argc, char *argv[]) {
char *program;
google::protobuf::ConformanceTestSuite suite;
+ string failure_list_filename;
vector<string> failure_list;
for (int arg = 1; arg < argc; ++arg) {
if (strcmp(argv[arg], "--failure_list") == 0) {
if (++arg == argc) UsageError();
+ failure_list_filename = argv[arg];
ParseFailureList(argv[arg], &failure_list);
} else if (strcmp(argv[arg], "--verbose") == 0) {
suite.SetVerbose(true);
@@ -300,7 +302,7 @@ int main(int argc, char *argv[]) {
}
}
- suite.SetFailureList(failure_list);
+ suite.SetFailureList(failure_list_filename, failure_list);
ForkPipeRunner runner(program);
std::string output;
diff --git a/conformance/failure_list_cpp.txt b/conformance/failure_list_cpp.txt
index 2ddf831c..839e5210 100644
--- a/conformance/failure_list_cpp.txt
+++ b/conformance/failure_list_cpp.txt
@@ -12,21 +12,11 @@ FieldMaskPathsDontRoundTrip.JsonOutput
FieldMaskTooManyUnderscore.JsonOutput
JsonInput.AnyUnorderedTypeTag.JsonOutput
JsonInput.AnyUnorderedTypeTag.ProtobufOutput
-JsonInput.AnyWithValueForInteger.JsonOutput
-JsonInput.AnyWithValueForInteger.ProtobufOutput
-JsonInput.AnyWithValueForJsonObject.JsonOutput
-JsonInput.AnyWithValueForJsonObject.ProtobufOutput
JsonInput.BoolFieldDoubleQuotedFalse
JsonInput.BoolFieldDoubleQuotedTrue
-JsonInput.BoolFieldIntegerOne
-JsonInput.BoolFieldIntegerZero
-JsonInput.BytesFieldInvalidBase64Characters
JsonInput.BytesFieldNoPadding
JsonInput.DoubleFieldTooSmall
JsonInput.DurationHasZeroFractionalDigit.Validator
-JsonInput.DurationJsonInputTooLarge
-JsonInput.DurationJsonInputTooSmall
-JsonInput.DurationMissingS
JsonInput.EnumFieldUnknownValue.Validator
JsonInput.FieldMaskInvalidCharacter
JsonInput.FieldNameDuplicate
@@ -36,58 +26,12 @@ JsonInput.FieldNameInLowerCamelCase.Validator
JsonInput.FieldNameInSnakeCase.JsonOutput
JsonInput.FieldNameInSnakeCase.ProtobufOutput
JsonInput.FieldNameNotQuoted
-JsonInput.FloatFieldTooLarge
-JsonInput.FloatFieldTooSmall
-JsonInput.Int32FieldLeadingSpace
-JsonInput.Int32FieldLeadingZero
-JsonInput.Int32FieldMinFloatValue.JsonOutput
-JsonInput.Int32FieldMinFloatValue.ProtobufOutput
-JsonInput.Int32FieldMinValue.JsonOutput
-JsonInput.Int32FieldMinValue.ProtobufOutput
-JsonInput.Int32FieldNegativeWithLeadingZero
-JsonInput.Int32FieldNotInteger
-JsonInput.Int32FieldNotNumber
-JsonInput.Int32FieldTooLarge
-JsonInput.Int32FieldTooSmall
-JsonInput.Int32FieldTrailingSpace
-JsonInput.Int64FieldNotInteger
-JsonInput.Int64FieldNotNumber
-JsonInput.Int64FieldTooLarge
-JsonInput.Int64FieldTooSmall
JsonInput.MapFieldValueIsNull
-JsonInput.OneofFieldDuplicate
JsonInput.RepeatedFieldMessageElementIsNull
JsonInput.RepeatedFieldPrimitiveElementIsNull
JsonInput.RepeatedFieldTrailingComma
-JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
-JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotMessage
-JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotString
-JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotBool
-JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotInt
-JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotString
-JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool
-JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt
-JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotMessage
-JsonInput.StringFieldNotAString
-JsonInput.StringFieldSurrogateInWrongOrder
-JsonInput.StringFieldSurrogatePair.JsonOutput
-JsonInput.StringFieldSurrogatePair.ProtobufOutput
-JsonInput.StringFieldUnpairedHighSurrogate
-JsonInput.StringFieldUnpairedLowSurrogate
JsonInput.StringFieldUppercaseEscapeLetter
-JsonInput.TimestampJsonInputLowercaseT
-JsonInput.TimestampJsonInputLowercaseZ
-JsonInput.TimestampJsonInputMissingT
-JsonInput.TimestampJsonInputMissingZ
-JsonInput.TimestampJsonInputTooLarge
-JsonInput.TimestampJsonInputTooSmall
JsonInput.TrailingCommaInAnObject
-JsonInput.Uint32FieldNotInteger
-JsonInput.Uint32FieldNotNumber
-JsonInput.Uint32FieldTooLarge
-JsonInput.Uint64FieldNotInteger
-JsonInput.Uint64FieldNotNumber
-JsonInput.Uint64FieldTooLarge
JsonInput.WrapperTypesWithNullValue.JsonOutput
JsonInput.WrapperTypesWithNullValue.ProtobufOutput
ProtobufInput.PrematureEofBeforeKnownRepeatedValue.MESSAGE
@@ -102,5 +46,3 @@ ProtobufInput.PrematureEofInPackedField.SINT64
ProtobufInput.PrematureEofInPackedField.UINT32
ProtobufInput.PrematureEofInPackedField.UINT64
ProtobufInput.PrematureEofInsideKnownRepeatedValue.MESSAGE
-TimestampProtoInputTooLarge.JsonOutput
-TimestampProtoInputTooSmall.JsonOutput
diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt
index 552c0cc9..99232ad2 100644
--- a/conformance/failure_list_java.txt
+++ b/conformance/failure_list_java.txt
@@ -8,8 +8,6 @@ FieldMaskNumbersDontRoundTrip.JsonOutput
FieldMaskPathsDontRoundTrip.JsonOutput
FieldMaskTooManyUnderscore.JsonOutput
JsonInput.AnyWithFieldMask.ProtobufOutput
-JsonInput.AnyWithValueForInteger.JsonOutput
-JsonInput.AnyWithValueForJsonObject.JsonOutput
JsonInput.BoolFieldAllCapitalFalse
JsonInput.BoolFieldAllCapitalTrue
JsonInput.BoolFieldCamelCaseFalse
@@ -30,8 +28,6 @@ JsonInput.FloatFieldInfinityNotQuoted
JsonInput.FloatFieldNanNotQuoted
JsonInput.FloatFieldNegativeInfinityNotQuoted
JsonInput.Int32FieldLeadingZero
-JsonInput.Int32FieldMinFloatValue.JsonOutput
-JsonInput.Int32FieldMinValue.JsonOutput
JsonInput.Int32FieldNegativeWithLeadingZero
JsonInput.Int32FieldPlusSign
JsonInput.Int32MapFieldKeyNotQuoted
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-08 23:25:13 +0000