From 35a1cc7a7c7cfd205641f15258ca991f6d3ec2bc Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Wed, 1 Apr 2015 17:23:48 -0700 Subject: Added first version of conformance tests. Change-Id: Ib75664194491643f8e4f1503a2ed942a2d1e1655 --- Makefile.am | 6 +- configure.ac | 2 +- conformance/Makefile.am | 50 +++++ conformance/README.md | 45 +++++ conformance/conformance.proto | 219 ++++++++++++++++++++++ conformance/conformance_cpp.cc | 156 ++++++++++++++++ conformance/conformance_test.cc | 395 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 871 insertions(+), 2 deletions(-) create mode 100644 conformance/Makefile.am create mode 100644 conformance/README.md create mode 100644 conformance/conformance.proto create mode 100644 conformance/conformance_cpp.cc create mode 100644 conformance/conformance_test.cc diff --git a/Makefile.am b/Makefile.am index e7820f2a..3cabcf57 100644 --- a/Makefile.am +++ b/Makefile.am @@ -9,7 +9,7 @@ AUTOMAKE_OPTIONS = foreign SUBDIRS = . src # Always include gtest in distributions. -DIST_SUBDIRS = $(subdirs) src +DIST_SUBDIRS = $(subdirs) src conformance # Build gtest before we build protobuf tests. We don't add gtest to SUBDIRS # because then "make check" would also build and run all of gtest's own tests, @@ -30,6 +30,10 @@ clean-local: @if test -e gtest/Makefile; then \ echo "Making clean in gtest"; \ cd gtest && $(MAKE) $(AM_MAKEFLAGS) clean; \ + fi; \ + if test -e conformance/Makefile; then \ + echo "Making clean in conformance"; \ + cd conformance && $(MAKE) $(AM_MAKEFLAGS) clean; \ fi pkgconfigdir = $(libdir)/pkgconfig diff --git a/configure.ac b/configure.ac index 14351a8b..0615cd85 100644 --- a/configure.ac +++ b/configure.ac @@ -164,5 +164,5 @@ export CFLAGS export CXXFLAGS AC_CONFIG_SUBDIRS([gtest]) -AC_CONFIG_FILES([Makefile src/Makefile protobuf.pc protobuf-lite.pc]) +AC_CONFIG_FILES([Makefile src/Makefile conformance/Makefile protobuf.pc protobuf-lite.pc]) AC_OUTPUT diff --git a/conformance/Makefile.am b/conformance/Makefile.am new file mode 100644 index 00000000..2e1de17d --- /dev/null +++ b/conformance/Makefile.am @@ -0,0 +1,50 @@ +## Process this file with automake to produce Makefile.in + +protoc_inputs = \ + conformance.proto + +protoc_outputs = \ + conformance.pb.cc \ + conformance.pb.h + +bin_PROGRAMS = conformance-test conformance-cpp + +conformance_test_LDADD = $(top_srcdir)/src/libprotobuf.la +conformance_test_SOURCES = conformance_test.cc +nodist_conformance_test_SOURCES = conformance.pb.cc +conformance_test_CPPFLAGS = -I$(top_srcdir)/src + +conformance_cpp_LDADD = $(top_srcdir)/src/libprotobuf.la +conformance_cpp_SOURCES = conformance_cpp.cc +nodist_conformance_cpp_SOURCES = conformance.pb.cc +conformance_cpp_CPPFLAGS = -I$(top_srcdir)/src + +if USE_EXTERNAL_PROTOC + +unittest_proto_middleman: $(protoc_inputs) + $(PROTOC) -I$(srcdir) --cpp_out=. $^ + touch unittest_proto_middleman + +else + +# We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is +# relative to srcdir, which may not be the same as the current directory when +# building out-of-tree. +unittest_proto_middleman: $(top_srcdir)/src/protoc$(EXEEXT) $(protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. --cpp_out=$$oldpwd $(protoc_inputs) ) + touch unittest_proto_middleman + +endif + +$(protoc_outputs): unittest_proto_middleman + +BUILT_SOURCES = $(protoc_outputs) + +CLEANFILES = $(protoc_outputs) unittest_proto_middleman + +MAINTAINERCLEANFILES = \ + Makefile.in + +# Targets for actually running tests. +test_cpp: unittest_proto_middleman conformance-test conformance-cpp + ./conformance-test ./conformance-cpp diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 00000000..9388055f --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,45 @@ +Protocol Buffers - Google's data interchange format +=================================================== + +[![Build Status](https://travis-ci.org/google/protobuf.svg?branch=master)](https://travis-ci.org/google/protobuf) + +Copyright 2008 Google Inc. + +This directory contains conformance tests for testing completeness and +correctness of Protocol Buffers implementations. These tests are designed +to be easy to run against any Protocol Buffers implementation. + +This directory contains the tester process `conformance-test`, which +contains all of the tests themselves. Then separate programs written +in whatever language you want to test communicate with the tester +program over a pipe. + +Before running any of these tests, make sure you run `make` in the base +directory to build `protoc`, since all the tests depend on it. + + $ make + +Then to run the tests against the C++ implementation, run: + + $ cd conformance && make test_cpp + +More tests and languages will be added soon! + +Testing other Protocol Buffer implementations +--------------------------------------------- + +To run these tests against a new Protocol Buffers implementation, write a +program in your language that uses the protobuf implementation you want +to test. This program should implement the testing protocol defined in +[conformance.proto](https://github.com/google/protobuf/blob/master/conformance/conformance.proto). +This is designed to be as easy as possible: the C++ version is only +150 lines and is a good example for what this program should look like +(see [conformance_cpp.cc](https://github.com/google/protobuf/blob/master/conformance/conformance_cpp.cc)). +The program only needs to be able to read from stdin and write to stdout. + +Portability +----------- + +Note that the test runner currently does not work on Windows. Patches +to fix this are welcome! (But please get in touch first to settle on +a general implementation strategy). diff --git a/conformance/conformance.proto b/conformance/conformance.proto new file mode 100644 index 00000000..34fec45f --- /dev/null +++ b/conformance/conformance.proto @@ -0,0 +1,219 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; +package conformance; + +// This defines the conformance testing protocol. This protocol exists between +// the conformance tester process (the "tester") and the process whose protobuf +// implemention is being tested (the "testee"). The tester forks the testee and +// communicates with it over its stdin/stdout: +// +// +--------+ pipe +----------+ +// | tester | <------> | testee | +// | | | | +// | C++ | | any lang | +// +--------+ +----------+ +// +// The tester contains all of the test cases and their expected output. +// The testee is a simple program written in the target language that reads +// each test case and attempts to produce acceptable output for it. +// +// Every test consists of a ConformanceRequest/ConformanceResponse +// request/reply pair. The protocol on the pipe is simply: +// +// 1. tester sends 4-byte length N +// 2. tester sends N bytes representing a ConformanceRequest proto +// 3. testee sends 4-byte length M +// 4. testee sends M bytes representing a ConformanceResponse proto + +// Represents a single test case's input. The testee should: +// +// 1. parse this proto (which should always succeed) +// 2. parse the protobuf or JSON payload in "payload" (which may fail) +// 3. if the parse succeeded, serialize the message in the requested format. +message ConformanceRequest { + // The payload (whether protobuf of JSON) is always for a TestAllTypes proto + // (see below). + oneof payload { + bytes protobuf_payload = 1; + string json_payload = 2; + } + + enum RequestedOutput { + UNSPECIFIED = 0; + PROTOBUF = 1; + JSON = 2; + } + + // Which format should the testee serialize its message to? + optional RequestedOutput requested_output = 3; +} + +// Represents a single test case's output. +message ConformanceResponse { + oneof result { + // This string should be set to indicate parsing failed. The string can + // provide more information about the parse error if it is available. + // + // Setting this string does not necessarily mean the testee failed the + // test. Some of the test cases are intentionally invalid input. + string parse_error = 1; + + // This should be set if some other error occurred. This will always + // indicate that the test failed. The string can provide more information + // about the failure. + string runtime_error = 2; + + // If the input was successfully parsed and the requested output was + // protobuf, serialize it to protobuf and set it in this field. + bytes protobuf_payload = 3; + + // If the input was successfully parsed and the requested output was JSON, + // serialize to JSON and set it in this field. + string json_payload = 4; + } +} + +// This proto includes every type of field in both singular and repeated +// forms. +message TestAllTypes { + message NestedMessage { + optional int32 a = 1; + optional TestAllTypes corecursive = 2; + } + + enum NestedEnum { + FOO = 0; + BAR = 1; + BAZ = 2; + NEG = -1; // Intentionally negative. + } + + // Singular + optional int32 optional_int32 = 1; + optional int64 optional_int64 = 2; + optional uint32 optional_uint32 = 3; + optional uint64 optional_uint64 = 4; + optional sint32 optional_sint32 = 5; + optional sint64 optional_sint64 = 6; + optional fixed32 optional_fixed32 = 7; + optional fixed64 optional_fixed64 = 8; + optional sfixed32 optional_sfixed32 = 9; + optional sfixed64 optional_sfixed64 = 10; + optional float optional_float = 11; + optional double optional_double = 12; + optional bool optional_bool = 13; + optional string optional_string = 14; + optional bytes optional_bytes = 15; + + optional group OptionalGroup = 16 { + optional int32 a = 17; + } + + optional NestedMessage optional_nested_message = 18; + optional ForeignMessage optional_foreign_message = 19; + + optional NestedEnum optional_nested_enum = 21; + optional ForeignEnum optional_foreign_enum = 22; + + optional string optional_string_piece = 24 [ctype=STRING_PIECE]; + optional string optional_cord = 25 [ctype=CORD]; + + optional TestAllTypes recursive_message = 27; + + // Repeated + repeated int32 repeated_int32 = 31; + repeated int64 repeated_int64 = 32; + repeated uint32 repeated_uint32 = 33; + repeated uint64 repeated_uint64 = 34; + repeated sint32 repeated_sint32 = 35; + repeated sint64 repeated_sint64 = 36; + repeated fixed32 repeated_fixed32 = 37; + repeated fixed64 repeated_fixed64 = 38; + repeated sfixed32 repeated_sfixed32 = 39; + repeated sfixed64 repeated_sfixed64 = 40; + repeated float repeated_float = 41; + repeated double repeated_double = 42; + repeated bool repeated_bool = 43; + repeated string repeated_string = 44; + repeated bytes repeated_bytes = 45; + + repeated group RepeatedGroup = 46 { + optional int32 a = 47; + } + + repeated NestedMessage repeated_nested_message = 48; + repeated ForeignMessage repeated_foreign_message = 49; + + repeated NestedEnum repeated_nested_enum = 51; + repeated ForeignEnum repeated_foreign_enum = 52; + + repeated string repeated_string_piece = 54 [ctype=STRING_PIECE]; + repeated string repeated_cord = 55 [ctype=CORD]; + + // Map + map < int32, int32> map_int32_int32 = 56; + map < int64, int64> map_int64_int64 = 57; + map < uint32, uint32> map_uint32_uint32 = 58; + map < uint64, uint64> map_uint64_uint64 = 59; + map < sint32, sint32> map_sint32_sint32 = 60; + map < sint64, sint64> map_sint64_sint64 = 61; + map < fixed32, fixed32> map_fixed32_fixed32 = 62; + map < fixed64, fixed64> map_fixed64_fixed64 = 63; + map map_sfixed32_sfixed32 = 64; + map map_sfixed64_sfixed64 = 65; + map < int32, float> map_int32_float = 66; + map < int32, double> map_int32_double = 67; + map < bool, bool> map_bool_bool = 68; + map < string, string> map_string_string = 69; + map < string, bytes> map_string_bytes = 70; + map < string, NestedMessage> map_string_nested_message = 71; + map < string, ForeignMessage> map_string_foreign_message = 72; + map < string, NestedEnum> map_string_nested_enum = 73; + map < string, ForeignEnum> map_string_foreign_enum = 74; + + oneof oneof_field { + uint32 oneof_uint32 = 111; + NestedMessage oneof_nested_message = 112; + string oneof_string = 113; + bytes oneof_bytes = 114; + } +} + +message ForeignMessage { + optional int32 c = 1; +} + +enum ForeignEnum { + FOREIGN_FOO = 0; + FOREIGN_BAR = 1; + FOREIGN_BAZ = 2; +} diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc new file mode 100644 index 00000000..ff47fbbf --- /dev/null +++ b/conformance/conformance_cpp.cc @@ -0,0 +1,156 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include + +#include "conformance.pb.h" + +using std::string; +using conformance::ConformanceRequest; +using conformance::ConformanceResponse; +using conformance::TestAllTypes; + +int test_count = 0; +bool verbose = false; + +bool CheckedRead(int fd, void *buf, size_t len) { + size_t ofs = 0; + while (len > 0) { + ssize_t bytes_read = read(fd, (char*)buf + ofs, len); + + if (bytes_read == 0) return false; + + if (bytes_read < 0) { + GOOGLE_LOG(FATAL) << "Error reading from test runner: " << strerror(errno); + } + + len -= bytes_read; + ofs += bytes_read; + } + + return true; +} + +void CheckedWrite(int fd, const void *buf, size_t len) { + if (write(fd, buf, len) != len) { + GOOGLE_LOG(FATAL) << "Error writing to test runner: " << strerror(errno); + } +} + +void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { + TestAllTypes test_message; + + switch (request.payload_case()) { + case ConformanceRequest::kProtobufPayload: + if (!test_message.ParseFromString(request.protobuf_payload())) { + // Getting parse details would involve something like: + // http://stackoverflow.com/questions/22121922/how-can-i-get-more-details-about-errors-generated-during-protobuf-parsing-c + response->set_parse_error("Parse error (no more details available)."); + return; + } + break; + + case ConformanceRequest::kJsonPayload: + response->set_runtime_error("JSON input is not yet supported."); + break; + + case ConformanceRequest::PAYLOAD_NOT_SET: + GOOGLE_LOG(FATAL) << "Request didn't have payload."; + break; + } + + switch (request.requested_output()) { + case ConformanceRequest::UNSPECIFIED: + GOOGLE_LOG(FATAL) << "Unspecified output format"; + break; + + case ConformanceRequest::PROTOBUF: + test_message.SerializeToString(response->mutable_protobuf_payload()); + break; + + case ConformanceRequest::JSON: + response->set_runtime_error("JSON output is not yet supported."); + break; + } +} + +bool DoTestIo() { + string serialized_input; + string serialized_output; + ConformanceRequest request; + ConformanceResponse response; + uint32_t bytes; + + if (!CheckedRead(STDIN_FILENO, &bytes, sizeof(uint32_t))) { + // EOF. + return false; + } + + serialized_input.resize(bytes); + + if (!CheckedRead(STDIN_FILENO, (char*)serialized_input.c_str(), bytes)) { + GOOGLE_LOG(ERROR) << "Unexpected EOF on stdin. " << strerror(errno); + } + + if (!request.ParseFromString(serialized_input)) { + GOOGLE_LOG(FATAL) << "Parse of ConformanceRequest proto failed."; + return false; + } + + DoTest(request, &response); + + response.SerializeToString(&serialized_output); + + bytes = serialized_output.size(); + CheckedWrite(STDOUT_FILENO, &bytes, sizeof(uint32_t)); + CheckedWrite(STDOUT_FILENO, serialized_output.c_str(), bytes); + + if (verbose) { + fprintf(stderr, "conformance-cpp: request=%s, response=%s\n", + request.ShortDebugString().c_str(), + response.ShortDebugString().c_str()); + } + + test_count++; + + return true; +} + +int main() { + while (1) { + if (!DoTestIo()) { + fprintf(stderr, "conformance-cpp: received EOF from test runner " + "after %d tests, exiting\n", test_count); + return 0; + } + } +} diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc new file mode 100644 index 00000000..ee75031b --- /dev/null +++ b/conformance/conformance_test.cc @@ -0,0 +1,395 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include + +#include "conformance.pb.h" +#include +#include + +using conformance::ConformanceRequest; +using conformance::ConformanceResponse; +using conformance::TestAllTypes; +using google::protobuf::Descriptor; +using google::protobuf::FieldDescriptor; +using google::protobuf::internal::WireFormatLite; +using std::string; + +int write_fd; +int read_fd; +int successes; +int failures; +bool verbose = false; + +string Escape(const string& str) { + // TODO. + return str; +} + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#define CHECK_SYSCALL(call) \ + if (call < 0) { \ + perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \ + exit(1); \ + } + +// TODO(haberman): make this work on Windows, instead of using these +// UNIX-specific APIs. +// +// There is a platform-agnostic API in +// src/google/protobuf/compiler/subprocess.h +// +// However that API only supports sending a single message to the subprocess. +// We really want to be able to send messages and receive responses one at a +// time: +// +// 1. Spawning a new process for each test would take way too long for thousands +// of tests and subprocesses like java that can take 100ms or more to start +// up. +// +// 2. Sending all the tests in one big message and receiving all results in one +// big message would take away our visibility about which test(s) caused a +// crash or other fatal error. It would also give us only a single failure +// instead of all of them. +void SpawnTestProgram(char *executable) { + int toproc_pipe_fd[2]; + int fromproc_pipe_fd[2]; + if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) { + perror("pipe"); + exit(1); + } + + pid_t pid = fork(); + if (pid < 0) { + perror("fork"); + exit(1); + } + + if (pid) { + // Parent. + CHECK_SYSCALL(close(toproc_pipe_fd[0])); + CHECK_SYSCALL(close(fromproc_pipe_fd[1])); + write_fd = toproc_pipe_fd[1]; + read_fd = fromproc_pipe_fd[0]; + } else { + // Child. + CHECK_SYSCALL(close(STDIN_FILENO)); + CHECK_SYSCALL(close(STDOUT_FILENO)); + CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO)); + CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO)); + + CHECK_SYSCALL(close(toproc_pipe_fd[0])); + CHECK_SYSCALL(close(fromproc_pipe_fd[1])); + CHECK_SYSCALL(close(toproc_pipe_fd[1])); + CHECK_SYSCALL(close(fromproc_pipe_fd[0])); + + char *const argv[] = {executable, NULL}; + CHECK_SYSCALL(execv(executable, argv)); // Never returns. + } +} + +/* Invoking of tests **********************************************************/ + +void ReportSuccess() { + successes++; +} + +void ReportFailure(const char *fmt, ...) { + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + failures++; +} + +void CheckedWrite(int fd, const void *buf, size_t len) { + if (write(fd, buf, len) != len) { + GOOGLE_LOG(FATAL) << "Error writing to test program: " << strerror(errno); + } +} + +void CheckedRead(int fd, void *buf, size_t len) { + size_t ofs = 0; + while (len > 0) { + ssize_t bytes_read = read(fd, (char*)buf + ofs, len); + + if (bytes_read == 0) { + GOOGLE_LOG(FATAL) << "Unexpected EOF from test program"; + } else if (bytes_read < 0) { + GOOGLE_LOG(FATAL) << "Error reading from test program: " << strerror(errno); + } + + len -= bytes_read; + ofs += bytes_read; + } +} + +void RunTest(const ConformanceRequest& request, ConformanceResponse* response) { + string serialized; + request.SerializeToString(&serialized); + uint32_t len = serialized.size(); + CheckedWrite(write_fd, &len, sizeof(uint32_t)); + CheckedWrite(write_fd, serialized.c_str(), serialized.size()); + CheckedRead(read_fd, &len, sizeof(uint32_t)); + serialized.resize(len); + CheckedRead(read_fd, (void*)serialized.c_str(), len); + if (!response->ParseFromString(serialized)) { + GOOGLE_LOG(FATAL) << "Could not parse response proto from tested process."; + } + + if (verbose) { + fprintf(stderr, "conformance_test: request=%s, response=%s\n", + request.ShortDebugString().c_str(), + response->ShortDebugString().c_str()); + } +} + +void DoExpectParseFailureForProto(const string& proto, int line) { + ConformanceRequest request; + ConformanceResponse response; + request.set_protobuf_payload(proto); + + // We don't expect output, but if the program erroneously accepts the protobuf + // we let it send its response as this. We must not leave it unspecified. + request.set_requested_output(ConformanceRequest::PROTOBUF); + + RunTest(request, &response); + if (response.result_case() == ConformanceResponse::kParseError) { + ReportSuccess(); + } else { + ReportFailure("Should have failed, but didn't. Line: %d, Request: %s, " + "response: %s\n", + line, + request.ShortDebugString().c_str(), + response.ShortDebugString().c_str()); + } +} + +// Expect that this precise protobuf will cause a parse error. +#define ExpectParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__) + +// Expect that this protobuf will cause a parse error, even if it is followed +// by valid protobuf data. We can try running this twice: once with this +// data verbatim and once with this data followed by some valid data. +// +// TODO(haberman): implement the second of these. +#define ExpectHardParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__) + + +/* Routines for building arbitrary protos *************************************/ + +// We would use CodedOutputStream except that we want more freedom to build +// arbitrary protos (even invalid ones). + +const string empty; + +string cat(const string& a, const string& b, + const string& c = empty, + const string& d = empty, + const string& e = empty, + const string& f = empty, + const string& g = empty, + const string& h = empty, + const string& i = empty, + const string& j = empty, + const string& k = empty, + const string& l = empty) { + string ret; + ret.reserve(a.size() + b.size() + c.size() + d.size() + e.size() + f.size() + + g.size() + h.size() + i.size() + j.size() + k.size() + l.size()); + ret.append(a); + ret.append(b); + ret.append(c); + ret.append(d); + ret.append(e); + ret.append(f); + ret.append(g); + ret.append(h); + ret.append(i); + ret.append(j); + ret.append(k); + ret.append(l); + return ret; +} + +// The maximum number of bytes that it takes to encode a 64-bit varint. +#define VARINT_MAX_LEN 10 + +size_t vencode64(uint64_t val, char *buf) { + if (val == 0) { buf[0] = 0; return 1; } + size_t i = 0; + while (val) { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + buf[i++] = byte; + } + return i; +} + +string varint(uint64_t x) { + char buf[VARINT_MAX_LEN]; + size_t len = vencode64(x, buf); + return string(buf, len); +} + +// TODO: proper byte-swapping for big-endian machines. +string fixed32(void *data) { return string(static_cast(data), 4); } +string fixed64(void *data) { return string(static_cast(data), 8); } + +string delim(const string& buf) { return cat(varint(buf.size()), buf); } +string uint32(uint32_t u32) { return fixed32(&u32); } +string uint64(uint64_t u64) { return fixed64(&u64); } +string flt(float f) { return fixed32(&f); } +string dbl(double d) { return fixed64(&d); } +string zz32(int32_t x) { return varint(WireFormatLite::ZigZagEncode32(x)); } +string zz64(int64_t x) { return varint(WireFormatLite::ZigZagEncode64(x)); } + +string tag(uint32_t fieldnum, char wire_type) { + return varint((fieldnum << 3) | wire_type); +} + +string submsg(uint32_t fn, const string& buf) { + return cat( tag(fn, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), delim(buf) ); +} + +#define UNKNOWN_FIELD 666 + +uint32_t GetFieldNumberForType(WireFormatLite::FieldType type, bool repeated) { + const Descriptor* d = TestAllTypes().GetDescriptor(); + for (int i = 0; i < d->field_count(); i++) { + const FieldDescriptor* f = d->field(i); + if (static_cast(f->type()) == type && + f->is_repeated() == repeated) { + return f->number(); + } + } + GOOGLE_LOG(FATAL) << "Couldn't find field with type " << (int)type; + return 0; +} + +void TestPrematureEOFForType(WireFormatLite::FieldType type) { + // Incomplete values for each wire type. + static const string incompletes[6] = { + string("\x80"), // VARINT + string("abcdefg"), // 64BIT + string("\x80"), // DELIMITED (partial length) + string(), // START_GROUP (no value required) + string(), // END_GROUP (no value required) + string("abc") // 32BIT + }; + + uint32_t fieldnum = GetFieldNumberForType(type, false); + uint32_t rep_fieldnum = GetFieldNumberForType(type, true); + WireFormatLite::WireType wire_type = + WireFormatLite::WireTypeForFieldType(type); + const string& incomplete = incompletes[wire_type]; + + // EOF before a known non-repeated value. + ExpectParseFailureForProto(tag(fieldnum, wire_type)); + + // EOF before a known repeated value. + ExpectParseFailureForProto(tag(rep_fieldnum, wire_type)); + + // EOF before an unknown value. + ExpectParseFailureForProto(tag(UNKNOWN_FIELD, wire_type)); + + // EOF inside a known non-repeated value. + ExpectParseFailureForProto( + cat( tag(fieldnum, wire_type), incomplete )); + + // EOF inside a known repeated value. + ExpectParseFailureForProto( + cat( tag(rep_fieldnum, wire_type), incomplete )); + + // EOF inside an unknown value. + ExpectParseFailureForProto( + cat( tag(UNKNOWN_FIELD, wire_type), incomplete )); + + if (wire_type == WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { + // EOF in the middle of delimited data for known non-repeated value. + ExpectParseFailureForProto( + cat( tag(fieldnum, wire_type), varint(1) )); + + // EOF in the middle of delimited data for known repeated value. + ExpectParseFailureForProto( + cat( tag(rep_fieldnum, wire_type), varint(1) )); + + // EOF in the middle of delimited data for unknown value. + ExpectParseFailureForProto( + cat( tag(UNKNOWN_FIELD, wire_type), varint(1) )); + + if (type == WireFormatLite::TYPE_MESSAGE) { + // Submessage ends in the middle of a value. + string incomplete_submsg = + cat( tag(WireFormatLite::TYPE_INT32, WireFormatLite::WIRETYPE_VARINT), + incompletes[WireFormatLite::WIRETYPE_VARINT] ); + ExpectHardParseFailureForProto( + cat( tag(fieldnum, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), + varint(incomplete_submsg.size()), + incomplete_submsg )); + } + } else if (type != WireFormatLite::TYPE_GROUP) { + // Non-delimited, non-group: eligible for packing. + + // Packed region ends in the middle of a value. + ExpectHardParseFailureForProto( + cat( tag(rep_fieldnum, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), + varint(incomplete.size()), + incomplete )); + + // EOF in the middle of packed region. + ExpectParseFailureForProto( + cat( tag(rep_fieldnum, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), + varint(1) )); + } +} + + +int main(int argc, char *argv[]) { + if (argc < 2) { + fprintf(stderr, "Usage: conformance_test \n"); + exit(1); + } + + SpawnTestProgram(argv[1]); + + for (int i = 1; i <= FieldDescriptor::MAX_TYPE; i++) { + TestPrematureEOFForType(static_cast(i)); + } + + fprintf(stderr, "conformance_test: completed %d tests for %s, %d successes, " + "%d failures.\n", successes + failures, argv[1], successes, + failures); +} -- cgit v1.2.3