From 2ae36f1e9671b40723dd06462b4a416e4baa5a57 Mon Sep 17 00:00:00 2001 From: rsgowman Date: Thu, 8 Mar 2018 11:53:37 -0500 Subject: [De]Serialize FieldValue map_values ("Objects") (#878) These can (recursively) contain other FieldValues. --- .../src/firebase/firestore/model/field_value.cc | 11 +- .../src/firebase/firestore/model/field_value.h | 13 +- .../src/firebase/firestore/remote/serializer.cc | 327 +++++++++++++++++++-- 3 files changed, 311 insertions(+), 40 deletions(-) (limited to 'Firestore/core/src/firebase') diff --git a/Firestore/core/src/firebase/firestore/model/field_value.cc b/Firestore/core/src/firebase/firestore/model/field_value.cc index 03cf1d4..a38a676 100644 --- a/Firestore/core/src/firebase/firestore/model/field_value.cc +++ b/Firestore/core/src/firebase/firestore/model/field_value.cc @@ -113,7 +113,7 @@ FieldValue& FieldValue::operator=(const FieldValue& value) { } case Type::Object: { // copy-and-swap - std::map tmp = value.object_value_; + std::map tmp = value.object_value_; std::swap(object_value_, tmp); break; } @@ -281,13 +281,12 @@ FieldValue FieldValue::ArrayValue(std::vector&& value) { } FieldValue FieldValue::ObjectValue( - const std::map& value) { - std::map copy(value); + const std::map& value) { + std::map copy(value); return ObjectValue(std::move(copy)); } -FieldValue FieldValue::ObjectValue( - std::map&& value) { +FieldValue FieldValue::ObjectValue(std::map&& value) { FieldValue result; result.SwitchTo(Type::Object); std::swap(result.object_value_, value); @@ -418,7 +417,7 @@ void FieldValue::SwitchTo(const Type type) { new (&array_value_) std::vector(); break; case Type::Object: - new (&object_value_) std::map(); + new (&object_value_) std::map(); break; default: {} // The other types where there is nothing to worry about. } diff --git a/Firestore/core/src/firebase/firestore/model/field_value.h b/Firestore/core/src/firebase/firestore/model/field_value.h index cb219f5..fc8619d 100644 --- a/Firestore/core/src/firebase/firestore/model/field_value.h +++ b/Firestore/core/src/firebase/firestore/model/field_value.h @@ -111,6 +111,11 @@ class FieldValue { return string_value_; } + const std::map& object_value() const { + FIREBASE_ASSERT(tag_ == Type::Object); + return object_value_; + } + /** factory methods. */ static const FieldValue& NullValue(); static const FieldValue& TrueValue(); @@ -134,10 +139,8 @@ class FieldValue { static FieldValue GeoPointValue(const GeoPoint& value); static FieldValue ArrayValue(const std::vector& value); static FieldValue ArrayValue(std::vector&& value); - static FieldValue ObjectValue( - const std::map& value); - static FieldValue ObjectValue( - std::map&& value); + static FieldValue ObjectValue(const std::map& value); + static FieldValue ObjectValue(std::map&& value); friend bool operator<(const FieldValue& lhs, const FieldValue& rhs); @@ -164,7 +167,7 @@ class FieldValue { firebase::firestore::model::ReferenceValue reference_value_; GeoPoint geo_point_value_; std::vector array_value_; - std::map object_value_; + std::map object_value_; }; }; diff --git a/Firestore/core/src/firebase/firestore/remote/serializer.cc b/Firestore/core/src/firebase/firestore/remote/serializer.cc index b7ab891..21b499e 100644 --- a/Firestore/core/src/firebase/firestore/remote/serializer.cc +++ b/Firestore/core/src/firebase/firestore/remote/serializer.cc @@ -19,14 +19,25 @@ #include #include +#include #include +#include + +#include "Firestore/core/src/firebase/firestore/util/firebase_assert.h" namespace firebase { namespace firestore { namespace remote { +using firebase::firestore::model::FieldValue; + namespace { +void EncodeObject(pb_ostream_t* stream, + const std::map& object_value); + +std::map DecodeObject(pb_istream_t* stream); + /** * Note that (despite the value parameter type) this works for bool, enum, * int32, int64, uint32 and uint64 proto field types. @@ -141,76 +152,76 @@ std::string DecodeString(pb_istream_t* stream) { return result; } -} // namespace - -using firebase::firestore::model::FieldValue; - -void Serializer::EncodeFieldValue(const FieldValue& field_value, - std::vector* out_bytes) { - // TODO(rsgowman): how large should the output buffer be? Do some - // investigation to see if we can get nanopb to tell us how much space it's - // going to need. - uint8_t buf[1024]; - pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); - +// Named '..Impl' so as to not conflict with Serializer::EncodeFieldValue. +// TODO(rsgowman): Refactor to use a helper class that wraps the stream struct. +// This will help with error handling, and should eliminate the issue of two +// 'EncodeFieldValue' methods. +void EncodeFieldValueImpl(pb_ostream_t* stream, const FieldValue& field_value) { // TODO(rsgowman): some refactoring is in order... but will wait until after a // non-varint, non-fixed-size (i.e. string) type is present before doing so. bool status = false; switch (field_value.type()) { case FieldValue::Type::Null: - status = pb_encode_tag(&stream, PB_WT_VARINT, + status = pb_encode_tag(stream, PB_WT_VARINT, google_firestore_v1beta1_Value_null_value_tag); if (!status) { // TODO(rsgowman): figure out error handling abort(); } - EncodeNull(&stream); + EncodeNull(stream); break; case FieldValue::Type::Boolean: - status = pb_encode_tag(&stream, PB_WT_VARINT, + status = pb_encode_tag(stream, PB_WT_VARINT, google_firestore_v1beta1_Value_boolean_value_tag); if (!status) { // TODO(rsgowman): figure out error handling abort(); } - EncodeBool(&stream, field_value.boolean_value()); + EncodeBool(stream, field_value.boolean_value()); break; case FieldValue::Type::Integer: - status = pb_encode_tag(&stream, PB_WT_VARINT, + status = pb_encode_tag(stream, PB_WT_VARINT, google_firestore_v1beta1_Value_integer_value_tag); if (!status) { // TODO(rsgowman): figure out error handling abort(); } - EncodeInteger(&stream, field_value.integer_value()); + EncodeInteger(stream, field_value.integer_value()); break; case FieldValue::Type::String: - status = pb_encode_tag(&stream, PB_WT_STRING, + status = pb_encode_tag(stream, PB_WT_STRING, google_firestore_v1beta1_Value_string_value_tag); if (!status) { // TODO(rsgowman): figure out error handling abort(); } - EncodeString(&stream, field_value.string_value()); + EncodeString(stream, field_value.string_value()); + break; + + case FieldValue::Type::Object: + status = pb_encode_tag(stream, PB_WT_STRING, + google_firestore_v1beta1_Value_map_value_tag); + if (!status) { + // TODO(rsgowman): figure out error handling + abort(); + } + EncodeObject(stream, field_value.object_value()); break; default: // TODO(rsgowman): implement the other types abort(); } - - out_bytes->insert(out_bytes->end(), buf, buf + stream.bytes_written); } -FieldValue Serializer::DecodeFieldValue(const uint8_t* bytes, size_t length) { - pb_istream_t stream = pb_istream_from_buffer(bytes, length); +FieldValue DecodeFieldValueImpl(pb_istream_t* stream) { pb_wire_type_t wire_type; uint32_t tag; bool eof; - bool status = pb_decode_tag(&stream, &wire_type, &tag, &eof); + bool status = pb_decode_tag(stream, &wire_type, &tag, &eof); if (!status) { // TODO(rsgowman): figure out error handling abort(); @@ -228,6 +239,7 @@ FieldValue Serializer::DecodeFieldValue(const uint8_t* bytes, size_t length) { break; case google_firestore_v1beta1_Value_string_value_tag: + case google_firestore_v1beta1_Value_map_value_tag: if (wire_type != PB_WT_STRING) { abort(); } @@ -239,14 +251,16 @@ FieldValue Serializer::DecodeFieldValue(const uint8_t* bytes, size_t length) { switch (tag) { case google_firestore_v1beta1_Value_null_value_tag: - DecodeNull(&stream); + DecodeNull(stream); return FieldValue::NullValue(); case google_firestore_v1beta1_Value_boolean_value_tag: - return FieldValue::BooleanValue(DecodeBool(&stream)); + return FieldValue::BooleanValue(DecodeBool(stream)); case google_firestore_v1beta1_Value_integer_value_tag: - return FieldValue::IntegerValue(DecodeInteger(&stream)); + return FieldValue::IntegerValue(DecodeInteger(stream)); case google_firestore_v1beta1_Value_string_value_tag: - return FieldValue::StringValue(DecodeString(&stream)); + return FieldValue::StringValue(DecodeString(stream)); + case google_firestore_v1beta1_Value_map_value_tag: + return FieldValue::ObjectValue(DecodeObject(stream)); default: // TODO(rsgowman): figure out error handling @@ -254,6 +268,261 @@ FieldValue Serializer::DecodeFieldValue(const uint8_t* bytes, size_t length) { } } +/** + * Encodes a FieldValue *and* its length. + * + * When encoding a top level message, protobuf doesn't include the length (since + * you can get that already from the length of the binary output.) But when + * encoding a sub/nested message, you must include the length in the + * serialization. + * + * Call this method when encoding a non top level FieldValue. Otherwise call + * EncodeFieldValue[Impl]. + */ +void EncodeNestedFieldValue(pb_ostream_t* stream, + const FieldValue& field_value) { + // Implementation note: This is roughly modeled on pb_encode_delimited (which + // is actually pb_encode_submessage), adjusted to account for the oneof in + // FieldValue. + + // First calculate the message size using a non-writing substream. + pb_ostream_t substream = PB_OSTREAM_SIZING; + EncodeFieldValueImpl(&substream, field_value); + size_t size = substream.bytes_written; + + // Write out the size to the output stream. + EncodeVarint(stream, size); + + // If stream is itself a sizing stream, then we don't need to actually parse + // field_value a second time; just update the bytes_written via a call to + // pb_write. (If we try to write the contents into a sizing stream, it'll + // fail since sizing streams don't actually have any buffer space.) + if (stream->callback == NULL) { + bool status = pb_write(stream, NULL, size); + if (!status) { + // TODO(rsgowman): figure out error handling + abort(); + } + return; + } + + // Ensure the output stream has enough space + if (stream->bytes_written + size > stream->max_size) { + // TODO(rsgowman): figure out error handling + abort(); + } + + // Use a substream to verify that a callback doesn't write more than what it + // did the first time. (Use an initializer rather than setting fields + // individually like nanopb does. This gives us a *chance* of noticing if + // nanopb adds new fields.) + substream = {stream->callback, stream->state, /*max_size=*/size, + /*bytes_written=*/0, /*errmsg=*/NULL}; + + EncodeFieldValueImpl(&substream, field_value); + stream->bytes_written += substream.bytes_written; + stream->state = substream.state; + stream->errmsg = substream.errmsg; + + if (substream.bytes_written != size) { + // submsg size changed + // TODO(rsgowman): figure out error handling + abort(); + } +} + +FieldValue DecodeNestedFieldValue(pb_istream_t* stream) { + // Implementation note: This is roughly modeled on pb_decode_delimited, + // adjusted to account for the oneof in FieldValue. + pb_istream_t substream; + bool status = pb_make_string_substream(stream, &substream); + if (!status) { + // TODO(rsgowman): figure out error handling + abort(); + } + + FieldValue fv = DecodeFieldValueImpl(&substream); + + // NB: future versions of nanopb read the remaining characters out of the + // substream (and return false if that fails) as an additional safety + // check within pb_close_string_substream. Unfortunately, that's not present + // in the current version (0.38). We'll make a stronger assertion and check + // to make sure there *are* no remaining characters in the substream. + if (substream.bytes_left != 0) { + // TODO(rsgowman): figure out error handling + abort(); + } + pb_close_string_substream(stream, &substream); + + return fv; +} + +/** + * Encodes a 'FieldsEntry' object, within a FieldValue's map_value type. + * + * In protobuf, maps are implemented as a repeated set of key/values. For + * instance, this: + * message Foo { + * map fields = 1; + * } + * would be encoded (in proto text format) as: + * { + * fields: {key:"key string 1", value:{}} + * fields: {key:"key string 2", value:{}} + * ... + * } + * + * This method encodes an individual entry from that list. It is expected that + * this method will be called once for each entry in the map. + * + * @param kv The individual key/value pair to encode. + */ +void EncodeFieldsEntry(pb_ostream_t* stream, + const std::pair& kv) { + // Encode the key (string) + bool status = + pb_encode_tag(stream, PB_WT_STRING, + google_firestore_v1beta1_MapValue_FieldsEntry_key_tag); + if (!status) { + // TODO(rsgowman): figure out error handling + abort(); + } + EncodeString(stream, kv.first); + + // Encode the value (FieldValue) + status = + pb_encode_tag(stream, PB_WT_STRING, + google_firestore_v1beta1_MapValue_FieldsEntry_value_tag); + if (!status) { + // TODO(rsgowman): figure out error handling + abort(); + } + EncodeNestedFieldValue(stream, kv.second); +} + +std::pair DecodeFieldsEntry(pb_istream_t* stream) { + pb_wire_type_t wire_type; + uint32_t tag; + bool eof; + bool status = pb_decode_tag(stream, &wire_type, &tag, &eof); + // TODO(rsgowman): figure out error handling: We can do better than a failed + // assertion. + FIREBASE_ASSERT(tag == google_firestore_v1beta1_MapValue_FieldsEntry_key_tag); + FIREBASE_ASSERT(wire_type == PB_WT_STRING); + FIREBASE_ASSERT(!eof); + FIREBASE_ASSERT(status); + std::string key = DecodeString(stream); + + status = pb_decode_tag(stream, &wire_type, &tag, &eof); + FIREBASE_ASSERT(tag == + google_firestore_v1beta1_MapValue_FieldsEntry_value_tag); + FIREBASE_ASSERT(wire_type == PB_WT_STRING); + FIREBASE_ASSERT(!eof); + FIREBASE_ASSERT(status); + + FieldValue value = DecodeNestedFieldValue(stream); + + return {key, value}; +} + +void EncodeObject(pb_ostream_t* stream, + const std::map& object_value) { + google_firestore_v1beta1_MapValue map_value = + google_firestore_v1beta1_MapValue_init_zero; + // NB: c-style callbacks can't use *capturing* lambdas, so we'll pass in the + // object_value via the arg field (and therefore need to do a bunch of + // casting). + map_value.fields.funcs.encode = [](pb_ostream_t* stream, const pb_field_t*, + void* const* arg) -> bool { + auto& object_value = + *static_cast*>(*arg); + + // Encode each FieldsEntry (i.e. key-value pair.) + for (const auto& kv : object_value) { + bool status = + pb_encode_tag(stream, PB_WT_STRING, + google_firestore_v1beta1_MapValue_FieldsEntry_key_tag); + if (!status) { + // TODO(rsgowman): figure out error handling + abort(); + } + + // Calculate the size of this FieldsEntry using a non-writing substream. + pb_ostream_t sizing_stream = PB_OSTREAM_SIZING; + EncodeFieldsEntry(&sizing_stream, kv); + size_t size = sizing_stream.bytes_written; + // Write out the size to the output stream. + EncodeVarint(stream, size); + + EncodeFieldsEntry(stream, kv); + } + + return true; + }; + map_value.fields.arg = + const_cast*>(&object_value); + + bool status = pb_encode_delimited( + stream, google_firestore_v1beta1_MapValue_fields, &map_value); + if (!status) { + // TODO(rsgowman): figure out error handling + abort(); + } +} + +std::map DecodeObject(pb_istream_t* stream) { + google_firestore_v1beta1_MapValue map_value = + google_firestore_v1beta1_MapValue_init_zero; + std::map result; + // NB: c-style callbacks can't use *capturing* lambdas, so we'll pass in the + // object_value via the arg field (and therefore need to do a bunch of + // casting). + map_value.fields.funcs.decode = [](pb_istream_t* stream, const pb_field_t*, + void** arg) -> bool { + auto& result = *static_cast*>(*arg); + + std::pair fv = DecodeFieldsEntry(stream); + + // Sanity check: ensure that this key doesn't already exist in the map. + // TODO(rsgowman): figure out error handling: We can do better than a failed + // assertion. + FIREBASE_ASSERT(result.find(fv.first) == result.end()); + + // Add this key,fieldvalue to the results map. + result.emplace(std::move(fv)); + + return true; + }; + map_value.fields.arg = &result; + + bool status = pb_decode_delimited( + stream, google_firestore_v1beta1_MapValue_fields, &map_value); + if (!status) { + // TODO(rsgowman): figure out error handling + abort(); + } + + return result; +} + +} // namespace + +void Serializer::EncodeFieldValue(const FieldValue& field_value, + std::vector* out_bytes) { + // TODO(rsgowman): how large should the output buffer be? Do some + // investigation to see if we can get nanopb to tell us how much space it's + // going to need. (Hint: use a sizing stream, i.e. PB_OSTREAM_SIZING) + uint8_t buf[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + EncodeFieldValueImpl(&stream, field_value); + out_bytes->insert(out_bytes->end(), buf, buf + stream.bytes_written); +} + +FieldValue Serializer::DecodeFieldValue(const uint8_t* bytes, size_t length) { + pb_istream_t stream = pb_istream_from_buffer(bytes, length); + return DecodeFieldValueImpl(&stream); +} + } // namespace remote } // namespace firestore } // namespace firebase -- cgit v1.2.3