From 0e7b58956684c7c2fc228eff9173ea2ae50de911 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Thu, 7 Dec 2017 14:18:38 -0800 Subject: Add discard unknown API in ruby. (#3990) * Add discard unknown API in ruby. * Add test for oneof message field. * Add TestUnknown to represent unknown field data clearly. * Only serialize the message with unknown fields itself in test. * Move discard_unknown from Message to Google.Protobuf --- ruby/ext/google/protobuf_c/encode_decode.c | 88 ++++++++++++++++++++++++++++++ ruby/ext/google/protobuf_c/protobuf.c | 2 + ruby/ext/google/protobuf_c/protobuf.h | 1 + 3 files changed, 91 insertions(+) (limited to 'ruby/ext/google') diff --git a/ruby/ext/google/protobuf_c/encode_decode.c b/ruby/ext/google/protobuf_c/encode_decode.c index d1b6e89e..12080d03 100644 --- a/ruby/ext/google/protobuf_c/encode_decode.c +++ b/ruby/ext/google/protobuf_c/encode_decode.c @@ -1305,3 +1305,91 @@ VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass) { } } +static void discard_unknown(VALUE msg_rb, const Descriptor* desc) { + MessageHeader* msg; + upb_msg_field_iter it; + + TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg); + + stringsink* unknown = msg->unknown_fields; + if (unknown != NULL) { + stringsink_uninit(unknown); + msg->unknown_fields = NULL; + } + + for (upb_msg_field_begin(&it, desc->msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { + upb_fielddef *f = upb_msg_iter_field(&it); + uint32_t offset = + desc->layout->fields[upb_fielddef_index(f)].offset + + sizeof(MessageHeader); + + if (upb_fielddef_containingoneof(f)) { + uint32_t oneof_case_offset = + desc->layout->fields[upb_fielddef_index(f)].case_offset + + sizeof(MessageHeader); + // For a oneof, check that this field is actually present -- skip all the + // below if not. + if (DEREF(msg, oneof_case_offset, uint32_t) != + upb_fielddef_number(f)) { + continue; + } + // Otherwise, fall through to the appropriate singular-field handler + // below. + } + + if (!upb_fielddef_issubmsg(f)) { + continue; + } + + if (is_map_field(f)) { + if (!upb_fielddef_issubmsg(map_field_value(f))) continue; + VALUE map = DEREF(msg, offset, VALUE); + if (map == Qnil) continue; + Map_iter map_it; + for (Map_begin(map, &map_it); !Map_done(&map_it); Map_next(&map_it)) { + VALUE submsg = Map_iter_value(&map_it); + VALUE descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned); + const Descriptor* subdesc = ruby_to_Descriptor(descriptor); + discard_unknown(submsg, subdesc); + } + } else if (upb_fielddef_isseq(f)) { + VALUE ary = DEREF(msg, offset, VALUE); + if (ary == Qnil) continue; + int size = NUM2INT(RepeatedField_length(ary)); + for (int i = 0; i < size; i++) { + void* memory = RepeatedField_index_native(ary, i); + VALUE submsg = *((VALUE *)memory); + VALUE descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned); + const Descriptor* subdesc = ruby_to_Descriptor(descriptor); + discard_unknown(submsg, subdesc); + } + } else { + VALUE submsg = DEREF(msg, offset, VALUE); + if (submsg == Qnil) continue; + VALUE descriptor = rb_ivar_get(submsg, descriptor_instancevar_interned); + const Descriptor* subdesc = ruby_to_Descriptor(descriptor); + discard_unknown(submsg, subdesc); + } + } +} + +/* + * call-seq: + * Google::Protobuf.discard_unknown(msg) + * + * Discard unknown fields in the given message object and recursively discard + * unknown fields in submessages. + */ +VALUE Google_Protobuf_discard_unknown(VALUE self, VALUE msg_rb) { + VALUE klass = CLASS_OF(msg_rb); + VALUE descriptor = rb_ivar_get(klass, descriptor_instancevar_interned); + Descriptor* desc = ruby_to_Descriptor(descriptor); + if (klass == cRepeatedField || klass == cMap) { + rb_raise(rb_eArgError, "Expected proto msg for discard unknown."); + } else { + discard_unknown(msg_rb, desc); + } + return Qnil; +} diff --git a/ruby/ext/google/protobuf_c/protobuf.c b/ruby/ext/google/protobuf_c/protobuf.c index c7750c44..db696426 100644 --- a/ruby/ext/google/protobuf_c/protobuf.c +++ b/ruby/ext/google/protobuf_c/protobuf.c @@ -103,6 +103,8 @@ void Init_protobuf_c() { cError = rb_const_get(protobuf, rb_intern("Error")); cParseError = rb_const_get(protobuf, rb_intern("ParseError")); + rb_define_singleton_method(protobuf, "discard_unknown", + Google_Protobuf_discard_unknown, 1); rb_define_singleton_method(protobuf, "deep_copy", Google_Protobuf_deep_copy, 1); diff --git a/ruby/ext/google/protobuf_c/protobuf.h b/ruby/ext/google/protobuf_c/protobuf.h index 1291ac59..5266aa8d 100644 --- a/ruby/ext/google/protobuf_c/protobuf.h +++ b/ruby/ext/google/protobuf_c/protobuf.h @@ -515,6 +515,7 @@ VALUE Message_encode(VALUE klass, VALUE msg_rb); VALUE Message_decode_json(VALUE klass, VALUE data); VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass); +VALUE Google_Protobuf_discard_unknown(VALUE self, VALUE msg_rb); VALUE Google_Protobuf_deep_copy(VALUE self, VALUE obj); VALUE build_module_from_enumdesc(EnumDescriptor* enumdef); -- cgit v1.2.3 From cf7c15e31a456f634d4e3deaf4ef74a6bfad825d Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 8 Dec 2017 12:38:25 -0800 Subject: Fix ruby gc_test in ruby 2.4 (#4011) * Fix ruby gc_test in ruby 2.4 * Initialize global variables to Qnil. --- ruby/ext/google/protobuf_c/defs.c | 22 +++++++++++----------- ruby/ext/google/protobuf_c/map.c | 2 +- ruby/ext/google/protobuf_c/repeated_field.c | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) (limited to 'ruby/ext/google') diff --git a/ruby/ext/google/protobuf_c/defs.c b/ruby/ext/google/protobuf_c/defs.c index 34d9663a..d9d2ebac 100644 --- a/ruby/ext/google/protobuf_c/defs.c +++ b/ruby/ext/google/protobuf_c/defs.c @@ -76,7 +76,7 @@ static upb_enumdef* check_enum_notfrozen(const upb_enumdef* def) { // ----------------------------------------------------------------------------- #define DEFINE_CLASS(name, string_name) \ - VALUE c ## name; \ + VALUE c ## name = Qnil; \ const rb_data_type_t _ ## name ## _type = { \ string_name, \ { name ## _mark, name ## _free, NULL }, \ @@ -126,11 +126,11 @@ void DescriptorPool_register(VALUE module) { rb_define_method(klass, "lookup", DescriptorPool_lookup, 1); rb_define_singleton_method(klass, "generated_pool", DescriptorPool_generated_pool, 0); - cDescriptorPool = klass; rb_gc_register_address(&cDescriptorPool); + cDescriptorPool = klass; - generated_pool = rb_class_new_instance(0, NULL, klass); rb_gc_register_address(&generated_pool); + generated_pool = rb_class_new_instance(0, NULL, klass); } static void add_descriptor_to_pool(DescriptorPool* self, @@ -299,8 +299,8 @@ void Descriptor_register(VALUE module) { rb_define_method(klass, "name", Descriptor_name, 0); rb_define_method(klass, "name=", Descriptor_name_set, 1); rb_include_module(klass, rb_mEnumerable); - cDescriptor = klass; rb_gc_register_address(&cDescriptor); + cDescriptor = klass; } /* @@ -518,8 +518,8 @@ void FieldDescriptor_register(VALUE module) { rb_define_method(klass, "subtype", FieldDescriptor_subtype, 0); rb_define_method(klass, "get", FieldDescriptor_get, 1); rb_define_method(klass, "set", FieldDescriptor_set, 2); - cFieldDescriptor = klass; rb_gc_register_address(&cFieldDescriptor); + cFieldDescriptor = klass; } /* @@ -916,8 +916,8 @@ void OneofDescriptor_register(VALUE module) { rb_define_method(klass, "add_field", OneofDescriptor_add_field, 1); rb_define_method(klass, "each", OneofDescriptor_each, 0); rb_include_module(klass, rb_mEnumerable); - cOneofDescriptor = klass; rb_gc_register_address(&cOneofDescriptor); + cOneofDescriptor = klass; } /* @@ -1037,8 +1037,8 @@ void EnumDescriptor_register(VALUE module) { rb_define_method(klass, "each", EnumDescriptor_each, 0); rb_define_method(klass, "enummodule", EnumDescriptor_enummodule, 0); rb_include_module(klass, rb_mEnumerable); - cEnumDescriptor = klass; rb_gc_register_address(&cEnumDescriptor); + cEnumDescriptor = klass; } /* @@ -1202,8 +1202,8 @@ void MessageBuilderContext_register(VALUE module) { rb_define_method(klass, "repeated", MessageBuilderContext_repeated, -1); rb_define_method(klass, "map", MessageBuilderContext_map, -1); rb_define_method(klass, "oneof", MessageBuilderContext_oneof, 1); - cMessageBuilderContext = klass; rb_gc_register_address(&cMessageBuilderContext); + cMessageBuilderContext = klass; } /* @@ -1491,8 +1491,8 @@ void OneofBuilderContext_register(VALUE module) { rb_define_method(klass, "initialize", OneofBuilderContext_initialize, 2); rb_define_method(klass, "optional", OneofBuilderContext_optional, -1); - cOneofBuilderContext = klass; rb_gc_register_address(&cOneofBuilderContext); + cOneofBuilderContext = klass; } /* @@ -1569,8 +1569,8 @@ void EnumBuilderContext_register(VALUE module) { rb_define_method(klass, "initialize", EnumBuilderContext_initialize, 1); rb_define_method(klass, "value", EnumBuilderContext_value, 2); - cEnumBuilderContext = klass; rb_gc_register_address(&cEnumBuilderContext); + cEnumBuilderContext = klass; } /* @@ -1645,8 +1645,8 @@ void Builder_register(VALUE module) { rb_define_method(klass, "add_enum", Builder_add_enum, 1); rb_define_method(klass, "initialize", Builder_initialize, 0); rb_define_method(klass, "finalize_to_pool", Builder_finalize_to_pool, 1); - cBuilder = klass; rb_gc_register_address(&cBuilder); + cBuilder = klass; } /* diff --git a/ruby/ext/google/protobuf_c/map.c b/ruby/ext/google/protobuf_c/map.c index 26e22dc7..8c2f6424 100644 --- a/ruby/ext/google/protobuf_c/map.c +++ b/ruby/ext/google/protobuf_c/map.c @@ -825,8 +825,8 @@ VALUE Map_iter_value(Map_iter* iter) { void Map_register(VALUE module) { VALUE klass = rb_define_class_under(module, "Map", rb_cObject); rb_define_alloc_func(klass, Map_alloc); - cMap = klass; rb_gc_register_address(&cMap); + cMap = klass; rb_define_method(klass, "initialize", Map_init, -1); rb_define_method(klass, "each", Map_each, 0); diff --git a/ruby/ext/google/protobuf_c/repeated_field.c b/ruby/ext/google/protobuf_c/repeated_field.c index 1c651c19..c6620ee6 100644 --- a/ruby/ext/google/protobuf_c/repeated_field.c +++ b/ruby/ext/google/protobuf_c/repeated_field.c @@ -626,8 +626,8 @@ void RepeatedField_register(VALUE module) { VALUE klass = rb_define_class_under( module, "RepeatedField", rb_cObject); rb_define_alloc_func(klass, RepeatedField_alloc); - cRepeatedField = klass; rb_gc_register_address(&cRepeatedField); + cRepeatedField = klass; rb_define_method(klass, "initialize", RepeatedField_init, -1); -- cgit v1.2.3