This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 707de6029 feat(rust): support skipping fields bytes when deserializing
in compatible mode (#2545)
707de6029 is described below
commit 707de6029dd84088eba48b436ddecdf231cd620c
Author: urlyy <[email protected]>
AuthorDate: Fri Aug 29 15:22:09 2025 +0800
feat(rust): support skipping fields bytes when deserializing in compatible
mode (#2545)
## What does this PR do?
Support skipping unneeded field bytes when deserializing in compatible
mode.
## Related issues
#2531
---
rust/fory-core/src/fory.rs | 2 +-
rust/fory-core/src/meta/type_meta.rs | 18 ++---
rust/fory-core/src/resolver/meta_resolver.rs | 2 +-
rust/fory-core/src/resolver/type_resolver.rs | 66 +++-------------
rust/fory-core/src/serializer/any.rs | 2 +-
rust/fory-core/src/serializer/mod.rs | 3 +-
rust/fory-core/src/serializer/nonexistent.rs | 108 +++++++++++++++++++++++++++
rust/fory-core/src/types.rs | 27 +++----
rust/fory-derive/src/object/derive_enum.rs | 2 +-
rust/fory-derive/src/object/misc.rs | 4 +-
rust/fory-derive/src/object/read.rs | 9 +--
rust/tests/tests/test_compatible.rs | 55 +++++++++++++-
12 files changed, 202 insertions(+), 96 deletions(-)
diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs
index 8a8b576e7..42c4492e1 100644
--- a/rust/fory-core/src/fory.rs
+++ b/rust/fory-core/src/fory.rs
@@ -90,7 +90,7 @@ impl Fory {
&self.type_resolver
}
- pub fn register<T: 'static + StructSerializer>(&mut self, id: u32) {
+ pub fn register<T: 'static + StructSerializer>(&mut self, id: i16) {
let type_info = TypeInfo::new::<T>(self, id);
self.type_resolver.register::<T>(type_info, id);
}
diff --git a/rust/fory-core/src/meta/type_meta.rs
b/rust/fory-core/src/meta/type_meta.rs
index eca526511..a436f2e89 100644
--- a/rust/fory-core/src/meta/type_meta.rs
+++ b/rust/fory-core/src/meta/type_meta.rs
@@ -39,10 +39,10 @@ static ENCODING_OPTIONS: &[Encoding] = &[
Encoding::LowerUpperDigitSpecial,
];
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
pub struct FieldType {
pub type_id: i16,
- generics: Vec<FieldType>,
+ pub generics: Vec<FieldType>,
}
impl FieldType {
@@ -128,7 +128,7 @@ impl FieldType {
}
}
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
pub struct FieldInfo {
pub field_name: String,
pub field_type: FieldType,
@@ -213,19 +213,19 @@ impl FieldInfo {
#[derive(Debug)]
pub struct TypeMetaLayer {
- type_id: u32,
+ type_id: i16,
field_infos: Vec<FieldInfo>,
}
impl TypeMetaLayer {
- pub fn new(type_id: u32, field_infos: Vec<FieldInfo>) -> TypeMetaLayer {
+ pub fn new(type_id: i16, field_infos: Vec<FieldInfo>) -> TypeMetaLayer {
TypeMetaLayer {
type_id,
field_infos,
}
}
- pub fn get_type_id(&self) -> u32 {
+ pub fn get_type_id(&self) -> i16 {
self.type_id
}
@@ -271,7 +271,7 @@ impl TypeMetaLayer {
if is_register_by_name {
todo!()
} else {
- type_id = reader.var_int32() as u32;
+ type_id = reader.var_int32() as i16;
}
let mut field_infos = Vec::with_capacity(num_fields);
for _ in 0..num_fields {
@@ -293,11 +293,11 @@ impl TypeMeta {
self.layers.first().unwrap().get_field_infos()
}
- pub fn get_type_id(&self) -> u32 {
+ pub fn get_type_id(&self) -> i16 {
self.layers.first().unwrap().get_type_id()
}
- pub fn from_fields(type_id: u32, field_infos: Vec<FieldInfo>) -> TypeMeta {
+ pub fn from_fields(type_id: i16, field_infos: Vec<FieldInfo>) -> TypeMeta {
TypeMeta {
// hash: 0,
layers: vec![TypeMetaLayer::new(type_id, field_infos)],
diff --git a/rust/fory-core/src/resolver/meta_resolver.rs
b/rust/fory-core/src/resolver/meta_resolver.rs
index 25eb0afd5..8423bd268 100644
--- a/rust/fory-core/src/resolver/meta_resolver.rs
+++ b/rust/fory-core/src/resolver/meta_resolver.rs
@@ -24,7 +24,7 @@ use std::rc::Rc;
#[derive(Default)]
pub struct MetaReaderResolver {
- reading_type_defs: Vec<Rc<TypeMeta>>,
+ pub reading_type_defs: Vec<Rc<TypeMeta>>,
}
impl MetaReaderResolver {
diff --git a/rust/fory-core/src/resolver/type_resolver.rs
b/rust/fory-core/src/resolver/type_resolver.rs
index eea1f347f..41fcdc557 100644
--- a/rust/fory-core/src/resolver/type_resolver.rs
+++ b/rust/fory-core/src/resolver/type_resolver.rs
@@ -18,9 +18,7 @@
use super::context::{ReadContext, WriteContext};
use crate::error::Error;
use crate::fory::Fory;
-use crate::serializer::{Serializer, StructSerializer};
-use crate::types::TypeId;
-use chrono::{NaiveDate, NaiveDateTime};
+use crate::serializer::StructSerializer;
use std::{any::Any, collections::HashMap};
pub struct Harness {
@@ -50,18 +48,18 @@ impl Harness {
pub struct TypeInfo {
type_def: Vec<u8>,
- type_id: u32,
+ type_id: i16,
}
impl TypeInfo {
- pub fn new<T: StructSerializer>(fory: &Fory, type_id: u32) -> TypeInfo {
+ pub fn new<T: StructSerializer>(fory: &Fory, type_id: i16) -> TypeInfo {
TypeInfo {
- type_def: T::type_def(fory),
+ type_def: T::type_def(fory, type_id),
type_id,
}
}
- pub fn get_type_id(&self) -> u32 {
+ pub fn get_type_id(&self) -> i16 {
self.type_id
}
@@ -70,56 +68,12 @@ impl TypeInfo {
}
}
+#[derive(Default)]
pub struct TypeResolver {
- serialize_map: HashMap<u32, Harness>,
- type_id_map: HashMap<std::any::TypeId, u32>,
+ serialize_map: HashMap<i16, Harness>,
+ type_id_map: HashMap<std::any::TypeId, i16>,
type_info_map: HashMap<std::any::TypeId, TypeInfo>,
}
-macro_rules! register_harness {
- ($ty:ty, $id:expr, $map:expr) => {{
- fn serializer(this: &dyn std::any::Any, context: &mut WriteContext) {
- let this = this.downcast_ref::<$ty>();
- match this {
- Some(v) => <$ty>::serialize(v, context),
- None => todo!(""),
- }
- }
-
- fn deserializer(context: &mut ReadContext) -> Result<Box<dyn
std::any::Any>, Error> {
- match <$ty>::deserialize(context) {
- Ok(v) => Ok(Box::new(v)),
- Err(e) => Err(e),
- }
- }
-
- $map.insert($id as u32, Harness::new(serializer, deserializer));
- }};
-}
-
-impl Default for TypeResolver {
- fn default() -> Self {
- let mut serialize_map = HashMap::new();
-
- register_harness!(bool, TypeId::BOOL, serialize_map);
- register_harness!(i8, TypeId::INT8, serialize_map);
- register_harness!(i16, TypeId::INT16, serialize_map);
- register_harness!(i32, TypeId::INT32, serialize_map);
- register_harness!(i64, TypeId::INT64, serialize_map);
- register_harness!(f32, TypeId::FLOAT32, serialize_map);
- register_harness!(f64, TypeId::FLOAT64, serialize_map);
-
- register_harness!(String, TypeId::STRING, serialize_map);
-
- register_harness!(NaiveDate, TypeId::LOCAL_DATE, serialize_map);
- register_harness!(NaiveDateTime, TypeId::TIMESTAMP, serialize_map);
-
- TypeResolver {
- serialize_map,
- type_id_map: HashMap::new(),
- type_info_map: HashMap::new(),
- }
- }
-}
impl TypeResolver {
pub fn get_type_info(&self, type_id: std::any::TypeId) -> &TypeInfo {
@@ -131,7 +85,7 @@ impl TypeResolver {
})
}
- pub fn register<T: StructSerializer>(&mut self, type_info: TypeInfo, id:
u32) {
+ pub fn register<T: StructSerializer>(&mut self, type_info: TypeInfo, id:
i16) {
fn serializer<T2: 'static + StructSerializer>(this: &dyn Any, context:
&mut WriteContext) {
let this = this.downcast_ref::<T2>();
match this {
@@ -161,7 +115,7 @@ impl TypeResolver {
self.get_harness(*self.type_id_map.get(&type_id).unwrap())
}
- pub fn get_harness(&self, id: u32) -> Option<&Harness> {
+ pub fn get_harness(&self, id: i16) -> Option<&Harness> {
self.serialize_map.get(&id)
}
}
diff --git a/rust/fory-core/src/serializer/any.rs
b/rust/fory-core/src/serializer/any.rs
index cf9ab52ce..321f75fc2 100644
--- a/rust/fory-core/src/serializer/any.rs
+++ b/rust/fory-core/src/serializer/any.rs
@@ -71,7 +71,7 @@ impl Serializer for Box<dyn Any> {
context
.get_fory()
.get_type_resolver()
- .get_harness(type_id as u32)
+ .get_harness(type_id)
.unwrap()
.get_deserializer()(context)
}
diff --git a/rust/fory-core/src/serializer/mod.rs
b/rust/fory-core/src/serializer/mod.rs
index be75e39f2..10a84175f 100644
--- a/rust/fory-core/src/serializer/mod.rs
+++ b/rust/fory-core/src/serializer/mod.rs
@@ -27,6 +27,7 @@ mod bool;
mod datetime;
mod list;
mod map;
+pub mod nonexistent;
mod number;
mod option;
mod primitive_list;
@@ -92,5 +93,5 @@ where
}
pub trait StructSerializer: Serializer + 'static {
- fn type_def(fory: &Fory) -> Vec<u8>;
+ fn type_def(fory: &Fory, type_id: i16) -> Vec<u8>;
}
diff --git a/rust/fory-core/src/serializer/nonexistent.rs
b/rust/fory-core/src/serializer/nonexistent.rs
new file mode 100644
index 000000000..b9732ea64
--- /dev/null
+++ b/rust/fory-core/src/serializer/nonexistent.rs
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::ensure;
+use crate::error::Error;
+use crate::meta::FieldType;
+use crate::resolver::context::ReadContext;
+use crate::serializer::Serializer;
+use crate::types::{RefFlag, TypeId, BASIC_TYPES, COLLECTION_TYPES};
+use anyhow::anyhow;
+use chrono::{NaiveDate, NaiveDateTime};
+
+macro_rules! basic_type_deserialize {
+ ($tid:expr, $context:expr; $(($ty:ty, $id:ident)),+ $(,)?) => {
+ $(
+ if $tid == TypeId::$id {
+ <$ty>::deserialize($context)?;
+ return Ok(());
+ }
+ )+else {
+ unreachable!()
+ }
+ };
+}
+
+pub fn skip_field_value(context: &mut ReadContext, field_type: &FieldType) ->
Result<(), Error> {
+ match TypeId::try_from(field_type.type_id) {
+ Ok(type_id) => {
+ if BASIC_TYPES.contains(&type_id) {
+ basic_type_deserialize!(type_id, context;
+ (bool, BOOL),
+ (i8, INT8),
+ (i16, INT16),
+ (i32, INT32),
+ (i64, INT64),
+ (f32, FLOAT32),
+ (f64, FLOAT64),
+ (String, STRING),
+ (NaiveDate, LOCAL_DATE),
+ (NaiveDateTime, TIMESTAMP),
+ );
+ } else if COLLECTION_TYPES.contains(&type_id) {
+ let ref_flag = context.reader.i8();
+ let actual_type_id = context.reader.i16();
+ let type_id_num = type_id.into();
+ ensure!(
+ actual_type_id == type_id_num,
+ anyhow!("Invalid field type, expected:{type_id_num},
actual:{actual_type_id}")
+ );
+ if ref_flag == (RefFlag::NotNullValue as i8)
+ || ref_flag == (RefFlag::RefValue as i8)
+ {
+ if type_id == TypeId::ARRAY || type_id == TypeId::SET {
+ let length = context.reader.var_int32() as usize;
+ println!("skipping array with length {}", length);
+ for _ in 0..length {
+ skip_field_value(context,
field_type.generics.first().unwrap())?;
+ }
+ } else if type_id == TypeId::MAP {
+ let length = context.reader.var_int32() as usize;
+ for _ in 0..length {
+ skip_field_value(context,
field_type.generics.first().unwrap())?;
+ skip_field_value(context,
field_type.generics.get(1).unwrap())?;
+ }
+ }
+ Ok(())
+ } else if ref_flag == (RefFlag::Null as i8) {
+ Err(anyhow!("Try to deserialize non-option type to null"))?
+ } else if ref_flag == (RefFlag::Ref as i8) {
+ Err(Error::Ref)
+ } else {
+ Err(anyhow!("Unknown ref flag, value:{ref_flag}"))?
+ }
+ } else {
+ unreachable!()
+ }
+ }
+ Err(_) => {
+ // skip ref_flag and meta_index
+ context.reader.i8();
+ context.reader.i16();
+ let type_defs: Vec<_> =
context.meta_resolver.reading_type_defs.to_vec();
+ for type_def in type_defs.iter() {
+ if type_def.get_type_id() == field_type.type_id {
+ let field_infos: Vec<_> =
type_def.get_field_infos().to_vec();
+ for field_info in field_infos.iter() {
+ skip_field_value(context, &field_info.field_type)?;
+ }
+ }
+ }
+ Ok(())
+ }
+ }
+}
diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs
index f8bddebdc..6128447f0 100644
--- a/rust/fory-core/src/types.rs
+++ b/rust/fory-core/src/types.rs
@@ -112,19 +112,20 @@ pub fn compute_string_hash(s: &str) -> u32 {
hash as u32
}
-// const BASIC_TYPES: [FieldType; 11] = [
-// FieldType::BOOL,
-// FieldType::INT8,
-// FieldType::INT16,
-// FieldType::INT32,
-// FieldType::INT64,
-// FieldType::FLOAT,
-// FieldType::DOUBLE,
-// FieldType::STRING,
-// FieldType::BINARY,
-// FieldType::DATE,
-// FieldType::TIMESTAMP,
-// ];
+pub const BASIC_TYPES: [TypeId; 10] = [
+ TypeId::BOOL,
+ TypeId::INT8,
+ TypeId::INT16,
+ TypeId::INT32,
+ TypeId::INT64,
+ TypeId::FLOAT32,
+ TypeId::FLOAT64,
+ TypeId::STRING,
+ TypeId::LOCAL_DATE,
+ TypeId::TIMESTAMP,
+];
+
+pub const COLLECTION_TYPES: [TypeId; 3] = [TypeId::ARRAY, TypeId::SET,
TypeId::MAP];
pub fn compute_field_hash(hash: u32, id: i16) -> u32 {
let mut new_hash: u64 = (hash as u64) * 31 + (id as u64);
diff --git a/rust/fory-derive/src/object/derive_enum.rs
b/rust/fory-derive/src/object/derive_enum.rs
index 2f76c4df9..a571dd441 100644
--- a/rust/fory-derive/src/object/derive_enum.rs
+++ b/rust/fory-derive/src/object/derive_enum.rs
@@ -21,7 +21,7 @@ use syn::DataEnum;
pub fn gen_type_def(_data_enum: &DataEnum) -> TokenStream {
quote! {
- fn type_def(fory: &fory_core::fory::Fory) -> Vec<u8> {
+ fn type_def(fory: &fory_core::fory::Fory, type_id: i16) -> Vec<u8> {
Vec::new()
}
}
diff --git a/rust/fory-derive/src/object/misc.rs
b/rust/fory-derive/src/object/misc.rs
index 0c8f83dbf..867dd255e 100644
--- a/rust/fory-derive/src/object/misc.rs
+++ b/rust/fory-derive/src/object/misc.rs
@@ -56,9 +56,9 @@ fn type_def(fields: &[&Field]) -> TokenStream {
}
});
quote! {
- fn type_def(fory: &fory_core::fory::Fory) -> Vec<u8> {
+ fn type_def(fory: &fory_core::fory::Fory, layer_id: i16) -> Vec<u8> {
fory_core::meta::TypeMeta::from_fields(
- 0,
+ layer_id,
vec![#(#field_infos),*]
).to_bytes().unwrap()
}
diff --git a/rust/fory-derive/src/object/read.rs
b/rust/fory-derive/src/object/read.rs
index 5652c11bd..3678a2697 100644
--- a/rust/fory-derive/src/object/read.rs
+++ b/rust/fory-derive/src/object/read.rs
@@ -113,14 +113,7 @@ fn deserialize_compatible(fields: &[&Field]) ->
TokenStream {
_ => {
// skip bytes
println!("no need to deserialize {:?}:{:?}",
_field.field_name.as_str(), _field.field_type);
- let _ = context
- .get_fory()
- .get_type_resolver()
- .get_harness((&_field.field_type).type_id as u32)
- .unwrap_or_else(|| {
- panic!("missing harness for type_id {}",
_field.field_type.type_id);
- })
- .get_deserializer()(context);
+
fory_core::serializer::nonexistent::skip_field_value(context,
&_field.field_type).unwrap();
}
}
}
diff --git a/rust/tests/tests/test_compatible.rs
b/rust/tests/tests/test_compatible.rs
index f31e5d4a1..d9d94d72b 100644
--- a/rust/tests/tests/test_compatible.rs
+++ b/rust/tests/tests/test_compatible.rs
@@ -30,7 +30,9 @@ fn simple() {
f3: Vec<i8>,
// f4: String,
f5: String,
- // f6: Vec<i8>,
+ f6: Vec<i8>,
+ f7: i8,
+ f8: i8,
}
#[derive(Fory, Debug)]
@@ -40,7 +42,9 @@ fn simple() {
f3: Vec<i8>,
f4: String,
f5: i8,
- // f6: Vec<i16>,
+ f6: Vec<i16>,
+ f7: i16,
+ f8: i8,
}
let mut fory1 = Fory::default().mode(Compatible);
@@ -52,7 +56,9 @@ fn simple() {
f2: String::from("hello"),
f3: vec![1, 2, 3],
f5: String::from("f5"),
- // f6: vec![42]
+ f6: vec![42],
+ f7: 43,
+ f8: 44,
};
let bin = fory1.serialize(&animal);
let obj: Animal2 = fory2.deserialize(&bin).unwrap();
@@ -61,6 +67,49 @@ fn simple() {
assert_eq!(animal.f3, obj.f3);
assert_eq!(obj.f4, String::default());
assert_eq!(obj.f5, i8::default());
+ assert_eq!(obj.f6, Vec::<i16>::default());
+ assert_eq!(obj.f7, i16::default());
+ assert_eq!(animal.f8, obj.f8);
+}
+
+#[test]
+fn nonexistent_struct() {
+ #[derive(Fory, Debug, Default)]
+ pub struct Item1 {
+ f1: i8,
+ }
+ #[derive(Fory, Debug, Default, PartialEq)]
+ pub struct Item2 {
+ f1: i64,
+ }
+ #[derive(Fory, Debug)]
+ struct Person1 {
+ f2: Item1,
+ f3: i8,
+ f4: String,
+ }
+ #[derive(Fory, Debug)]
+ struct Person2 {
+ f2: Item2,
+ f3: i64,
+ f4: String,
+ }
+ let mut fory1 = Fory::default().mode(Compatible);
+ let mut fory2 = Fory::default().mode(Compatible);
+ fory1.register::<Item1>(899);
+ fory1.register::<Person1>(999);
+ fory2.register::<Item2>(799);
+ fory2.register::<Person2>(999);
+ let person = Person1 {
+ f2: Item1 { f1: 42 },
+ f3: 24,
+ f4: String::from("foo"),
+ };
+ let bin = fory1.serialize(&person);
+ let obj: Person2 = fory2.deserialize(&bin).unwrap();
+ assert_eq!(obj.f2, Item2::default());
+ assert_eq!(obj.f3, i64::default());
+ assert_eq!(obj.f4, person.f4);
}
#[test]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
