This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 4507810d9 feat(rust): query type meta from parsed cache to speed up
deserialization (#2764)
4507810d9 is described below
commit 4507810d928b54a1af629354ad06d1095644af2c
Author: Shawn Yang <[email protected]>
AuthorDate: Tue Oct 14 06:55:21 2025 +0530
feat(rust): query type meta from parsed cache to speed up deserialization
(#2764)
## Why?
<!-- Describe the purpose of this PR. -->
## What does this PR do?
<!-- Describe the details of this PR. -->
## Related issues
Closes #2759
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.
Delete section if not applicable.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
Delete section if not applicable.
-->
---
rust/fory-core/src/meta/type_meta.rs | 44 ++++++++++++++++++++++++----
rust/fory-core/src/resolver/meta_resolver.rs | 21 +++++++++++--
rust/fory-derive/src/object/read.rs | 4 +--
3 files changed, 59 insertions(+), 10 deletions(-)
diff --git a/rust/fory-core/src/meta/type_meta.rs
b/rust/fory-core/src/meta/type_meta.rs
index 2dbbe06a9..b8f8c5d5f 100644
--- a/rust/fory-core/src/meta/type_meta.rs
+++ b/rust/fory-core/src/meta/type_meta.rs
@@ -577,25 +577,59 @@ impl TypeMeta {
layer: TypeMetaLayer::new(type_id, namespace, type_name,
register_by_name, field_infos),
}
}
- #[allow(unused_assignments)]
+
pub fn from_bytes(reader: &mut Reader, type_resolver: &TypeResolver) ->
TypeMeta {
let header = reader.read_i64();
- let mut meta_size = header & META_SIZE_MASK;
+ let meta_size = header & META_SIZE_MASK;
if meta_size == META_SIZE_MASK {
- meta_size += reader.read_varuint32() as i64;
+ // meta_size += reader.read_varuint32() as i64;
+ reader.read_varuint32();
}
// let write_fields_meta = (header & HAS_FIELDS_META_FLAG) != 0;
// let is_compressed: bool = (header & COMPRESS_META_FLAG) != 0;
- // let meta_hash = header >> (64 - NUM_HASH_BITS);
+ let meta_hash = header >> (64 - NUM_HASH_BITS);
// let current_meta_size = 0;
// while current_meta_size < meta_size {}
let layer = TypeMetaLayer::from_bytes(reader, type_resolver);
TypeMeta {
layer,
- hash: header,
+ hash: meta_hash,
+ }
+ }
+
+ pub fn from_bytes_with_header(
+ reader: &mut Reader,
+ type_resolver: &TypeResolver,
+ header: i64,
+ ) -> TypeMeta {
+ let meta_size = header & META_SIZE_MASK;
+ if meta_size == META_SIZE_MASK {
+ // meta_size += reader.read_varuint32() as i64;
+ reader.read_varuint32();
+ }
+
+ // let write_fields_meta = (header & HAS_FIELDS_META_FLAG) != 0;
+ // let is_compressed: bool = (header & COMPRESS_META_FLAG) != 0;
+ let meta_hash = header >> (64 - NUM_HASH_BITS);
+
+ // let current_meta_size = 0;
+ // while current_meta_size < meta_size {}
+ let layer = TypeMetaLayer::from_bytes(reader, type_resolver);
+ TypeMeta {
+ layer,
+ hash: meta_hash,
+ }
+ }
+
+ pub fn skip_bytes(reader: &mut Reader, header: i64) {
+ let mut meta_size = header & META_SIZE_MASK;
+ if meta_size == META_SIZE_MASK {
+ meta_size += reader.read_varuint32() as i64;
}
+ // TODO skio should return result and we need to return it to caller
+ reader.skip(meta_size as u32);
}
pub fn to_bytes(&self) -> Result<Vec<u8>, Error> {
diff --git a/rust/fory-core/src/resolver/meta_resolver.rs
b/rust/fory-core/src/resolver/meta_resolver.rs
index 0e15f653f..50a24635f 100644
--- a/rust/fory-core/src/resolver/meta_resolver.rs
+++ b/rust/fory-core/src/resolver/meta_resolver.rs
@@ -29,6 +29,8 @@ pub struct MetaWriterResolver {
type_id_index_map: HashMap<std::any::TypeId, usize>,
}
+const MAX_PARSED_NUM_TYPE_DEFS: usize = 8192;
+
#[allow(dead_code)]
impl MetaWriterResolver {
pub fn push(&mut self, type_id: std::any::TypeId, fory: &Fory) -> usize {
@@ -68,6 +70,7 @@ impl MetaWriterResolver {
#[derive(Default)]
pub struct MetaReaderResolver {
pub reading_type_defs: Vec<Arc<TypeMeta>>,
+ parsed_type_defs: HashMap<i64, Arc<TypeMeta>>,
}
impl MetaReaderResolver {
@@ -79,8 +82,22 @@ impl MetaReaderResolver {
let meta_size = reader.read_varuint32();
// self.reading_type_defs.reserve(meta_size as usize);
for _ in 0..meta_size {
- let type_meta = TypeMeta::from_bytes(reader, type_resolver);
- self.reading_type_defs.push(Arc::new(type_meta));
+ let meta_header = reader.read_i64();
+ if let Some(type_meta) = self.parsed_type_defs.get(&meta_header) {
+ self.reading_type_defs.push(type_meta.clone());
+ TypeMeta::skip_bytes(reader, meta_header);
+ } else {
+ let type_meta = Arc::new(TypeMeta::from_bytes_with_header(
+ reader,
+ type_resolver,
+ meta_header,
+ ));
+ if self.parsed_type_defs.len() < MAX_PARSED_NUM_TYPE_DEFS {
+ // avoid malicious type defs to OOM parsed_type_defs
+ self.parsed_type_defs.insert(meta_header,
type_meta.clone());
+ }
+ self.reading_type_defs.push(type_meta);
+ }
}
reader.get_cursor()
}
diff --git a/rust/fory-derive/src/object/read.rs
b/rust/fory-derive/src/object/read.rs
index 3aa53334d..82a3d8ec7 100644
--- a/rust/fory-derive/src/object/read.rs
+++ b/rust/fory-derive/src/object/read.rs
@@ -437,9 +437,7 @@ pub fn gen_read_compatible(fields: &[&Field]) ->
TokenStream {
};
#(#declare_ts)*
- let local_type_def =
fory.get_type_resolver().get_type_info(std::any::TypeId::of::<Self>()).get_type_def();
- let high_bytes = &local_type_def[..8];
- let local_type_hash =
i64::from_le_bytes(high_bytes.try_into().unwrap());
+ let local_type_hash =
fory.get_type_resolver().get_type_info(std::any::TypeId::of::<Self>()).get_type_meta().get_hash();
if meta.get_hash() == local_type_hash {
<Self as fory_core::serializer::Serializer>::fory_read_data(fory,
context, false)
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
