This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 8ab364ccf feat(rust): support btreemap serialization (#2743)
8ab364ccf is described below
commit 8ab364ccf2848a704815e768bb23e8d787878112
Author: Shawn Yang <[email protected]>
AuthorDate: Fri Oct 10 12:56:03 2025 +0530
feat(rust): support btreemap serialization (#2743)
## Why?
<!-- Describe the purpose of this PR. -->
## What does this PR do?
support btreemap serialization
## Related issues
<!--
Is there any related issue? If this PR closes them you say say
fix/closes:
- #xxxx0
- #xxxx1
- Fixes #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.
Delete section if not applicable.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
Delete section if not applicable.
-->
---
rust/fory-core/src/serializer/map.rs | 245 ++++++++++++++++++++++++-----------
rust/fory-derive/src/object/util.rs | 2 +-
rust/tests/tests/test_map.rs | 68 ++++++++++
3 files changed, 239 insertions(+), 76 deletions(-)
diff --git a/rust/fory-core/src/serializer/map.rs
b/rust/fory-core/src/serializer/map.rs
index 36e096173..7db4fc8f9 100644
--- a/rust/fory-core/src/serializer/map.rs
+++ b/rust/fory-core/src/serializer/map.rs
@@ -17,14 +17,13 @@
use crate::error::Error;
use crate::fory::Fory;
-use crate::resolver::context::ReadContext;
-use crate::resolver::context::WriteContext;
+use crate::resolver::context::{ReadContext, WriteContext};
use crate::serializer::{
read_ref_info_data, read_type_info, write_ref_info_data, write_type_info,
ForyDefault,
Serializer,
};
use crate::types::{TypeId, SIZE_OF_REF_AND_TYPE};
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
use std::mem;
const MAX_CHUNK_SIZE: u8 = 255;
@@ -36,7 +35,7 @@ const TRACKING_VALUE_REF: u8 = 0b1000;
pub const VALUE_NULL: u8 = 0b10000;
const DECL_VALUE_TYPE: u8 = 0b100000;
-fn check_and_write_null<K: Serializer + Eq + std::hash::Hash, V: Serializer>(
+fn check_and_write_null<K: Serializer, V: Serializer>(
context: &mut WriteContext,
is_field: bool,
key: &K,
@@ -83,81 +82,84 @@ fn write_chunk_size(context: &mut WriteContext,
header_offset: usize, size: u8)
context.writer.set_bytes(header_offset + 1, &[size]);
}
-impl<K: Serializer + ForyDefault + Eq + std::hash::Hash, V: Serializer +
ForyDefault> Serializer
- for HashMap<K, V>
+fn write_map_data<'a, K, V, I>(iter: I, length: usize, context: &mut
WriteContext, is_field: bool)
+where
+ K: Serializer + ForyDefault + 'a,
+ V: Serializer + ForyDefault + 'a,
+ I: Iterator<Item = (&'a K, &'a V)>,
{
- fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) {
- let length = self.len();
- context.writer.write_varuint32(length as u32);
- if length == 0 {
- return;
- }
- let reserved_space = (K::fory_reserved_space() + SIZE_OF_REF_AND_TYPE)
* self.len()
- + (V::fory_reserved_space() + SIZE_OF_REF_AND_TYPE) * self.len();
- context.writer.reserve(reserved_space);
+ context.writer.write_varuint32(length as u32);
+ if length == 0 {
+ return;
+ }
+ let reserved_space = (K::fory_reserved_space() + SIZE_OF_REF_AND_TYPE) *
length
+ + (V::fory_reserved_space() + SIZE_OF_REF_AND_TYPE) * length;
+ context.writer.reserve(reserved_space);
- let mut header_offset = 0;
- let mut pair_counter: u8 = 0;
- let mut need_write_header = true;
- let mut skip_key_ref_flag = false;
- let mut skip_val_ref_flag = false;
- for entry in self.iter() {
- let key = entry.0;
- let value = entry.1;
- if need_write_header {
- if check_and_write_null(context, is_field, key, value) {
- continue;
- }
- header_offset = context.writer.len();
- context.writer.write_i16(-1);
- let mut chunk_header = 0;
- if is_field {
- chunk_header |= DECL_KEY_TYPE;
- chunk_header |= DECL_VALUE_TYPE;
- }
- // skip_key_ref_flag =
crate::serializer::get_skip_ref_flag::<K>(context.get_fory());
- skip_key_ref_flag = !K::fory_is_polymorphic();
- // skip_val_ref_flag =
crate::serializer::get_skip_ref_flag::<V>(context.get_fory());
- skip_val_ref_flag = !V::fory_is_polymorphic();
- if !skip_key_ref_flag {
- chunk_header |= TRACKING_KEY_REF;
- }
- if !skip_val_ref_flag {
- chunk_header |= TRACKING_VALUE_REF;
- }
- K::fory_write_type_info(context, is_field);
- V::fory_write_type_info(context, is_field);
- context.writer.set_bytes(header_offset, &[chunk_header]);
- need_write_header = false;
- }
- if key.fory_is_none() || value.fory_is_none() {
- write_chunk_size(context, header_offset, pair_counter);
- pair_counter = 0;
- need_write_header = true;
- check_and_write_null(context, is_field, key, value);
+ let mut header_offset = 0;
+ let mut pair_counter: u8 = 0;
+ let mut need_write_header = true;
+ let mut skip_key_ref_flag = false;
+ let mut skip_val_ref_flag = false;
+ for (key, value) in iter {
+ if need_write_header {
+ if check_and_write_null(context, is_field, key, value) {
continue;
}
- if K::fory_is_polymorphic() || K::fory_is_shared_ref() {
- key.fory_write(context, is_field);
- } else {
- write_ref_info_data(key, context, is_field, skip_key_ref_flag,
true);
+ header_offset = context.writer.len();
+ context.writer.write_i16(-1);
+ let mut chunk_header = 0;
+ if is_field {
+ chunk_header |= DECL_KEY_TYPE | DECL_VALUE_TYPE;
}
- if V::fory_is_polymorphic() || V::fory_is_shared_ref() {
- value.fory_write(context, is_field);
- } else {
- write_ref_info_data(value, context, is_field,
skip_val_ref_flag, true);
+ skip_key_ref_flag = !K::fory_is_polymorphic();
+ skip_val_ref_flag = !V::fory_is_polymorphic();
+ if !skip_key_ref_flag {
+ chunk_header |= TRACKING_KEY_REF;
}
- pair_counter += 1;
- if pair_counter == MAX_CHUNK_SIZE {
- write_chunk_size(context, header_offset, pair_counter);
- pair_counter = 0;
- need_write_header = true;
+ if !skip_val_ref_flag {
+ chunk_header |= TRACKING_VALUE_REF;
}
+ K::fory_write_type_info(context, is_field);
+ V::fory_write_type_info(context, is_field);
+ context.writer.set_bytes(header_offset, &[chunk_header]);
+ need_write_header = false;
}
- if pair_counter > 0 {
+ if key.fory_is_none() || value.fory_is_none() {
write_chunk_size(context, header_offset, pair_counter);
+ pair_counter = 0;
+ need_write_header = true;
+ check_and_write_null(context, is_field, key, value);
+ continue;
+ }
+ if K::fory_is_polymorphic() || K::fory_is_shared_ref() {
+ key.fory_write(context, is_field);
+ } else {
+ write_ref_info_data(key, context, is_field, skip_key_ref_flag,
true);
+ }
+ if V::fory_is_polymorphic() || V::fory_is_shared_ref() {
+ value.fory_write(context, is_field);
+ } else {
+ write_ref_info_data(value, context, is_field, skip_val_ref_flag,
true);
+ }
+ pair_counter += 1;
+ if pair_counter == MAX_CHUNK_SIZE {
+ write_chunk_size(context, header_offset, pair_counter);
+ pair_counter = 0;
+ need_write_header = true;
}
}
+ if pair_counter > 0 {
+ write_chunk_size(context, header_offset, pair_counter);
+ }
+}
+
+impl<K: Serializer + ForyDefault + Eq + std::hash::Hash, V: Serializer +
ForyDefault> Serializer
+ for HashMap<K, V>
+{
+ fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) {
+ write_map_data(self.iter(), self.len(), context, is_field);
+ }
fn fory_read_data(context: &mut ReadContext, _is_field: bool) ->
Result<Self, Error> {
let len = context.reader.read_varuint32();
@@ -177,11 +179,8 @@ impl<K: Serializer + ForyDefault + Eq + std::hash::Hash,
V: Serializer + ForyDef
continue;
}
let key_declared = (header & DECL_KEY_TYPE) != 0;
- let _key_tracking_ref = (header & TRACKING_KEY_REF) != 0;
let value_declared = (header & DECL_VALUE_TYPE) != 0;
- let _value_tracking_ref = (header & TRACKING_VALUE_REF) != 0;
if header & KEY_NULL != 0 {
- // let skip_ref_flag =
crate::serializer::get_skip_ref_flag::<V>(context.get_fory());
let skip_ref_flag = if value_declared {
crate::serializer::get_skip_ref_flag::<V>(context.get_fory())
} else {
@@ -193,7 +192,6 @@ impl<K: Serializer + ForyDefault + Eq + std::hash::Hash, V:
Serializer + ForyDef
continue;
}
if header & VALUE_NULL != 0 {
- // let skip_ref_flag =
crate::serializer::get_skip_ref_flag::<K>(context.get_fory());
let skip_ref_flag = if key_declared {
crate::serializer::get_skip_ref_flag::<K>(context.get_fory())
} else {
@@ -208,17 +206,15 @@ impl<K: Serializer + ForyDefault + Eq + std::hash::Hash,
V: Serializer + ForyDef
K::fory_read_type_info(context, key_declared);
V::fory_read_type_info(context, value_declared);
assert!(len_counter + chunk_size as u32 <= len);
- for _ in (0..chunk_size).enumerate() {
+ for _ in 0..chunk_size {
let key = if K::fory_is_polymorphic() {
K::fory_read(context, key_declared)?
} else {
- // let skip_ref_flag =
crate::serializer::get_skip_ref_flag::<K>(context.get_fory());
read_ref_info_data(context, key_declared, true, true)?
};
let value = if V::fory_is_polymorphic() {
V::fory_read(context, value_declared)?
} else {
- // let skip_ref_flag =
crate::serializer::get_skip_ref_flag::<V>(context.get_fory());
read_ref_info_data(context, value_declared, true, true)?
};
map.insert(key, value);
@@ -258,3 +254,102 @@ impl<K, V> ForyDefault for HashMap<K, V> {
HashMap::new()
}
}
+
+impl<K: Serializer + ForyDefault + Ord, V: Serializer + ForyDefault>
Serializer for BTreeMap<K, V> {
+ fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) {
+ write_map_data(self.iter(), self.len(), context, is_field);
+ }
+
+ fn fory_read_data(context: &mut ReadContext, _is_field: bool) ->
Result<Self, Error> {
+ let len = context.reader.read_varuint32();
+ let mut map = BTreeMap::<K, V>::new();
+ if len == 0 {
+ return Ok(map);
+ }
+ let mut len_counter = 0;
+ loop {
+ if len_counter == len {
+ break;
+ }
+ let header = context.reader.read_u8();
+ if header & KEY_NULL != 0 && header & VALUE_NULL != 0 {
+ map.insert(K::fory_default(), V::fory_default());
+ len_counter += 1;
+ continue;
+ }
+ let key_declared = (header & DECL_KEY_TYPE) != 0;
+ let value_declared = (header & DECL_VALUE_TYPE) != 0;
+ if header & KEY_NULL != 0 {
+ let skip_ref_flag = if value_declared {
+
crate::serializer::get_skip_ref_flag::<V>(context.get_fory())
+ } else {
+ false
+ };
+ let value = read_ref_info_data(context, value_declared,
skip_ref_flag, false)?;
+ map.insert(K::fory_default(), value);
+ len_counter += 1;
+ continue;
+ }
+ if header & VALUE_NULL != 0 {
+ let skip_ref_flag = if key_declared {
+
crate::serializer::get_skip_ref_flag::<K>(context.get_fory())
+ } else {
+ false
+ };
+ let key = read_ref_info_data(context, key_declared,
skip_ref_flag, false)?;
+ map.insert(key, V::fory_default());
+ len_counter += 1;
+ continue;
+ }
+ let chunk_size = context.reader.read_u8();
+ K::fory_read_type_info(context, key_declared);
+ V::fory_read_type_info(context, value_declared);
+ assert!(len_counter + chunk_size as u32 <= len);
+ for _ in 0..chunk_size {
+ let key = if K::fory_is_polymorphic() {
+ K::fory_read(context, key_declared)?
+ } else {
+ read_ref_info_data(context, key_declared, true, true)?
+ };
+ let value = if V::fory_is_polymorphic() {
+ V::fory_read(context, value_declared)?
+ } else {
+ read_ref_info_data(context, value_declared, true, true)?
+ };
+ map.insert(key, value);
+ }
+ len_counter += chunk_size as u32;
+ }
+ Ok(map)
+ }
+
+ fn fory_reserved_space() -> usize {
+ mem::size_of::<i32>()
+ }
+
+ fn fory_get_type_id(_fory: &Fory) -> u32 {
+ TypeId::MAP as u32
+ }
+
+ fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 {
+ TypeId::MAP as u32
+ }
+
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+
+ fn fory_write_type_info(context: &mut WriteContext, is_field: bool) {
+ write_type_info::<Self>(context, is_field);
+ }
+
+ fn fory_read_type_info(context: &mut ReadContext, is_field: bool) {
+ read_type_info::<Self>(context, is_field);
+ }
+}
+
+impl<K, V> ForyDefault for BTreeMap<K, V> {
+ fn fory_default() -> Self {
+ BTreeMap::new()
+ }
+}
diff --git a/rust/fory-derive/src/object/util.rs
b/rust/fory-derive/src/object/util.rs
index 655bbfa91..40687c96f 100644
--- a/rust/fory-derive/src/object/util.rs
+++ b/rust/fory-derive/src/object/util.rs
@@ -881,7 +881,7 @@ pub(super) fn get_sort_fields_ts(fields: &[&Field]) ->
TokenStream {
collection_fields.push((ident, ty.to_string(), TypeId::LIST as
u32));
} else if ty.starts_with("HashSet<") {
collection_fields.push((ident, ty.to_string(), TypeId::SET as
u32));
- } else if ty.starts_with("HashMap<") {
+ } else if ty.starts_with("HashMap<") ||
ty.starts_with("BTreeMap<") {
map_fields.push((ident, ty.to_string(), TypeId::MAP as u32));
} else {
struct_or_enum_fields.push((ident, ty.to_string(), 0));
diff --git a/rust/tests/tests/test_map.rs b/rust/tests/tests/test_map.rs
new file mode 100644
index 000000000..38b2c18f7
--- /dev/null
+++ b/rust/tests/tests/test_map.rs
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use fory_core::fory::Fory;
+use fory_derive::ForyObject;
+use std::collections::{BTreeMap, HashMap};
+
+#[test]
+fn test_hashmap_string() {
+ let fory = Fory::default();
+ let mut map = HashMap::new();
+ map.insert("key1".to_string(), "value1".to_string());
+ map.insert("key2".to_string(), "value2".to_string());
+ let bin = fory.serialize(&map);
+ let obj: HashMap<String, String> =
fory.deserialize(&bin).expect("deserialize");
+ assert_eq!(map, obj);
+}
+
+#[test]
+fn test_btreemap_string() {
+ let fory = Fory::default();
+ let mut map = BTreeMap::new();
+ map.insert("key1".to_string(), "value1".to_string());
+ map.insert("key2".to_string(), "value2".to_string());
+ let bin = fory.serialize(&map);
+ let obj: BTreeMap<String, String> =
fory.deserialize(&bin).expect("deserialize");
+ assert_eq!(map, obj);
+}
+
+#[derive(ForyObject, PartialEq, Debug)]
+struct MapContainer {
+ hash_map: HashMap<String, String>,
+ btree_map: BTreeMap<String, i32>,
+}
+
+#[test]
+fn test_struct_with_maps() {
+ let mut fory = Fory::default();
+ fory.register_by_name::<MapContainer>("MapContainer");
+ let mut hash_map = HashMap::new();
+ hash_map.insert("foo".to_string(), "bar".to_string());
+ let mut btree_map = BTreeMap::new();
+ btree_map.insert("a".to_string(), 1);
+ btree_map.insert("b".to_string(), 2);
+
+ let container = MapContainer {
+ hash_map,
+ btree_map,
+ };
+
+ let bin = fory.serialize(&container);
+ let obj: MapContainer = fory.deserialize(&bin).expect("deserialize");
+ assert_eq!(container, obj);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
