This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 724242469 perf(Rust): remove clone()/to_owned() on
MetaString/MetaStringBytes in MetaStringResolver to improve performance && fix
xlang test (#2791)
724242469 is described below
commit 7242424697221353ef37eb2e17214500f4a037d9
Author: urlyy <[email protected]>
AuthorDate: Tue Oct 21 10:49:57 2025 +0800
perf(Rust): remove clone()/to_owned() on MetaString/MetaStringBytes in
MetaStringResolver to improve performance && fix xlang test (#2791)
## Why?
`clone()/to_owned()` on `MetaString/MetaStringBytes` is expensive.
## What does this PR do?
1. use `Rc` in `MetaStringWriterResolver `.
2. aligned `metastring_resolver` logic with Java
## Related issues
close #2762
---------
Co-authored-by: Shawn Yang <[email protected]>
---
.../test/java/org/apache/fory/RustXlangTest.java | 36 +-
rust/fory-core/src/fory.rs | 6 +-
rust/fory-core/src/meta/meta_string.rs | 14 +
rust/fory-core/src/meta/type_meta.rs | 29 +-
rust/fory-core/src/resolver/context.rs | 27 +-
rust/fory-core/src/resolver/meta_resolver.rs | 27 +-
rust/fory-core/src/resolver/metastring_resolver.rs | 412 +++++++++------------
rust/fory-core/src/resolver/type_resolver.rs | 40 +-
rust/fory-core/src/serializer/enum_.rs | 12 +-
rust/fory-core/src/serializer/skip.rs | 2 +-
rust/fory-core/src/serializer/struct_.rs | 12 +-
rust/tests/tests/test_cross_language.rs | 81 ++--
rust/tests/tests/test_metastring_resolver.rs | 122 ++++++
13 files changed, 415 insertions(+), 405 deletions(-)
diff --git a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
index eecd52eb0..6fa5310b1 100644
--- a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
+++ b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
@@ -395,15 +395,6 @@ public class RustXlangTest extends ForyTestBase {
fory.serialize(buffer, strSet);
fory.serialize(buffer, strMap);
fory.serialize(buffer, color);
- // Map<Object, Object> map = new HashMap<>();
- // for (int i = 0; i < list.size(); i++) {
- // map.put("k" + i, list.get(i));
- // map.put(list.get(i), list.get(i));
- // }
- // fory.serialize(buffer, map);
-
- // Set<Object> set = new HashSet<>(list);
- // fory.serialize(buffer, set);
BiConsumer<MemoryBuffer, Boolean> function =
(MemoryBuffer buf, Boolean useToString) -> {
@@ -433,9 +424,6 @@ public class RustXlangTest extends ForyTestBase {
assertStringEquals(fory.deserialize(buf), strSet, useToString);
assertStringEquals(fory.deserialize(buf), strMap, useToString);
assertStringEquals(fory.deserialize(buf), color, useToString);
- // assertStringEquals(fory.deserialize(buf), list,
useToString);
- // assertStringEquals(fory.deserialize(buf), map,
useToString);
- // assertStringEquals(fory.deserialize(buf), set,
useToString);
};
function.accept(buffer, false);
Path dataFile = Files.createTempFile("test_cross_language_serializer",
"data");
@@ -817,25 +805,25 @@ public class RustXlangTest extends ForyTestBase {
MyStruct myStruct = new MyStruct(42);
MyExt myExt = new MyExt(43);
MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32);
- fory.serialize(buffer, Color.White);
- fory.serialize(buffer, Color.White);
- fory.serialize(buffer, Color.White);
+ for (int i = 0; i < 3; i++) {
+ fory.serialize(buffer, Color.White);
+ }
// todo: checkVersion
// fory.serialize(buffer, myStruct);
- fory.serialize(buffer, myExt);
- fory.serialize(buffer, myExt);
- fory.serialize(buffer, myExt);
+ for (int i = 0; i < 3; i++) {
+ fory.serialize(buffer, myExt);
+ }
byte[] bytes = buffer.getBytes(0, buffer.writerIndex());
Path dataFile = Files.createTempFile("test_consistent_named", "data");
Pair<Map<String, String>, File> env_workdir = setFilePath(language,
command, dataFile, bytes);
Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(),
env_workdir.getRight()));
MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile));
- Assert.assertEquals(fory.deserialize(buffer2), Color.White);
- Assert.assertEquals(fory.deserialize(buffer2), Color.White);
- Assert.assertEquals(fory.deserialize(buffer2), Color.White);
- Assert.assertEquals(fory.deserialize(buffer2), myExt);
- Assert.assertEquals(fory.deserialize(buffer2), myExt);
- Assert.assertEquals(fory.deserialize(buffer2), myExt);
+ for (int i = 0; i < 3; i++) {
+ Assert.assertEquals(fory.deserialize(buffer2), Color.White);
+ }
+ for (int i = 0; i < 3; i++) {
+ Assert.assertEquals(fory.deserialize(buffer2), myExt);
+ }
}
/**
diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs
index ca3f50096..ec34cbd57 100644
--- a/rust/fory-core/src/fory.rs
+++ b/rust/fory-core/src/fory.rs
@@ -403,7 +403,7 @@ impl Fory {
if result.is_ok() {
assert_eq!(context.reader.slice_after_cursor().len(), 0);
}
- context.reset();
+ context.reader.reset();
pool.put(context);
result
}
@@ -428,6 +428,7 @@ impl Fory {
context.reader.skip(bytes_to_skip)?;
}
context.ref_reader.resolve_callbacks();
+ context.reset();
result
}
@@ -481,7 +482,7 @@ impl Fory {
});
let mut context = pool.get();
let result = self.serialize_with_context(record, &mut context)?;
- context.reset();
+ context.writer.reset();
pool.put(context);
Ok(result)
}
@@ -503,6 +504,7 @@ impl Fory {
context.write_meta(meta_start_offset);
}
}
+ context.reset();
Ok(context.writer.dump())
}
diff --git a/rust/fory-core/src/meta/meta_string.rs
b/rust/fory-core/src/meta/meta_string.rs
index 9391b94e4..23b42f5c8 100644
--- a/rust/fory-core/src/meta/meta_string.rs
+++ b/rust/fory-core/src/meta/meta_string.rs
@@ -18,6 +18,7 @@
use crate::ensure;
use crate::error::Error;
use crate::meta::string_util;
+use std::sync::OnceLock;
// equal to "std::i16::MAX"
const SHORT_MAX_VALUE: usize = 32767;
@@ -66,6 +67,8 @@ impl std::hash::Hash for MetaString {
}
}
+static EMPTY: OnceLock<MetaString> = OnceLock::new();
+
impl MetaString {
pub fn new(
original: String,
@@ -95,6 +98,17 @@ impl MetaString {
writer.write_varuint32(self.bytes.len() as u32);
writer.write_bytes(&self.bytes);
}
+
+ pub fn get_empty() -> &'static MetaString {
+ EMPTY.get_or_init(|| MetaString {
+ original: "".to_string(),
+ encoding: Encoding::default(),
+ bytes: vec![],
+ strip_last_char: false,
+ special_char1: '\0',
+ special_char2: '\0',
+ })
+ }
}
#[derive(Clone)]
diff --git a/rust/fory-core/src/meta/type_meta.rs
b/rust/fory-core/src/meta/type_meta.rs
index a52f94d4a..d7fb33864 100644
--- a/rust/fory-core/src/meta/type_meta.rs
+++ b/rust/fory-core/src/meta/type_meta.rs
@@ -26,6 +26,7 @@ use crate::types::{TypeId, PRIMITIVE_TYPES};
use std::clone::Clone;
use std::cmp::min;
use std::collections::HashMap;
+use std::rc::Rc;
const SMALL_NUM_FIELDS_THRESHOLD: usize = 0b11111;
const REGISTER_BY_NAME_FLAG: u8 = 0b100000;
@@ -236,8 +237,8 @@ impl PartialEq for FieldType {
#[derive(Debug)]
pub struct TypeMetaLayer {
type_id: u32,
- namespace: MetaString,
- type_name: MetaString,
+ namespace: Rc<MetaString>,
+ type_name: Rc<MetaString>,
register_by_name: bool,
field_infos: Vec<FieldInfo>,
}
@@ -252,8 +253,8 @@ impl TypeMetaLayer {
) -> TypeMetaLayer {
TypeMetaLayer {
type_id,
- namespace,
- type_name,
+ namespace: Rc::from(namespace),
+ type_name: Rc::from(type_name),
register_by_name,
field_infos,
}
@@ -262,8 +263,8 @@ impl TypeMetaLayer {
pub fn empty() -> TypeMetaLayer {
TypeMetaLayer {
type_id: 0,
- namespace: MetaString::default(),
- type_name: MetaString::default(),
+ namespace: Rc::from(MetaString::get_empty().clone()),
+ type_name: Rc::from(MetaString::get_empty().clone()),
register_by_name: false,
field_infos: vec![],
}
@@ -273,12 +274,12 @@ impl TypeMetaLayer {
self.type_id
}
- pub fn get_type_name(&self) -> &MetaString {
- &self.type_name
+ pub fn get_type_name(&self) -> Rc<MetaString> {
+ self.type_name.clone()
}
- pub fn get_namespace(&self) -> &MetaString {
- &self.namespace
+ pub fn get_namespace(&self) -> Rc<MetaString> {
+ self.namespace.clone()
}
pub fn get_field_infos(&self) -> &Vec<FieldInfo> {
@@ -556,12 +557,12 @@ impl TypeMeta {
self.hash
}
- pub fn get_type_name(&self) -> MetaString {
- self.layer.get_type_name().clone()
+ pub fn get_type_name(&self) -> Rc<MetaString> {
+ self.layer.get_type_name()
}
- pub fn get_namespace(&self) -> MetaString {
- self.layer.get_namespace().clone()
+ pub fn get_namespace(&self) -> Rc<MetaString> {
+ self.layer.get_namespace()
}
pub fn empty() -> TypeMeta {
diff --git a/rust/fory-core/src/resolver/context.rs
b/rust/fory-core/src/resolver/context.rs
index a347d54a9..f3f4025ad 100644
--- a/rust/fory-core/src/resolver/context.rs
+++ b/rust/fory-core/src/resolver/context.rs
@@ -21,9 +21,7 @@ use crate::error::Error;
use crate::fory::Fory;
use crate::meta::MetaString;
use crate::resolver::meta_resolver::{MetaReaderResolver, MetaWriterResolver};
-use crate::resolver::metastring_resolver::{
- MetaStringBytes, MetaStringReaderResolver, MetaStringWriterResolver,
-};
+use crate::resolver::metastring_resolver::{MetaStringReaderResolver,
MetaStringWriterResolver};
use crate::resolver::ref_resolver::{RefReader, RefWriter};
use crate::resolver::type_resolver::{TypeInfo, TypeResolver};
use crate::types;
@@ -170,8 +168,8 @@ impl WriteContext {
as u32;
self.writer.write_varuint32(meta_index);
} else {
- namespace.write_to(&mut self.writer);
- type_name.write_to(&mut self.writer);
+ self.write_meta_string_bytes(namespace)?;
+ self.write_meta_string_bytes(type_name)?;
}
}
_ => {
@@ -182,7 +180,7 @@ impl WriteContext {
}
#[inline(always)]
- pub fn write_meta_string_bytes(&mut self, ms: &MetaString) -> Result<(),
Error> {
+ pub fn write_meta_string_bytes(&mut self, ms: Rc<MetaString>) ->
Result<(), Error> {
self.meta_string_resolver
.write_meta_string_bytes(&mut self.writer, ms)
}
@@ -190,8 +188,8 @@ impl WriteContext {
#[inline(always)]
pub fn reset(&mut self) {
self.meta_resolver.reset();
+ self.meta_string_resolver.reset();
self.ref_writer.reset();
- self.writer.reset();
}
}
@@ -332,10 +330,12 @@ impl ReadContext {
let type_info =
self.get_type_info_by_index(meta_index)?.clone();
Ok(type_info)
} else {
- let namespace = self.meta_resolver.read_metastring(&mut
self.reader)?;
- let type_name = self.meta_resolver.read_metastring(&mut
self.reader)?;
+ let namespace = self.read_meta_string()?.to_owned();
+ let type_name = self.read_meta_string()?.to_owned();
+ let rc_namespace = Rc::from(namespace);
+ let rc_type_name = Rc::from(type_name);
self.type_resolver
- .get_type_info_by_msname(&namespace, &type_name)
+ .get_type_info_by_msname(rc_namespace, rc_type_name)
.ok_or_else(|| Error::type_error("Name harness not
found"))
}
}
@@ -351,9 +351,8 @@ impl ReadContext {
self.type_resolver.get_type_info(type_id)
}
- pub fn read_meta_string_bytes(&mut self) -> Result<MetaStringBytes, Error>
{
- self.meta_string_resolver
- .read_meta_string_bytes(&mut self.reader)
+ pub fn read_meta_string(&mut self) -> Result<&MetaString, Error> {
+ self.meta_string_resolver.read_meta_string(&mut self.reader)
}
#[inline(always)]
@@ -378,8 +377,8 @@ impl ReadContext {
#[inline(always)]
pub fn reset(&mut self) {
- self.reader.reset();
self.meta_resolver.reset();
+ self.meta_string_resolver.reset();
self.ref_reader.reset();
}
}
diff --git a/rust/fory-core/src/resolver/meta_resolver.rs
b/rust/fory-core/src/resolver/meta_resolver.rs
index dbd4db05c..9504c6388 100644
--- a/rust/fory-core/src/resolver/meta_resolver.rs
+++ b/rust/fory-core/src/resolver/meta_resolver.rs
@@ -17,7 +17,7 @@
use crate::buffer::{Reader, Writer};
use crate::error::Error;
-use crate::meta::{Encoding, MetaString, TypeMeta, NAMESPACE_DECODER};
+use crate::meta::TypeMeta;
use crate::resolver::type_resolver::TypeInfo;
use crate::TypeResolver;
use std::collections::HashMap;
@@ -140,31 +140,6 @@ impl MetaReaderResolver {
Ok(reader.get_cursor())
}
- pub fn read_metastring(&self, reader: &mut Reader) -> Result<MetaString,
Error> {
- let len = reader.read_varuint32()? as usize;
- if len == 0 {
- return Ok(MetaString {
- bytes: vec![],
- encoding: Encoding::Utf8,
- original: String::new(),
- strip_last_char: false,
- special_char1: '\0',
- special_char2: '\0',
- });
- }
- let bytes = reader.read_bytes(len)?;
- let encoding_byte = bytes[0] & 0x07;
- let encoding = match encoding_byte {
- 0x00 => Encoding::Utf8,
- 0x01 => Encoding::LowerSpecial,
- 0x02 => Encoding::LowerUpperDigitSpecial,
- 0x03 => Encoding::FirstToLowerSpecial,
- 0x04 => Encoding::AllToLowerSpecial,
- _ => Encoding::Utf8,
- };
- NAMESPACE_DECODER.decode(bytes, encoding)
- }
-
pub fn reset(&mut self) {
self.reading_type_infos.clear();
}
diff --git a/rust/fory-core/src/resolver/metastring_resolver.rs
b/rust/fory-core/src/resolver/metastring_resolver.rs
index 975d27e67..64ca15ad2 100644
--- a/rust/fory-core/src/resolver/metastring_resolver.rs
+++ b/rust/fory-core/src/resolver/metastring_resolver.rs
@@ -15,14 +15,17 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::convert::TryInto;
+use std::rc::Rc;
+use std::sync::OnceLock;
use crate::buffer::Writer;
use crate::error::Error;
-use crate::meta::murmurhash3_x64_128;
+use crate::meta::{murmurhash3_x64_128, NAMESPACE_DECODER};
use crate::meta::{Encoding, MetaString};
-use crate::Reader;
+use crate::{ensure, Reader};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct MetaStringBytes {
@@ -31,7 +34,6 @@ pub struct MetaStringBytes {
pub encoding: Encoding,
pub first8: u64,
pub second8: u64,
- pub dynamic_write_id: i16,
}
const HEADER_MASK: i64 = 0xff;
@@ -47,240 +49,181 @@ fn byte_to_encoding(byte: u8) -> Encoding {
}
}
+static EMPTY: OnceLock<MetaStringBytes> = OnceLock::new();
+
impl MetaStringBytes {
pub const DEFAULT_DYNAMIC_WRITE_STRING_ID: i16 = -1;
- pub const EMPTY: MetaStringBytes = MetaStringBytes {
- bytes: Vec::new(),
- hash_code: 0,
- encoding: Encoding::Utf8,
- first8: 0,
- second8: 0,
- dynamic_write_id: MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID,
- };
-
- pub fn new(
- bytes: Vec<u8>,
- hash_code: i64,
- encoding: Encoding,
- first8: u64,
- second8: u64,
- ) -> Self {
+
+ pub fn new(bytes: Vec<u8>, hash_code: i64) -> Self {
+ let header = (hash_code & HEADER_MASK) as u8;
+ let encoding = byte_to_encoding(header);
+ let mut data = bytes.clone();
+ if bytes.len() < 16 {
+ data.resize(16, 0);
+ }
+ let first8 = u64::from_le_bytes(data[0..8].try_into().unwrap());
+ let second8 = u64::from_le_bytes(data[8..16].try_into().unwrap());
MetaStringBytes {
bytes,
hash_code,
encoding,
first8,
second8,
- dynamic_write_id: MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID,
}
}
- pub fn decode_lossy(&self) -> String {
- String::from_utf8_lossy(&self.bytes).into_owned()
+ pub fn to_metastring(&self) -> Result<MetaString, Error> {
+ let ms = NAMESPACE_DECODER.decode(&self.bytes, self.encoding)?;
+ Ok(ms)
}
pub(crate) fn from_metastring(meta_string: &MetaString) -> Result<Self,
Error> {
- let mut bytes = meta_string.bytes.to_vec();
- let len = bytes.len();
- // follow python/java implementation: small strings use quick v1/v2
based hash,
- // large strings use murmurhash, then mask lower 8 bits for encoding
- let encoding_val = meta_string.encoding as i64 & HEADER_MASK;
- let hash_code: i64;
- if len <= MetaStringReaderResolver::SMALL_STRING_THRESHOLD {
- // compute v1 and v2 from bytes little-endian
- let mut v1: u64 = 0;
- let mut v2: u64 = 0;
- for (i, b) in bytes.iter().take(8).enumerate() {
- v1 |= (*b as u64) << (8 * i);
- }
- if bytes.len() > 8 {
- for j in 0..usize::min(8, bytes.len() - 8) {
- v2 |= (bytes[8 + j] as u64) << (8 * j);
- }
- }
- // ((v1 * 31 + v2) >> 8 << 8) | encoding
- let tmp: u128 = (v1 as u128).wrapping_mul(31u128).wrapping_add(v2
as u128);
- let masked = ((tmp >> 8) << 8) as u64;
- hash_code = masked as i64 | encoding_val;
- } else {
- let mut hc = murmurhash3_x64_128(&bytes, 47).0 as i64;
- hc = hc.abs();
- if hc == 0 {
- hc += 256;
- }
- hc = (hc as u64 & 0xffffffffffffff00) as i64;
- hash_code = hc | encoding_val;
- }
-
- // ensure we have 16 bytes to extract first8/second8
- if bytes.len() < 16 {
- bytes.resize(16, 0);
+ let bytes = meta_string.bytes.to_vec();
+ let mut hash_code = murmurhash3_x64_128(&bytes, 47).0 as i64;
+ hash_code = hash_code.abs();
+ if hash_code == 0 {
+ hash_code += 256;
}
+ hash_code = (hash_code as u64 & 0xffffffffffffff00) as i64;
+ let encoding = meta_string.encoding;
+ let header = encoding as i64 & HEADER_MASK;
+ hash_code |= header;
+ Ok(Self::new(bytes, hash_code))
+ }
- let first8: [u8; 8] = bytes[0..8].try_into().map_err(|_| {
- Error::invalid_data(format!("expected at least 8 bytes, got {}",
bytes.len()))
- })?;
- let first8 = u64::from_le_bytes(first8);
-
- let second8: [u8; 8] = bytes[8..16].try_into().map_err(|_| {
- Error::invalid_data(format!("expected at least 16 bytes, got {}",
bytes.len()))
- })?;
- let second8 = u64::from_le_bytes(second8);
-
- Ok(Self::new(
- bytes,
- hash_code,
- byte_to_encoding((hash_code & HEADER_MASK) as u8),
- first8,
- second8,
- ))
+ pub fn get_empty() -> &'static MetaStringBytes {
+ EMPTY.get_or_init(||
MetaStringBytes::from_metastring(MetaString::get_empty()).unwrap())
}
}
-#[derive(Default)]
pub struct MetaStringWriterResolver {
- meta_string_to_bytes: HashMap<MetaString, MetaStringBytes>,
- dynamic_written: Vec<Option<MetaStringBytes>>,
+ meta_string_to_bytes: HashMap<Rc<MetaString>, MetaStringBytes>,
+ dynamic_written: Vec<*const MetaStringBytes>,
dynamic_write_id: usize,
+ bytes_id_map: HashMap<*const MetaStringBytes, i16>,
}
-impl MetaStringWriterResolver {
- const INITIAL_CAPACITY: usize = 8;
- const SMALL_STRING_THRESHOLD: usize = 16;
-
- pub fn new() -> Self {
+impl Default for MetaStringWriterResolver {
+ fn default() -> Self {
Self {
meta_string_to_bytes:
HashMap::with_capacity(Self::INITIAL_CAPACITY),
- dynamic_written: vec![None; 32],
+ dynamic_written: vec![std::ptr::null(); 32],
dynamic_write_id: 0,
+ bytes_id_map: HashMap::with_capacity(Self::INITIAL_CAPACITY),
}
}
+}
- pub fn get_or_create_meta_string_bytes(&mut self, m: &MetaString) ->
MetaStringBytes {
- if let Some(b) = self.meta_string_to_bytes.get(m) {
- return b.clone();
- }
- // create via from_metastring to keep same hash/encoding rules
- let msb =
MetaStringBytes::from_metastring(m).expect("from_metastring");
- self.meta_string_to_bytes.insert(m.clone(), msb.clone());
- msb
- }
-
- pub fn write_meta_string_bytes_with_flag(&mut self, w: &mut Writer, mut
mb: MetaStringBytes) {
- let id = mb.dynamic_write_id;
- if id == MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID {
- let id_usize = self.dynamic_write_id;
- self.dynamic_write_id += 1;
- mb.dynamic_write_id = id_usize as i16;
- if id_usize >= self.dynamic_written.len() {
- self.dynamic_written.resize(id_usize * 2, None);
- }
- self.dynamic_written[id_usize] = Some(mb.clone());
-
- let len = mb.bytes.len();
- let header = ((len as u32) << 2) | 0b1;
- w.write_varuint32(header);
- if len > Self::SMALL_STRING_THRESHOLD {
- w.write_i64(mb.hash_code);
- } else {
- w.write_u8(mb.encoding as i16 as u8);
- }
- w.write_bytes(&mb.bytes);
- } else {
- let header = ((id as u32 + 1) << 2) | 0b11;
- w.write_varuint32(header);
- }
- }
+impl MetaStringWriterResolver {
+ const INITIAL_CAPACITY: usize = 8;
+ const SMALL_STRING_THRESHOLD: usize = 16;
pub fn write_meta_string_bytes(
&mut self,
writer: &mut Writer,
- ms: &MetaString,
+ ms: Rc<MetaString>,
) -> Result<(), Error> {
- let mut mb = MetaStringBytes::from_metastring(ms)?;
- let id = mb.dynamic_write_id;
- if id == MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID {
- let id_usize = self.dynamic_write_id;
- self.dynamic_write_id += 1;
- mb.dynamic_write_id = id_usize as i16;
- if id_usize >= self.dynamic_written.len() {
- self.dynamic_written.resize(id_usize * 2 + 1, None);
+ // get_or_create_meta_string_bytes
+ let mb_ref = {
+ let entry = self.meta_string_to_bytes.entry(ms.clone());
+ match entry {
+ Entry::Occupied(o) => o.into_mut(),
+ Entry::Vacant(v) =>
v.insert(MetaStringBytes::from_metastring(&ms)?),
}
- self.dynamic_written[id_usize] = Some(mb.clone());
+ };
- let len = mb.bytes.len();
- writer.write_varuint32((len as u32) << 1);
- if len > Self::SMALL_STRING_THRESHOLD {
- writer.write_i64(mb.hash_code);
- } else {
- writer.write_u8(mb.encoding as i16 as u8);
+ let mb_ptr: *const MetaStringBytes = mb_ref as *const _;
+ let id = if let Some(exist_id) = self.bytes_id_map.get_mut(&mb_ptr) {
+ if *exist_id != MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID {
+ writer.write_varuint32(((*exist_id as u32 + 1) << 1) | 1);
+ return Ok(());
}
- writer.write_bytes(&mb.bytes);
+ let id = self.dynamic_write_id;
+ *exist_id = id as i16;
+ id
} else {
- let header = ((id as u32 + 1) << 1) | 1;
- writer.write_varuint32(header);
+ let id = self.dynamic_write_id;
+ self.bytes_id_map.insert(mb_ptr, id as i16);
+ id
+ };
+ // // update dynamic_write
+ self.dynamic_write_id += 1;
+ if id >= self.dynamic_written.len() {
+ self.dynamic_written.resize(id * 2, std::ptr::null());
}
+ self.dynamic_written[id] = mb_ptr;
+
+ let len = mb_ref.bytes.len();
+ writer.write_varuint32((len as u32) << 1);
+ if len > Self::SMALL_STRING_THRESHOLD {
+ writer.write_i64(mb_ref.hash_code);
+ } else {
+ writer.write_u8(mb_ref.encoding as i16 as u8);
+ }
+ writer.write_bytes(&mb_ref.bytes);
Ok(())
}
- pub fn reset_write(&mut self) {
+ pub fn reset(&mut self) {
if self.dynamic_write_id != 0 {
for i in 0..self.dynamic_write_id {
- if let Some(ref mut mb) = self.dynamic_written[i] {
- mb.dynamic_write_id =
MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID;
+ let key = self.dynamic_written[i];
+ if !key.is_null() {
+ if let Some(v) = self.bytes_id_map.get_mut(&key) {
+ *v = MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID;
+ }
+ self.dynamic_written[i] = std::ptr::null();
}
- self.dynamic_written[i] = None;
}
self.dynamic_write_id = 0;
}
}
}
-#[derive(Default)]
pub struct MetaStringReaderResolver {
- meta_string_bytes_to_string: HashMap<MetaStringBytes, String>,
- hash_to_meta: HashMap<i64, MetaStringBytes>,
- small_map: HashMap<(u64, u64, u8), MetaStringBytes>,
- dynamic_read: Vec<Option<MetaStringBytes>>,
+ meta_string_bytes_to_string: HashMap<*const MetaStringBytes, MetaString>,
+ hash_to_meta_string_bytes: HashMap<i64, MetaStringBytes>,
+ long_long_byte_map: HashMap<(u64, u64, u8), MetaStringBytes>,
+ dynamic_read: Vec<Option<*const MetaStringBytes>>,
dynamic_read_id: usize,
}
-impl MetaStringReaderResolver {
- const INITIAL_CAPACITY: usize = 8;
- const SMALL_STRING_THRESHOLD: usize = 16;
-
- pub fn new() -> Self {
+impl Default for MetaStringReaderResolver {
+ fn default() -> Self {
Self {
meta_string_bytes_to_string:
HashMap::with_capacity(Self::INITIAL_CAPACITY),
- hash_to_meta: HashMap::with_capacity(Self::INITIAL_CAPACITY),
- small_map: HashMap::with_capacity(Self::INITIAL_CAPACITY),
+ hash_to_meta_string_bytes:
HashMap::with_capacity(Self::INITIAL_CAPACITY),
+ long_long_byte_map: HashMap::with_capacity(Self::INITIAL_CAPACITY),
dynamic_read: vec![None; 32],
dynamic_read_id: 0,
}
}
+}
+
+impl MetaStringReaderResolver {
+ const INITIAL_CAPACITY: usize = 8;
+ const SMALL_STRING_THRESHOLD: usize = 16;
pub fn read_meta_string_bytes_with_flag(
&mut self,
reader: &mut Reader,
header: u32,
- ) -> Result<MetaStringBytes, Error> {
+ ) -> Result<&MetaStringBytes, Error> {
let len = (header >> 2) as usize;
+
if (header & 0b10) == 0 {
if len <= Self::SMALL_STRING_THRESHOLD {
- let mb = self.read_small_meta_string_bytes(reader, len)?;
- self.update_dynamic_string(mb.clone());
- Ok(mb)
+ self.read_small_meta_string_bytes_and_update(reader, len)
} else {
let hash_code = reader.read_i64()?;
- let mb = self.read_big_meta_string_bytes(reader, len,
hash_code)?;
- self.update_dynamic_string(mb.clone());
- Ok(mb)
+ self.read_big_meta_string_bytes_and_update(reader, len,
hash_code)
}
} else {
let idx = len - 1;
self.dynamic_read
.get(idx)
- .and_then(|opt| opt.clone())
+ .and_then(|opt| opt.as_ref())
+ .map(|ptr| unsafe { &**ptr })
.ok_or_else(|| Error::invalid_data("dynamic id not found"))
}
}
@@ -288,66 +231,72 @@ impl MetaStringReaderResolver {
pub fn read_meta_string_bytes(
&mut self,
reader: &mut Reader,
- ) -> Result<MetaStringBytes, Error> {
+ ) -> Result<&MetaStringBytes, Error> {
let header = reader.read_varuint32()?;
let len = (header >> 1) as usize;
+
if (header & 0b1) == 0 {
if len > Self::SMALL_STRING_THRESHOLD {
let hash_code = reader.read_i64()?;
- let mb = self.read_big_meta_string_bytes(reader, len,
hash_code)?;
- self.update_dynamic_string(mb.clone());
- Ok(mb)
+ self.read_big_meta_string_bytes_and_update(reader, len,
hash_code)
} else {
- let mb = self.read_small_meta_string_bytes(reader, len)?;
- self.update_dynamic_string(mb.clone());
- Ok(mb)
+ self.read_small_meta_string_bytes_and_update(reader, len)
}
} else {
let idx = len - 1;
self.dynamic_read
.get(idx)
- .and_then(|opt| opt.clone())
+ .and_then(|opt| opt.as_ref())
+ .map(|ptr| unsafe { &**ptr })
.ok_or_else(|| Error::invalid_data("dynamic id not found"))
}
}
- fn read_big_meta_string_bytes(
+ fn read_big_meta_string_bytes_and_update(
&mut self,
reader: &mut Reader,
len: usize,
hash_code: i64,
- ) -> Result<MetaStringBytes, Error> {
- if let Some(existing) = self.hash_to_meta.get(&hash_code) {
- reader.skip(len)?;
- Ok(existing.clone())
- } else {
- let bytes = reader.read_bytes(len)?.to_vec();
- let mut first8 = 0;
- let mut second8 = 0;
- for (i, b) in bytes.iter().enumerate().take(8) {
- first8 |= (*b as u64) << (8 * i);
+ ) -> Result<&MetaStringBytes, Error> {
+ let mb_ref: &mut MetaStringBytes = match
self.hash_to_meta_string_bytes.entry(hash_code) {
+ Entry::Occupied(entry) => {
+ reader.skip(len)?;
+ entry.into_mut()
}
- if bytes.len() > 8 {
- for j in 0..usize::min(8, bytes.len() - 8) {
- second8 |= (bytes[8 + j] as u64) << (8 * j);
- }
+ Entry::Vacant(entry) => {
+ let bytes = reader.read_bytes(len)?.to_vec();
+ let mb = MetaStringBytes::new(bytes, hash_code);
+ entry.insert(mb)
}
- let mb = MetaStringBytes::new(bytes, hash_code, Encoding::Utf8,
first8, second8);
- self.hash_to_meta.insert(hash_code, mb.clone());
- Ok(mb)
+ };
+
+ // update dynamic_read
+ let id = self.dynamic_read_id;
+ self.dynamic_read_id += 1;
+ if id >= self.dynamic_read.len() {
+ self.dynamic_read.resize(id * 2 + 1, None);
}
+ let ptr = mb_ref as *const MetaStringBytes;
+ self.dynamic_read[id] = Some(ptr);
+ Ok(mb_ref)
}
- fn read_small_meta_string_bytes(
+ fn read_small_meta_string_bytes_and_update(
&mut self,
reader: &mut Reader,
len: usize,
- ) -> Result<MetaStringBytes, Error> {
+ ) -> Result<&MetaStringBytes, Error> {
let encoding_val = reader.read_u8()?;
if len == 0 {
- debug_assert_eq!(encoding_val, Encoding::Utf8 as i16 as u8);
- return Ok(MetaStringBytes::EMPTY.clone());
+ ensure!(
+ encoding_val == Encoding::Utf8 as u8,
+ Error::EncodingError(format!("wrong encoding value: {}",
encoding_val).into())
+ );
+ let empty = MetaStringBytes::get_empty();
+ // empty must be a static or globally unique instance
+ return Ok(empty);
}
+
let (v1, v2) = if len <= 8 {
let v1 = Self::read_bytes_as_u64(reader, len)?;
(v1, 0)
@@ -357,32 +306,31 @@ impl MetaStringReaderResolver {
(v1, v2)
};
let key = (v1, v2, encoding_val);
- if let Some(existing) = self.small_map.get(&key) {
- Ok(existing.clone())
- } else {
- let mut data = Vec::with_capacity(len);
- for i in 0..usize::min(8, len) {
- data.push(((v1 >> (8 * i)) & 0xFF) as u8);
- }
- if len > 8 {
- for j in 0..(len - 8) {
- data.push(((v2 >> (8 * j)) & 0xFF) as u8);
- }
+
+ let mb_ref = match self.long_long_byte_map.entry(key) {
+ Entry::Occupied(entry) => entry.into_mut(),
+ Entry::Vacant(entry) => {
+ let mut data = vec![0u8; 16];
+ data[0..8].copy_from_slice(&v1.to_le_bytes());
+ data[8..16].copy_from_slice(&v2.to_le_bytes());
+ data.truncate(len);
+
+ let hash_code = (murmurhash3_x64_128(&data, 47).0 as
i64).abs();
+ let hash_code =
+ (hash_code as u64 & 0xffffffffffffff00_u64) as i64 |
(encoding_val as i64);
+ let mb = MetaStringBytes::new(data, hash_code);
+ entry.insert(mb)
}
- data.truncate(len);
- let hash_code = (murmurhash3_x64_128(&data, 47).0 as i64).abs();
- let hash_code =
- (hash_code as u64 & 0xffffffffffffff00_u64) as i64 |
(encoding_val as i64);
- let mb = MetaStringBytes::new(
- data.clone(),
- hash_code,
- byte_to_encoding(encoding_val),
- v1,
- v2,
- );
- self.small_map.insert(key, mb.clone());
- Ok(mb)
+ };
+ // update dynamic_read
+ let ptr = mb_ref as *const MetaStringBytes;
+ let id = self.dynamic_read_id;
+ self.dynamic_read_id += 1;
+ if id >= self.dynamic_read.len() {
+ self.dynamic_read.resize(id * 2, None);
}
+ self.dynamic_read[id] = Some(ptr);
+ Ok(mb_ref)
}
fn read_bytes_as_u64(reader: &mut Reader, len: usize) -> Result<u64,
Error> {
@@ -394,16 +342,7 @@ impl MetaStringReaderResolver {
Ok(v)
}
- fn update_dynamic_string(&mut self, mb: MetaStringBytes) {
- let id = self.dynamic_read_id;
- self.dynamic_read_id += 1;
- if id >= self.dynamic_read.len() {
- self.dynamic_read.resize(id * 2 + 1, None);
- }
- self.dynamic_read[id] = Some(mb);
- }
-
- pub fn reset_read(&mut self) {
+ pub fn reset(&mut self) {
if self.dynamic_read_id != 0 {
for i in 0..self.dynamic_read_id {
self.dynamic_read[i] = None;
@@ -412,14 +351,19 @@ impl MetaStringReaderResolver {
}
}
- pub fn read_meta_string(&mut self, reader: &mut Reader) -> Result<String,
Error> {
- let mb = self.read_meta_string_bytes(reader)?;
- Ok(if let Some(s) = self.meta_string_bytes_to_string.get(&mb) {
- s.clone()
- } else {
- let s = mb.decode_lossy();
- self.meta_string_bytes_to_string.insert(mb, s.clone());
- s
- })
+ pub fn read_meta_string(&mut self, reader: &mut Reader) ->
Result<&MetaString, Error> {
+ let ptr = {
+ let mb_ref = self.read_meta_string_bytes(reader)?;
+ mb_ref as *const MetaStringBytes
+ };
+ let ms_ref = self
+ .meta_string_bytes_to_string
+ .entry(ptr)
+ .or_insert_with(|| {
+ let mb_ref = unsafe { &*ptr };
+ mb_ref.to_metastring().unwrap()
+ });
+
+ Ok(ms_ref)
}
}
diff --git a/rust/fory-core/src/resolver/type_resolver.rs
b/rust/fory-core/src/resolver/type_resolver.rs
index 7da594172..3ee153418 100644
--- a/rust/fory-core/src/resolver/type_resolver.rs
+++ b/rust/fory-core/src/resolver/type_resolver.rs
@@ -95,8 +95,8 @@ pub struct TypeInfo {
type_def: Rc<Vec<u8>>,
type_meta: Rc<TypeMeta>,
type_id: u32,
- namespace: MetaString,
- type_name: MetaString,
+ namespace: Rc<MetaString>,
+ type_name: Rc<MetaString>,
register_by_name: bool,
harness: Harness,
}
@@ -147,8 +147,8 @@ impl TypeInfo {
type_def: Rc::from(type_def_bytes),
type_meta,
type_id,
- namespace: namespace_metastring,
- type_name: type_name_metastring,
+ namespace: Rc::from(namespace_metastring),
+ type_name: Rc::from(type_name_metastring),
register_by_name,
harness,
})
@@ -179,8 +179,8 @@ impl TypeInfo {
type_def: Rc::from(type_def),
type_meta: Rc::new(meta),
type_id,
- namespace: namespace_metastring,
- type_name: type_name_metastring,
+ namespace: Rc::from(namespace_metastring),
+ type_name: Rc::from(type_name_metastring),
register_by_name,
harness,
})
@@ -190,12 +190,12 @@ impl TypeInfo {
self.type_id
}
- pub fn get_namespace(&self) -> &MetaString {
- &self.namespace
+ pub fn get_namespace(&self) -> Rc<MetaString> {
+ self.namespace.clone()
}
- pub fn get_type_name(&self) -> &MetaString {
- &self.type_name
+ pub fn get_type_name(&self) -> Rc<MetaString> {
+ self.type_name.clone()
}
pub fn get_type_def(&self) -> Rc<Vec<u8>> {
@@ -294,7 +294,7 @@ pub struct TypeResolver {
type_info_map_by_id: HashMap<u32, Rc<TypeInfo>>,
type_info_map: HashMap<std::any::TypeId, Rc<TypeInfo>>,
type_info_map_by_name: HashMap<(String, String), Rc<TypeInfo>>,
- type_info_map_by_ms_name: HashMap<(MetaString, MetaString), Rc<TypeInfo>>,
+ type_info_map_by_ms_name: HashMap<(Rc<MetaString>, Rc<MetaString>),
Rc<TypeInfo>>,
// Fast lookup by numeric ID for common types
type_id_index: Vec<u32>,
compatible: bool,
@@ -347,11 +347,11 @@ impl TypeResolver {
pub fn get_type_info_by_msname(
&self,
- namespace: &MetaString,
- type_name: &MetaString,
+ namespace: Rc<MetaString>,
+ type_name: Rc<MetaString>,
) -> Option<Rc<TypeInfo>> {
self.type_info_map_by_ms_name
- .get(&(namespace.clone(), type_name.clone()))
+ .get(&(namespace, type_name))
.cloned()
}
@@ -378,10 +378,10 @@ impl TypeResolver {
pub fn get_name_harness(
&self,
- namespace: &MetaString,
- type_name: &MetaString,
+ namespace: Rc<MetaString>,
+ type_name: Rc<MetaString>,
) -> Option<Rc<Harness>> {
- let key = (namespace.clone(), type_name.clone());
+ let key = (namespace, type_name);
self.type_info_map_by_ms_name
.get(&key)
.map(|info| Rc::new(info.get_harness().clone()))
@@ -396,10 +396,10 @@ impl TypeResolver {
pub fn get_ext_name_harness(
&self,
- namespace: &MetaString,
- type_name: &MetaString,
+ namespace: Rc<MetaString>,
+ type_name: Rc<MetaString>,
) -> Result<Rc<Harness>, Error> {
- let key = (namespace.clone(), type_name.clone());
+ let key = (namespace, type_name);
self.type_info_map_by_ms_name
.get(&key)
.map(|info| Rc::new(info.get_harness().clone()))
diff --git a/rust/fory-core/src/serializer/enum_.rs
b/rust/fory-core/src/serializer/enum_.rs
index 2d07ec109..46755f210 100644
--- a/rust/fory-core/src/serializer/enum_.rs
+++ b/rust/fory-core/src/serializer/enum_.rs
@@ -60,10 +60,10 @@ pub fn write_type_info<T: Serializer>(context: &mut
WriteContext) -> Result<(),
context.writer.write_varuint32(meta_index);
} else {
let type_info =
context.get_type_resolver().get_type_info(&rs_type_id)?;
- let namespace = type_info.get_namespace().to_owned();
- let type_name = type_info.get_type_name().to_owned();
- context.write_meta_string_bytes(&namespace)?;
- context.write_meta_string_bytes(&type_name)?;
+ let namespace = type_info.get_namespace();
+ let type_name = type_info.get_type_name();
+ context.write_meta_string_bytes(namespace)?;
+ context.write_meta_string_bytes(type_name)?;
}
Ok(())
}
@@ -111,8 +111,8 @@ pub fn read_type_info<T: Serializer>(context: &mut
ReadContext) -> Result<(), Er
if context.is_share_meta() {
let _meta_index = context.reader.read_varuint32()?;
} else {
- let _namespace_msb = context.read_meta_string_bytes()?;
- let _type_name_msb = context.read_meta_string_bytes()?;
+ let _namespace_msb = context.read_meta_string()?;
+ let _type_name_msb = context.read_meta_string()?;
}
Ok(())
}
diff --git a/rust/fory-core/src/serializer/skip.rs
b/rust/fory-core/src/serializer/skip.rs
index 8cb0d421a..524d0ab26 100644
--- a/rust/fory-core/src/serializer/skip.rs
+++ b/rust/fory-core/src/serializer/skip.rs
@@ -181,7 +181,7 @@ pub fn skip_value(
let type_resolver = context.get_type_resolver();
let type_meta = type_info.get_type_meta();
type_resolver
- .get_ext_name_harness(&type_meta.get_namespace(),
&type_meta.get_type_name())?
+ .get_ext_name_harness(type_meta.get_namespace(),
type_meta.get_type_name())?
.get_read_data_fn()(context)?;
Ok(())
} else {
diff --git a/rust/fory-core/src/serializer/struct_.rs
b/rust/fory-core/src/serializer/struct_.rs
index 98597e60b..4f821bd9c 100644
--- a/rust/fory-core/src/serializer/struct_.rs
+++ b/rust/fory-core/src/serializer/struct_.rs
@@ -50,10 +50,10 @@ pub fn write_type_info<T: Serializer>(context: &mut
WriteContext) -> Result<(),
context.writer.write_varuint32(meta_index);
} else {
let type_info =
context.get_type_resolver().get_type_info(&rs_type_id)?;
- let namespace = type_info.get_namespace().to_owned();
- let type_name = type_info.get_type_name().to_owned();
- context.write_meta_string_bytes(&namespace)?;
- context.write_meta_string_bytes(&type_name)?;
+ let namespace = type_info.get_namespace();
+ let type_name = type_info.get_type_name();
+ context.write_meta_string_bytes(namespace)?;
+ context.write_meta_string_bytes(type_name)?;
}
} else if type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32
|| type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32
@@ -77,8 +77,8 @@ pub fn read_type_info<T: Serializer>(context: &mut
ReadContext) -> Result<(), Er
if context.is_share_meta() {
let _meta_index = context.reader.read_varuint32()?;
} else {
- let _namespace_msb = context.read_meta_string_bytes()?;
- let _type_name_msb = context.read_meta_string_bytes()?;
+ let _namespace_msb = context.read_meta_string()?;
+ let _type_name_msb = context.read_meta_string()?;
}
} else if local_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32
|| local_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32
diff --git a/rust/tests/tests/test_cross_language.rs
b/rust/tests/tests/test_cross_language.rs
index 9a61cc55c..207507eb4 100644
--- a/rust/tests/tests/test_cross_language.rs
+++ b/rust/tests/tests/test_cross_language.rs
@@ -662,25 +662,6 @@ fn _test_skip_custom(fory1: &Fory, fory2: &Fory) {
fs::write(&data_file_path, bytes).unwrap();
}
-#[test]
-fn test_kankankan() {
- let mut fory1 = Fory::default().compatible(true);
- fory1.register_serializer::<MyExt>(103).unwrap();
- fory1.register::<Empty>(104).unwrap();
- let mut fory2 = Fory::default().compatible(true);
- fory2.register::<Color>(101).unwrap();
- fory2.register::<MyStruct>(102).unwrap();
- fory2.register_serializer::<MyExt>(103).unwrap();
- fory2.register::<MyWrapper>(104).unwrap();
- let wrapper = MyWrapper {
- color: Color::White,
- my_struct: MyStruct { id: 42 },
- my_ext: MyExt { id: 43 },
- };
- let bytes = fory2.serialize(&wrapper).unwrap();
- fory1.deserialize::<Empty>(&bytes).unwrap();
-}
-
#[test]
#[ignore]
fn test_skip_id_custom() {
@@ -716,7 +697,7 @@ fn test_skip_name_custom() {
#[test]
#[ignore]
fn test_consistent_named() {
- let mut fory = Fory::default().compatible(true);
+ let mut fory = Fory::default().compatible(false);
fory.register_by_name::<Color>("color").unwrap();
fory.register_by_name::<MyStruct>("my_struct").unwrap();
fory.register_serializer_by_name::<MyExt>("my_ext").unwrap();
@@ -730,47 +711,31 @@ fn test_consistent_named() {
let reader = Reader::new(bytes.as_slice());
let mut context = ReadContext::new_from_fory(reader, &fory);
- assert_eq!(
- fory.deserialize_with_context::<Color>(&mut context)
- .unwrap(),
- color
- );
- assert_eq!(
- fory.deserialize_with_context::<Color>(&mut context)
- .unwrap(),
- color
- );
- assert_eq!(
- fory.deserialize_with_context::<Color>(&mut context)
- .unwrap(),
- color
- );
+ for _ in 0..3 {
+ assert_eq!(
+ fory.deserialize_with_context::<Color>(&mut context)
+ .unwrap(),
+ color
+ );
+ }
+ for _ in 0..3 {
+ assert_eq!(
+ fory.deserialize_with_context::<MyExt>(&mut context)
+ .unwrap(),
+ my_ext
+ );
+ }
// assert_eq!(fory.deserialize_with_context::<MyStruct>(&mut
context).unwrap(), my_struct);
- assert_eq!(
- fory.deserialize_with_context::<MyExt>(&mut context)
- .unwrap(),
- my_ext
- );
- assert_eq!(
- fory.deserialize_with_context::<MyExt>(&mut context)
- .unwrap(),
- my_ext
- );
- assert_eq!(
- fory.deserialize_with_context::<MyExt>(&mut context)
- .unwrap(),
- my_ext
- );
let writer = Writer::default();
let mut context = WriteContext::new_from_fory(writer, &fory);
- fory.serialize_with_context(&color, &mut context).unwrap();
- fory.serialize_with_context(&color, &mut context).unwrap();
- fory.serialize_with_context(&color, &mut context).unwrap();
- // todo: checkVersion
- // fory.serialize_with_context(&my_struct, &mut context);
- fory.serialize_with_context(&my_ext, &mut context).unwrap();
- fory.serialize_with_context(&my_ext, &mut context).unwrap();
- fory.serialize_with_context(&my_ext, &mut context).unwrap();
+ for _ in 0..3 {
+ fory.serialize_with_context(&color, &mut context).unwrap();
+ }
+ for _ in 0..3 {
+ fory.serialize_with_context(&my_ext, &mut context).unwrap();
+ }
+ // // todo: checkVersion
+ // // fory.serialize_with_context(&my_struct, &mut context);
fs::write(&data_file_path, context.writer.dump()).unwrap();
}
diff --git a/rust/tests/tests/test_metastring_resolver.rs
b/rust/tests/tests/test_metastring_resolver.rs
new file mode 100644
index 000000000..a3f6ff3d6
--- /dev/null
+++ b/rust/tests/tests/test_metastring_resolver.rs
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use fory_core::meta::NAMESPACE_ENCODER;
+use fory_core::resolver::metastring_resolver::{
+ MetaStringReaderResolver, MetaStringWriterResolver,
+};
+use fory_core::{Reader, Writer};
+use std::rc::Rc;
+
+#[test]
+pub fn empty() {
+ let mut ms_writer = MetaStringWriterResolver::default();
+ let mut ms_reader = MetaStringReaderResolver::default();
+
+ for _ in 0..3 {
+ let ms = NAMESPACE_ENCODER.encode("").unwrap();
+ let rc_ms = Rc::from(ms);
+
+ let mut writer = Writer::default();
+ ms_writer
+ .write_meta_string_bytes(&mut writer, rc_ms.clone())
+ .unwrap();
+
+ let binding = writer.dump();
+ let mut reader = Reader::new(binding.as_slice());
+
+ let read_ms = ms_reader.read_meta_string(&mut reader).unwrap();
+ assert_eq!(&*rc_ms, read_ms);
+ ms_writer.reset();
+ ms_reader.reset();
+ }
+}
+
+#[test]
+pub fn small_ms() {
+ let mut ms_writer = MetaStringWriterResolver::default();
+ let mut ms_reader = MetaStringReaderResolver::default();
+ // test reset
+ for _ in 0..3 {
+ // write
+ let mut data = Vec::new();
+ for i in 0..20 {
+ let ms = NAMESPACE_ENCODER.encode(&format!("ms_{i}")).unwrap();
+ let rc_ms = Rc::from(ms);
+ // test cache
+ for _ in 0..3 {
+ data.push(rc_ms.clone());
+ }
+ }
+ let mut writer = Writer::default();
+ for ms in data.iter() {
+ ms_writer
+ .write_meta_string_bytes(&mut writer, ms.clone())
+ .unwrap();
+ }
+ // read
+ let binding = writer.dump();
+ let mut reader = Reader::new(binding.as_slice());
+ let read_data: Vec<_> = (0..60)
+ .map(|_| ms_reader.read_meta_string(&mut reader).unwrap().clone())
+ .collect();
+ for i in 0..60 {
+ assert_eq!(*data[i], read_data[i]);
+ }
+ ms_writer.reset();
+ ms_reader.reset();
+ }
+}
+
+#[test]
+pub fn big_ms() {
+ let long_string = "a".repeat(50);
+ let mut ms_writer = MetaStringWriterResolver::default();
+ let mut ms_reader = MetaStringReaderResolver::default();
+ // test reset
+ for _ in 0..3 {
+ // write
+ let mut data = Vec::new();
+ for i in 0..20 {
+ let ms = NAMESPACE_ENCODER
+ .encode(&format!("{long_string}_{i}"))
+ .unwrap();
+ let rc_ms = Rc::from(ms);
+ // test cache
+ for _ in 0..3 {
+ data.push(rc_ms.clone());
+ }
+ }
+ let mut writer = Writer::default();
+ for ms in data.iter() {
+ ms_writer
+ .write_meta_string_bytes(&mut writer, ms.clone())
+ .unwrap();
+ }
+ // read
+ let binding = writer.dump();
+ let mut reader = Reader::new(binding.as_slice());
+ let read_data: Vec<_> = (0..60)
+ .map(|_| ms_reader.read_meta_string(&mut reader).unwrap().clone())
+ .collect();
+ for i in 0..60 {
+ assert_eq!(*data[i], read_data[i]);
+ }
+ ms_writer.reset();
+ ms_reader.reset();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
