This is an automated email from the ASF dual-hosted git repository.
pandalee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 024a7a2a2 fix(rust): fix rust xlang tests (#2788)
024a7a2a2 is described below
commit 024a7a2a2e15e07c927644cce0d1f7fe1319d747
Author: Shawn Yang <[email protected]>
AuthorDate: Mon Oct 20 23:15:41 2025 +0800
fix(rust): fix rust xlang tests (#2788)
## Why?
<!-- Describe the purpose of this PR. -->
## What does this PR do?
- fix rust map read
- fix java map read
- enable rust xlang tests
## Related issues
<!--
Is there any related issue? If this PR closes them you say say
fix/closes:
- #xxxx0
- #xxxx1
- Fixes #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.
Delete section if not applicable.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
Delete section if not applicable.
-->
---
.../src/main/java/org/apache/fory/Fory.java | 5 ++
.../serializer/collection/MapLikeSerializer.java | 12 +++-
.../collection/SerializationBinding.java | 12 ++++
.../test/java/org/apache/fory/RustXlangTest.java | 2 +-
rust/fory-core/src/resolver/metastring_resolver.rs | 64 +++++++++++++---------
rust/fory-core/src/serializer/map.rs | 57 +++++++++++--------
6 files changed, 100 insertions(+), 52 deletions(-)
diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java
b/java/fory-core/src/main/java/org/apache/fory/Fory.java
index 43b7b74df..3ce595398 100644
--- a/java/fory-core/src/main/java/org/apache/fory/Fory.java
+++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java
@@ -1124,6 +1124,11 @@ public final class Fory implements BaseFory {
}
}
+ public Object xreadNonRef(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder) {
+ ClassInfo classInfo = xtypeResolver.readClassInfo(buffer, classInfoHolder);
+ return xreadNonRef(buffer, classInfo);
+ }
+
public Object xreadNullable(MemoryBuffer buffer, Serializer<Object>
serializer) {
byte headFlag = buffer.readByte();
if (headFlag == Fory.NULL_FLAG) {
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java
index c46e1d7fb..1794317b9 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java
@@ -699,7 +699,11 @@ public abstract class MapLikeSerializer<T> extends
Serializer<T> {
fory.decDepth();
}
} else {
- key = binding.readRef(buffer, keyClassInfoReadCache);
+ if (trackKeyRef) {
+ key = binding.readRef(buffer, keyClassInfoReadCache);
+ } else {
+ key = binding.readNonRef(buffer, keyClassInfoReadCache);
+ }
}
map.put(key, null);
}
@@ -741,7 +745,11 @@ public abstract class MapLikeSerializer<T> extends
Serializer<T> {
fory.decDepth();
}
} else {
- value = binding.readRef(buffer, valueClassInfoReadCache);
+ if (trackValueRef) {
+ value = binding.readRef(buffer, valueClassInfoReadCache);
+ } else {
+ value = binding.readNonRef(buffer, valueClassInfoReadCache);
+ }
}
map.put(null, value);
} else {
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java
index 6b461f3b1..86dda6849 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java
@@ -49,6 +49,8 @@ interface SerializationBinding {
Object readNonRef(MemoryBuffer buffer);
+ Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder);
+
static SerializationBinding createBinding(Fory fory) {
if (fory.isCrossLanguage()) {
return new XlangSerializationBinding(fory);
@@ -99,6 +101,11 @@ interface SerializationBinding {
return fory.readNonRef(buffer);
}
+ @Override
+ public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder) {
+ return fory.readNonRef(buffer, classInfoHolder);
+ }
+
@Override
public void write(MemoryBuffer buffer, Serializer serializer, Object
value) {
serializer.write(buffer, value);
@@ -158,6 +165,11 @@ interface SerializationBinding {
return fory.xreadNonRef(buffer);
}
+ @Override
+ public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder
classInfoHolder) {
+ return fory.xreadNonRef(buffer, classInfoHolder);
+ }
+
@Override
public void write(MemoryBuffer buffer, Serializer serializer, Object
value) {
serializer.xwrite(buffer, value);
diff --git a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
index f19367d6f..eecd52eb0 100644
--- a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
+++ b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
@@ -77,7 +77,7 @@ public class RustXlangTest extends ForyTestBase {
@BeforeClass
public void isRustJavaCIEnabled() {
- String enabled = "0";
+ String enabled = System.getenv("RUST_TESTCASE_ENABLED");
if (enabled == null || !enabled.equals("1")) {
throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not
set to 1");
}
diff --git a/rust/fory-core/src/resolver/metastring_resolver.rs
b/rust/fory-core/src/resolver/metastring_resolver.rs
index beb6f4445..975d27e67 100644
--- a/rust/fory-core/src/resolver/metastring_resolver.rs
+++ b/rust/fory-core/src/resolver/metastring_resolver.rs
@@ -81,18 +81,38 @@ impl MetaStringBytes {
pub(crate) fn from_metastring(meta_string: &MetaString) -> Result<Self,
Error> {
let mut bytes = meta_string.bytes.to_vec();
- let mut hash_code = murmurhash3_x64_128(&bytes, 47).0 as i64;
- hash_code = hash_code.abs();
- if hash_code == 0 {
- hash_code += 256;
+ let len = bytes.len();
+ // follow python/java implementation: small strings use quick v1/v2
based hash,
+ // large strings use murmurhash, then mask lower 8 bits for encoding
+ let encoding_val = meta_string.encoding as i64 & HEADER_MASK;
+ let hash_code: i64;
+ if len <= MetaStringReaderResolver::SMALL_STRING_THRESHOLD {
+ // compute v1 and v2 from bytes little-endian
+ let mut v1: u64 = 0;
+ let mut v2: u64 = 0;
+ for (i, b) in bytes.iter().take(8).enumerate() {
+ v1 |= (*b as u64) << (8 * i);
+ }
+ if bytes.len() > 8 {
+ for j in 0..usize::min(8, bytes.len() - 8) {
+ v2 |= (bytes[8 + j] as u64) << (8 * j);
+ }
+ }
+ // ((v1 * 31 + v2) >> 8 << 8) | encoding
+ let tmp: u128 = (v1 as u128).wrapping_mul(31u128).wrapping_add(v2
as u128);
+ let masked = ((tmp >> 8) << 8) as u64;
+ hash_code = masked as i64 | encoding_val;
+ } else {
+ let mut hc = murmurhash3_x64_128(&bytes, 47).0 as i64;
+ hc = hc.abs();
+ if hc == 0 {
+ hc += 256;
+ }
+ hc = (hc as u64 & 0xffffffffffffff00) as i64;
+ hash_code = hc | encoding_val;
}
- hash_code = (hash_code as u64 & 0xffffffffffffff00) as i64;
- let encoding = meta_string.encoding;
- let header = encoding as i64 & HEADER_MASK;
- hash_code |= header;
- let header = (hash_code & HEADER_MASK) as u8;
- let encoding = byte_to_encoding(header);
+ // ensure we have 16 bytes to extract first8/second8
if bytes.len() < 16 {
bytes.resize(16, 0);
}
@@ -107,7 +127,13 @@ impl MetaStringBytes {
})?;
let second8 = u64::from_le_bytes(second8);
- Ok(Self::new(bytes, hash_code, encoding, first8, second8))
+ Ok(Self::new(
+ bytes,
+ hash_code,
+ byte_to_encoding((hash_code & HEADER_MASK) as u8),
+ first8,
+ second8,
+ ))
}
}
@@ -134,20 +160,8 @@ impl MetaStringWriterResolver {
if let Some(b) = self.meta_string_to_bytes.get(m) {
return b.clone();
}
- let bytes = m.bytes.to_vec();
- let hash_code = murmurhash3_x64_128(&bytes, 47).0 as i64;
- let encoding = m.encoding;
- let mut first8: u64 = 0;
- let mut second8: u64 = 0;
- for (i, b) in bytes.iter().take(8).enumerate() {
- first8 |= (*b as u64) << (8 * i);
- }
- if bytes.len() > 8 {
- for j in 0..usize::min(8, bytes.len() - 8) {
- second8 |= (bytes[8 + j] as u64) << (8 * j);
- }
- }
- let msb = MetaStringBytes::new(bytes, hash_code, encoding, first8,
second8);
+ // create via from_metastring to keep same hash/encoding rules
+ let msb =
MetaStringBytes::from_metastring(m).expect("from_metastring");
self.meta_string_to_bytes.insert(m.clone(), msb.clone());
msb
}
diff --git a/rust/fory-core/src/serializer/map.rs
b/rust/fory-core/src/serializer/map.rs
index 876a0980a..f3b2fd818 100644
--- a/rust/fory-core/src/serializer/map.rs
+++ b/rust/fory-core/src/serializer/map.rs
@@ -559,20 +559,16 @@ impl<K: Serializer + ForyDefault + Eq + std::hash::Hash,
V: Serializer + ForyDef
}
let key_declared = (header & DECL_KEY_TYPE) != 0;
let value_declared = (header & DECL_VALUE_TYPE) != 0;
+ let track_key_ref = (header & TRACKING_KEY_REF) != 0;
+ let track_value_ref = (header & TRACKING_VALUE_REF) != 0;
if header & KEY_NULL != 0 {
- if !value_declared {
- V::fory_read_type_info(context)?;
- }
- let value = V::fory_read_data(context)?;
+ let value = V::fory_read(context, track_value_ref,
!value_declared)?;
map.insert(K::fory_default(), value);
len_counter += 1;
continue;
}
if header & VALUE_NULL != 0 {
- if !key_declared {
- K::fory_read_type_info(context)?;
- }
- let key = K::fory_read_data(context)?;
+ let key = K::fory_read(context, track_key_ref, !key_declared)?;
map.insert(key, V::fory_default());
len_counter += 1;
continue;
@@ -592,11 +588,20 @@ impl<K: Serializer + ForyDefault + Eq + std::hash::Hash,
V: Serializer + ForyDef
cur_len, len
))
);
- for _ in 0..chunk_size {
- let key = K::fory_read_data(context)?;
- let value = V::fory_read_data(context)?;
- map.insert(key, value);
+ if !track_key_ref && !track_value_ref {
+ for _ in 0..chunk_size {
+ let key = K::fory_read_data(context)?;
+ let value = V::fory_read_data(context)?;
+ map.insert(key, value);
+ }
+ } else {
+ for _ in 0..chunk_size {
+ let key = K::fory_read(context, track_key_ref, false)?;
+ let value = V::fory_read(context, track_value_ref, false)?;
+ map.insert(key, value);
+ }
}
+ // advance the counter after processing the chunk
len_counter += chunk_size as u32;
}
Ok(map)
@@ -683,20 +688,16 @@ impl<K: Serializer + ForyDefault + Ord + std::hash::Hash,
V: Serializer + ForyDe
}
let key_declared = (header & DECL_KEY_TYPE) != 0;
let value_declared = (header & DECL_VALUE_TYPE) != 0;
+ let track_key_ref = (header & TRACKING_KEY_REF) != 0;
+ let track_value_ref = (header & TRACKING_VALUE_REF) != 0;
if header & KEY_NULL != 0 {
- if !value_declared {
- V::fory_read_type_info(context)?;
- }
- let value = V::fory_read_data(context)?;
+ let value = V::fory_read(context, track_value_ref,
!value_declared)?;
map.insert(K::fory_default(), value);
len_counter += 1;
continue;
}
if header & VALUE_NULL != 0 {
- if !key_declared {
- K::fory_read_type_info(context)?;
- }
- let key = K::fory_read_data(context)?;
+ let key = K::fory_read(context, track_key_ref, !key_declared)?;
map.insert(key, V::fory_default());
len_counter += 1;
continue;
@@ -716,10 +717,18 @@ impl<K: Serializer + ForyDefault + Ord + std::hash::Hash,
V: Serializer + ForyDe
cur_len, len
))
);
- for _ in 0..chunk_size {
- let key = K::fory_read_data(context)?;
- let value = V::fory_read_data(context)?;
- map.insert(key, value);
+ if !track_key_ref && !track_value_ref {
+ for _ in 0..chunk_size {
+ let key = K::fory_read_data(context)?;
+ let value = V::fory_read_data(context)?;
+ map.insert(key, value);
+ }
+ } else {
+ for _ in 0..chunk_size {
+ let key = K::fory_read(context, track_key_ref, false)?;
+ let value = V::fory_read(context, track_value_ref, false)?;
+ map.insert(key, value);
+ }
}
len_counter += chunk_size as u32;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
