This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new cedba3007 feat(rust): support static struct version hash check for
rust (#2793)
cedba3007 is described below
commit cedba3007c084c588c404dec706efa3a651155de
Author: Shawn Yang <[email protected]>
AuthorDate: Tue Oct 21 16:26:30 2025 +0800
feat(rust): support static struct version hash check for rust (#2793)
## Why?
<!-- Describe the purpose of this PR. -->
## What does this PR do?
support struct version hash check for rust
## Related issues
<!--
Is there any related issue? If this PR closes them you say say
fix/closes:
- #xxxx0
- #xxxx1
- Fixes #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.
Delete section if not applicable.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
Delete section if not applicable.
-->
---
docs/specification/xlang_serialization_spec.md | 34 ++--
.../apache/fory/builder/ObjectCodecBuilder.java | 3 +-
.../main/java/org/apache/fory/config/Config.java | 7 +
.../java/org/apache/fory/config/ForyBuilder.java | 6 +
.../java/org/apache/fory/resolver/ClassInfo.java | 2 +-
.../org/apache/fory/resolver/XtypeResolver.java | 6 +-
.../serializer/NonexistentClassSerializers.java | 7 +-
.../apache/fory/serializer/ObjectSerializer.java | 107 +++++++-----
.../java/org/apache/fory/CrossLanguageTest.java | 8 +-
.../test/java/org/apache/fory/RustXlangTest.java | 48 +++++-
python/pyfory/_serialization.pyx | 2 +-
python/pyfory/_struct.py | 164 ++++++++++--------
python/pyfory/serializer.py | 28 +--
python/pyfory/tests/test_cross_language.py | 11 +-
rust/fory-core/src/error.rs | 27 +++
rust/fory-core/src/fory.rs | 61 +++++++
rust/fory-core/src/meta/type_meta.rs | 16 +-
rust/fory-core/src/resolver/context.rs | 20 +++
rust/fory-derive/src/object/read.rs | 13 +-
rust/fory-derive/src/object/util.rs | 188 ++++++++++++++++++++-
rust/fory-derive/src/object/write.rs | 11 +-
rust/tests/tests/test_cross_language.rs | 32 ++++
22 files changed, 634 insertions(+), 167 deletions(-)
diff --git a/docs/specification/xlang_serialization_spec.md
b/docs/specification/xlang_serialization_spec.md
index 4b562de27..7252f7bbf 100644
--- a/docs/specification/xlang_serialization_spec.md
+++ b/docs/specification/xlang_serialization_spec.md
@@ -815,28 +815,42 @@ nullable primitive field value:
+-----------+---------------+
| null flag | field value |
+-----------+---------------+
-field value of final type with ref tracking:
+other interal types supported by fory
| var bytes | var objects |
+-----------+-------------+
-| ref meta | value data |
+| null flag | value data |
+-----------+-------------+
-field value of final type without ref tracking:
+list field type:
| one byte | var objects |
+-----------+-------------+
-| null flag | field value |
+| ref meta | value data |
+set field type:
+| one byte | var objects |
+-----------+-------------+
-field value of non-final type with ref tracking:
-| one byte | var bytes | var objects |
-+-----------+-------------+-------------+
-| ref meta | type meta | value data |
+| ref meta | value data |
+map field type:
+| one byte | var objects |
++-----------+-------------+
+| ref meta | value data |
+-----------+-------------+-------------+
-field value of non-final type without ref tracking:
+other types such as enum/struct/ext
| one byte | var bytes | var objects |
+-----------+------------+------------+
-| null flag | type meta | value data |
+| ref flag | type meta | value data |
+-----------+------------+------------+
```
+Type hash algorithm:
+
+- Sort fields by fields sort algorithm
+- Start with string `""`
+- Iterate every field, append string by:
+ - `snow_case(field_name),`. For camelcase name, convert it to snow_case
first.
+ - `$type_id,`, for other fields, use type id `TypeId::UNKNOWN` instead.
+ - `$nullable;`, `1` if nullable, `0` otherwise.
+- Then convert string to utf8 bytes
+- Compute murmurhash3_x64_128, and use first 32 bits
+
#### Schema evolution
Schema evolution have similar format as schema consistent mode for object
except:
diff --git
a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java
b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java
index de6fc10cd..c472aee6b 100644
---
a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java
+++
b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java
@@ -101,10 +101,9 @@ public class ObjectCodecBuilder extends
BaseObjectCodecBuilder {
}
Collection<Descriptor> p = descriptors;
DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(p,
false));
- descriptors = grouper.getSortedDescriptors();
classVersionHash =
fory.checkClassVersion()
- ? new Literal(ObjectSerializer.computeStructHash(fory,
descriptors), PRIMITIVE_INT_TYPE)
+ ? new Literal(ObjectSerializer.computeStructHash(fory, grouper),
PRIMITIVE_INT_TYPE)
: null;
objectCodecOptimizer =
new ObjectCodecOptimizer(beanClass, grouper,
!fory.isBasicTypesRefIgnored(), ctx);
diff --git a/java/fory-core/src/main/java/org/apache/fory/config/Config.java
b/java/fory-core/src/main/java/org/apache/fory/config/Config.java
index 3f7c8f8bd..46888240b 100644
--- a/java/fory-core/src/main/java/org/apache/fory/config/Config.java
+++ b/java/fory-core/src/main/java/org/apache/fory/config/Config.java
@@ -66,6 +66,9 @@ public class Config implements Serializable {
private final boolean serializeEnumByName;
private final int bufferSizeLimitBytes;
private final int maxDepth;
+ boolean foryDebugOutputEnabled =
+ "1".equals(System.getenv("ENABLE_FORY_DEBUG_OUTPUT"))
+ || "true".equals(System.getenv("ENABLE_FORY_DEBUG_OUTPUT"));
public Config(ForyBuilder builder) {
name = builder.name;
@@ -294,6 +297,10 @@ public class Config implements Serializable {
return scalaOptimizationEnabled;
}
+ public boolean isForyDebugOutputEnabled() {
+ return foryDebugOutputEnabled;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git
a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
index 38de05e15..a72d58c85 100644
--- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
+++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
@@ -264,6 +264,12 @@ public final class ForyBuilder {
* class won't evolve.
*/
public ForyBuilder withClassVersionCheck(boolean checkClassVersion) {
+ if (language == Language.XLANG
+ && compatibleMode == CompatibleMode.SCHEMA_CONSISTENT
+ && !checkClassVersion) {
+ throw new IllegalArgumentException(
+ "XLANG Schema consistent mode must enable class version check");
+ }
this.checkClassVersion = checkClassVersion;
return this;
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java
b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java
index 1d7b8fb97..6c0f943ce 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java
@@ -58,7 +58,7 @@ public class ClassInfo {
boolean isDynamicGeneratedClass,
Serializer<?> serializer,
short classId,
- short xtypeId) {
+ int xtypeId) {
this.cls = cls;
this.fullNameBytes = fullNameBytes;
this.namespaceBytes = namespaceBytes;
diff --git
a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
index 9d56c4829..34ac51918 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
@@ -251,7 +251,7 @@ public class XtypeResolver extends TypeResolver {
private void register(
Class<?> type, Serializer<?> serializer, String namespace, String
typeName, int xtypeId) {
- ClassInfo classInfo = newClassInfo(type, serializer, namespace, typeName,
(short) xtypeId);
+ ClassInfo classInfo = newClassInfo(type, serializer, namespace, typeName,
xtypeId);
String qualifiedName = qualifiedName(namespace, typeName);
qualifiedType2ClassInfo.put(qualifiedName, classInfo);
extRegistry.registeredClasses.put(qualifiedName, type);
@@ -316,7 +316,7 @@ public class XtypeResolver extends TypeResolver {
return serializer instanceof DeferedLazyObjectSerializer;
}
- private ClassInfo newClassInfo(Class<?> type, Serializer<?> serializer,
short xtypeId) {
+ private ClassInfo newClassInfo(Class<?> type, Serializer<?> serializer, int
xtypeId) {
return newClassInfo(
type,
serializer,
@@ -326,7 +326,7 @@ public class XtypeResolver extends TypeResolver {
}
private ClassInfo newClassInfo(
- Class<?> type, Serializer<?> serializer, String namespace, String
typeName, short xtypeId) {
+ Class<?> type, Serializer<?> serializer, String namespace, String
typeName, int xtypeId) {
MetaStringBytes fullClassNameBytes =
metaStringResolver.getOrCreateMetaStringBytes(
GENERIC_ENCODER.encode(type.getName(), MetaString.Encoding.UTF_8));
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java
index 04fe92e77..9b12e543e 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java
@@ -163,17 +163,16 @@ public final class NonexistentClassSerializers {
Collection<Descriptor> descriptors =
MetaSharedSerializer.consolidateFields(
resolver, NonexistentClass.NonexistentSkip.class, classDef);
- DescriptorGrouper descriptorGrouper =
+ DescriptorGrouper grouper =
fory.getClassResolver().createDescriptorGrouper(descriptors,
false);
Tuple3<
Tuple2<ObjectSerializer.FinalTypeField[], boolean[]>,
ObjectSerializer.GenericTypeField[],
ObjectSerializer.GenericTypeField[]>
- tuple = AbstractObjectSerializer.buildFieldInfos(fory,
descriptorGrouper);
- descriptors = descriptorGrouper.getSortedDescriptors();
+ tuple = AbstractObjectSerializer.buildFieldInfos(fory, grouper);
int classVersionHash = 0;
if (fory.checkClassVersion()) {
- classVersionHash = ObjectSerializer.computeStructHash(fory,
descriptors);
+ classVersionHash = ObjectSerializer.computeStructHash(fory, grouper);
}
fieldsInfo =
new ClassFieldsInfo(tuple.f0.f0, tuple.f0.f1, tuple.f1, tuple.f2,
classVersionHash);
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
index db1cf2dcc..e44d00744 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
@@ -19,20 +19,21 @@
package org.apache.fory.serializer;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
import org.apache.fory.Fory;
import org.apache.fory.collection.Tuple2;
import org.apache.fory.collection.Tuple3;
import org.apache.fory.exception.ForyException;
+import org.apache.fory.logging.Logger;
+import org.apache.fory.logging.LoggerFactory;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.meta.ClassDef;
import org.apache.fory.reflect.FieldAccessor;
import org.apache.fory.reflect.ReflectionUtils;
-import org.apache.fory.reflect.TypeRef;
import org.apache.fory.resolver.ClassInfo;
import org.apache.fory.resolver.ClassResolver;
import org.apache.fory.resolver.RefResolver;
@@ -40,10 +41,9 @@ import org.apache.fory.resolver.TypeResolver;
import org.apache.fory.type.Descriptor;
import org.apache.fory.type.DescriptorGrouper;
import org.apache.fory.type.Generics;
-import org.apache.fory.type.TypeUtils;
import org.apache.fory.type.Types;
-import org.apache.fory.util.ExceptionUtils;
-import org.apache.fory.util.Preconditions;
+import org.apache.fory.util.MurmurHash3;
+import org.apache.fory.util.StringUtils;
import org.apache.fory.util.record.RecordInfo;
import org.apache.fory.util.record.RecordUtils;
@@ -63,6 +63,8 @@ import org.apache.fory.util.record.RecordUtils;
// TODO(chaokunyang) support generics optimization for {@code SomeClass<T>}
@SuppressWarnings({"unchecked"})
public final class ObjectSerializer<T> extends AbstractObjectSerializer<T> {
+ private static final Logger LOG =
LoggerFactory.getLogger(ObjectSerializer.class);
+
private final RecordInfo recordInfo;
private final FinalTypeField[] finalFields;
@@ -101,9 +103,9 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
} else {
descriptors = typeResolver.getFieldDescriptors(cls, resolveParent);
}
- DescriptorGrouper descriptorGrouper =
typeResolver.createDescriptorGrouper(descriptors, false);
- descriptors = descriptorGrouper.getSortedDescriptors();
- System.out.println(descriptors.stream().map(f ->
f.getName()).collect(Collectors.toList()));
+ DescriptorGrouper grouper =
typeResolver.createDescriptorGrouper(descriptors, false);
+ descriptors = grouper.getSortedDescriptors();
+
System.out.println(descriptors.stream().map(Descriptor::getName).collect(Collectors.toList()));
if (isRecord) {
List<String> fieldNames =
descriptors.stream().map(Descriptor::getName).collect(Collectors.toList());
@@ -112,12 +114,12 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
recordInfo = null;
}
if (fory.checkClassVersion()) {
- classVersionHash = computeStructHash(fory, descriptors);
+ classVersionHash = computeStructHash(fory, grouper);
} else {
classVersionHash = 0;
}
Tuple3<Tuple2<FinalTypeField[], boolean[]>, GenericTypeField[],
GenericTypeField[]> infos =
- buildFieldInfos(fory, descriptorGrouper);
+ buildFieldInfos(fory, grouper);
finalFields = infos.f0.f0;
isFinal = infos.f0.f1;
otherFields = infos.f1;
@@ -354,45 +356,68 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
return obj;
}
- public static int computeStructHash(Fory fory, Collection<Descriptor>
descriptors) {
- int hash = 17;
- for (Descriptor descriptor : descriptors) {
- hash = computeFieldHash(hash, fory, descriptor.getTypeRef());
+ public static int computeStructHash(Fory fory, DescriptorGrouper grouper) {
+ StringBuilder builder = new StringBuilder();
+ List<Descriptor> sorted = grouper.getSortedDescriptors();
+ for (Descriptor descriptor : sorted) {
+ Class<?> rawType = descriptor.getTypeRef().getRawType();
+ int typeId = getTypeId(fory, rawType);
+ String underscore =
StringUtils.lowerCamelToLowerUnderscore(descriptor.getName());
+ char nullable = rawType.isPrimitive() ? '0' : '1';
+ builder
+ .append(underscore)
+ .append(',')
+ .append(typeId)
+ .append(',')
+ .append(nullable)
+ .append(';');
+ }
+
+ String fingerprint = builder.toString();
+ byte[] bytes = fingerprint.getBytes(StandardCharsets.UTF_8);
+ long hashLong = MurmurHash3.murmurhash3_x64_128(bytes, 0, bytes.length,
47)[0];
+ int hash = (int) (hashLong & 0xffffffffL);
+ if (fory.getConfig().isForyDebugOutputEnabled()) {
+ String className =
+ sorted.isEmpty() ? "<unknown>" :
String.valueOf(sorted.get(0).getDeclaringClass());
+ LOG.info(
+ "[fory-debug] struct "
+ + className
+ + " version fingerprint=\""
+ + fingerprint
+ + "\" version hash="
+ + hash);
}
- Preconditions.checkState(hash != 0);
return hash;
}
- private static int computeFieldHash(int hash, Fory fory, TypeRef<?> typeRef)
{
- int id = 0;
- if (typeRef.isSubtypeOf(List.class)) {
- // TODO(chaokunyang) add list element type into schema hash
- id = Types.LIST;
- } else if (typeRef.isSubtypeOf(Map.class)) {
- // TODO(chaokunyang) add map key&value type into schema hash
- id = Types.MAP;
+ private static int getTypeId(Fory fory, Class<?> cls) {
+ TypeResolver resolver = fory._getTypeResolver();
+ if (resolver.isSet(cls)) {
+ return Types.SET;
+ } else if (resolver.isCollection(cls)) {
+ return Types.LIST;
+ } else if (resolver.isMap(cls)) {
+ return Types.MAP;
} else {
- try {
- TypeResolver resolver = fory._getTypeResolver();
- Class<?> cls = typeRef.getRawType();
- if (!ReflectionUtils.isAbstract(cls) && !cls.isInterface()) {
- ClassInfo classInfo = resolver.getClassInfo(cls);
- int xtypeId = id = classInfo.getXtypeId();
- if (Types.isNamedType(xtypeId & 0xff)) {
- id =
- TypeUtils.computeStringHash(
- classInfo.decodeNamespace() + classInfo.decodeTypeName());
- }
+ if (ReflectionUtils.isAbstract(cls) || cls.isInterface() ||
cls.isEnum()) {
+ return Types.UNKNOWN;
+ }
+ ClassInfo classInfo = resolver.getClassInfo(cls, false);
+ if (classInfo == null) {
+ return Types.UNKNOWN;
+ }
+ int typeId;
+ if (fory.isCrossLanguage()) {
+ typeId = classInfo.getXtypeId();
+ if (Types.isUserDefinedType((byte) typeId)) {
+ return Types.UNKNOWN;
}
- } catch (Exception e) {
- ExceptionUtils.ignore(e);
+ } else {
+ typeId = classInfo.getClassId();
}
+ return typeId;
}
- long newHash = ((long) hash) * 31 + id;
- while (newHash >= Integer.MAX_VALUE) {
- newHash /= 7;
- }
- return (int) newHash;
}
public static void checkClassVersion(Fory fory, int readHash, int
classVersionHash) {
diff --git
a/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java
b/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java
index ce357e273..a5111f3a7 100644
--- a/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java
+++ b/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java
@@ -72,6 +72,7 @@ import org.apache.fory.serializer.ObjectSerializer;
import org.apache.fory.serializer.Serializer;
import org.apache.fory.test.TestUtils;
import org.apache.fory.type.Descriptor;
+import org.apache.fory.type.DescriptorGrouper;
import org.apache.fory.util.DateTimeUtils;
import org.apache.fory.util.MurmurHash3;
import org.testng.Assert;
@@ -475,12 +476,13 @@ public class CrossLanguageTest extends ForyTestBase {
fory.serialize(new ComplexObject1()); // trigger serializer update
ObjectSerializer serializer = (ObjectSerializer)
fory.getSerializer(ComplexObject1.class);
Method method =
- ObjectSerializer.class.getDeclaredMethod("computeStructHash",
Fory.class, Collection.class);
+ ObjectSerializer.class.getDeclaredMethod(
+ "computeStructHash", Fory.class, DescriptorGrouper.class);
method.setAccessible(true);
TypeResolver resolver = fory._getTypeResolver();
Collection<Descriptor> descriptors =
resolver.getFieldDescriptors(ComplexObject1.class, false);
- descriptors = resolver.createDescriptorGrouper(descriptors,
false).getSortedDescriptors();
- Integer hash = (Integer) method.invoke(serializer, fory, descriptors);
+ DescriptorGrouper grouper = resolver.createDescriptorGrouper(descriptors,
false);
+ Integer hash = (Integer) method.invoke(serializer, fory, grouper);
MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(4);
buffer.writeInt32(hash);
roundBytes("test_struct_hash", buffer.getBytes(0, 4));
diff --git a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
index 6fa5310b1..09c5dd512 100644
--- a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
+++ b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
@@ -123,6 +123,8 @@ public class RustXlangTest extends ForyTestBase {
testSkipIdCustom(Language.RUST, command);
command.set(RUST_TESTCASE_INDEX, "test_skip_name_custom");
testSkipNameCustom(Language.RUST, command);
+ command.set(RUST_TESTCASE_INDEX, "test_struct_version_check");
+ testStructVersionCheck(Language.RUST, command);
command.set(RUST_TESTCASE_INDEX, "test_consistent_named");
testConsistentNamed(Language.RUST, command);
}
@@ -795,7 +797,7 @@ public class RustXlangTest extends ForyTestBase {
.withLanguage(Language.XLANG)
.withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT)
.withCodegen(false)
- .withClassVersionCheck(false)
+ .withClassVersionCheck(true)
.build();
fory.register(Color.class, "color");
fory.register(MyStruct.class, "my_struct");
@@ -826,6 +828,44 @@ public class RustXlangTest extends ForyTestBase {
}
}
+ @Data
+ static class VersionCheckStruct {
+ int f1;
+ String f2;
+ double f3;
+ }
+
+ private void testStructVersionCheck(Language language, List<String> command)
+ throws java.io.IOException {
+ Fory fory =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT)
+ .withCodegen(false)
+ .withClassVersionCheck(true)
+ .build();
+ fory.register(VersionCheckStruct.class, 201);
+
+ VersionCheckStruct obj = new VersionCheckStruct();
+ obj.f1 = 10;
+ obj.f2 = "test";
+ obj.f3 = 3.2;
+
+ MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32);
+ fory.serialize(buffer, obj);
+ byte[] bytes = buffer.getBytes(0, buffer.writerIndex());
+ // Debug: print first 30 bytes
+ System.out.println(
+ "Java serialized bytes (first 30): "
+ + java.util.Arrays.toString(
+ java.util.Arrays.copyOf(bytes, Math.min(30, bytes.length))));
+ Path dataFile = Files.createTempFile("test_struct_version_check", "data");
+ Pair<Map<String, String>, File> env_workdir = setFilePath(language,
command, dataFile, bytes);
+ Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(),
env_workdir.getRight()));
+ MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile));
+ Assert.assertEquals(fory.deserialize(buffer2), obj);
+ }
+
/**
* Execute an external command.
*
@@ -850,7 +890,11 @@ public class RustXlangTest extends ForyTestBase {
if (peerLanguage == Language.RUST) {
return Pair.of(
ImmutableMap.of(
- "DATA_FILE", dataFile.toAbsolutePath().toString(), "RUSTFLAGS",
"-Awarnings"),
+ "DATA_FILE", dataFile.toAbsolutePath().toString(),
+ "RUSTFLAGS", "-Awarnings",
+ "RUST_BACKTRACE", "1",
+ "ENABLE_FORY_DEBUG_OUTPUT", "1",
+ "FORY_PANIC_ON_ERROR", "1"),
new File("../../rust"));
} else {
return Pair.of(Collections.emptyMap(), new File("../../python"));
diff --git a/python/pyfory/_serialization.pyx b/python/pyfory/_serialization.pyx
index 9b163fdf4..188c48320 100644
--- a/python/pyfory/_serialization.pyx
+++ b/python/pyfory/_serialization.pyx
@@ -411,7 +411,7 @@ cdef class TypeInfo:
when serializing the actual data.
"""
cdef public object cls
- cdef public int16_t type_id
+ cdef public int32_t type_id
cdef public Serializer serializer
cdef public MetaStringBytes namespace_bytes
cdef public MetaStringBytes typename_bytes
diff --git a/python/pyfory/_struct.py b/python/pyfory/_struct.py
index 481e695fe..31755f587 100644
--- a/python/pyfory/_struct.py
+++ b/python/pyfory/_struct.py
@@ -18,8 +18,10 @@
import datetime
import enum
import logging
+import os
import typing
+from pyfory.lib.mmh3 import hash_buffer
from pyfory.type import (
TypeVisitor,
infer_field,
@@ -31,7 +33,6 @@ from pyfory.type import (
Float32Type,
Float64Type,
is_py_array_type,
- compute_string_hash,
is_primitive_type,
)
@@ -47,6 +48,37 @@ from pyfory.type import is_subclass
logger = logging.getLogger(__name__)
+_TYPE_HASH_SEED = 47
+
+
+def _extract_primary_type_id(type_ids):
+ if isinstance(type_ids, (list, tuple)):
+ if not type_ids:
+ return TypeId.UNKNOWN
+ return type_ids[0]
+ return type_ids
+
+
+def _normalize_type_id(raw_type_id):
+ if not isinstance(raw_type_id, int):
+ return TypeId.UNKNOWN
+ base_type = raw_type_id & 0xFF
+ if base_type >= TypeId.BOUND:
+ return TypeId.UNKNOWN
+ if TypeId.is_namespaced_type(base_type):
+ return TypeId.UNKNOWN
+ return base_type
+
+
+def _to_snow_case(name: str) -> str:
+ chars = []
+ for index, ch in enumerate(name):
+ if ch.isupper() and index > 0:
+ chars.append("_")
+ chars.append(ch.lower())
+ return "".join(chars)
+
+
basic_types = {
bool,
Int8Type,
@@ -108,20 +140,19 @@ class StructFieldSerializerVisitor(TypeVisitor):
return serializer
-def _get_hash(fory, field_names: list, type_hints: dict):
- visitor = StructHashVisitor(fory)
- for index, key in enumerate(field_names):
- infer_field(key, type_hints[key], visitor, types_path=[])
- hash_ = visitor.get_hash()
- assert hash_ != 0
- return hash_
-
-
_UNKNOWN_TYPE_ID = -1
_time_types = {datetime.date, datetime.datetime, datetime.timedelta}
def _sort_fields(type_resolver, field_names, serializers, nullable_map=None):
+ (boxed_types, nullable_boxed_types, internal_types, collection_types,
set_types, map_types, other_types) = group_fields(
+ type_resolver, field_names, serializers, nullable_map
+ )
+ all_types = boxed_types + nullable_boxed_types + internal_types +
collection_types + set_types + map_types + other_types
+ return [t[2] for t in all_types], [t[1] for t in all_types]
+
+
+def group_fields(type_resolver, field_names, serializers, nullable_map=None):
nullable_map = nullable_map or {}
boxed_types = []
nullable_boxed_types = []
@@ -182,67 +213,62 @@ def _sort_fields(type_resolver, field_names, serializers,
nullable_map=None):
internal_types = sorted(internal_types, key=sorter)
map_types = sorted(map_types, key=sorter)
other_types = sorted(other_types, key=lambda item: item[2])
- all_types = boxed_types + nullable_boxed_types + internal_types +
collection_types + set_types + map_types + other_types
- return [t[2] for t in all_types], [t[1] for t in all_types]
-
-
-class StructHashVisitor(TypeVisitor):
- def __init__(
- self,
- fory,
+ return (boxed_types, nullable_boxed_types, internal_types,
collection_types, set_types, map_types, other_types)
+
+
+def compute_struct_meta(type_resolver, field_names, serializers,
nullable_map=None):
+ (boxed_types, nullable_boxed_types, internal_types, collection_types,
set_types, map_types, other_types) = group_fields(
+ type_resolver, field_names, serializers, nullable_map
+ )
+
+ # Build hash string
+ hash_parts = []
+
+ # boxed_types => non-nullable
+ for field in boxed_types:
+ type_id = field[0]
+ field_name = field[2] # already snake_case
+ nullable_flag = "0"
+ hash_parts.append(f"{field_name},{type_id},{nullable_flag};")
+
+ # All other groups => nullable
+ for group in (
+ nullable_boxed_types,
+ internal_types,
+ collection_types,
+ set_types,
+ map_types,
):
- self.fory = fory
- self._hash = 17
-
- def visit_list(self, field_name, elem_type, types_path=None):
- # TODO add list element type to hash.
- xtype_id = self.fory.type_resolver.get_typeinfo(list).type_id
- self._hash = self._compute_field_hash(self._hash, abs(xtype_id))
-
- def visit_set(self, field_name, elem_type, types_path=None):
- # TODO add set element type to hash.
- xtype_id = self.fory.type_resolver.get_typeinfo(set).type_id
- self._hash = self._compute_field_hash(self._hash, abs(xtype_id))
-
- def visit_dict(self, field_name, key_type, value_type, types_path=None):
- # TODO add map key/value type to hash.
- xtype_id = self.fory.type_resolver.get_typeinfo(dict).type_id
- self._hash = self._compute_field_hash(self._hash, abs(xtype_id))
-
- def visit_customized(self, field_name, type_, types_path=None):
- typeinfo = self.fory.type_resolver.get_typeinfo(type_, create=False)
- hash_value = 0
- if typeinfo is not None:
- hash_value = typeinfo.type_id
- if TypeId.is_namespaced_type(typeinfo.type_id):
- namespace_str = typeinfo.decode_namespace()
- typename_str = typeinfo.decode_typename()
- hash_value = compute_string_hash(namespace_str + typename_str)
- self._hash = self._compute_field_hash(self._hash, hash_value)
+ for field in group:
+ type_id = field[0]
+ field_name = field[2]
+ nullable_flag = "1"
+ hash_parts.append(f"{field_name},{type_id},{nullable_flag};")
+ for field in other_types:
+ type_id = TypeId.UNKNOWN
+ field_name = field[2]
+ nullable_flag = "1"
+ hash_parts.append(f"{field_name},{type_id},{nullable_flag};")
+
+ hash_str = "".join(hash_parts)
+ hash_bytes = hash_str.encode("utf-8")
+
+ full_hash = hash_buffer(hash_bytes, seed=47)[0]
+ type_hash_32 = full_hash & 0xFFFFFFFF
+ if full_hash & 0x80000000:
+ # If the sign bit is set, it's a negative number in 2's complement
+ # Subtract 2^32 to get the correct negative value
+ type_hash_32 = type_hash_32 - 0x100000000
+ assert type_hash_32 != 0
+ if os.environ.get("ENABLE_FORY_DEBUG_OUTPUT", "").lower() in ("1", "true"):
+ print(f'[fory-debug] struct version fingerprint="{hash_str}" version
hash={type_hash_32}')
+
+ # Flatten all groups in correct order (already sorted from group_fields)
+ all_types = boxed_types + nullable_boxed_types + internal_types +
collection_types + set_types + map_types + other_types
+ sorted_field_names = [f[2] for f in all_types]
+ sorted_serializers = [f[1] for f in all_types]
- def visit_other(self, field_name, type_, types_path=None):
- typeinfo = self.fory.type_resolver.get_typeinfo(type_, create=False)
- if typeinfo is None:
- id_ = 0
- else:
- serializer = typeinfo.serializer
- id_ = typeinfo.type_id
- assert id_ is not None, serializer
- if TypeId.is_namespaced_type(typeinfo.type_id):
- namespace_str = typeinfo.decode_namespace()
- typename_str = typeinfo.decode_typename()
- id_ = compute_string_hash(namespace_str + typename_str)
- self._hash = self._compute_field_hash(self._hash, id_)
-
- @staticmethod
- def _compute_field_hash(hash_, id_):
- new_hash = hash_ * 31 + id_
- while new_hash >= 2**31 - 1:
- new_hash = new_hash // 7
- return new_hash
-
- def get_hash(self):
- return self._hash
+ return type_hash_32, sorted_field_names, sorted_serializers
class StructTypeIdVisitor(TypeVisitor):
diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py
index b009a0bda..893aa12c1 100644
--- a/python/pyfory/serializer.py
+++ b/python/pyfory/serializer.py
@@ -356,7 +356,7 @@ _ENABLE_FORY_PYTHON_JIT =
os.environ.get("ENABLE_FORY_PYTHON_JIT", "True").lower
)
-from pyfory._struct import _get_hash, _sort_fields,
StructFieldSerializerVisitor
+from pyfory._struct import compute_struct_meta, StructFieldSerializerVisitor
class DataClassSerializer(Serializer):
@@ -396,8 +396,9 @@ class DataClassSerializer(Serializer):
unwrapped_type, _ = unwrap_optional(self._type_hints[key])
serializer = infer_field(key, unwrapped_type, visitor,
types_path=[])
self._serializers[index] = serializer
- self._field_names, self._serializers =
_sort_fields(fory.type_resolver, self._field_names, self._serializers,
self._nullable_fields)
- self._hash = 0 # Will be computed on first xwrite/xread
+ self._hash, self._field_names, self._serializers =
compute_struct_meta(
+ fory.type_resolver, self._field_names, self._serializers,
self._nullable_fields
+ )
self._generated_xwrite_method = self._gen_xwrite_method()
self._generated_xread_method = self._gen_xread_method()
if _ENABLE_FORY_PYTHON_JIT:
@@ -423,8 +424,9 @@ class DataClassSerializer(Serializer):
# In compatible mode, maintain stable field ordering (don't sort)
# In non-compatible mode, sort fields for consistent serialization
if not fory.compatible:
- self._field_names, self._serializers =
_sort_fields(fory.type_resolver, self._field_names, self._serializers,
self._nullable_fields)
- self._hash = 0 # Will be computed on first write/read
+ self._hash, self._field_names, self._serializers =
compute_struct_meta(
+ fory.type_resolver, self._field_names, self._serializers,
self._nullable_fields
+ )
self._generated_write_method = self._gen_write_method()
self._generated_read_method = self._gen_read_method()
if _ENABLE_FORY_PYTHON_JIT:
@@ -453,16 +455,10 @@ class DataClassSerializer(Serializer):
return {field_name: unwrap_optional(hint)[0] for field_name, hint in
self._type_hints.items()}
- def _ensure_hash_computed(self):
- """Lazily compute and cache the hash if not already computed."""
- if self._hash == 0:
- self._hash = _get_hash(self.fory, self._field_names,
self._unwrapped_hints)
- return self._hash
-
def _write_header(self, buffer):
"""Write serialization header (hash or field count based on compatible
mode)."""
if not self.fory.compatible:
- buffer.write_int32(self._ensure_hash_computed())
+ buffer.write_int32(self._hash)
else:
buffer.write_varuint32(len(self._field_names))
@@ -477,7 +473,7 @@ class DataClassSerializer(Serializer):
"""
if not self.fory.compatible:
hash_ = buffer.read_int32()
- expected_hash = self._ensure_hash_computed()
+ expected_hash = self._hash
if hash_ != expected_hash:
raise TypeNotCompatibleError(f"Hash {hash_} is not consistent
with {expected_hash} for type {self.type_}")
return len(self._field_names)
@@ -602,7 +598,6 @@ class DataClassSerializer(Serializer):
]
# Write hash only in non-compatible mode; in compatible mode, write
field count
- self._ensure_hash_computed()
if not self.fory.compatible:
stmts.append(f"{buffer}.write_int32({self._hash})")
else:
@@ -672,7 +667,6 @@ class DataClassSerializer(Serializer):
]
# Read hash only in non-compatible mode; in compatible mode, read
field count
- self._ensure_hash_computed()
if not self.fory.compatible:
stmts.extend(
[
@@ -764,7 +758,6 @@ class DataClassSerializer(Serializer):
f'"""xwrite method for {self.type_}"""',
]
if not self.fory.compatible:
- self._ensure_hash_computed()
stmts.append(f"{buffer}.write_int32({self._hash})")
if not self._has_slots:
stmts.append(f"{value_dict} = {value}.__dict__")
@@ -823,7 +816,6 @@ class DataClassSerializer(Serializer):
f'"""xread method for {self.type_}"""',
]
if not self.fory.compatible:
- self._ensure_hash_computed()
stmts.extend(
[
f"read_hash = {buffer}.read_int32()",
@@ -924,7 +916,6 @@ class DataClassSerializer(Serializer):
"""Write dataclass instance to buffer in cross-language format."""
if not self._xlang:
raise TypeError("xwrite can only be called when
DataClassSerializer is in xlang mode")
- self._ensure_hash_computed()
if not self.fory.compatible:
buffer.write_int32(self._hash)
for index, field_name in enumerate(self._field_names):
@@ -940,7 +931,6 @@ class DataClassSerializer(Serializer):
"""Read dataclass instance from buffer in cross-language format."""
if not self._xlang:
raise TypeError("xread can only be called when DataClassSerializer
is in xlang mode")
- self._ensure_hash_computed()
if not self.fory.compatible:
hash_ = buffer.read_int32()
if hash_ != self._hash:
diff --git a/python/pyfory/tests/test_cross_language.py
b/python/pyfory/tests/test_cross_language.py
index 3cb9c9daa..99fdc3bf7 100644
--- a/python/pyfory/tests/test_cross_language.py
+++ b/python/pyfory/tests/test_cross_language.py
@@ -542,9 +542,14 @@ def test_struct_hash(data_file_path):
fory = pyfory.Fory(xlang=True, ref=True)
fory.register_type(ComplexObject1, typename="ComplexObject1")
serializer = fory.type_resolver.get_serializer(ComplexObject1)._replace()
- from pyfory._struct import _get_hash
-
- v = _get_hash(fory, serializer._field_names, serializer._type_hints)
+ from pyfory._struct import compute_struct_meta
+
+ v = compute_struct_meta(
+ fory.type_resolver,
+ serializer._field_names,
+ serializer._serializers,
+ serializer._nullable_fields,
+ )[0]
assert read_hash == v, (read_hash, v)
diff --git a/rust/fory-core/src/error.rs b/rust/fory-core/src/error.rs
index 84606e603..08cc3d568 100644
--- a/rust/fory-core/src/error.rs
+++ b/rust/fory-core/src/error.rs
@@ -173,6 +173,12 @@ pub enum Error {
/// Do not construct this variant directly; use [`Error::unknown`] instead.
#[error("{0}")]
Unknown(Cow<'static, str>),
+
+ /// Struct version mismatch between local and remote schemas.
+ ///
+ /// Do not construct this variant directly; use
[`Error::struct_version_mismatch`] instead.
+ #[error("{0}")]
+ StructVersionMismatch(Cow<'static, str>),
}
impl Error {
@@ -405,6 +411,27 @@ impl Error {
err
}
+ /// Creates a new [`Error::StructVersionMismatch`] from a string or static
message.
+ ///
+ /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic
with the error message.
+ ///
+ /// # Example
+ /// ```
+ /// use fory_core::error::Error;
+ ///
+ /// let err = Error::struct_version_mismatch("Version mismatch");
+ /// let err = Error::struct_version_mismatch(format!("Class {} version
mismatch", "Foo"));
+ /// ```
+ #[inline(always)]
+ #[track_caller]
+ pub fn struct_version_mismatch<S: Into<Cow<'static, str>>>(s: S) -> Self {
+ let err = Error::StructVersionMismatch(s.into());
+ if should_panic_on_error() {
+ panic!("FORY_PANIC_ON_ERROR: {}", err);
+ }
+ err
+ }
+
/// Creates a new [`Error::Unknown`] from a string or static message.
///
/// This function is a convenient way to produce an error message
diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs
index ec34cbd57..0f7528d78 100644
--- a/rust/fory-core/src/fory.rs
+++ b/rust/fory-core/src/fory.rs
@@ -80,6 +80,7 @@ pub struct Fory {
type_resolver: TypeResolver,
compress_string: bool,
max_dyn_depth: u32,
+ check_struct_version: bool,
// Lazy-initialized pools (thread-safe, one-time initialization)
write_context_pool: OnceLock<Pool<WriteContext>>,
read_context_pool: OnceLock<Pool<ReadContext>>,
@@ -94,6 +95,7 @@ impl Default for Fory {
type_resolver: TypeResolver::default(),
compress_string: false,
max_dyn_depth: 5,
+ check_struct_version: false,
write_context_pool: OnceLock::new(),
read_context_pool: OnceLock::new(),
}
@@ -133,6 +135,9 @@ impl Fory {
self.share_meta = compatible;
self.compatible = compatible;
self.type_resolver.set_compatible(compatible);
+ if compatible {
+ self.check_struct_version = false;
+ }
self
}
@@ -166,6 +171,9 @@ impl Fory {
/// ```
pub fn xlang(mut self, xlang: bool) -> Self {
self.xlang = xlang;
+ if !self.check_struct_version {
+ self.check_struct_version = !self.compatible;
+ }
self
}
@@ -202,6 +210,46 @@ impl Fory {
self
}
+ /// Enables or disables class version checking for schema consistency.
+ ///
+ /// # Arguments
+ ///
+ /// * `check_struct_version` - If `true`, enables class version checking
to ensure
+ /// schema consistency between serialization and deserialization. When
enabled,
+ /// a version hash computed from field types is written/read to detect
schema mismatches.
+ /// If `false`, no version checking is performed.
+ ///
+ /// # Returns
+ ///
+ /// Returns `self` for method chaining.
+ ///
+ /// # Default
+ ///
+ /// The default value is `false`.
+ ///
+ /// # Note
+ ///
+ /// This feature is only effective when `compatible` mode is `false`. In
compatible mode,
+ /// schema evolution is supported and version checking is not needed.
+ ///
+ /// # Examples
+ ///
+ /// ```rust
+ /// use fory_core::Fory;
+ ///
+ /// let fory = Fory::default()
+ /// .compatible(false)
+ /// .check_struct_version(true);
+ /// ```
+ pub fn check_struct_version(mut self, check_struct_version: bool) -> Self {
+ if self.compatible && check_struct_version {
+ // ignore setting if compatible mode is on
+ return self;
+ }
+ self.check_struct_version = check_struct_version;
+ self
+ }
+
/// Sets the maximum depth for nested dynamic object serialization.
///
/// # Arguments
@@ -276,6 +324,15 @@ impl Fory {
self.max_dyn_depth
}
+ /// Returns whether class version checking is enabled.
+ ///
+ /// # Returns
+ ///
+ /// `true` if class version checking is enabled, `false` otherwise.
+ pub fn is_check_struct_version(&self) -> bool {
+ self.check_struct_version
+ }
+
/// Returns a type resolver for type lookups.
pub(crate) fn get_type_resolver(&self) -> &TypeResolver {
&self.type_resolver
@@ -383,6 +440,7 @@ impl Fory {
let share_meta = self.share_meta;
let xlang = self.xlang;
let max_dyn_depth = self.max_dyn_depth;
+ let check_struct_version = self.check_struct_version;
let factory = move || {
let reader = Reader::new(&[]);
@@ -393,6 +451,7 @@ impl Fory {
share_meta,
xlang,
max_dyn_depth,
+ check_struct_version,
)
};
Pool::new(factory)
@@ -466,6 +525,7 @@ impl Fory {
let share_meta = self.share_meta;
let compress_string = self.compress_string;
let xlang = self.xlang;
+ let check_struct_version = self.check_struct_version;
let factory = move || {
let writer = Writer::default();
@@ -476,6 +536,7 @@ impl Fory {
share_meta,
compress_string,
xlang,
+ check_struct_version,
)
};
Pool::new(factory)
diff --git a/rust/fory-core/src/meta/type_meta.rs
b/rust/fory-core/src/meta/type_meta.rs
index d7fb33864..3a2ecfa68 100644
--- a/rust/fory-core/src/meta/type_meta.rs
+++ b/rust/fory-core/src/meta/type_meta.rs
@@ -556,7 +556,6 @@ impl TypeMeta {
pub fn get_hash(&self) -> i64 {
self.hash
}
-
pub fn get_type_name(&self) -> Rc<MetaString> {
self.layer.get_type_name()
}
@@ -641,6 +640,21 @@ impl TypeMeta {
reader.skip(meta_size as usize)
}
+ /// Check class version consistency, similar to Java's checkClassVersion
+ pub fn check_struct_version(
+ read_version: i32,
+ local_version: i32,
+ type_name: &str,
+ ) -> Result<(), Error> {
+ if read_version != local_version {
+ return Err(Error::struct_version_mismatch(format!(
+ "Read class {} version {} is not consistent with {}",
+ type_name, read_version, local_version
+ )));
+ }
+ Ok(())
+ }
+
pub fn to_bytes(&self) -> Result<Vec<u8>, Error> {
// | global_binary_header | layers_bytes |
let mut result = Writer::default();
diff --git a/rust/fory-core/src/resolver/context.rs
b/rust/fory-core/src/resolver/context.rs
index f3f4025ad..48e91af28 100644
--- a/rust/fory-core/src/resolver/context.rs
+++ b/rust/fory-core/src/resolver/context.rs
@@ -37,6 +37,7 @@ pub struct WriteContext {
share_meta: bool,
compress_string: bool,
xlang: bool,
+ check_struct_version: bool,
// Context-specific fields
pub writer: Writer,
@@ -53,6 +54,7 @@ impl WriteContext {
share_meta: bool,
compress_string: bool,
xlang: bool,
+ check_struct_version: bool,
) -> WriteContext {
WriteContext {
type_resolver,
@@ -60,6 +62,7 @@ impl WriteContext {
share_meta,
compress_string,
xlang,
+ check_struct_version,
writer,
meta_resolver: MetaWriterResolver::default(),
meta_string_resolver: MetaStringWriterResolver::default(),
@@ -74,6 +77,7 @@ impl WriteContext {
share_meta: fory.is_share_meta(),
compress_string: fory.is_compress_string(),
xlang: fory.is_xlang(),
+ check_struct_version: fory.is_check_struct_version(),
writer,
meta_resolver: MetaWriterResolver::default(),
meta_string_resolver: MetaStringWriterResolver::default(),
@@ -116,6 +120,12 @@ impl WriteContext {
self.xlang
}
+ /// Check if class version checking is enabled
+ #[inline(always)]
+ pub fn is_check_struct_version(&self) -> bool {
+ self.check_struct_version
+ }
+
#[inline(always)]
pub fn empty(&mut self) -> bool {
self.meta_resolver.empty()
@@ -209,6 +219,7 @@ pub struct ReadContext {
share_meta: bool,
xlang: bool,
max_dyn_depth: u32,
+ check_struct_version: bool,
// Context-specific fields
pub reader: Reader,
@@ -233,6 +244,7 @@ impl ReadContext {
share_meta: bool,
xlang: bool,
max_dyn_depth: u32,
+ check_struct_version: bool,
) -> ReadContext {
ReadContext {
type_resolver,
@@ -240,6 +252,7 @@ impl ReadContext {
share_meta,
xlang,
max_dyn_depth,
+ check_struct_version,
reader,
meta_resolver: MetaReaderResolver::default(),
meta_string_resolver: MetaStringReaderResolver::default(),
@@ -255,6 +268,7 @@ impl ReadContext {
share_meta: fory.is_share_meta(),
xlang: fory.is_xlang(),
max_dyn_depth: fory.get_max_dyn_depth(),
+ check_struct_version: fory.is_check_struct_version(),
reader,
meta_resolver: MetaReaderResolver::default(),
meta_string_resolver: MetaStringReaderResolver::default(),
@@ -287,6 +301,12 @@ impl ReadContext {
self.xlang
}
+ /// Check if class version checking is enabled
+ #[inline(always)]
+ pub fn is_check_struct_version(&self) -> bool {
+ self.check_struct_version
+ }
+
/// Get maximum dynamic depth
#[inline(always)]
pub fn max_dyn_depth(&self) -> u32 {
diff --git a/rust/fory-derive/src/object/read.rs
b/rust/fory-derive/src/object/read.rs
index b288e0ecd..8bc8a4bc7 100644
--- a/rust/fory-derive/src/object/read.rs
+++ b/rust/fory-derive/src/object/read.rs
@@ -20,9 +20,9 @@ use quote::{format_ident, quote};
use syn::{Field, Type};
use super::util::{
- classify_trait_object_field, create_wrapper_types_arc,
create_wrapper_types_rc,
- extract_type_name, get_struct_name, is_debug_enabled, is_primitive_type,
- should_skip_type_info_for_field, skip_ref_flag, StructField,
+ classify_trait_object_field, compute_struct_version_hash,
create_wrapper_types_arc,
+ create_wrapper_types_rc, extract_type_name, get_struct_name,
is_debug_enabled,
+ is_primitive_type, should_skip_type_info_for_field, skip_ref_flag,
StructField,
};
fn create_private_field_name(field: &Field) -> Ident {
@@ -244,6 +244,7 @@ fn get_fields_loop_ts(fields: &[&Field]) -> TokenStream {
}
pub fn gen_read_data(fields: &[&Field]) -> TokenStream {
+ let version_hash = compute_struct_version_hash(fields);
let sorted_read = if fields.is_empty() {
quote! {}
} else {
@@ -260,6 +261,12 @@ pub fn gen_read_data(fields: &[&Field]) -> TokenStream {
}
});
quote! {
+ // Read and check version hash when class version checking is enabled
+ if context.is_check_struct_version() {
+ let read_version = context.reader.read_i32()?;
+ let type_name = std::any::type_name::<Self>();
+ fory_core::meta::TypeMeta::check_struct_version(read_version,
#version_hash, type_name)?;
+ }
#sorted_read
Ok(Self {
#(#field_idents),*
diff --git a/rust/fory-derive/src/object/util.rs
b/rust/fory-derive/src/object/util.rs
index 0ef41a93e..4baef0546 100644
--- a/rust/fory-derive/src/object/util.rs
+++ b/rust/fory-derive/src/object/util.rs
@@ -23,6 +23,7 @@ use fory_core::types::{TypeId, PRIMITIVE_ARRAY_TYPE_MAP};
use proc_macro2::{Ident, TokenStream};
use quote::{format_ident, quote, ToTokens};
use std::cell::RefCell;
+use std::collections::HashMap;
use std::fmt;
use syn::{Field, GenericArgument, PathArguments, Type};
@@ -540,6 +541,8 @@ fn is_internal_type_id(type_id: u32) -> bool {
.contains(&type_id)
}
+/// Group fields into serialization categories while normalizing field names
to snake_case.
+/// The returned groups preserve the ordering rules required by the
serialization layout.
fn group_fields_by_type(fields: &[&Field]) -> FieldGroups {
let mut primitive_fields = Vec::new();
let mut nullable_primitive_fields = Vec::new();
@@ -552,7 +555,8 @@ fn group_fields_by_type(fields: &[&Field]) -> FieldGroups {
// First handle Forward fields separately to avoid borrow checker issues
for field in fields {
if is_forward_field(&field.ty) {
- let ident = field.ident.as_ref().unwrap().to_string();
+ let raw_ident = field.ident.as_ref().unwrap().to_string();
+ let ident = to_snake_case(&raw_ident);
other_fields.push((ident, "Forward".to_string(), TypeId::UNKNOWN
as u32));
}
}
@@ -577,7 +581,8 @@ fn group_fields_by_type(fields: &[&Field]) -> FieldGroups {
};
for field in fields {
- let ident = field.ident.as_ref().unwrap().to_string();
+ let raw_ident = field.ident.as_ref().unwrap().to_string();
+ let ident = to_snake_case(&raw_ident);
// Skip if already handled as Forward field
if is_forward_field(&field.ty) {
@@ -678,6 +683,90 @@ pub(super) fn get_sort_fields_ts(fields: &[&Field]) ->
TokenStream {
}
}
+fn to_snake_case(name: &str) -> String {
+ if name
+ .chars()
+ .all(|c| c.is_lowercase() || c.is_ascii_digit() || c == '_')
+ {
+ return name.to_string();
+ }
+
+ let mut result = String::with_capacity(name.len() * 2);
+ let mut chars = name.chars().peekable();
+ let mut prev: Option<char> = None;
+
+ while let Some(ch) = chars.next() {
+ if ch == '_' {
+ result.push('_');
+ prev = Some(ch);
+ continue;
+ }
+
+ if ch.is_uppercase() {
+ if let Some(prev_ch) = prev {
+ let need_underscore = (prev_ch.is_lowercase() ||
prev_ch.is_ascii_digit())
+ || (prev_ch.is_uppercase()
+ && chars
+ .peek()
+ .map(|next| next.is_lowercase())
+ .unwrap_or(false));
+ if need_underscore && !result.ends_with('_') {
+ result.push('_');
+ }
+ }
+ result.push(ch.to_ascii_lowercase());
+ } else {
+ result.push(ch);
+ }
+ prev = Some(ch);
+ }
+
+ result
+}
+
+pub(crate) fn compute_struct_version_hash(fields: &[&Field]) -> i32 {
+ let mut field_info_map: HashMap<String, (u32, bool)> =
HashMap::with_capacity(fields.len());
+ for field in fields {
+ let name = field.ident.as_ref().unwrap().to_string();
+ let type_id = get_type_id_by_type_ast(&field.ty);
+ let nullable = is_option(&field.ty);
+ field_info_map.insert(name, (type_id, nullable));
+ }
+
+ let mut fingerprint = String::new();
+ for name in get_sorted_field_names(fields).iter() {
+ let (type_id, nullable) = field_info_map
+ .get(name)
+ .expect("Field metadata missing during struct hash computation");
+ fingerprint.push_str(&to_snake_case(name));
+ fingerprint.push(',');
+ let effective_type_id = if *type_id == TypeId::UNKNOWN as u32 {
+ TypeId::UNKNOWN as u32
+ } else {
+ *type_id
+ };
+ fingerprint.push_str(&effective_type_id.to_string());
+ fingerprint.push(',');
+ fingerprint.push_str(if *nullable { "1;" } else { "0;" });
+ }
+
+ let seed: u64 = 47;
+ let (hash, _) =
fory_core::meta::murmurhash3_x64_128(fingerprint.as_bytes(), seed);
+ let version = (hash & 0xFFFF_FFFF) as u32;
+ let version = version as i32;
+
+ if is_debug_enabled() {
+ if let Some(struct_name) = get_struct_name() {
+ println!(
+ "[fory-debug] struct {struct_name} version
fingerprint=\"{fingerprint}\" hash={version}"
+ );
+ } else {
+ println!("[fory-debug] struct version
fingerprint=\"{fingerprint}\" hash={version}");
+ }
+ }
+ version
+}
+
pub(crate) fn skip_ref_flag(ty: &Type) -> bool {
// !T::fory_is_option() && PRIMITIVE_TYPES.contains(&elem_type_id)
PRIMITIVE_TYPE_NAMES.contains(&extract_type_name(ty).as_str())
@@ -714,3 +803,98 @@ pub(crate) fn should_skip_type_info_for_field(ty: &Type)
-> bool {
// Primitive, nullable primitive, internal types, List/Set/Map skip type
info
true
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use syn::parse_quote;
+
+ #[test]
+ fn to_snake_case_handles_common_patterns() {
+ assert_eq!(to_snake_case("lowercase"), "lowercase");
+ assert_eq!(to_snake_case("camelCase"), "camel_case");
+ assert_eq!(to_snake_case("HTTPRequest"), "http_request");
+ assert_eq!(to_snake_case("withNumbers123"), "with_numbers123");
+ assert_eq!(to_snake_case("snake_case"), "snake_case");
+ }
+
+ #[test]
+ fn group_fields_normalizes_names_and_preserves_ordering() {
+ let fields: Vec<syn::Field> = vec![
+ parse_quote!(pub camelCase: i32),
+ parse_quote!(pub optionalValue: Option<i64>),
+ parse_quote!(pub simpleString: String),
+ parse_quote!(pub listItems: Vec<String>),
+ parse_quote!(pub setItems: HashSet<i32>),
+ parse_quote!(pub mapValues: HashMap<String, i32>),
+ parse_quote!(pub customType: CustomType),
+ ];
+ let field_refs: Vec<&syn::Field> = fields.iter().collect();
+
+ let (
+ primitive_fields,
+ nullable_primitive_fields,
+ internal_type_fields,
+ list_fields,
+ set_fields,
+ map_fields,
+ other_fields,
+ ) = group_fields_by_type(&field_refs);
+
+ let primitive_names: Vec<&str> = primitive_fields
+ .iter()
+ .map(|(name, _, _)| name.as_str())
+ .collect();
+ assert_eq!(primitive_names, vec!["camel_case"]);
+
+ let nullable_names: Vec<&str> = nullable_primitive_fields
+ .iter()
+ .map(|(name, _, _)| name.as_str())
+ .collect();
+ assert_eq!(nullable_names, vec!["optional_value"]);
+
+ let internal_names: Vec<&str> = internal_type_fields
+ .iter()
+ .map(|(name, _, _)| name.as_str())
+ .collect();
+ assert_eq!(internal_names, vec!["simple_string"]);
+
+ let list_names: Vec<&str> = list_fields
+ .iter()
+ .map(|(name, _, _)| name.as_str())
+ .collect();
+ assert_eq!(list_names, vec!["list_items"]);
+
+ let set_names: Vec<&str> = set_fields
+ .iter()
+ .map(|(name, _, _)| name.as_str())
+ .collect();
+ assert_eq!(set_names, vec!["set_items"]);
+
+ let map_names: Vec<&str> = map_fields
+ .iter()
+ .map(|(name, _, _)| name.as_str())
+ .collect();
+ assert_eq!(map_names, vec!["map_values"]);
+
+ let other_names: Vec<&str> = other_fields
+ .iter()
+ .map(|(name, _, _)| name.as_str())
+ .collect();
+ assert_eq!(other_names, vec!["custom_type"]);
+
+ let sorted_names = get_sorted_field_names(&field_refs);
+ assert_eq!(
+ sorted_names,
+ vec![
+ "camel_case".to_string(),
+ "optional_value".to_string(),
+ "simple_string".to_string(),
+ "list_items".to_string(),
+ "set_items".to_string(),
+ "map_values".to_string(),
+ "custom_type".to_string(),
+ ]
+ );
+ }
+}
diff --git a/rust/fory-derive/src/object/write.rs
b/rust/fory-derive/src/object/write.rs
index f242da28d..ae42d8d1e 100644
--- a/rust/fory-derive/src/object/write.rs
+++ b/rust/fory-derive/src/object/write.rs
@@ -16,9 +16,9 @@
// under the License.
use super::util::{
- classify_trait_object_field, create_wrapper_types_arc,
create_wrapper_types_rc,
- get_struct_name, get_type_id_by_type_ast, is_debug_enabled,
should_skip_type_info_for_field,
- skip_ref_flag, StructField,
+ classify_trait_object_field, compute_struct_version_hash,
create_wrapper_types_arc,
+ create_wrapper_types_rc, get_struct_name, get_type_id_by_type_ast,
is_debug_enabled,
+ should_skip_type_info_for_field, skip_ref_flag, StructField,
};
use fory_core::types::TypeId;
use proc_macro2::TokenStream;
@@ -244,7 +244,12 @@ fn gen_write_field(field: &Field) -> TokenStream {
pub fn gen_write_data(fields: &[&Field]) -> TokenStream {
let write_fields_ts: Vec<_> = fields.iter().map(|field|
gen_write_field(field)).collect();
+ let version_hash = compute_struct_version_hash(fields);
quote! {
+ // Write version hash when class version checking is enabled
+ if context.is_check_struct_version() {
+ context.writer.write_i32(#version_hash);
+ }
#(#write_fields_ts)*
Ok(())
}
diff --git a/rust/tests/tests/test_cross_language.rs
b/rust/tests/tests/test_cross_language.rs
index 207507eb4..a1d716faa 100644
--- a/rust/tests/tests/test_cross_language.rs
+++ b/rust/tests/tests/test_cross_language.rs
@@ -739,3 +739,35 @@ fn test_consistent_named() {
// // fory.serialize_with_context(&my_struct, &mut context);
fs::write(&data_file_path, context.writer.dump()).unwrap();
}
+
+#[derive(ForyObject, Debug, PartialEq)]
+#[fory_debug]
+struct VersionCheckStruct {
+ f1: i32,
+ f2: Option<String>,
+ f3: f64,
+}
+
+#[test]
+#[ignore]
+fn test_struct_version_check() {
+ let data_file_path = get_data_file();
+ let bytes = fs::read(&data_file_path).unwrap();
+ let mut fory = Fory::default()
+ .compatible(false)
+ .xlang(true)
+ .check_struct_version(true);
+ fory.register::<VersionCheckStruct>(201).unwrap();
+
+ let local_obj = VersionCheckStruct {
+ f1: 10,
+ f2: Some("test".to_string()),
+ f3: 3.2,
+ };
+ let remote_obj: VersionCheckStruct = fory.deserialize(&bytes).unwrap();
+ assert_eq!(remote_obj, local_obj);
+ let new_bytes = fory.serialize(&remote_obj).unwrap();
+ let new_local_obj: VersionCheckStruct =
fory.deserialize(&new_bytes).unwrap();
+ assert_eq!(new_local_obj, local_obj);
+ fs::write(&data_file_path, new_bytes).unwrap();
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
