This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new d60de394b feat(go): add go struct field tag support (#3082)
d60de394b is described below
commit d60de394b010bf972e315cdf77f91233df954174
Author: Shawn Yang <[email protected]>
AuthorDate: Wed Dec 24 23:16:52 2025 +0800
feat(go): add go struct field tag support (#3082)
## Why?
## What does this PR do?
Closes #3005
#2982
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---
cpp/fory/serialization/type_resolver.cc | 45 +-
cpp/fory/serialization/type_resolver.h | 41 +-
go/fory/codegen/decoder.go | 11 +-
go/fory/codegen/encoder.go | 11 +-
go/fory/codegen/generator.go | 23 +-
go/fory/codegen/utils.go | 145 -----
go/fory/errors.go | 18 +
go/fory/struct.go | 260 +++++++--
go/fory/tag.go | 208 +++++++
go/fory/tag_test.go | 613 +++++++++++++++++++++
go/fory/tests/structs_fory_gen.go | 154 ++++--
go/fory/type_def.go | 153 +++--
.../org/apache/fory/resolver/XtypeResolver.java | 5 +-
.../apache/fory/serializer/ObjectSerializer.java | 108 +++-
.../org/apache/fory/type/DescriptorGrouper.java | 38 +-
.../test/java/org/apache/fory/XlangTestBase.java | 32 --
python/pyfory/struct.py | 119 ++--
python/pyfory/tests/xlang_test_main.py | 239 ++++++++
rust/fory-derive/src/object/util.rs | 46 +-
19 files changed, 1874 insertions(+), 395 deletions(-)
diff --git a/cpp/fory/serialization/type_resolver.cc
b/cpp/fory/serialization/type_resolver.cc
index f93a0e06f..900eac39e 100644
--- a/cpp/fory/serialization/type_resolver.cc
+++ b/cpp/fory/serialization/type_resolver.cc
@@ -928,15 +928,37 @@ std::string ToSnakeCase(const std::string &name) {
} // anonymous namespace
-int32_t TypeMeta::compute_struct_version(const TypeMeta &meta) {
- // Build fingerprint string as in Rust's compute_struct_version_hash:
- // snake_case(field_name),effective_type_id,nullable; ...
+std::string TypeMeta::compute_struct_fingerprint(
+ const std::vector<FieldInfo> &field_infos) {
+ // Computes the fingerprint string for a struct type used in schema
+ // versioning.
+ //
+ // Fingerprint Format:
+ // Each field contributes: <field_name>,<type_id>,<ref>,<nullable>;
+ // Fields are sorted lexicographically by field name (not by type
category).
+ //
+ // Field Components:
+ // - field_name: snake_case field name (C++ doesn't support field tag IDs
+ // yet)
+ // - type_id: Fory TypeId as decimal string (e.g., "4" for INT32)
+ // - ref: "1" if reference tracking enabled, "0" otherwise (always "0" in
+ // C++)
+ // - nullable: "1" if null flag is written, "0" otherwise
+ //
+ // Example fingerprint: "age,4,0,0;name,12,0,1;"
+
+ // Copy fields and sort lexicographically by snake_case name for fingerprint
+ std::vector<FieldInfo> sorted_fields = field_infos;
+ std::sort(sorted_fields.begin(), sorted_fields.end(),
+ [](const FieldInfo &a, const FieldInfo &b) {
+ return ToSnakeCase(a.field_name) < ToSnakeCase(b.field_name);
+ });
+
std::string fingerprint;
- const auto &fields = meta.field_infos;
// Reserve a rough estimate to avoid reallocations
- fingerprint.reserve(fields.size() * 24);
+ fingerprint.reserve(sorted_fields.size() * 24);
- for (const auto &fi : fields) {
+ for (const auto &fi : sorted_fields) {
std::string snake = ToSnakeCase(fi.field_name);
fingerprint.append(snake);
fingerprint.push_back(',');
@@ -951,14 +973,23 @@ int32_t TypeMeta::compute_struct_version(const TypeMeta
&meta) {
}
fingerprint.append(std::to_string(effective_type_id));
fingerprint.push_back(',');
+ // ref flag: currently always 0 in C++ (no ref tracking support yet)
+ fingerprint.push_back('0');
+ fingerprint.push_back(',');
fingerprint.append(fi.field_type.nullable ? "1;" : "0;");
}
+ return fingerprint;
+}
+
+int32_t TypeMeta::compute_struct_version(const TypeMeta &meta) {
+ std::string fingerprint = compute_struct_fingerprint(meta.field_infos);
+
int64_t hash_out[2] = {0, 0};
MurmurHash3_x64_128(reinterpret_cast<const uint8_t *>(fingerprint.data()),
static_cast<int>(fingerprint.size()), 47, hash_out);
- // Rust uses the low 64 bits and then keeps low 32 bits as i32.
+ // Use the low 64 bits and then keep low 32 bits as i32.
uint64_t low = static_cast<uint64_t>(hash_out[0]);
uint32_t version = static_cast<uint32_t>(low & 0xFFFF'FFFFu);
#ifdef FORY_DEBUG
diff --git a/cpp/fory/serialization/type_resolver.h
b/cpp/fory/serialization/type_resolver.h
index a4a605735..3e40189ab 100644
--- a/cpp/fory/serialization/type_resolver.h
+++ b/cpp/fory/serialization/type_resolver.h
@@ -215,23 +215,36 @@ public:
const std::string &get_type_name() const { return type_name; }
const std::string &get_namespace() const { return namespace_str; }
- /// Compute struct version hash from field metadata.
+ /// Computes the fingerprint string for a struct type used in schema
+ /// versioning.
+ ///
+ /// Fingerprint Format:
+ /// Each field contributes: `<field_name>,<type_id>,<ref>,<nullable>;`
+ /// Fields are sorted lexicographically by field name (not by type
+ /// category).
+ ///
+ /// Field Components:
+ /// - field_name: snake_case field name (C++ doesn't support field tag IDs
+ /// yet)
+ /// - type_id: Fory TypeId as decimal string (e.g., "4" for INT32)
+ /// - ref: "1" if reference tracking enabled, "0" otherwise (always "0" in
+ /// C++)
+ /// - nullable: "1" if null flag is written, "0" otherwise
///
- /// This mirrors Rust's
- /// `compute_struct_version_hash` in
- /// rust/fory-derive/src/object/util.rs:
+ /// Example fingerprint: "age,4,0,0;name,12,0,1;"
+ ///
+ /// This format is consistent across Go, Java, Rust, and C++ implementations.
+ static std::string
+ compute_struct_fingerprint(const std::vector<FieldInfo> &field_infos);
+
+ /// Compute struct version hash from field metadata.
///
- /// - Build a fingerprint string using
- /// `snake_case(field_name),effective_type_id,nullable;`
- /// for each field in the sorted order defined by the
- /// xlang spec (primitive, nullable primitive, internal,
- /// list, set, map, other).
- /// - Hash the fingerprint with MurmurHash3_x64_128 using
- /// seed 47, then take the low 32 bits as signed i32.
+ /// Uses compute_struct_fingerprint to build the fingerprint string,
+ /// then hashes it with MurmurHash3_x64_128 using seed 47, and takes
+ /// the low 32 bits as signed i32.
///
- /// Java's ObjectSerializer.computeStructHash uses the same
- /// algorithm, so this function provides the cross-language
- /// struct version ID used by class version checking.
+ /// This provides the cross-language struct version ID used by class
+ /// version checking, consistent with Go, Java, and Rust implementations.
static int32_t compute_struct_version(const TypeMeta &meta);
private:
diff --git a/go/fory/codegen/decoder.go b/go/fory/codegen/decoder.go
index 2d708ee6d..0f0960408 100644
--- a/go/fory/codegen/decoder.go
+++ b/go/fory/codegen/decoder.go
@@ -27,20 +27,18 @@ import (
// generateReadTyped generates the strongly-typed ReadData method
func generateReadTyped(buf *bytes.Buffer, s *StructInfo) error {
- hash := computeStructHash(s)
-
fmt.Fprintf(buf, "// ReadTyped provides strongly-typed deserialization
with no reflection overhead\n")
- fmt.Fprintf(buf, "func (g %s_ForyGenSerializer) ReadTyped(ctx
*fory.ReadContext, v *%s) error {\n", s.Name, s.Name)
+ fmt.Fprintf(buf, "func (g *%s_ForyGenSerializer) ReadTyped(ctx
*fory.ReadContext, v *%s) error {\n", s.Name, s.Name)
fmt.Fprintf(buf, "\tbuf := ctx.Buffer()\n")
fmt.Fprintf(buf, "\terr := ctx.Err() // Get error pointer for deferred
error checking\n\n")
// ReadData and verify struct hash
fmt.Fprintf(buf, "\t// ReadData and verify struct hash\n")
- fmt.Fprintf(buf, "\tif got := buf.ReadInt32(err); got != %d {\n", hash)
+ fmt.Fprintf(buf, "\tif got := buf.ReadInt32(err); got != g.structHash
{\n")
fmt.Fprintf(buf, "\t\tif ctx.HasError() {\n")
fmt.Fprintf(buf, "\t\t\treturn ctx.TakeError()\n")
fmt.Fprintf(buf, "\t\t}\n")
- fmt.Fprintf(buf, "\t\treturn fory.HashMismatchError(got, %d,
\"%s\")\n", hash, s.Name)
+ fmt.Fprintf(buf, "\t\treturn fory.HashMismatchError(got, g.structHash,
\"%s\")\n", s.Name)
fmt.Fprintf(buf, "\t}\n\n")
// ReadData fields in sorted order
@@ -65,7 +63,8 @@ func generateReadTyped(buf *bytes.Buffer, s *StructInfo)
error {
func generateReadInterface(buf *bytes.Buffer, s *StructInfo) error {
// Generate ReadData method (reflect.Value-based API)
fmt.Fprintf(buf, "// ReadData provides reflect.Value interface
compatibility (implements fory.Serializer)\n")
- fmt.Fprintf(buf, "func (g %s_ForyGenSerializer) ReadData(ctx
*fory.ReadContext, type_ reflect.Type, value reflect.Value) {\n", s.Name)
+ fmt.Fprintf(buf, "func (g *%s_ForyGenSerializer) ReadData(ctx
*fory.ReadContext, type_ reflect.Type, value reflect.Value) {\n", s.Name)
+ fmt.Fprintf(buf, "\tg.initHash(ctx.TypeResolver())\n")
fmt.Fprintf(buf, "\t// Convert reflect.Value to concrete type and
delegate to typed method\n")
fmt.Fprintf(buf, "\tvar v *%s\n", s.Name)
fmt.Fprintf(buf, "\tif value.Kind() == reflect.Ptr {\n")
diff --git a/go/fory/codegen/encoder.go b/go/fory/codegen/encoder.go
index b90e230b5..17f5b6b42 100644
--- a/go/fory/codegen/encoder.go
+++ b/go/fory/codegen/encoder.go
@@ -27,15 +27,13 @@ import (
// generateWriteTyped generates the strongly-typed WriteData method
func generateWriteTyped(buf *bytes.Buffer, s *StructInfo) error {
- hash := computeStructHash(s)
-
fmt.Fprintf(buf, "// WriteTyped provides strongly-typed serialization
with no reflection overhead\n")
- fmt.Fprintf(buf, "func (g %s_ForyGenSerializer) WriteTyped(ctx
*fory.WriteContext, v *%s) error {\n", s.Name, s.Name)
+ fmt.Fprintf(buf, "func (g *%s_ForyGenSerializer) WriteTyped(ctx
*fory.WriteContext, v *%s) error {\n", s.Name, s.Name)
fmt.Fprintf(buf, "\tbuf := ctx.Buffer()\n")
// WriteData struct hash
- fmt.Fprintf(buf, "\t// WriteData precomputed struct hash for
compatibility checking\n")
- fmt.Fprintf(buf, "\tbuf.WriteInt32(%d) // hash of %s structure\n\n",
hash, s.Name)
+ fmt.Fprintf(buf, "\t// WriteData struct hash for compatibility
checking\n")
+ fmt.Fprintf(buf, "\tbuf.WriteInt32(g.structHash)\n\n")
// WriteData fields in sorted order
fmt.Fprintf(buf, "\t// WriteData fields in sorted order\n")
@@ -54,7 +52,8 @@ func generateWriteTyped(buf *bytes.Buffer, s *StructInfo)
error {
func generateWriteInterface(buf *bytes.Buffer, s *StructInfo) error {
// Generate WriteData method (reflect.Value-based API)
fmt.Fprintf(buf, "// WriteData provides reflect.Value interface
compatibility (implements fory.Serializer)\n")
- fmt.Fprintf(buf, "func (g %s_ForyGenSerializer) WriteData(ctx
*fory.WriteContext, value reflect.Value) {\n", s.Name)
+ fmt.Fprintf(buf, "func (g *%s_ForyGenSerializer) WriteData(ctx
*fory.WriteContext, value reflect.Value) {\n", s.Name)
+ fmt.Fprintf(buf, "\tg.initHash(ctx.TypeResolver())\n")
fmt.Fprintf(buf, "\t// Convert reflect.Value to concrete type and
delegate to typed method\n")
fmt.Fprintf(buf, "\tvar v *%s\n", s.Name)
fmt.Fprintf(buf, "\tif value.Kind() == reflect.Ptr {\n")
diff --git a/go/fory/codegen/generator.go b/go/fory/codegen/generator.go
index f289a40c1..0d7cdfd45 100644
--- a/go/fory/codegen/generator.go
+++ b/go/fory/codegen/generator.go
@@ -390,12 +390,21 @@ func cleanupGeneratedFiles(opts *GeneratorOptions) error {
// generateStructSerializer generates a complete serializer for a struct
func generateStructSerializer(buf *bytes.Buffer, s *StructInfo) error {
- // Generate struct serializer type
- fmt.Fprintf(buf, "type %s_ForyGenSerializer struct {}\n\n", s.Name)
+ // Generate struct serializer type with lazy hash computation
+ fmt.Fprintf(buf, "type %s_ForyGenSerializer struct {\n", s.Name)
+ fmt.Fprintf(buf, "\tstructHash int32\n")
+ fmt.Fprintf(buf, "}\n\n")
// Generate factory function
fmt.Fprintf(buf, "func NewSerializerFor_%s() fory.Serializer {\n",
s.Name)
- fmt.Fprintf(buf, "\treturn %s_ForyGenSerializer{}\n", s.Name)
+ fmt.Fprintf(buf, "\treturn &%s_ForyGenSerializer{}\n", s.Name)
+ fmt.Fprintf(buf, "}\n\n")
+
+ // Generate hash initialization method
+ fmt.Fprintf(buf, "func (g *%s_ForyGenSerializer) initHash(resolver
*fory.TypeResolver) {\n", s.Name)
+ fmt.Fprintf(buf, "\tif g.structHash == 0 {\n")
+ fmt.Fprintf(buf, "\t\tg.structHash =
fory.GetStructHash(reflect.TypeOf(%s{}), resolver)\n", s.Name)
+ fmt.Fprintf(buf, "\t}\n")
fmt.Fprintf(buf, "}\n\n")
// Generate Write method (entry point with ref/type handling)
@@ -438,7 +447,8 @@ func generateStructSerializer(buf *bytes.Buffer, s
*StructInfo) error {
// generateWriteMethod generates the Write method that handles ref/type flags
func generateWriteMethod(buf *bytes.Buffer, s *StructInfo) error {
fmt.Fprintf(buf, "// Write is the entry point for serialization with
ref/type handling\n")
- fmt.Fprintf(buf, "func (g %s_ForyGenSerializer) Write(ctx
*fory.WriteContext, refMode fory.RefMode, writeType bool, hasGenerics bool,
value reflect.Value) {\n", s.Name)
+ fmt.Fprintf(buf, "func (g *%s_ForyGenSerializer) Write(ctx
*fory.WriteContext, refMode fory.RefMode, writeType bool, hasGenerics bool,
value reflect.Value) {\n", s.Name)
+ fmt.Fprintf(buf, "\tg.initHash(ctx.TypeResolver())\n")
fmt.Fprintf(buf, "\t_ = hasGenerics // not used for struct
serializers\n")
fmt.Fprintf(buf, "\tswitch refMode {\n")
fmt.Fprintf(buf, "\tcase fory.RefModeTracking:\n")
@@ -472,7 +482,8 @@ func generateWriteMethod(buf *bytes.Buffer, s *StructInfo)
error {
// generateReadMethod generates the Read method that handles ref/type flags
func generateReadMethod(buf *bytes.Buffer, s *StructInfo) error {
fmt.Fprintf(buf, "// Read is the entry point for deserialization with
ref/type handling\n")
- fmt.Fprintf(buf, "func (g %s_ForyGenSerializer) Read(ctx
*fory.ReadContext, refMode fory.RefMode, readType bool, hasGenerics bool, value
reflect.Value) {\n", s.Name)
+ fmt.Fprintf(buf, "func (g *%s_ForyGenSerializer) Read(ctx
*fory.ReadContext, refMode fory.RefMode, readType bool, hasGenerics bool, value
reflect.Value) {\n", s.Name)
+ fmt.Fprintf(buf, "\tg.initHash(ctx.TypeResolver())\n")
fmt.Fprintf(buf, "\t_ = hasGenerics // not used for struct
serializers\n")
fmt.Fprintf(buf, "\terr := ctx.Err() // Get error pointer for deferred
error checking\n")
fmt.Fprintf(buf, "\tswitch refMode {\n")
@@ -506,7 +517,7 @@ func generateReadMethod(buf *bytes.Buffer, s *StructInfo)
error {
// generateReadWithTypeInfoMethod generates the ReadWithTypeInfo method
func generateReadWithTypeInfoMethod(buf *bytes.Buffer, s *StructInfo) error {
fmt.Fprintf(buf, "// ReadWithTypeInfo deserializes with pre-read type
information\n")
- fmt.Fprintf(buf, "func (g %s_ForyGenSerializer) ReadWithTypeInfo(ctx
*fory.ReadContext, refMode fory.RefMode, typeInfo *fory.TypeInfo, value
reflect.Value) {\n", s.Name)
+ fmt.Fprintf(buf, "func (g *%s_ForyGenSerializer) ReadWithTypeInfo(ctx
*fory.ReadContext, refMode fory.RefMode, typeInfo *fory.TypeInfo, value
reflect.Value) {\n", s.Name)
fmt.Fprintf(buf, "\tg.Read(ctx, refMode, false, false, value)\n")
fmt.Fprintf(buf, "}\n\n")
return nil
diff --git a/go/fory/codegen/utils.go b/go/fory/codegen/utils.go
index 937467408..d8cdf2e55 100644
--- a/go/fory/codegen/utils.go
+++ b/go/fory/codegen/utils.go
@@ -18,14 +18,11 @@
package codegen
import (
- "fmt"
"go/types"
"sort"
- "strings"
"unicode"
"github.com/apache/fory/go/fory"
- "github.com/spaolacci/murmur3"
)
// FieldInfo contains metadata about a struct field
@@ -397,148 +394,6 @@ func getFieldGroup(field *FieldInfo) int {
return groupOther
}
-// computeStructHash computes a hash for struct schema compatibility
-// This implementation follows the new xlang serialization spec:
-// 1. Sort fields by fields sort algorithm (already done in s.Fields)
-// 2. Build string: snake_case(field_name),$type_id,$nullable;
-// 3. For "other fields", use TypeId::UNKNOWN
-// 4. Convert to UTF8 bytes
-// 5. Compute murmurhash3_x64_128, use first 32 bits
-func computeStructHash(s *StructInfo) int32 {
- var hashString strings.Builder
-
- // Iterate through sorted fields
- for _, field := range s.Fields {
- // Append snake_case field name
- hashString.WriteString(field.SnakeName)
- hashString.WriteString(",")
-
- // Append type_id
- typeID := getTypeIDForHash(field)
- hashString.WriteString(fmt.Sprintf("%d", typeID))
- hashString.WriteString(",")
-
- // Append nullable (1 if nullable, 0 otherwise)
- // nullable is determined by field type (matching reflection's
nullable() function)
- nullable := 0
- if isNullableType(field.Type) {
- nullable = 1
- }
- hashString.WriteString(fmt.Sprintf("%d", nullable))
- hashString.WriteString(";")
- }
-
- // Convert to UTF8 bytes
- hashBytes := []byte(hashString.String())
-
- // Compute murmurhash3_x64_128 with seed 47, and use first 32 bits
- // This matches the reflection implementation
- h1, _ := murmur3.Sum128WithSeed(hashBytes, 47)
- hash := int32(h1 & 0xFFFFFFFF)
-
- if hash == 0 {
- panic(fmt.Errorf("hash for type %v is 0", s.Name))
- }
-
- return hash
-}
-
-// isNullableType checks if a type should have nullable=1 in hash computation
-// This matches Java's behavior where primitives have nullable=0 and objects
have nullable=1
-func isNullableType(t types.Type) bool {
- // Check basic types - only primitives return false
- if basic, ok := t.(*types.Basic); ok {
- switch basic.Kind() {
- case types.Bool,
- types.Int8, types.Int16, types.Int32, types.Int64,
- types.Uint8, types.Uint16, types.Uint32, types.Uint64,
- types.Float32, types.Float64,
- types.Int, types.Uint:
- // These are primitive types - not nullable
- return false
- case types.String:
- // String is an object type in Java - nullable
- return true
- }
- }
-
- // All other types (pointers, slices, maps, interfaces, arrays,
structs) are nullable
- return true
-}
-
-// getTypeIDForHash returns the TypeId for hash calculation according to new
spec
-// For "other fields" (groupOther), returns UNKNOWN (63)
-func getTypeIDForHash(field *FieldInfo) int16 {
- // Determine field group
- group := getFieldGroup(field)
-
- // For "other fields", use UNKNOWN
- if group == groupOther {
- return fory.UNKNOWN
- }
-
- // For struct fields declared with concrete slice types,
- // use typeID = LIST uniformly for hash calculation to align
cross-language behavior
- // This matches the reflection implementation
- if field.TypeID == "LIST" {
- return fory.LIST
- }
-
- // Map field TypeID string to Fory TypeId value
- switch field.TypeID {
- case "BOOL":
- return fory.BOOL
- case "INT8":
- return fory.INT8
- case "INT16":
- return fory.INT16
- case "INT32":
- return fory.INT32
- case "INT64":
- return fory.INT64
- case "UINT8":
- return fory.UINT8
- case "UINT16":
- return fory.UINT16
- case "UINT32":
- return fory.UINT32
- case "UINT64":
- return fory.UINT64
- case "FLOAT32":
- return fory.FLOAT
- case "FLOAT64":
- return fory.DOUBLE
- case "STRING":
- return fory.STRING
- case "TIMESTAMP":
- return fory.TIMESTAMP
- case "LOCAL_DATE":
- return fory.LOCAL_DATE
- case "NAMED_STRUCT":
- return fory.NAMED_STRUCT
- case "STRUCT":
- return fory.STRUCT
- case "SET":
- return fory.SET
- case "MAP":
- return fory.MAP
- case "BINARY":
- return fory.BINARY
- case "ENUM":
- return fory.ENUM
- case "NAMED_ENUM":
- return fory.NAMED_ENUM
- case "EXT":
- return fory.EXT
- case "NAMED_EXT":
- return fory.NAMED_EXT
- case "INTERFACE":
- return fory.UNKNOWN // interface{} treated as UNKNOWN
- default:
- return fory.UNKNOWN
- }
-}
-
// getStructNames extracts struct names from StructInfo slice
func getStructNames(structs []*StructInfo) []string {
names := make([]string, len(structs))
diff --git a/go/fory/errors.go b/go/fory/errors.go
index e1d549944..9f8cde95a 100644
--- a/go/fory/errors.go
+++ b/go/fory/errors.go
@@ -44,6 +44,8 @@ const (
ErrKindInvalidRefId
// ErrKindHashMismatch indicates struct hash mismatch
ErrKindHashMismatch
+ // ErrKindInvalidTag indicates invalid fory struct tag configuration
+ ErrKindInvalidTag
)
// Error is a lightweight error type optimized for hot path performance.
@@ -200,6 +202,22 @@ func InvalidRefIdError(refId int32) Error {
}
}
+// InvalidTagError creates an invalid fory struct tag error
+func InvalidTagError(msg string) Error {
+ return Error{
+ kind: ErrKindInvalidTag,
+ message: msg,
+ }
+}
+
+// InvalidTagErrorf creates a formatted invalid fory struct tag error
+func InvalidTagErrorf(format string, args ...interface{}) Error {
+ return Error{
+ kind: ErrKindInvalidTag,
+ message: fmt.Sprintf(format, args...),
+ }
+}
+
// WrapError wraps a standard error into a fory Error
func WrapError(err error, kind ErrorKind) Error {
if err == nil {
diff --git a/go/fory/struct.go b/go/fory/struct.go
index aa506c777..849a86ca6 100644
--- a/go/fory/struct.go
+++ b/go/fory/struct.go
@@ -53,6 +53,12 @@ type FieldInfo struct {
RefMode RefMode // ref mode for serializer.Write/Read
WriteType bool // whether to write type info (true for struct
fields in compatible mode)
HasGenerics bool // whether element types are known from TypeDef
(for container fields)
+
+ // Tag-based configuration (from fory struct tags)
+ TagID int // -1 = use field name, >=0 = use tag ID
+ HasForyTag bool // Whether field has explicit fory tag
+ TagRefSet bool // Whether ref was explicitly set via fory tag
+ TagRef bool // The ref value from fory tag (only valid if TagRefSet
is true)
}
// fieldHasNonPrimitiveSerializer returns true if the field has a serializer
with a non-primitive type ID.
@@ -995,6 +1001,7 @@ func (s *structSerializer)
initFieldsFromTypeResolver(typeResolver *TypeResolver
var serializers []Serializer
var typeIds []TypeId
var nullables []bool
+ var tagIDs []int
for i := 0; i < type_.NumField(); i++ {
field := type_.Field(i)
@@ -1003,6 +1010,12 @@ func (s *structSerializer)
initFieldsFromTypeResolver(typeResolver *TypeResolver
continue // skip unexported fields
}
+ // Parse fory struct tag and check for ignore
+ foryTag := ParseForyTag(field)
+ if foryTag.Ignore {
+ continue // skip ignored fields
+ }
+
fieldType := field.Type
var fieldSerializer Serializer
@@ -1035,9 +1048,20 @@ func (s *structSerializer)
initFieldsFromTypeResolver(typeResolver *TypeResolver
if isUserDefinedType(int16(internalId)) || internalId == ENUM
|| internalId == NAMED_ENUM {
nullableFlag = true
}
- // Pre-compute RefMode based on trackRef and nullable flag
+ // Override nullable flag if explicitly set in fory tag
+ if foryTag.NullableSet {
+ nullableFlag = foryTag.Nullable
+ }
+
+ // Calculate ref tracking - use tag override if explicitly set
+ trackRef := typeResolver.TrackRef()
+ if foryTag.RefSet {
+ trackRef = foryTag.Ref
+ }
+
+ // Pre-compute RefMode based on (possibly overridden) trackRef
and nullable
refMode := RefModeNone
- if typeResolver.TrackRef() && nullableFlag {
+ if trackRef && nullableFlag {
refMode = RefModeTracking
} else if nullableFlag {
refMode = RefModeNullOnly
@@ -1069,16 +1093,21 @@ func (s *structSerializer)
initFieldsFromTypeResolver(typeResolver *TypeResolver
RefMode: refMode,
WriteType: writeType,
HasGenerics: isCollectionType(fieldTypeId), //
Container fields have declared element types
+ TagID: foryTag.ID,
+ HasForyTag: foryTag.HasTag,
+ TagRefSet: foryTag.RefSet,
+ TagRef: foryTag.Ref,
}
fields = append(fields, fieldInfo)
fieldNames = append(fieldNames, fieldInfo.Name)
serializers = append(serializers, fieldSerializer)
typeIds = append(typeIds, fieldTypeId)
nullables = append(nullables, nullableFlag)
+ tagIDs = append(tagIDs, foryTag.ID)
}
- // Sort fields according to specification using nullable info for
consistent ordering
- serializers, fieldNames = sortFields(typeResolver, fieldNames,
serializers, typeIds, nullables)
+ // Sort fields according to specification using nullable info and tag
IDs for consistent ordering
+ serializers, fieldNames = sortFields(typeResolver, fieldNames,
serializers, typeIds, nullables, tagIDs)
order := make(map[string]int, len(fieldNames))
for idx, name := range fieldNames {
order[name] = idx
@@ -1195,16 +1224,35 @@ func (s *structSerializer)
initFieldsFromDefsWithResolver(typeResolver *TypeReso
return nil
}
- // Build map from field names to struct field indices
+ // Build maps from field names and tag IDs to struct field indices
fieldNameToIndex := make(map[string]int)
fieldNameToOffset := make(map[string]uintptr)
fieldNameToType := make(map[string]reflect.Type)
+ fieldTagIDToIndex := make(map[int]int) // tag ID -> struct
field index
+ fieldTagIDToOffset := make(map[int]uintptr) // tag ID -> field offset
+ fieldTagIDToType := make(map[int]reflect.Type) // tag ID -> field type
+ fieldTagIDToName := make(map[int]string) // tag ID -> snake_case
field name
for i := 0; i < type_.NumField(); i++ {
field := type_.Field(i)
+
+ // Parse fory tag and skip ignored fields
+ foryTag := ParseForyTag(field)
+ if foryTag.Ignore {
+ continue
+ }
+
name := SnakeCase(field.Name)
fieldNameToIndex[name] = i
fieldNameToOffset[name] = field.Offset
fieldNameToType[name] = field.Type
+
+ // Also index by tag ID if present
+ if foryTag.ID >= 0 {
+ fieldTagIDToIndex[foryTag.ID] = i
+ fieldTagIDToOffset[foryTag.ID] = field.Offset
+ fieldTagIDToType[foryTag.ID] = field.Type
+ fieldTagIDToName[foryTag.ID] = name
+ }
}
var fields []*FieldInfo
@@ -1237,12 +1285,43 @@ func (s *structSerializer)
initFieldsFromDefsWithResolver(typeResolver *TypeReso
isStructLikeField := isStructFieldType(def.fieldType)
// Try to find corresponding local field
+ // First try to match by tag ID (if remote def uses tag ID)
+ // Then fall back to matching by field name
fieldIndex := -1
var offset uintptr
var fieldType reflect.Type
+ var localFieldName string
+ var localType reflect.Type
+ var exists bool
+
+ if def.tagID >= 0 {
+ // Try to match by tag ID
+ if idx, ok := fieldTagIDToIndex[def.tagID]; ok {
+ exists = true
+ fieldIndex = idx // Will be overwritten if
types are compatible
+ localType = fieldTagIDToType[def.tagID]
+ offset = fieldTagIDToOffset[def.tagID]
+ localFieldName = fieldTagIDToName[def.tagID]
+ _ = fieldIndex // Use to avoid compiler
warning, will be set properly below
+ }
+ }
+
+ // Fall back to name-based matching if tag ID match failed
+ if !exists && def.name != "" {
+ if idx, ok := fieldNameToIndex[def.name]; ok {
+ exists = true
+ localType = fieldNameToType[def.name]
+ offset = fieldNameToOffset[def.name]
+ localFieldName = def.name
+ _ = idx // Will be set properly below
+ }
+ }
- if idx, exists := fieldNameToIndex[def.name]; exists {
- localType := fieldNameToType[def.name]
+ if exists {
+ idx := fieldNameToIndex[localFieldName]
+ if def.tagID >= 0 {
+ idx = fieldTagIDToIndex[def.tagID]
+ }
// Check if types are compatible
// For primitive types: skip if types don't match
// For struct-like types: allow read even if TypeDef
lookup failed,
@@ -1306,7 +1385,7 @@ func (s *structSerializer)
initFieldsFromDefsWithResolver(typeResolver *TypeReso
if shouldRead {
fieldIndex = idx
- offset = fieldNameToOffset[def.name]
+ // offset was already set above when matching
by tag ID or field name
// For struct-like fields with failed type
lookup, get the serializer for the local type
if typeLookupFailed && isStructLikeField &&
fieldSerializer == nil {
fieldSerializer, _ =
typeResolver.getSerializerByType(localType, true)
@@ -1398,8 +1477,14 @@ func (s *structSerializer)
initFieldsFromDefsWithResolver(typeResolver *TypeReso
}
}
+ // Determine field name: use local field name if matched,
otherwise use def.name
+ fieldName := def.name
+ if localFieldName != "" {
+ fieldName = localFieldName
+ }
+
fieldInfo := &FieldInfo{
- Name: def.name,
+ Name: fieldName,
Offset: offset,
Type: fieldType,
StaticId: staticId,
@@ -1411,6 +1496,8 @@ func (s *structSerializer)
initFieldsFromDefsWithResolver(typeResolver *TypeReso
RefMode: refMode,
WriteType: writeType,
HasGenerics: isCollectionType(fieldTypeId), //
Container fields have declared element types
+ TagID: def.tagID,
+ HasForyTag: def.tagID >= 0,
}
fields = append(fields, fieldInfo)
}
@@ -1518,13 +1605,91 @@ func isStructFieldType(ft FieldType) bool {
return false
}
-func (s *structSerializer) computeHash() int32 {
- var sb strings.Builder
+// FieldFingerprintInfo contains the information needed to compute a field's
fingerprint.
+type FieldFingerprintInfo struct {
+ // FieldID is the tag ID if configured (>= 0), or -1 to use field name
+ FieldID int
+ // FieldName is the snake_case field name (used when FieldID < 0)
+ FieldName string
+ // TypeID is the Fory type ID for the field
+ TypeID TypeId
+ // Ref is true if reference tracking is enabled for this field
+ Ref bool
+ // Nullable is true if null flag is written for this field
+ Nullable bool
+}
- for _, field := range s.fields {
- sb.WriteString(SnakeCase(field.Name))
+// ComputeStructFingerprint computes the fingerprint string for a struct type.
+//
+// Fingerprint Format:
+//
+// Each field contributes: "<field_id_or_name>,<type_id>,<ref>,<nullable>;"
+// Fields are sorted by field_id_or_name (lexicographically as strings)
+//
+// Field Components:
+// - field_id_or_name: Tag ID as string if configured (e.g., "0", "1"),
otherwise snake_case field name
+// - type_id: Fory TypeId as decimal string (e.g., "4" for INT32)
+// - ref: "1" if reference tracking enabled, "0" otherwise
+// - nullable: "1" if null flag is written, "0" otherwise
+//
+// Example fingerprints:
+// - With tag IDs: "0,4,0,0;1,4,0,1;2,9,0,1;"
+// - With field names: "age,4,0,0;name,9,0,1;"
+//
+// The fingerprint is used to compute a hash for struct schema versioning.
+// Different nullable/ref settings will produce different fingerprints,
+// ensuring schema compatibility is properly validated.
+func ComputeStructFingerprint(fields []FieldFingerprintInfo) string {
+ // Sort fields by their identifier (field ID or name)
+ type fieldWithKey struct {
+ field FieldFingerprintInfo
+ sortKey string
+ }
+ fieldsWithKeys := make([]fieldWithKey, 0, len(fields))
+ for _, field := range fields {
+ var sortKey string
+ if field.FieldID >= 0 {
+ sortKey = fmt.Sprintf("%d", field.FieldID)
+ } else {
+ sortKey = field.FieldName
+ }
+ fieldsWithKeys = append(fieldsWithKeys, fieldWithKey{field:
field, sortKey: sortKey})
+ }
+
+ sort.Slice(fieldsWithKeys, func(i, j int) bool {
+ return fieldsWithKeys[i].sortKey < fieldsWithKeys[j].sortKey
+ })
+
+ var sb strings.Builder
+ for _, fw := range fieldsWithKeys {
+ // Field identifier
+ sb.WriteString(fw.sortKey)
+ sb.WriteString(",")
+ // Type ID
+ sb.WriteString(fmt.Sprintf("%d", fw.field.TypeID))
sb.WriteString(",")
+ // Ref flag
+ if fw.field.Ref {
+ sb.WriteString("1")
+ } else {
+ sb.WriteString("0")
+ }
+ sb.WriteString(",")
+ // Nullable flag
+ if fw.field.Nullable {
+ sb.WriteString("1")
+ } else {
+ sb.WriteString("0")
+ }
+ sb.WriteString(";")
+ }
+ return sb.String()
+}
+func (s *structSerializer) computeHash() int32 {
+ // Build FieldFingerprintInfo for each field
+ fields := make([]FieldFingerprintInfo, 0, len(s.fields))
+ for _, field := range s.fields {
var typeId TypeId
isEnumField := false
if field.Serializer == nil {
@@ -1534,17 +1699,14 @@ func (s *structSerializer) computeHash() int32 {
// Check if this is an enum serializer (directly or
wrapped in ptrToValueSerializer)
if _, ok := field.Serializer.(*enumSerializer); ok {
isEnumField = true
- // Java uses UNKNOWN (0) for enum types in
fingerprint computation
typeId = UNKNOWN
} else if ptrSer, ok :=
field.Serializer.(*ptrToValueSerializer); ok {
if _, ok :=
ptrSer.valueSerializer.(*enumSerializer); ok {
isEnumField = true
- // Java uses UNKNOWN (0) for enum types
in fingerprint computation
typeId = UNKNOWN
}
}
// For fixed-size arrays with primitive elements, use
primitive array type IDs
- // This matches Python's int8_array, int16_array, etc.
types
if field.Type.Kind() == reflect.Array {
elemKind := field.Type.Elem().Kind()
switch elemKind {
@@ -1564,26 +1726,29 @@ func (s *structSerializer) computeHash() int32 {
typeId = LIST
}
} else if field.Type.Kind() == reflect.Slice {
- // Slices use LIST type ID (maps to Python
List[T])
typeId = LIST
}
}
- sb.WriteString(fmt.Sprintf("%d", typeId))
- sb.WriteString(",")
- // For cross-language hash compatibility, nullable=0 only for
primitive non-pointer types
- // This matches Java's behavior where isPrimitive() returns
true only for int, long, boolean, etc.
- // Go strings and other non-primitive types should have
nullable=1
- // Enum types are always nullable (like Java enums which are
objects)
- nullableFlag := "1"
- if isNonNullablePrimitiveKind(field.Type.Kind()) &&
!field.Referencable && !isEnumField {
- nullableFlag = "0"
+ // Determine nullable flag based on field type
+ // Use the same logic as codegen for consistency
+ nullable := true
+ if isNonNullablePrimitiveKind(field.Type.Kind()) &&
!isEnumField {
+ nullable = false
}
- sb.WriteString(nullableFlag)
- sb.WriteString(";")
+
+ fields = append(fields, FieldFingerprintInfo{
+ FieldID: field.TagID,
+ FieldName: SnakeCase(field.Name),
+ TypeID: typeId,
+ // Ref is based on explicit tag annotation only, NOT
runtime ref_tracking config
+ // This allows fingerprint to be computed at compile
time for C++/Rust
+ Ref: field.TagRefSet && field.TagRef,
+ Nullable: nullable,
+ })
}
- hashString := sb.String()
+ hashString := ComputeStructFingerprint(fields)
data := []byte(hashString)
h1, _ := murmur3.Sum128WithSeed(data, 47)
hash := int32(h1 & 0xFFFFFFFF)
@@ -1598,6 +1763,16 @@ func (s *structSerializer) computeHash() int32 {
return hash
}
+// GetStructHash returns the struct hash for a given type using the provided
TypeResolver.
+// This is used by codegen serializers to get the hash at runtime.
+func GetStructHash(type_ reflect.Type, resolver *TypeResolver) int32 {
+ ser := newStructSerializer(type_, "", nil)
+ if err := ser.initialize(resolver); err != nil {
+ panic(fmt.Errorf("failed to initialize struct serializer for
hash computation: %v", err))
+ }
+ return ser.structHash
+}
+
// Field sorting helpers
type triple struct {
@@ -1605,17 +1780,30 @@ type triple struct {
serializer Serializer
name string
nullable bool
+ tagID int // -1 = use field name, >=0 = use tag ID for sorting
+}
+
+// getFieldSortKey returns the sort key for a field.
+// If tagID >= 0, returns the tag ID as string (for tag-based sorting).
+// Otherwise returns the snake_case field name.
+func (t triple) getSortKey() string {
+ if t.tagID >= 0 {
+ return fmt.Sprintf("%d", t.tagID)
+ }
+ return SnakeCase(t.name)
}
// sortFields sorts fields with nullable information to match Java's field
ordering.
// Java separates primitive types (int, long) from boxed types (Integer, Long).
// In Go, this corresponds to non-pointer primitives vs pointer-to-primitive.
+// When tagIDs are provided (>= 0), fields are sorted by tag ID instead of
field name.
func sortFields(
typeResolver *TypeResolver,
fieldNames []string,
serializers []Serializer,
typeIds []TypeId,
nullables []bool,
+ tagIDs []int,
) ([]Serializer, []string) {
var (
typeTriples []triple
@@ -1625,11 +1813,15 @@ func sortFields(
for i, name := range fieldNames {
ser := serializers[i]
+ tagID := TagIDUseFieldName // default: use field name
+ if tagIDs != nil && i < len(tagIDs) {
+ tagID = tagIDs[i]
+ }
if ser == nil {
- others = append(others, triple{UNKNOWN, nil, name,
nullables[i]})
+ others = append(others, triple{UNKNOWN, nil, name,
nullables[i], tagID})
continue
}
- typeTriples = append(typeTriples, triple{typeIds[i], ser, name,
nullables[i]})
+ typeTriples = append(typeTriples, triple{typeIds[i], ser, name,
nullables[i], tagID})
}
// Java orders: primitives, boxed, finals, others, collections, maps
// primitives = non-nullable primitive types (int, long, etc.)
@@ -1674,7 +1866,7 @@ func sortFields(
if szI != szJ {
return szI > szJ
}
- return SnakeCase(ai.name) < SnakeCase(aj.name)
+ return ai.getSortKey() < aj.getSortKey()
})
}
sortPrimitiveSlice(primitives)
@@ -1684,12 +1876,12 @@ func sortFields(
if s[i].typeID != s[j].typeID {
return s[i].typeID < s[j].typeID
}
- return SnakeCase(s[i].name) < SnakeCase(s[j].name)
+ return s[i].getSortKey() < s[j].getSortKey()
})
}
sortTuple := func(s []triple) {
sort.Slice(s, func(i, j int) bool {
- return SnakeCase(s[i].name) < SnakeCase(s[j].name)
+ return s[i].getSortKey() < s[j].getSortKey()
})
}
sortByTypeIDThenName(otherInternalTypeFields)
diff --git a/go/fory/tag.go b/go/fory/tag.go
new file mode 100644
index 000000000..d07154374
--- /dev/null
+++ b/go/fory/tag.go
@@ -0,0 +1,208 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package fory
+
+import (
+ "reflect"
+ "strconv"
+ "strings"
+)
+
+const (
+ // TagIDUseFieldName indicates field name should be used instead of tag
ID
+ TagIDUseFieldName = -1
+)
+
+// ForyTag represents parsed fory struct tag options.
+//
+// Tag format: `fory:"id=N,nullable=bool,ref=bool,ignore=bool"` or `fory:"-"`
+//
+// Options:
+// - id: Field tag ID. -1 (default) uses field name, >=0 uses numeric tag ID
for compact encoding
+// - nullable: Whether to write null flag. Default false (skip null flag for
non-nullable fields)
+// - ref: Whether to enable reference tracking. Default false (skip ref
tracking overhead)
+// - ignore: Whether to skip this field during serialization. Default false
+//
+// Examples:
+//
+// type Example struct {
+// Name string `fory:"id=0"` // Use tag
ID 0
+// Age int `fory:"id=1,nullable=false"` // Explicit
nullable=false
+// Email *string `fory:"id=2,nullable=true,ref=false"` // Nullable
pointer, no ref tracking
+// Parent *Node `fory:"id=3,ref=true,nullable=true"` // With
reference tracking
+// Secret string `fory:"ignore"` // Skip this
field
+// Hidden string `fory:"-"` // Skip this
field (shorthand)
+// }
+type ForyTag struct {
+ ID int // Field tag ID (-1 = use field name, >=0 = use tag ID)
+ Nullable bool // Whether to write null flag (default: false)
+ Ref bool // Whether to enable reference tracking (default: false)
+ Ignore bool // Whether to ignore this field during serialization
(default: false)
+ HasTag bool // Whether field has fory tag at all
+
+ // Track which options were explicitly set (for override logic)
+ NullableSet bool
+ RefSet bool
+ IgnoreSet bool
+}
+
+// ParseForyTag parses a fory struct tag from reflect.StructField.Tag.
+//
+// Tag format: `fory:"id=N,nullable=bool,ref=bool,ignore=bool"` or `fory:"-"`
+//
+// Supported syntaxes:
+// - Key-value: `nullable=true`, `ref=false`, `ignore=true`, `id=0`
+// - Standalone flags: `nullable`, `ref`, `ignore` (equivalent to =true)
+// - Shorthand: `-` (equivalent to `ignore=true`)
+func ParseForyTag(field reflect.StructField) ForyTag {
+ tag := ForyTag{
+ ID: TagIDUseFieldName,
+ Nullable: false,
+ Ref: false,
+ Ignore: false,
+ HasTag: false,
+ }
+
+ tagValue, ok := field.Tag.Lookup("fory")
+ if !ok {
+ return tag
+ }
+
+ tag.HasTag = true
+
+ // Handle "-" shorthand for ignore
+ if tagValue == "-" {
+ tag.Ignore = true
+ tag.IgnoreSet = true
+ return tag
+ }
+
+ // Parse comma-separated options
+ parts := strings.Split(tagValue, ",")
+ for _, part := range parts {
+ part = strings.TrimSpace(part)
+ if part == "" {
+ continue
+ }
+
+ // Handle key=value pairs and standalone flags
+ if idx := strings.Index(part, "="); idx >= 0 {
+ key := strings.TrimSpace(part[:idx])
+ value := strings.TrimSpace(part[idx+1:])
+
+ switch key {
+ case "id":
+ if id, err := strconv.Atoi(value); err == nil {
+ tag.ID = id
+ }
+ case "nullable":
+ tag.Nullable = parseBool(value)
+ tag.NullableSet = true
+ case "ref":
+ tag.Ref = parseBool(value)
+ tag.RefSet = true
+ case "ignore":
+ tag.Ignore = parseBool(value)
+ tag.IgnoreSet = true
+ }
+ } else {
+ // Handle standalone flags (presence means true)
+ switch part {
+ case "nullable":
+ tag.Nullable = true
+ tag.NullableSet = true
+ case "ref":
+ tag.Ref = true
+ tag.RefSet = true
+ case "ignore":
+ tag.Ignore = true
+ tag.IgnoreSet = true
+ }
+ }
+ }
+
+ return tag
+}
+
+// parseBool parses a boolean value from string.
+// Accepts: "true", "1", "yes" as true; everything else as false.
+func parseBool(s string) bool {
+ s = strings.ToLower(strings.TrimSpace(s))
+ return s == "true" || s == "1" || s == "yes"
+}
+
+// ValidateForyTags validates all fory tags in a struct type.
+// Returns an error if validation fails.
+//
+// Validation rules:
+// - Tag ID must be >= -1
+// - Tag IDs must be unique within a struct (except -1)
+// - Ignored fields are not validated for ID uniqueness
+func ValidateForyTags(t reflect.Type) error {
+ if t.Kind() == reflect.Ptr {
+ t = t.Elem()
+ }
+ if t.Kind() != reflect.Struct {
+ return nil
+ }
+
+ tagIDs := make(map[int]string) // id -> field name
+
+ for i := 0; i < t.NumField(); i++ {
+ field := t.Field(i)
+ tag := ParseForyTag(field)
+
+ // Skip ignored fields for ID uniqueness validation
+ if tag.Ignore {
+ continue
+ }
+
+ // Validate tag ID range
+ if tag.ID < TagIDUseFieldName {
+ return InvalidTagErrorf("invalid fory tag id=%d on
field %s: id must be >= -1",
+ tag.ID, field.Name)
+ }
+
+ // Check for duplicate tag IDs (except -1 which means use field
name)
+ if tag.ID >= 0 {
+ if existing, ok := tagIDs[tag.ID]; ok {
+ return InvalidTagErrorf("duplicate fory tag
id=%d on fields %s and %s",
+ tag.ID, existing, field.Name)
+ }
+ tagIDs[tag.ID] = field.Name
+ }
+ }
+
+ return nil
+}
+
+// ShouldIncludeField returns true if the field should be serialized.
+// A field is excluded if:
+// - It's unexported (starts with lowercase)
+// - It has `fory:"-"` tag
+// - It has `fory:"ignore"` or `fory:"ignore=true"` tag
+func ShouldIncludeField(field reflect.StructField) bool {
+ // Skip unexported fields
+ if field.PkgPath != "" {
+ return false
+ }
+
+ // Check for ignore tag
+ tag := ParseForyTag(field)
+ return !tag.Ignore
+}
diff --git a/go/fory/tag_test.go b/go/fory/tag_test.go
new file mode 100644
index 000000000..3c7f52e2b
--- /dev/null
+++ b/go/fory/tag_test.go
@@ -0,0 +1,613 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package fory
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestParseForyTag(t *testing.T) {
+ type TestStruct struct {
+ Field1 string `fory:"id=0"`
+ Field2 int
`fory:"id=1,nullable=false,ref=false,ignore=false"`
+ Field3 *string `fory:"id=2,nullable=true,ref=true"`
+ Field4 string `fory:"nullable,ref"`
+ Field5 string `fory:"ignore"`
+ Field6 string `fory:"ignore=true"`
+ Field7 string `fory:"ignore=false"`
+ Field8 string `fory:"-"`
+ Field9 string
+ Field10 string `fory:"id=3,ignore=false"`
+ Field11 string `fory:"id=-1"`
+ Field12 string `fory:"nullable=true,ref=false"`
+ }
+
+ typ := reflect.TypeOf(TestStruct{})
+
+ // Test Field1: id=0
+ tag1 := ParseForyTag(typ.Field(0))
+ require.True(t, tag1.HasTag)
+ require.Equal(t, 0, tag1.ID)
+ require.False(t, tag1.Nullable)
+ require.False(t, tag1.Ref)
+ require.False(t, tag1.Ignore)
+ require.False(t, tag1.NullableSet)
+ require.False(t, tag1.RefSet)
+ require.False(t, tag1.IgnoreSet)
+
+ // Test Field2: all explicit false values
+ tag2 := ParseForyTag(typ.Field(1))
+ require.Equal(t, 1, tag2.ID)
+ require.False(t, tag2.Nullable)
+ require.False(t, tag2.Ref)
+ require.False(t, tag2.Ignore)
+ require.True(t, tag2.NullableSet)
+ require.True(t, tag2.RefSet)
+ require.True(t, tag2.IgnoreSet)
+
+ // Test Field3: explicit true values
+ tag3 := ParseForyTag(typ.Field(2))
+ require.Equal(t, 2, tag3.ID)
+ require.True(t, tag3.Nullable)
+ require.True(t, tag3.Ref)
+ require.False(t, tag3.Ignore)
+
+ // Test Field4: standalone flags (presence = true)
+ tag4 := ParseForyTag(typ.Field(3))
+ require.Equal(t, TagIDUseFieldName, tag4.ID)
+ require.True(t, tag4.Nullable)
+ require.True(t, tag4.Ref)
+ require.True(t, tag4.NullableSet)
+ require.True(t, tag4.RefSet)
+
+ // Test Field5: standalone ignore
+ tag5 := ParseForyTag(typ.Field(4))
+ require.True(t, tag5.Ignore)
+ require.True(t, tag5.IgnoreSet)
+
+ // Test Field6: explicit ignore=true
+ tag6 := ParseForyTag(typ.Field(5))
+ require.True(t, tag6.Ignore)
+ require.True(t, tag6.IgnoreSet)
+
+ // Test Field7: explicit ignore=false
+ tag7 := ParseForyTag(typ.Field(6))
+ require.False(t, tag7.Ignore)
+ require.True(t, tag7.IgnoreSet)
+
+ // Test Field8: "-" shorthand
+ tag8 := ParseForyTag(typ.Field(7))
+ require.True(t, tag8.Ignore)
+ require.True(t, tag8.IgnoreSet)
+
+ // Test Field9: no tag
+ tag9 := ParseForyTag(typ.Field(8))
+ require.False(t, tag9.HasTag)
+ require.False(t, tag9.Ignore)
+ require.Equal(t, TagIDUseFieldName, tag9.ID)
+
+ // Test Field10: has ID but not ignored
+ tag10 := ParseForyTag(typ.Field(9))
+ require.Equal(t, 3, tag10.ID)
+ require.False(t, tag10.Ignore)
+ require.True(t, tag10.IgnoreSet)
+
+ // Test Field11: explicit id=-1 (use field name)
+ tag11 := ParseForyTag(typ.Field(10))
+ require.Equal(t, TagIDUseFieldName, tag11.ID)
+ require.True(t, tag11.HasTag)
+
+ // Test Field12: nullable=true,ref=false
+ tag12 := ParseForyTag(typ.Field(11))
+ require.True(t, tag12.Nullable)
+ require.False(t, tag12.Ref)
+ require.True(t, tag12.NullableSet)
+ require.True(t, tag12.RefSet)
+}
+
+func TestShouldIncludeField(t *testing.T) {
+ type TestStruct struct {
+ Included1 string `fory:"id=0"`
+ Included2 string `fory:"ignore=false"`
+ Ignored1 string `fory:"ignore"`
+ Ignored2 string `fory:"ignore=true"`
+ Ignored3 string `fory:"-"`
+ NoTag string
+ }
+
+ typ := reflect.TypeOf(TestStruct{})
+
+ require.True(t, ShouldIncludeField(typ.Field(0))) // Included1
+ require.True(t, ShouldIncludeField(typ.Field(1))) // Included2
(ignore=false)
+ require.False(t, ShouldIncludeField(typ.Field(2))) // Ignored1
+ require.False(t, ShouldIncludeField(typ.Field(3))) // Ignored2
+ require.False(t, ShouldIncludeField(typ.Field(4))) // Ignored3
+ require.True(t, ShouldIncludeField(typ.Field(5))) // NoTag (default:
include)
+}
+
+func TestValidateForyTags(t *testing.T) {
+ // Test valid struct
+ type ValidStruct struct {
+ Field1 string `fory:"id=0"`
+ Field2 string `fory:"id=1"`
+ Field3 string `fory:"id=-1"`
+ Field4 string // No tag
+ }
+ err := ValidateForyTags(reflect.TypeOf(ValidStruct{}))
+ require.NoError(t, err)
+
+ // Test duplicate tag IDs
+ type DuplicateIDs struct {
+ Field1 string `fory:"id=0"`
+ Field2 string `fory:"id=0"`
+ }
+ err = ValidateForyTags(reflect.TypeOf(DuplicateIDs{}))
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "duplicate")
+ foryErr, ok := err.(Error)
+ require.True(t, ok, "error should be fory.Error type")
+ require.Equal(t, ErrKindInvalidTag, foryErr.Kind())
+
+ // Test invalid ID (< -1)
+ type InvalidID struct {
+ Field1 string `fory:"id=-2"`
+ }
+ err = ValidateForyTags(reflect.TypeOf(InvalidID{}))
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "invalid")
+ foryErr, ok = err.(Error)
+ require.True(t, ok, "error should be fory.Error type")
+ require.Equal(t, ErrKindInvalidTag, foryErr.Kind())
+
+ // Test that ignored fields don't count for ID uniqueness
+ type IgnoredFields struct {
+ Field1 string `fory:"id=0"`
+ Field2 string `fory:"id=0,ignore"` // Same ID but ignored
+ Field3 string `fory:"id=1"`
+ }
+ err = ValidateForyTags(reflect.TypeOf(IgnoredFields{}))
+ require.NoError(t, err)
+}
+
+func TestParseForyTagEdgeCases(t *testing.T) {
+ // Test whitespace handling
+ type WhitespaceStruct struct {
+ Field1 string `fory:" id = 0 , nullable = true "`
+ }
+ typ := reflect.TypeOf(WhitespaceStruct{})
+ tag := ParseForyTag(typ.Field(0))
+ require.Equal(t, 0, tag.ID)
+ require.True(t, tag.Nullable)
+
+ // Test empty tag value
+ type EmptyTagStruct struct {
+ Field1 string `fory:""`
+ }
+ typ2 := reflect.TypeOf(EmptyTagStruct{})
+ tag2 := ParseForyTag(typ2.Field(0))
+ require.True(t, tag2.HasTag)
+ require.Equal(t, TagIDUseFieldName, tag2.ID)
+
+ // Test boolean values
+ type BoolValuesStruct struct {
+ Field1 string `fory:"nullable=1"`
+ Field2 string `fory:"nullable=yes"`
+ Field3 string `fory:"nullable=TRUE"`
+ Field4 string `fory:"nullable=no"`
+ }
+ typ3 := reflect.TypeOf(BoolValuesStruct{})
+
+ tag3 := ParseForyTag(typ3.Field(0))
+ require.True(t, tag3.Nullable) // "1" -> true
+
+ tag4 := ParseForyTag(typ3.Field(1))
+ require.True(t, tag4.Nullable) // "yes" -> true
+
+ tag5 := ParseForyTag(typ3.Field(2))
+ require.True(t, tag5.Nullable) // "TRUE" -> true
+
+ tag6 := ParseForyTag(typ3.Field(3))
+ require.False(t, tag6.Nullable) // "no" -> false
+}
+
+// Test struct with tags for serialization tests
+type PersonWithTags struct {
+ Name string `fory:"id=0"`
+ Age int32 `fory:"id=1"`
+ Email string `fory:"id=2,nullable=true"`
+ Secret string `fory:"-"`
+}
+
+// Test struct without tags
+type PersonWithoutTags struct {
+ Name string
+ Age int32
+ Email string
+}
+
+func TestSerializationWithTags(t *testing.T) {
+ fory := NewFory(WithRefTracking(false), WithCompatible(true))
+ err := fory.RegisterByName(PersonWithTags{}, "test.PersonWithTags")
+ require.NoError(t, err)
+
+ person := PersonWithTags{
+ Name: "John",
+ Age: 30,
+ Email: "[email protected]",
+ Secret: "should-be-ignored",
+ }
+
+ // Serialize
+ data, err := fory.Marshal(person)
+ require.NoError(t, err)
+
+ // Deserialize
+ var result PersonWithTags
+ err = fory.Unmarshal(data, &result)
+ require.NoError(t, err)
+
+ require.Equal(t, person.Name, result.Name)
+ require.Equal(t, person.Age, result.Age)
+ require.Equal(t, person.Email, result.Email)
+ require.Empty(t, result.Secret) // Secret should be empty (not
serialized)
+}
+
+func TestSerializationWithoutTags(t *testing.T) {
+ fory := NewFory(WithRefTracking(false), WithCompatible(true))
+ err := fory.RegisterByName(PersonWithoutTags{},
"test.PersonWithoutTags")
+ require.NoError(t, err)
+
+ person := PersonWithoutTags{
+ Name: "Jane",
+ Age: 25,
+ Email: "[email protected]",
+ }
+
+ // Serialize
+ data, err := fory.Marshal(person)
+ require.NoError(t, err)
+
+ // Deserialize
+ var result PersonWithoutTags
+ err = fory.Unmarshal(data, &result)
+ require.NoError(t, err)
+
+ require.Equal(t, person.Name, result.Name)
+ require.Equal(t, person.Age, result.Age)
+ require.Equal(t, person.Email, result.Email)
+}
+
+func TestTagIDReducesPayloadSize(t *testing.T) {
+ fory1 := NewFory(WithRefTracking(false), WithCompatible(true))
+ err := fory1.RegisterByName(PersonWithTags{}, "test.PersonWithTags")
+ require.NoError(t, err)
+
+ fory2 := NewFory(WithRefTracking(false), WithCompatible(true))
+ err = fory2.RegisterByName(PersonWithoutTags{},
"test.PersonWithoutTags")
+ require.NoError(t, err)
+
+ // Create comparable data
+ personWithTags := PersonWithTags{
+ Name: "John",
+ Age: 30,
+ Email: "[email protected]",
+ }
+ personWithoutTags := PersonWithoutTags{
+ Name: "John",
+ Age: 30,
+ Email: "[email protected]",
+ }
+
+ // Serialize both
+ dataWithTags, err := fory1.Marshal(personWithTags)
+ require.NoError(t, err)
+
+ dataWithoutTags, err := fory2.Marshal(personWithoutTags)
+ require.NoError(t, err)
+
+ // Tag IDs should produce smaller or equal payload
+ // (Tag IDs use compact integer encoding vs field name strings)
+ t.Logf("With tags: %d bytes, Without tags: %d bytes",
len(dataWithTags), len(dataWithoutTags))
+ require.LessOrEqual(t, len(dataWithTags), len(dataWithoutTags),
+ "Tag IDs should produce smaller or equal payload size")
+}
+
+// Test struct with numeric fields and tag IDs for compact encoding
+type NumericStructWithTags struct {
+ FieldA *int32 `fory:"id=0,nullable=false"`
+ FieldB *int32 `fory:"id=1,nullable=false"`
+ FieldC *int32 `fory:"id=2,nullable=false"`
+ FieldD *int32 `fory:"id=3,nullable=false"`
+ FieldE *int32 `fory:"id=4,nullable=false"`
+}
+
+type NumericStructWithoutTags struct {
+ FieldA *int32
+ FieldB *int32
+ FieldC *int32
+ FieldD *int32
+ FieldE *int32
+}
+
+func TestNumericStructTagIDReducesSize(t *testing.T) {
+ fory1 := NewFory(WithRefTracking(false), WithCompatible(true))
+ err := fory1.RegisterByName(NumericStructWithTags{},
"test.NumericStructWithTags")
+ require.NoError(t, err)
+
+ fory2 := NewFory(WithRefTracking(false), WithCompatible(true))
+ err = fory2.RegisterByName(NumericStructWithoutTags{},
"test.NumericStructWithoutTags")
+ require.NoError(t, err)
+
+ // Create small int32 values
+ v1, v2, v3, v4, v5 := int32(1), int32(2), int32(3), int32(4), int32(5)
+
+ objWithTags := NumericStructWithTags{
+ FieldA: &v1,
+ FieldB: &v2,
+ FieldC: &v3,
+ FieldD: &v4,
+ FieldE: &v5,
+ }
+ objWithoutTags := NumericStructWithoutTags{
+ FieldA: &v1,
+ FieldB: &v2,
+ FieldC: &v3,
+ FieldD: &v4,
+ FieldE: &v5,
+ }
+
+ // Serialize both
+ dataWithTags, err := fory1.Marshal(objWithTags)
+ require.NoError(t, err)
+
+ dataWithoutTags, err := fory2.Marshal(objWithoutTags)
+ require.NoError(t, err)
+
+ t.Logf("Numeric with tags: %d bytes, without tags: %d bytes, saved: %d
bytes",
+ len(dataWithTags), len(dataWithoutTags),
len(dataWithoutTags)-len(dataWithTags))
+
+ // Tag IDs + nullable=false should produce significantly smaller payload
+ require.Less(t, len(dataWithTags), len(dataWithoutTags),
+ "Tag IDs with nullable=false should produce smaller payload")
+
+ // Deserialize and verify
+ var result NumericStructWithTags
+ err = fory1.Unmarshal(dataWithTags, &result)
+ require.NoError(t, err)
+
+ require.Equal(t, *objWithTags.FieldA, *result.FieldA)
+ require.Equal(t, *objWithTags.FieldB, *result.FieldB)
+ require.Equal(t, *objWithTags.FieldC, *result.FieldC)
+ require.Equal(t, *objWithTags.FieldD, *result.FieldD)
+ require.Equal(t, *objWithTags.FieldE, *result.FieldE)
+}
+
+// Test structs at package level for consistent naming
+type TestStructNoNull struct {
+ A *int32 `fory:"id=0,nullable=false,ref=false"`
+ B *int32 `fory:"id=1,nullable=false,ref=false"`
+ C *int32 `fory:"id=2,nullable=false,ref=false"`
+ D *int32 `fory:"id=3,nullable=false,ref=false"`
+ E *int32 `fory:"id=4,nullable=false,ref=false"`
+}
+
+type TestStructDefalt struct {
+ A *int32 `fory:"id=0"`
+ B *int32 `fory:"id=1"`
+ C *int32 `fory:"id=2"`
+ D *int32 `fory:"id=3"`
+ E *int32 `fory:"id=4"`
+}
+
+func TestNullableRefFlagsRespected(t *testing.T) {
+ // Debug: verify tag parsing
+ typ1 := reflect.TypeOf(TestStructNoNull{})
+ for i := 0; i < typ1.NumField(); i++ {
+ field := typ1.Field(i)
+ tag := ParseForyTag(field)
+ t.Logf("Field %s: ID=%d, Nullable=%v (set=%v), Ref=%v (set=%v)",
+ field.Name, tag.ID, tag.Nullable, tag.NullableSet,
tag.Ref, tag.RefSet)
+ }
+
+ fory1 := NewFory(WithRefTracking(false), WithCompatible(true))
+ err := fory1.RegisterByName(TestStructNoNull{}, "test.TestStructNoNull")
+ require.NoError(t, err)
+
+ fory2 := NewFory(WithRefTracking(false), WithCompatible(true))
+ err = fory2.RegisterByName(TestStructDefalt{}, "test.TestStructDefalt")
+ require.NoError(t, err)
+
+ v1, v2, v3, v4, v5 := int32(1), int32(2), int32(3), int32(4), int32(5)
+
+ objNoFlags := TestStructNoNull{A: &v1, B: &v2, C: &v3, D: &v4, E: &v5}
+ objDefault := TestStructDefalt{A: &v1, B: &v2, C: &v3, D: &v4, E: &v5}
+
+ dataNoFlags, err := fory1.Marshal(objNoFlags)
+ require.NoError(t, err)
+
+ dataDefault, err := fory2.Marshal(objDefault)
+ require.NoError(t, err)
+
+ // With nullable=false, we should save 5 bytes (1 null flag per field)
+ t.Logf("No nullable/ref flags: %d bytes, Default flags: %d bytes,
saved: %d bytes",
+ len(dataNoFlags), len(dataDefault),
len(dataDefault)-len(dataNoFlags))
+
+ require.Less(t, len(dataNoFlags), len(dataDefault),
+ "nullable=false,ref=false should produce smaller payload by
skipping null flags")
+
+ // Verify deserialization works
+ var result TestStructNoNull
+ err = fory1.Unmarshal(dataNoFlags, &result)
+ require.NoError(t, err)
+ require.Equal(t, *objNoFlags.A, *result.A)
+ require.Equal(t, *objNoFlags.B, *result.B)
+ require.Equal(t, *objNoFlags.C, *result.C)
+ require.Equal(t, *objNoFlags.D, *result.D)
+ require.Equal(t, *objNoFlags.E, *result.E)
+}
+
+func TestTypeDefEncodingSizeWithTagIDs(t *testing.T) {
+ fory1 := NewFory(WithRefTracking(false), WithCompatible(true))
+ err := fory1.RegisterByName(NumericStructWithTags{},
"test.NumericStructWithTags")
+ require.NoError(t, err)
+
+ fory2 := NewFory(WithRefTracking(false), WithCompatible(true))
+ err = fory2.RegisterByName(NumericStructWithoutTags{},
"test.NumericStructWithoutTags")
+ require.NoError(t, err)
+
+ // Build TypeDef for struct with tags
+ typeDefWithTags, err := buildTypeDef(fory1,
reflect.ValueOf(NumericStructWithTags{}))
+ require.NoError(t, err)
+
+ // Build TypeDef for struct without tags
+ typeDefWithoutTags, err := buildTypeDef(fory2,
reflect.ValueOf(NumericStructWithoutTags{}))
+ require.NoError(t, err)
+
+ // Encode TypeDef with tags
+ bufferWithTags := NewByteBuffer(make([]byte, 0, 256))
+ writeErr := &Error{}
+ typeDefWithTags.writeTypeDef(bufferWithTags, writeErr)
+ require.False(t, writeErr.HasError())
+ sizeWithTags := bufferWithTags.WriterIndex()
+
+ // Encode TypeDef without tags
+ bufferWithoutTags := NewByteBuffer(make([]byte, 0, 256))
+ typeDefWithoutTags.writeTypeDef(bufferWithoutTags, writeErr)
+ require.False(t, writeErr.HasError())
+ sizeWithoutTags := bufferWithoutTags.WriterIndex()
+
+ t.Logf("TypeDef with tags: %d bytes, without tags: %d bytes, saved: %d
bytes",
+ sizeWithTags, sizeWithoutTags, sizeWithoutTags-sizeWithTags)
+
+ // TypeDef with tag IDs should be smaller (tag IDs use 1 byte vs field
names)
+ require.Less(t, sizeWithTags, sizeWithoutTags,
+ "TypeDef with tag IDs should be smaller than with field names")
+}
+
+// Test struct with large tag IDs (>15 requires varint encoding)
+type StructWithLargeTagIDs struct {
+ Field1 string `fory:"id=0"`
+ Field2 string `fory:"id=15"`
+ Field3 string `fory:"id=100"`
+ Field4 string `fory:"id=1000"`
+}
+
+func TestLargeTagIDs(t *testing.T) {
+ fory := NewFory(WithRefTracking(false), WithCompatible(true))
+ err := fory.RegisterByName(StructWithLargeTagIDs{},
"test.StructWithLargeTagIDs")
+ require.NoError(t, err)
+
+ obj := StructWithLargeTagIDs{
+ Field1: "value1",
+ Field2: "value2",
+ Field3: "value3",
+ Field4: "value4",
+ }
+
+ // Serialize
+ data, err := fory.Marshal(obj)
+ require.NoError(t, err)
+
+ // Deserialize
+ var result StructWithLargeTagIDs
+ err = fory.Unmarshal(data, &result)
+ require.NoError(t, err)
+
+ require.Equal(t, obj.Field1, result.Field1)
+ require.Equal(t, obj.Field2, result.Field2)
+ require.Equal(t, obj.Field3, result.Field3)
+ require.Equal(t, obj.Field4, result.Field4)
+}
+
+// Test struct with mixed tag and no-tag fields
+type MixedTagStruct struct {
+ TaggedField string `fory:"id=0"`
+ UntaggedField string
+ AnotherTagged int32 `fory:"id=1,nullable=false"`
+}
+
+func TestMixedTagFields(t *testing.T) {
+ fory := NewFory(WithRefTracking(false), WithCompatible(true))
+ err := fory.RegisterByName(MixedTagStruct{}, "test.MixedTagStruct")
+ require.NoError(t, err)
+
+ obj := MixedTagStruct{
+ TaggedField: "tagged",
+ UntaggedField: "untagged",
+ AnotherTagged: 42,
+ }
+
+ // Serialize
+ data, err := fory.Marshal(obj)
+ require.NoError(t, err)
+
+ // Deserialize
+ var result MixedTagStruct
+ err = fory.Unmarshal(data, &result)
+ require.NoError(t, err)
+
+ require.Equal(t, obj.TaggedField, result.TaggedField)
+ require.Equal(t, obj.UntaggedField, result.UntaggedField)
+ require.Equal(t, obj.AnotherTagged, result.AnotherTagged)
+}
+
+// Test nested struct with tags
+type InnerWithTags struct {
+ Value string `fory:"id=0"`
+ Count int32 `fory:"id=1"`
+}
+
+type OuterWithTags struct {
+ Name string `fory:"id=0"`
+ Inner InnerWithTags `fory:"id=1"`
+ Items []string `fory:"id=2"`
+}
+
+func TestNestedStructWithTags(t *testing.T) {
+ fory := NewFory(WithRefTracking(false), WithCompatible(true))
+ err := fory.RegisterByName(InnerWithTags{}, "test.InnerWithTags")
+ require.NoError(t, err)
+ err = fory.RegisterByName(OuterWithTags{}, "test.OuterWithTags")
+ require.NoError(t, err)
+
+ obj := OuterWithTags{
+ Name: "outer",
+ Inner: InnerWithTags{
+ Value: "inner-value",
+ Count: 10,
+ },
+ Items: []string{"a", "b", "c"},
+ }
+
+ // Serialize
+ data, err := fory.Marshal(obj)
+ require.NoError(t, err)
+
+ // Deserialize
+ var result OuterWithTags
+ err = fory.Unmarshal(data, &result)
+ require.NoError(t, err)
+
+ require.Equal(t, obj.Name, result.Name)
+ require.Equal(t, obj.Inner.Value, result.Inner.Value)
+ require.Equal(t, obj.Inner.Count, result.Inner.Count)
+ require.Equal(t, obj.Items, result.Items)
+}
diff --git a/go/fory/tests/structs_fory_gen.go
b/go/fory/tests/structs_fory_gen.go
index 553748487..ae3faa5c8 100644
--- a/go/fory/tests/structs_fory_gen.go
+++ b/go/fory/tests/structs_fory_gen.go
@@ -1,6 +1,6 @@
// Code generated by forygen. DO NOT EDIT.
// source: ./structs.go
-// generated at: 2025-12-23T11:00:56+08:00
+// generated at: 2025-12-24T18:00:19+08:00
package fory
@@ -17,14 +17,23 @@ func init() {
fory.RegisterSerializerFactory((*ValidationDemo)(nil),
NewSerializerFor_ValidationDemo)
}
-type DynamicSliceDemo_ForyGenSerializer struct{}
+type DynamicSliceDemo_ForyGenSerializer struct {
+ structHash int32
+}
func NewSerializerFor_DynamicSliceDemo() fory.Serializer {
- return DynamicSliceDemo_ForyGenSerializer{}
+ return &DynamicSliceDemo_ForyGenSerializer{}
+}
+
+func (g *DynamicSliceDemo_ForyGenSerializer) initHash(resolver
*fory.TypeResolver) {
+ if g.structHash == 0 {
+ g.structHash =
fory.GetStructHash(reflect.TypeOf(DynamicSliceDemo{}), resolver)
+ }
}
// Write is the entry point for serialization with ref/type handling
-func (g DynamicSliceDemo_ForyGenSerializer) Write(ctx *fory.WriteContext,
refMode fory.RefMode, writeType bool, hasGenerics bool, value reflect.Value) {
+func (g *DynamicSliceDemo_ForyGenSerializer) Write(ctx *fory.WriteContext,
refMode fory.RefMode, writeType bool, hasGenerics bool, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
_ = hasGenerics // not used for struct serializers
switch refMode {
case fory.RefModeTracking:
@@ -54,10 +63,10 @@ func (g DynamicSliceDemo_ForyGenSerializer) Write(ctx
*fory.WriteContext, refMod
}
// WriteTyped provides strongly-typed serialization with no reflection overhead
-func (g DynamicSliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext,
v *DynamicSliceDemo) error {
+func (g *DynamicSliceDemo_ForyGenSerializer) WriteTyped(ctx
*fory.WriteContext, v *DynamicSliceDemo) error {
buf := ctx.Buffer()
- // WriteData precomputed struct hash for compatibility checking
- buf.WriteInt32(659991945) // hash of DynamicSliceDemo structure
+ // WriteData struct hash for compatibility checking
+ buf.WriteInt32(g.structHash)
// WriteData fields in sorted order
// Field: DynamicSlice ([]interface{})
@@ -81,7 +90,8 @@ func (g DynamicSliceDemo_ForyGenSerializer) WriteTyped(ctx
*fory.WriteContext, v
}
// WriteData provides reflect.Value interface compatibility (implements
fory.Serializer)
-func (g DynamicSliceDemo_ForyGenSerializer) WriteData(ctx *fory.WriteContext,
value reflect.Value) {
+func (g *DynamicSliceDemo_ForyGenSerializer) WriteData(ctx *fory.WriteContext,
value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
// Convert reflect.Value to concrete type and delegate to typed method
var v *DynamicSliceDemo
if value.Kind() == reflect.Ptr {
@@ -98,7 +108,8 @@ func (g DynamicSliceDemo_ForyGenSerializer) WriteData(ctx
*fory.WriteContext, va
}
// Read is the entry point for deserialization with ref/type handling
-func (g DynamicSliceDemo_ForyGenSerializer) Read(ctx *fory.ReadContext,
refMode fory.RefMode, readType bool, hasGenerics bool, value reflect.Value) {
+func (g *DynamicSliceDemo_ForyGenSerializer) Read(ctx *fory.ReadContext,
refMode fory.RefMode, readType bool, hasGenerics bool, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
_ = hasGenerics // not used for struct serializers
err := ctx.Err() // Get error pointer for deferred error checking
switch refMode {
@@ -128,16 +139,16 @@ func (g DynamicSliceDemo_ForyGenSerializer) Read(ctx
*fory.ReadContext, refMode
}
// ReadTyped provides strongly-typed deserialization with no reflection
overhead
-func (g DynamicSliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v
*DynamicSliceDemo) error {
+func (g *DynamicSliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext,
v *DynamicSliceDemo) error {
buf := ctx.Buffer()
err := ctx.Err() // Get error pointer for deferred error checking
// ReadData and verify struct hash
- if got := buf.ReadInt32(err); got != 659991945 {
+ if got := buf.ReadInt32(err); got != g.structHash {
if ctx.HasError() {
return ctx.TakeError()
}
- return fory.HashMismatchError(got, 659991945,
"DynamicSliceDemo")
+ return fory.HashMismatchError(got, g.structHash,
"DynamicSliceDemo")
}
// ReadData fields in same order as write
@@ -168,7 +179,8 @@ func (g DynamicSliceDemo_ForyGenSerializer) ReadTyped(ctx
*fory.ReadContext, v *
}
// ReadData provides reflect.Value interface compatibility (implements
fory.Serializer)
-func (g DynamicSliceDemo_ForyGenSerializer) ReadData(ctx *fory.ReadContext,
type_ reflect.Type, value reflect.Value) {
+func (g *DynamicSliceDemo_ForyGenSerializer) ReadData(ctx *fory.ReadContext,
type_ reflect.Type, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
// Convert reflect.Value to concrete type and delegate to typed method
var v *DynamicSliceDemo
if value.Kind() == reflect.Ptr {
@@ -188,18 +200,27 @@ func (g DynamicSliceDemo_ForyGenSerializer) ReadData(ctx
*fory.ReadContext, type
}
// ReadWithTypeInfo deserializes with pre-read type information
-func (g DynamicSliceDemo_ForyGenSerializer) ReadWithTypeInfo(ctx
*fory.ReadContext, refMode fory.RefMode, typeInfo *fory.TypeInfo, value
reflect.Value) {
+func (g *DynamicSliceDemo_ForyGenSerializer) ReadWithTypeInfo(ctx
*fory.ReadContext, refMode fory.RefMode, typeInfo *fory.TypeInfo, value
reflect.Value) {
g.Read(ctx, refMode, false, false, value)
}
-type MapDemo_ForyGenSerializer struct{}
+type MapDemo_ForyGenSerializer struct {
+ structHash int32
+}
func NewSerializerFor_MapDemo() fory.Serializer {
- return MapDemo_ForyGenSerializer{}
+ return &MapDemo_ForyGenSerializer{}
+}
+
+func (g *MapDemo_ForyGenSerializer) initHash(resolver *fory.TypeResolver) {
+ if g.structHash == 0 {
+ g.structHash = fory.GetStructHash(reflect.TypeOf(MapDemo{}),
resolver)
+ }
}
// Write is the entry point for serialization with ref/type handling
-func (g MapDemo_ForyGenSerializer) Write(ctx *fory.WriteContext, refMode
fory.RefMode, writeType bool, hasGenerics bool, value reflect.Value) {
+func (g *MapDemo_ForyGenSerializer) Write(ctx *fory.WriteContext, refMode
fory.RefMode, writeType bool, hasGenerics bool, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
_ = hasGenerics // not used for struct serializers
switch refMode {
case fory.RefModeTracking:
@@ -229,10 +250,10 @@ func (g MapDemo_ForyGenSerializer) Write(ctx
*fory.WriteContext, refMode fory.Re
}
// WriteTyped provides strongly-typed serialization with no reflection overhead
-func (g MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v
*MapDemo) error {
+func (g *MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v
*MapDemo) error {
buf := ctx.Buffer()
- // WriteData precomputed struct hash for compatibility checking
- buf.WriteInt32(-1565547955) // hash of MapDemo structure
+ // WriteData struct hash for compatibility checking
+ buf.WriteInt32(g.structHash)
// WriteData fields in sorted order
// Field: IntMap (map[int]int)
@@ -365,7 +386,8 @@ func (g MapDemo_ForyGenSerializer) WriteTyped(ctx
*fory.WriteContext, v *MapDemo
}
// WriteData provides reflect.Value interface compatibility (implements
fory.Serializer)
-func (g MapDemo_ForyGenSerializer) WriteData(ctx *fory.WriteContext, value
reflect.Value) {
+func (g *MapDemo_ForyGenSerializer) WriteData(ctx *fory.WriteContext, value
reflect.Value) {
+ g.initHash(ctx.TypeResolver())
// Convert reflect.Value to concrete type and delegate to typed method
var v *MapDemo
if value.Kind() == reflect.Ptr {
@@ -382,7 +404,8 @@ func (g MapDemo_ForyGenSerializer) WriteData(ctx
*fory.WriteContext, value refle
}
// Read is the entry point for deserialization with ref/type handling
-func (g MapDemo_ForyGenSerializer) Read(ctx *fory.ReadContext, refMode
fory.RefMode, readType bool, hasGenerics bool, value reflect.Value) {
+func (g *MapDemo_ForyGenSerializer) Read(ctx *fory.ReadContext, refMode
fory.RefMode, readType bool, hasGenerics bool, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
_ = hasGenerics // not used for struct serializers
err := ctx.Err() // Get error pointer for deferred error checking
switch refMode {
@@ -412,16 +435,16 @@ func (g MapDemo_ForyGenSerializer) Read(ctx
*fory.ReadContext, refMode fory.RefM
}
// ReadTyped provides strongly-typed deserialization with no reflection
overhead
-func (g MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v
*MapDemo) error {
+func (g *MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v
*MapDemo) error {
buf := ctx.Buffer()
err := ctx.Err() // Get error pointer for deferred error checking
// ReadData and verify struct hash
- if got := buf.ReadInt32(err); got != -1565547955 {
+ if got := buf.ReadInt32(err); got != g.structHash {
if ctx.HasError() {
return ctx.TakeError()
}
- return fory.HashMismatchError(got, -1565547955, "MapDemo")
+ return fory.HashMismatchError(got, g.structHash, "MapDemo")
}
// ReadData fields in same order as write
@@ -542,7 +565,8 @@ func (g MapDemo_ForyGenSerializer) ReadTyped(ctx
*fory.ReadContext, v *MapDemo)
}
// ReadData provides reflect.Value interface compatibility (implements
fory.Serializer)
-func (g MapDemo_ForyGenSerializer) ReadData(ctx *fory.ReadContext, type_
reflect.Type, value reflect.Value) {
+func (g *MapDemo_ForyGenSerializer) ReadData(ctx *fory.ReadContext, type_
reflect.Type, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
// Convert reflect.Value to concrete type and delegate to typed method
var v *MapDemo
if value.Kind() == reflect.Ptr {
@@ -562,18 +586,27 @@ func (g MapDemo_ForyGenSerializer) ReadData(ctx
*fory.ReadContext, type_ reflect
}
// ReadWithTypeInfo deserializes with pre-read type information
-func (g MapDemo_ForyGenSerializer) ReadWithTypeInfo(ctx *fory.ReadContext,
refMode fory.RefMode, typeInfo *fory.TypeInfo, value reflect.Value) {
+func (g *MapDemo_ForyGenSerializer) ReadWithTypeInfo(ctx *fory.ReadContext,
refMode fory.RefMode, typeInfo *fory.TypeInfo, value reflect.Value) {
g.Read(ctx, refMode, false, false, value)
}
-type SliceDemo_ForyGenSerializer struct{}
+type SliceDemo_ForyGenSerializer struct {
+ structHash int32
+}
func NewSerializerFor_SliceDemo() fory.Serializer {
- return SliceDemo_ForyGenSerializer{}
+ return &SliceDemo_ForyGenSerializer{}
+}
+
+func (g *SliceDemo_ForyGenSerializer) initHash(resolver *fory.TypeResolver) {
+ if g.structHash == 0 {
+ g.structHash = fory.GetStructHash(reflect.TypeOf(SliceDemo{}),
resolver)
+ }
}
// Write is the entry point for serialization with ref/type handling
-func (g SliceDemo_ForyGenSerializer) Write(ctx *fory.WriteContext, refMode
fory.RefMode, writeType bool, hasGenerics bool, value reflect.Value) {
+func (g *SliceDemo_ForyGenSerializer) Write(ctx *fory.WriteContext, refMode
fory.RefMode, writeType bool, hasGenerics bool, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
_ = hasGenerics // not used for struct serializers
switch refMode {
case fory.RefModeTracking:
@@ -603,10 +636,10 @@ func (g SliceDemo_ForyGenSerializer) Write(ctx
*fory.WriteContext, refMode fory.
}
// WriteTyped provides strongly-typed serialization with no reflection overhead
-func (g SliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v
*SliceDemo) error {
+func (g *SliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v
*SliceDemo) error {
buf := ctx.Buffer()
- // WriteData precomputed struct hash for compatibility checking
- buf.WriteInt32(-1393614469) // hash of SliceDemo structure
+ // WriteData struct hash for compatibility checking
+ buf.WriteInt32(g.structHash)
// WriteData fields in sorted order
// Field: BoolSlice ([]bool)
@@ -644,7 +677,8 @@ func (g SliceDemo_ForyGenSerializer) WriteTyped(ctx
*fory.WriteContext, v *Slice
}
// WriteData provides reflect.Value interface compatibility (implements
fory.Serializer)
-func (g SliceDemo_ForyGenSerializer) WriteData(ctx *fory.WriteContext, value
reflect.Value) {
+func (g *SliceDemo_ForyGenSerializer) WriteData(ctx *fory.WriteContext, value
reflect.Value) {
+ g.initHash(ctx.TypeResolver())
// Convert reflect.Value to concrete type and delegate to typed method
var v *SliceDemo
if value.Kind() == reflect.Ptr {
@@ -661,7 +695,8 @@ func (g SliceDemo_ForyGenSerializer) WriteData(ctx
*fory.WriteContext, value ref
}
// Read is the entry point for deserialization with ref/type handling
-func (g SliceDemo_ForyGenSerializer) Read(ctx *fory.ReadContext, refMode
fory.RefMode, readType bool, hasGenerics bool, value reflect.Value) {
+func (g *SliceDemo_ForyGenSerializer) Read(ctx *fory.ReadContext, refMode
fory.RefMode, readType bool, hasGenerics bool, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
_ = hasGenerics // not used for struct serializers
err := ctx.Err() // Get error pointer for deferred error checking
switch refMode {
@@ -691,16 +726,16 @@ func (g SliceDemo_ForyGenSerializer) Read(ctx
*fory.ReadContext, refMode fory.Re
}
// ReadTyped provides strongly-typed deserialization with no reflection
overhead
-func (g SliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v
*SliceDemo) error {
+func (g *SliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v
*SliceDemo) error {
buf := ctx.Buffer()
err := ctx.Err() // Get error pointer for deferred error checking
// ReadData and verify struct hash
- if got := buf.ReadInt32(err); got != -1393614469 {
+ if got := buf.ReadInt32(err); got != g.structHash {
if ctx.HasError() {
return ctx.TakeError()
}
- return fory.HashMismatchError(got, -1393614469, "SliceDemo")
+ return fory.HashMismatchError(got, g.structHash, "SliceDemo")
}
// ReadData fields in same order as write
@@ -774,7 +809,8 @@ func (g SliceDemo_ForyGenSerializer) ReadTyped(ctx
*fory.ReadContext, v *SliceDe
}
// ReadData provides reflect.Value interface compatibility (implements
fory.Serializer)
-func (g SliceDemo_ForyGenSerializer) ReadData(ctx *fory.ReadContext, type_
reflect.Type, value reflect.Value) {
+func (g *SliceDemo_ForyGenSerializer) ReadData(ctx *fory.ReadContext, type_
reflect.Type, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
// Convert reflect.Value to concrete type and delegate to typed method
var v *SliceDemo
if value.Kind() == reflect.Ptr {
@@ -794,18 +830,27 @@ func (g SliceDemo_ForyGenSerializer) ReadData(ctx
*fory.ReadContext, type_ refle
}
// ReadWithTypeInfo deserializes with pre-read type information
-func (g SliceDemo_ForyGenSerializer) ReadWithTypeInfo(ctx *fory.ReadContext,
refMode fory.RefMode, typeInfo *fory.TypeInfo, value reflect.Value) {
+func (g *SliceDemo_ForyGenSerializer) ReadWithTypeInfo(ctx *fory.ReadContext,
refMode fory.RefMode, typeInfo *fory.TypeInfo, value reflect.Value) {
g.Read(ctx, refMode, false, false, value)
}
-type ValidationDemo_ForyGenSerializer struct{}
+type ValidationDemo_ForyGenSerializer struct {
+ structHash int32
+}
func NewSerializerFor_ValidationDemo() fory.Serializer {
- return ValidationDemo_ForyGenSerializer{}
+ return &ValidationDemo_ForyGenSerializer{}
+}
+
+func (g *ValidationDemo_ForyGenSerializer) initHash(resolver
*fory.TypeResolver) {
+ if g.structHash == 0 {
+ g.structHash =
fory.GetStructHash(reflect.TypeOf(ValidationDemo{}), resolver)
+ }
}
// Write is the entry point for serialization with ref/type handling
-func (g ValidationDemo_ForyGenSerializer) Write(ctx *fory.WriteContext,
refMode fory.RefMode, writeType bool, hasGenerics bool, value reflect.Value) {
+func (g *ValidationDemo_ForyGenSerializer) Write(ctx *fory.WriteContext,
refMode fory.RefMode, writeType bool, hasGenerics bool, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
_ = hasGenerics // not used for struct serializers
switch refMode {
case fory.RefModeTracking:
@@ -835,10 +880,10 @@ func (g ValidationDemo_ForyGenSerializer) Write(ctx
*fory.WriteContext, refMode
}
// WriteTyped provides strongly-typed serialization with no reflection overhead
-func (g ValidationDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v
*ValidationDemo) error {
+func (g *ValidationDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext,
v *ValidationDemo) error {
buf := ctx.Buffer()
- // WriteData precomputed struct hash for compatibility checking
- buf.WriteInt32(728169998) // hash of ValidationDemo structure
+ // WriteData struct hash for compatibility checking
+ buf.WriteInt32(g.structHash)
// WriteData fields in sorted order
// Field: D (float64)
@@ -858,7 +903,8 @@ func (g ValidationDemo_ForyGenSerializer) WriteTyped(ctx
*fory.WriteContext, v *
}
// WriteData provides reflect.Value interface compatibility (implements
fory.Serializer)
-func (g ValidationDemo_ForyGenSerializer) WriteData(ctx *fory.WriteContext,
value reflect.Value) {
+func (g *ValidationDemo_ForyGenSerializer) WriteData(ctx *fory.WriteContext,
value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
// Convert reflect.Value to concrete type and delegate to typed method
var v *ValidationDemo
if value.Kind() == reflect.Ptr {
@@ -875,7 +921,8 @@ func (g ValidationDemo_ForyGenSerializer) WriteData(ctx
*fory.WriteContext, valu
}
// Read is the entry point for deserialization with ref/type handling
-func (g ValidationDemo_ForyGenSerializer) Read(ctx *fory.ReadContext, refMode
fory.RefMode, readType bool, hasGenerics bool, value reflect.Value) {
+func (g *ValidationDemo_ForyGenSerializer) Read(ctx *fory.ReadContext, refMode
fory.RefMode, readType bool, hasGenerics bool, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
_ = hasGenerics // not used for struct serializers
err := ctx.Err() // Get error pointer for deferred error checking
switch refMode {
@@ -905,16 +952,16 @@ func (g ValidationDemo_ForyGenSerializer) Read(ctx
*fory.ReadContext, refMode fo
}
// ReadTyped provides strongly-typed deserialization with no reflection
overhead
-func (g ValidationDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v
*ValidationDemo) error {
+func (g *ValidationDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v
*ValidationDemo) error {
buf := ctx.Buffer()
err := ctx.Err() // Get error pointer for deferred error checking
// ReadData and verify struct hash
- if got := buf.ReadInt32(err); got != 728169998 {
+ if got := buf.ReadInt32(err); got != g.structHash {
if ctx.HasError() {
return ctx.TakeError()
}
- return fory.HashMismatchError(got, 728169998, "ValidationDemo")
+ return fory.HashMismatchError(got, g.structHash,
"ValidationDemo")
}
// ReadData fields in same order as write
@@ -940,7 +987,8 @@ func (g ValidationDemo_ForyGenSerializer) ReadTyped(ctx
*fory.ReadContext, v *Va
}
// ReadData provides reflect.Value interface compatibility (implements
fory.Serializer)
-func (g ValidationDemo_ForyGenSerializer) ReadData(ctx *fory.ReadContext,
type_ reflect.Type, value reflect.Value) {
+func (g *ValidationDemo_ForyGenSerializer) ReadData(ctx *fory.ReadContext,
type_ reflect.Type, value reflect.Value) {
+ g.initHash(ctx.TypeResolver())
// Convert reflect.Value to concrete type and delegate to typed method
var v *ValidationDemo
if value.Kind() == reflect.Ptr {
@@ -960,7 +1008,7 @@ func (g ValidationDemo_ForyGenSerializer) ReadData(ctx
*fory.ReadContext, type_
}
// ReadWithTypeInfo deserializes with pre-read type information
-func (g ValidationDemo_ForyGenSerializer) ReadWithTypeInfo(ctx
*fory.ReadContext, refMode fory.RefMode, typeInfo *fory.TypeInfo, value
reflect.Value) {
+func (g *ValidationDemo_ForyGenSerializer) ReadWithTypeInfo(ctx
*fory.ReadContext, refMode fory.RefMode, typeInfo *fory.TypeInfo, value
reflect.Value) {
g.Read(ctx, refMode, false, false, value)
}
diff --git a/go/fory/type_def.go b/go/fory/type_def.go
index 1509388d6..cff3443e0 100644
--- a/go/fory/type_def.go
+++ b/go/fory/type_def.go
@@ -365,6 +365,7 @@ type FieldDef struct {
nullable bool
trackingRef bool
fieldType FieldType
+ tagID int // -1 = use field name, >=0 = use tag ID
}
// String returns a string representation of FieldDef for debugging
@@ -375,6 +376,10 @@ func (fd FieldDef) String() string {
} else {
fieldTypeStr = "nil"
}
+ if fd.tagID >= 0 {
+ return fmt.Sprintf("FieldDef{tagID=%d, nullable=%v,
trackingRef=%v, fieldType=%s}",
+ fd.tagID, fd.nullable, fd.trackingRef, fieldTypeStr)
+ }
return fmt.Sprintf("FieldDef{name=%s, nullable=%v, trackingRef=%v,
fieldType=%s}",
fd.name, fd.nullable, fd.trackingRef, fieldTypeStr)
}
@@ -405,9 +410,19 @@ func buildFieldDefs(fory *Fory, value reflect.Value)
([]FieldDef, error) {
type_ := value.Type()
for i := 0; i < type_.NumField(); i++ {
field := type_.Field(i)
- fieldValue := value.Field(i)
- var fieldInfo FieldDef
+ // Skip unexported fields
+ if field.PkgPath != "" {
+ continue
+ }
+
+ // Parse fory struct tag and check for ignore
+ foryTag := ParseForyTag(field)
+ if foryTag.Ignore {
+ continue // skip ignored fields
+ }
+
+ fieldValue := value.Field(i)
fieldName := SnakeCase(field.Name)
nameEncoding :=
fory.typeResolver.typeNameEncoder.ComputeEncodingWith(fieldName,
fieldNameEncodings)
@@ -423,24 +438,36 @@ func buildFieldDefs(fory *Fory, value reflect.Value)
([]FieldDef, error) {
if isUserDefinedType(int16(internalId)) || internalId == ENUM
|| internalId == NAMED_ENUM {
nullableFlag = true
}
+ // Override nullable flag if explicitly set in fory tag
+ if foryTag.NullableSet {
+ nullableFlag = foryTag.Nullable
+ }
+
+ // Calculate ref tracking - use tag override if explicitly set
+ trackingRef := fory.config.TrackRef
+ if foryTag.RefSet {
+ trackingRef = foryTag.Ref
+ }
- fieldInfo = FieldDef{
+ fieldInfo := FieldDef{
name: fieldName,
nameEncoding: nameEncoding,
nullable: nullableFlag,
- trackingRef: fory.config.TrackRef,
+ trackingRef: trackingRef,
fieldType: ft,
+ tagID: foryTag.ID,
}
fieldDefs = append(fieldDefs, fieldInfo)
}
// Sort field definitions
if len(fieldDefs) > 1 {
- // Extract serializers, names, typeIds and nullable info for
sorting
+ // Extract serializers, names, typeIds, nullable info and
tagIDs for sorting
serializers := make([]Serializer, len(fieldDefs))
fieldNames := make([]string, len(fieldDefs))
typeIds := make([]TypeId, len(fieldDefs))
nullables := make([]bool, len(fieldDefs))
+ tagIDs := make([]int, len(fieldDefs))
for i, fieldDef := range fieldDefs {
serializer, err := getFieldTypeSerializer(fory,
fieldDef.fieldType)
if err != nil {
@@ -452,11 +479,12 @@ func buildFieldDefs(fory *Fory, value reflect.Value)
([]FieldDef, error) {
fieldNames[i] = fieldDef.name
typeIds[i] = fieldDef.fieldType.TypeId()
nullables[i] = fieldDef.nullable
+ tagIDs[i] = fieldDef.tagID
}
// Use sortFields to match Java's field ordering
- // (primitives before boxed/nullable primitives)
- _, sortedNames := sortFields(fory.typeResolver, fieldNames,
serializers, typeIds, nullables)
+ // (primitives before boxed/nullable primitives, sorted by tag
ID if available)
+ _, sortedNames := sortFields(fory.typeResolver, fieldNames,
serializers, typeIds, nullables, tagIDs)
// Rebuild fieldInfos in the sorted order
nameToFieldInfo := make(map[string]FieldDef)
@@ -895,12 +923,20 @@ const (
FieldNameSizeThreshold = 15
)
-// Encoding `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL/TAG_ID` for
fieldName
+// Field name encoding flags (2 bits in header)
+const (
+ FieldNameEncodingUTF8 = 0 // UTF-8 encoding
+ FieldNameEncodingAllToLowerSpecial = 1 // ALL_TO_LOWER_SPECIAL encoding
+ FieldNameEncodingLowerUpperDigit = 2 // LOWER_UPPER_DIGIT_SPECIAL
encoding
+ FieldNameEncodingTagID = 3 // Use tag ID instead of field
name
+)
+
+// Encoding `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` for fieldName
+// Note: TAG_ID (0b11) is a special encoding that uses tag ID instead of field
name
var fieldNameEncodings = []meta.Encoding{
meta.UTF_8,
meta.ALL_TO_LOWER_SPECIAL,
meta.LOWER_UPPER_DIGIT_SPECIAL,
- // todo: add support for TAG_ID encoding
}
func getFieldNameEncodingIndex(encoding meta.Encoding) int {
@@ -1125,29 +1161,50 @@ func writeFieldDef(typeResolver *TypeResolver, buffer
*ByteBuffer, field FieldDe
if field.nullable {
header |= 0b10
}
- // store index of encoding in the 2 highest bits
- encodingFlag := byte(getFieldNameEncodingIndex(field.nameEncoding))
- header |= encodingFlag << 6
- metaString, err :=
typeResolver.typeNameEncoder.EncodeWithEncoding(field.name, field.nameEncoding)
- if err != nil {
- return err
- }
- nameLen := len(metaString.GetEncodedBytes())
- if nameLen < FieldNameSizeThreshold {
- header |= uint8((nameLen-1)&0x0F) << 2 // 1-based encoding
+
+ if field.tagID >= 0 {
+ // Use TAG_ID encoding (encoding flag = 3)
+ header |= FieldNameEncodingTagID << 6
+ // For tag ID, we encode the tag ID value in the size bits (4
bits)
+ // If tagID < 15, encode directly in header; otherwise use
varint
+ if field.tagID < FieldNameSizeThreshold {
+ header |= uint8(field.tagID&0x0F) << 2
+ } else {
+ header |= 0x0F << 2 // Max value, actual tag ID will
follow
+ }
+ buffer.PutUint8(offset, header)
+
+ // Write extra varint for large tag IDs
+ if field.tagID >= FieldNameSizeThreshold {
+ buffer.WriteVaruint32(uint32(field.tagID -
FieldNameSizeThreshold))
+ }
+
+ // Write field type
+ field.fieldType.write(buffer)
} else {
- header |= 0x0F << 2 // Max value, actual length will follow
- buffer.WriteVaruint32(uint32(nameLen - FieldNameSizeThreshold))
- }
- buffer.PutUint8(offset, header)
+ // Use field name encoding
+ encodingFlag :=
byte(getFieldNameEncodingIndex(field.nameEncoding))
+ header |= encodingFlag << 6
+ metaString, err :=
typeResolver.typeNameEncoder.EncodeWithEncoding(field.name, field.nameEncoding)
+ if err != nil {
+ return err
+ }
+ nameLen := len(metaString.GetEncodedBytes())
+ if nameLen < FieldNameSizeThreshold {
+ header |= uint8((nameLen-1)&0x0F) << 2 // 1-based
encoding
+ } else {
+ header |= 0x0F << 2 // Max value, actual length will
follow
+ buffer.WriteVaruint32(uint32(nameLen -
FieldNameSizeThreshold))
+ }
+ buffer.PutUint8(offset, header)
- // WriteData field type
- field.fieldType.write(buffer)
+ // Write field type
+ field.fieldType.write(buffer)
- // todo: support tag id
- // write field name
- if _, err := buffer.Write(metaString.GetEncodedBytes()); err != nil {
- return err
+ // Write field name
+ if _, err := buffer.Write(metaString.GetEncodedBytes()); err !=
nil {
+ return err
+ }
}
return nil
}
@@ -1361,24 +1418,51 @@ func readFieldDef(typeResolver *TypeResolver, buffer
*ByteBuffer) (FieldDef, err
}
// Resolve the header
- nameEncodingFlag := (headerByte >> 6) & 0b11
- nameEncoding := fieldNameEncodings[nameEncodingFlag]
- nameLen := int((headerByte >> 2) & 0x0F)
+ nameEncodingFlag := int((headerByte >> 6) & 0b11)
+ sizeBits := int((headerByte >> 2) & 0x0F)
refTracking := (headerByte & 0b1) != 0
isNullable := (headerByte & 0b10) != 0
+
+ // Check if using TAG_ID encoding
+ if nameEncodingFlag == FieldNameEncodingTagID {
+ // Read tag ID
+ tagID := sizeBits
+ if sizeBits == 0x0F {
+ tagID = FieldNameSizeThreshold +
int(buffer.ReadVaruint32(&bufErr))
+ }
+
+ // Read field type
+ ft, err := readFieldType(buffer, &bufErr)
+ if err != nil {
+ return FieldDef{}, err
+ }
+
+ return FieldDef{
+ name: "", // No field name when using tag ID
+ nameEncoding: meta.UTF_8,
+ fieldType: ft,
+ nullable: isNullable,
+ trackingRef: refTracking,
+ tagID: tagID,
+ }, nil
+ }
+
+ // Use field name encoding
+ nameEncoding := fieldNameEncodings[nameEncodingFlag]
+ nameLen := sizeBits
if nameLen == 0x0F {
nameLen = FieldNameSizeThreshold +
int(buffer.ReadVaruint32(&bufErr))
} else {
nameLen++ // Adjust for 1-based encoding
}
- // reading field type
+ // Read field type
ft, err := readFieldType(buffer, &bufErr)
if err != nil {
return FieldDef{}, err
}
- // Reading field name based on encoding
+ // Read field name based on encoding
nameBytes := buffer.ReadBinary(nameLen, &bufErr)
fieldName, err := typeResolver.typeNameDecoder.Decode(nameBytes,
nameEncoding)
if err != nil {
@@ -1391,5 +1475,6 @@ func readFieldDef(typeResolver *TypeResolver, buffer
*ByteBuffer) (FieldDef, err
fieldType: ft,
nullable: isNullable,
trackingRef: refTracking,
+ tagID: TagIDUseFieldName, // -1 indicates using field
name
}, nil
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
index 94e1a2774..08abaa201 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
@@ -942,12 +942,13 @@ public class XtypeResolver extends TypeResolver {
int xtypeId = getXtypeId(o1.getRawType());
int xtypeId2 = getXtypeId(o2.getRawType());
if (xtypeId == xtypeId2) {
- return o1.getSnakeCaseName().compareTo(o2.getSnakeCaseName());
+ return DescriptorGrouper.getFieldSortKey(o1)
+ .compareTo(DescriptorGrouper.getFieldSortKey(o2));
} else {
return xtypeId - xtypeId2;
}
})
-
.setOtherDescriptorComparator(Comparator.comparing(Descriptor::getSnakeCaseName))
+
.setOtherDescriptorComparator(Comparator.comparing(DescriptorGrouper::getFieldSortKey))
.sort();
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
index eeec93946..c12c18410 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
@@ -20,11 +20,13 @@
package org.apache.fory.serializer;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.fory.Fory;
+import org.apache.fory.annotation.ForyField;
import org.apache.fory.collection.Tuple2;
import org.apache.fory.collection.Tuple3;
import org.apache.fory.exception.ForyException;
@@ -43,7 +45,6 @@ import org.apache.fory.type.DescriptorGrouper;
import org.apache.fory.type.Generics;
import org.apache.fory.type.Types;
import org.apache.fory.util.MurmurHash3;
-import org.apache.fory.util.StringUtils;
import org.apache.fory.util.record.RecordInfo;
import org.apache.fory.util.record.RecordUtils;
@@ -356,23 +357,8 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
}
public static int computeStructHash(Fory fory, DescriptorGrouper grouper) {
- StringBuilder builder = new StringBuilder();
List<Descriptor> sorted = grouper.getSortedDescriptors();
- for (Descriptor descriptor : sorted) {
- Class<?> rawType = descriptor.getTypeRef().getRawType();
- int typeId = getTypeId(fory, rawType);
- String underscore =
StringUtils.lowerCamelToLowerUnderscore(descriptor.getName());
- char nullable = rawType.isPrimitive() ? '0' : '1';
- builder
- .append(underscore)
- .append(',')
- .append(typeId)
- .append(',')
- .append(nullable)
- .append(';');
- }
-
- String fingerprint = builder.toString();
+ String fingerprint = computeStructFingerprint(fory, sorted);
byte[] bytes = fingerprint.getBytes(StandardCharsets.UTF_8);
long hashLong = MurmurHash3.murmurhash3_x64_128(bytes, 0, bytes.length,
47)[0];
int hash = (int) (hashLong & 0xffffffffL);
@@ -390,6 +376,94 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
return hash;
}
+ /**
+ * Computes the fingerprint string for a struct type used in schema
versioning.
+ *
+ * <p><b>Fingerprint Format:</b>
+ *
+ * <p>Each field contributes: {@code
<field_id_or_name>,<type_id>,<ref>,<nullable>;}
+ *
+ * <p>Fields are sorted by their identifier (field ID or name)
lexicographically as strings.
+ *
+ * <p><b>Field Components:</b>
+ *
+ * <ul>
+ * <li><b>field_id_or_name</b>: Tag ID as string if configured via {@link
ForyField#id()} (e.g.,
+ * "0", "1"), otherwise snake_case field name
+ * <li><b>type_id</b>: Fory TypeId as decimal string (e.g., "4" for INT32)
+ * <li><b>ref</b>: "1" if reference tracking enabled, "0" otherwise
+ * <li><b>nullable</b>: "1" if null flag is written, "0" otherwise
+ * </ul>
+ *
+ * <p><b>Example fingerprints:</b>
+ *
+ * <ul>
+ * <li>With tag IDs: "0,4,0,0;1,4,0,1;2,9,0,1;"
+ * <li>With field names: "age,4,0,0;name,9,0,1;"
+ * </ul>
+ *
+ * <p>The fingerprint is used to compute a hash for struct schema
versioning. Different
+ * nullable/ref settings will produce different fingerprints, ensuring
schema compatibility is
+ * properly validated.
+ *
+ * @param fory the Fory instance for type resolution
+ * @param descriptors the sorted list of field descriptors
+ * @return the fingerprint string
+ */
+ public static String computeStructFingerprint(Fory fory, List<Descriptor>
descriptors) {
+ // Build fingerprint info for each field
+ List<String[]> fieldInfos = new ArrayList<>(descriptors.size());
+ for (Descriptor descriptor : descriptors) {
+ Class<?> rawType = descriptor.getTypeRef().getRawType();
+ int typeId = getTypeId(fory, rawType);
+
+ // Get field identifier: tag ID if configured, otherwise snake_case name
+ String fieldIdentifier;
+ ForyField foryField = descriptor.getForyField();
+ if (foryField != null && foryField.id() >= 0) {
+ fieldIdentifier = String.valueOf(foryField.id());
+ } else {
+ fieldIdentifier = descriptor.getSnakeCaseName();
+ }
+
+ // Get ref flag from @ForyField annotation only (compile-time info)
+ // If annotation is absent or ref not explicitly set to true, ref is 0
+ // This allows fingerprint to be computed at compile time for C++/Rust
+ char ref = (foryField != null && foryField.ref()) ? '1' : '0';
+
+ // Get nullable flag: primitives are non-nullable, others depend on
descriptor
+ char nullable;
+ if (rawType.isPrimitive()) {
+ nullable = '0';
+ } else {
+ nullable = descriptor.isNullable() ? '1' : '0';
+ }
+
+ fieldInfos.add(
+ new String[] {
+ fieldIdentifier, String.valueOf(typeId), String.valueOf(ref),
String.valueOf(nullable)
+ });
+ }
+
+ // Sort by field identifier (lexicographically as strings)
+ fieldInfos.sort((a, b) -> a[0].compareTo(b[0]));
+
+ // Build fingerprint string
+ StringBuilder builder = new StringBuilder();
+ for (String[] info : fieldInfos) {
+ builder
+ .append(info[0])
+ .append(',')
+ .append(info[1])
+ .append(',')
+ .append(info[2])
+ .append(',')
+ .append(info[3])
+ .append(';');
+ }
+ return builder.toString();
+ }
+
private static int getTypeId(Fory fory, Class<?> cls) {
TypeResolver resolver = fory._getTypeResolver();
if (resolver.isSet(cls)) {
diff --git
a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java
b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java
index 95213b893..1b6435ffe 100644
--- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java
+++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.TreeSet;
import java.util.function.Function;
import java.util.function.Predicate;
+import org.apache.fory.annotation.ForyField;
import org.apache.fory.util.Preconditions;
import org.apache.fory.util.record.RecordUtils;
@@ -42,16 +43,40 @@ import org.apache.fory.util.record.RecordUtils;
* <li>other fields
*/
public class DescriptorGrouper {
+
+ /**
+ * Gets the sort key for a field descriptor.
+ *
+ * <p>If the field has a {@link ForyField} annotation with id >= 0, returns
the id as a string.
+ * Otherwise, returns the snake_case field name. This ensures fields are
sorted by tag ID when
+ * configured, matching the fingerprint computation order.
+ *
+ * @param descriptor the field descriptor
+ * @return the sort key (tag ID as string or snake_case name)
+ */
+ public static String getFieldSortKey(Descriptor descriptor) {
+ ForyField foryField = descriptor.getForyField();
+ if (foryField != null && foryField.id() >= 0) {
+ return String.valueOf(foryField.id());
+ }
+ return descriptor.getSnakeCaseName();
+ }
+
static final Comparator<Descriptor> COMPARATOR_BY_PRIMITIVE_TYPE_ID =
(d1, d2) -> {
int c =
Types.getPrimitiveTypeId(TypeUtils.unwrap(d2.getRawType()))
- Types.getPrimitiveTypeId(TypeUtils.unwrap(d1.getRawType()));
if (c == 0) {
- c = d1.getSnakeCaseName().compareTo(d2.getSnakeCaseName());
+ c = getFieldSortKey(d1).compareTo(getFieldSortKey(d2));
if (c == 0) {
// Field name duplicate in super/child classes.
c = d1.getDeclaringClass().compareTo(d2.getDeclaringClass());
+ if (c == 0) {
+ // Final tie-breaker: use actual field name to distinguish
fields with same tag ID.
+ // This ensures TreeSet never treats different fields as
duplicates.
+ c = d1.getName().compareTo(d2.getName());
+ }
}
}
return c;
@@ -115,11 +140,11 @@ public class DescriptorGrouper {
return false;
}
- /** Comparator based on field type, name and declaring class. */
+ /** Comparator based on field type, name/id and declaring class. */
public static final Comparator<Descriptor> COMPARATOR_BY_TYPE_AND_NAME =
(d1, d2) -> {
// sort by type so that we can hit class info cache more possibly.
- // sort by field name to fix order if type is same.
+ // sort by field id/name to fix order if type is same.
int c =
d1
// Use type name instead of generic type so that fields with
type ref
@@ -128,10 +153,15 @@ public class DescriptorGrouper {
.getTypeName()
.compareTo(d2.getTypeName());
if (c == 0) {
- c = d1.getName().compareTo(d2.getName());
+ c = getFieldSortKey(d1).compareTo(getFieldSortKey(d2));
if (c == 0) {
// Field name duplicate in super/child classes.
c = d1.getDeclaringClass().compareTo(d2.getDeclaringClass());
+ if (c == 0) {
+ // Final tie-breaker: use actual field name to distinguish
fields with same tag ID.
+ // This ensures TreeSet never treats different fields as
duplicates.
+ c = d1.getName().compareTo(d2.getName());
+ }
}
}
return c;
diff --git a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java
b/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java
index 516a9ac9e..b8801bfa3 100644
--- a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java
+++ b/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java
@@ -1365,30 +1365,12 @@ public abstract class XlangTestBase extends
ForyTestBase {
buffer = MemoryBuffer.newHeapBuffer(64);
fory1.serialize(buffer, obj1);
-
- // Debug: print Java output bytes
byte[] javaBytes = buffer.getBytes(0, buffer.writerIndex());
- System.out.println("Java OneStringFieldStruct output " + javaBytes.length
+ " bytes:");
- for (int i = 0; i < javaBytes.length; i++) {
- if (i > 0 && i % 16 == 0) System.out.println();
- System.out.printf("%02x ", javaBytes[i] & 0xFF);
- }
- System.out.println();
-
String caseName2 = "test_schema_evolution_compatible_reverse";
ExecutionContext ctx2 = prepareExecution(caseName2, javaBytes);
runPeer(ctx2);
MemoryBuffer buffer3 = readBuffer(ctx2.dataFile());
- // Debug: print Go output bytes
- byte[] goBytes = buffer3.getBytes(0, buffer3.size());
- System.out.println("Go output " + goBytes.length + " bytes:");
- for (int i = 0; i < goBytes.length; i++) {
- if (i > 0 && i % 16 == 0) System.out.println();
- System.out.printf("%02x ", goBytes[i] & 0xFF);
- }
- System.out.println();
-
TwoStringFieldStruct result2 = (TwoStringFieldStruct)
fory2.deserialize(buffer3);
Assert.assertEquals(result2.f1, "only_one");
// Go uses empty string for missing fields (Go string can't be null)
@@ -1548,13 +1530,6 @@ public abstract class XlangTestBase extends ForyTestBase
{
// Debug: print Java output bytes
byte[] javaBytes = buffer.getBytes(0, buffer.writerIndex());
- System.out.println("Java OneEnumFieldStruct output " + javaBytes.length +
" bytes:");
- for (int i = 0; i < javaBytes.length; i++) {
- if (i > 0 && i % 16 == 0) System.out.println();
- System.out.printf("%02x ", javaBytes[i] & 0xFF);
- }
- System.out.println();
-
String caseName2 = "test_enum_schema_evolution_compatible_reverse";
ExecutionContext ctx2 = prepareExecution(caseName2, javaBytes);
runPeer(ctx2);
@@ -1562,13 +1537,6 @@ public abstract class XlangTestBase extends ForyTestBase
{
MemoryBuffer buffer3 = readBuffer(ctx2.dataFile());
// Debug: print Go output bytes
byte[] goBytes = buffer3.getBytes(0, buffer3.size());
- System.out.println("Go output " + goBytes.length + " bytes:");
- for (int i = 0; i < goBytes.length; i++) {
- if (i > 0 && i % 16 == 0) System.out.println();
- System.out.printf("%02x ", goBytes[i] & 0xFF);
- }
- System.out.println();
-
TwoEnumFieldStruct result2 = (TwoEnumFieldStruct)
fory2.deserialize(buffer3);
Assert.assertEquals(result2.f1, TestEnum.VALUE_C);
// Go uses zero value for missing enum fields (first enum value)
diff --git a/python/pyfory/struct.py b/python/pyfory/struct.py
index 3d3dcf8f7..12c2b7ba5 100644
--- a/python/pyfory/struct.py
+++ b/python/pyfory/struct.py
@@ -485,11 +485,19 @@ class DataClassSerializer(Serializer):
serializer_var = f"serializer{index}"
serializer = self._serializers[index]
context[serializer_var] = serializer
+ is_nullable = self._nullable_fields.get(field_name, False)
+ # For nullable fields, use safe access with None default to handle
+ # schema evolution cases where the field might not exist on the
object
if not self._has_slots:
- stmts.append(f"{field_value} = {value_dict}['{field_name}']")
+ if is_nullable:
+ stmts.append(f"{field_value} =
{value_dict}.get('{field_name}')")
+ else:
+ stmts.append(f"{field_value} =
{value_dict}['{field_name}']")
else:
- stmts.append(f"{field_value} = {value}.{field_name}")
- is_nullable = self._nullable_fields.get(field_name, False)
+ if is_nullable:
+ stmts.append(f"{field_value} = getattr({value},
'{field_name}', None)")
+ else:
+ stmts.append(f"{field_value} = {value}.{field_name}")
if is_nullable:
if isinstance(serializer, StringSerializer):
stmts.extend(
@@ -841,41 +849,86 @@ def group_fields(type_resolver, field_names, serializers,
nullable_map=None):
return (boxed_types, nullable_boxed_types, internal_types,
collection_types, set_types, map_types, other_types)
+def compute_struct_fingerprint(type_resolver, field_names, serializers,
nullable_map=None):
+ """
+ Computes the fingerprint string for a struct type used in schema
versioning.
+
+ Fingerprint Format:
+ Each field contributes: <field_name>,<type_id>,<ref>,<nullable>;
+ Fields are sorted lexicographically by field name (not by type
category).
+
+ Field Components:
+ - field_name: snake_case field name (Python doesn't support field tag
IDs yet)
+ - type_id: Fory TypeId as decimal string (e.g., "4" for INT32)
+ - ref: "1" if field has explicit ref annotation, "0" otherwise
+ (always "0" in Python since field annotations are not supported)
+ - nullable: "1" if null flag is written, "0" otherwise
+
+ Example fingerprint: "age,4,0,0;name,12,0,1;"
+
+ This format is consistent across Go, Java, Rust, C++, and Python
implementations.
+ The ref flag is based on compile-time annotations only, NOT runtime
ref_tracking config.
+ """
+ if nullable_map is None:
+ nullable_map = {}
+
+ # Build field info list: (field_name, type_id, nullable)
+ field_infos = []
+ for i, field_name in enumerate(field_names):
+ serializer = serializers[i]
+ if serializer is None:
+ # For dynamic/polymorphic fields (like Any/Object), use UNKNOWN
type_id
+ # These fields are included in fingerprint with type_id=0,
nullable=1
+ type_id = TypeId.UNKNOWN
+ nullable_flag = "1"
+ else:
+ type_id = type_resolver.get_typeinfo(serializer.type_).type_id &
0xFF
+ is_nullable = nullable_map.get(field_name, False)
+
+ # Determine nullable flag based on type category (matching Java
behavior)
+ if is_primitive_type(type_id) and not is_nullable:
+ nullable_flag = "0"
+ elif is_polymorphic_type(type_id) or type_id in {TypeId.ENUM,
TypeId.NAMED_ENUM}:
+ # For polymorphic/enum types, use UNKNOWN type_id
+ type_id = TypeId.UNKNOWN
+ nullable_flag = "1"
+ else:
+ nullable_flag = "1"
+
+ field_infos.append((field_name, type_id, nullable_flag))
+
+ # Sort fields lexicographically by field name for fingerprint computation
+ # This matches Java/Go/Rust/C++ behavior
+ field_infos.sort(key=lambda x: x[0])
+
+ # Build fingerprint string
+ # Format: <field_name>,<type_id>,<ref>,<nullable>;
+ hash_parts = []
+ for field_name, type_id, nullable_flag in field_infos:
+ # ref flag: always "0" in Python since field annotations are not
supported
+ # In Java/Go, ref is "1" only if explicitly annotated with
@ForyField(ref=true) or fory:"trackRef"
+ ref_flag = "0"
+
hash_parts.append(f"{field_name},{type_id},{ref_flag},{nullable_flag};")
+
+ return "".join(hash_parts)
+
+
def compute_struct_meta(type_resolver, field_names, serializers,
nullable_map=None):
+ """
+ Computes struct metadata including version hash, sorted field names, and
serializers.
+
+ Uses compute_struct_fingerprint to build the fingerprint string, then
hashes it
+ with MurmurHash3 using seed 47, and takes the low 32 bits as signed int32.
+
+ This provides the cross-language struct version ID used by class version
checking,
+ consistent with Go, Java, Rust, and C++ implementations.
+ """
(boxed_types, nullable_boxed_types, internal_types, collection_types,
set_types, map_types, other_types) = group_fields(
type_resolver, field_names, serializers, nullable_map
)
- # Build hash string
- hash_parts = []
-
- # boxed_types => non-nullable
- for field in boxed_types:
- type_id = field[0]
- field_name = field[2] # already snake_case
- nullable_flag = "0"
- hash_parts.append(f"{field_name},{type_id},{nullable_flag};")
-
- # All other groups => nullable
- for group in (
- nullable_boxed_types,
- internal_types,
- collection_types,
- set_types,
- map_types,
- ):
- for field in group:
- type_id = field[0]
- field_name = field[2]
- nullable_flag = "1"
- hash_parts.append(f"{field_name},{type_id},{nullable_flag};")
- for field in other_types:
- type_id = TypeId.UNKNOWN
- field_name = field[2]
- nullable_flag = "1"
- hash_parts.append(f"{field_name},{type_id},{nullable_flag};")
-
- hash_str = "".join(hash_parts)
+ # Compute fingerprint string using the new format
+ hash_str = compute_struct_fingerprint(type_resolver, field_names,
serializers, nullable_map)
hash_bytes = hash_str.encode("utf-8")
# Handle empty hash_bytes (no fields or all fields are unknown/dynamic)
diff --git a/python/pyfory/tests/xlang_test_main.py
b/python/pyfory/tests/xlang_test_main.py
index d2782c552..e1355aab2 100644
--- a/python/pyfory/tests/xlang_test_main.py
+++ b/python/pyfory/tests/xlang_test_main.py
@@ -140,6 +140,39 @@ class EmptyWrapper:
pass
+@dataclass
+class EmptyStruct:
+ pass
+
+
+@dataclass
+class OneStringFieldStruct:
+ f1: Optional[str] = None
+
+
+@dataclass
+class TwoStringFieldStruct:
+ f1: Optional[str] = None
+ f2: Optional[str] = None
+
+
+class TestEnum(enum.Enum):
+ VALUE_A = 0
+ VALUE_B = 1
+ VALUE_C = 2
+
+
+@dataclass
+class OneEnumFieldStruct:
+ f1: TestEnum = None
+
+
+@dataclass
+class TwoEnumFieldStruct:
+ f1: TestEnum = None
+ f2: TestEnum = None
+
+
# ============================================================================
# Test Functions - Each function handles read -> verify -> write back
# ============================================================================
@@ -448,6 +481,212 @@ def test_polymorphic_map():
f.write(new_buffer.get_bytes(0, new_buffer.writer_index))
+def test_one_string_field_schema():
+ """Test one string field struct with schema consistent mode."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ fory = pyfory.Fory(xlang=True, compatible=False)
+ fory.register_type(OneStringFieldStruct, type_id=200)
+
+ expected = OneStringFieldStruct(f1="hello")
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized: {obj}")
+ assert obj == expected, f"Mismatch: {obj} != {expected}"
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
+def test_one_string_field_compatible():
+ """Test one string field struct with compatible mode."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ fory = pyfory.Fory(xlang=True, compatible=True)
+ fory.register_type(OneStringFieldStruct, type_id=200)
+
+ expected = OneStringFieldStruct(f1="hello")
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized: {obj}")
+ assert obj == expected, f"Mismatch: {obj} != {expected}"
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
+def test_two_string_field_compatible():
+ """Test two string field struct with compatible mode."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ fory = pyfory.Fory(xlang=True, compatible=True)
+ fory.register_type(TwoStringFieldStruct, type_id=201)
+
+ expected = TwoStringFieldStruct(f1="first", f2="second")
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized: {obj}")
+ assert obj == expected, f"Mismatch: {obj} != {expected}"
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
+def test_schema_evolution_compatible():
+ """Test schema evolution: deserialize TwoStringFieldStruct as
EmptyStruct."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ # Deserialize TwoStringFieldStruct as EmptyStruct (should skip all fields)
+ fory = pyfory.Fory(xlang=True, compatible=True)
+ fory.register_type(EmptyStruct, type_id=200)
+
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized as EmptyStruct: {obj}")
+ assert isinstance(obj, EmptyStruct), f"Expected EmptyStruct, got
{type(obj)}"
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
+def test_schema_evolution_compatible_reverse():
+ """Test schema evolution: deserialize OneStringFieldStruct as
TwoStringFieldStruct."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ # Deserialize OneStringFieldStruct as TwoStringFieldStruct
+ fory = pyfory.Fory(xlang=True, compatible=True)
+ fory.register_type(TwoStringFieldStruct, type_id=200)
+
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized as TwoStringFieldStruct: {obj}")
+ assert isinstance(obj, TwoStringFieldStruct), f"Expected
TwoStringFieldStruct, got {type(obj)}"
+ assert obj.f1 == "only_one", f"Expected f1='only_one', got f1='{obj.f1}'"
+ # f2 should be None (missing field)
+ assert obj.f2 is None or obj.f2 == "", f"Expected f2=None or empty, got
f2='{obj.f2}'"
+
+ # Set f2 to empty string for serialization (match Go behavior)
+ if obj.f2 is None:
+ obj.f2 = ""
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
+def test_one_enum_field_schema():
+ """Test one enum field struct with schema consistent mode."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ fory = pyfory.Fory(xlang=True, compatible=False)
+ fory.register_type(TestEnum, type_id=210)
+ fory.register_type(OneEnumFieldStruct, type_id=211)
+
+ expected = OneEnumFieldStruct(f1=TestEnum.VALUE_B)
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized: {obj}")
+ assert obj == expected, f"Mismatch: {obj} != {expected}"
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
+def test_one_enum_field_compatible():
+ """Test one enum field struct with compatible mode."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ fory = pyfory.Fory(xlang=True, compatible=True)
+ fory.register_type(TestEnum, type_id=210)
+ fory.register_type(OneEnumFieldStruct, type_id=211)
+
+ expected = OneEnumFieldStruct(f1=TestEnum.VALUE_A)
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized: {obj}")
+ assert obj == expected, f"Mismatch: {obj} != {expected}"
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
+def test_two_enum_field_compatible():
+ """Test two enum field struct with compatible mode."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ fory = pyfory.Fory(xlang=True, compatible=True)
+ fory.register_type(TestEnum, type_id=210)
+ fory.register_type(TwoEnumFieldStruct, type_id=212)
+
+ expected = TwoEnumFieldStruct(f1=TestEnum.VALUE_A, f2=TestEnum.VALUE_C)
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized: {obj}")
+ assert obj == expected, f"Mismatch: {obj} != {expected}"
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
+def test_enum_schema_evolution_compatible():
+ """Test enum schema evolution: deserialize TwoEnumFieldStruct as
EmptyStruct."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ # Deserialize TwoEnumFieldStruct as EmptyStruct (should skip all fields)
+ fory = pyfory.Fory(xlang=True, compatible=True)
+ fory.register_type(TestEnum, type_id=210)
+ fory.register_type(EmptyStruct, type_id=211)
+
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized as EmptyStruct: {obj}")
+ assert isinstance(obj, EmptyStruct), f"Expected EmptyStruct, got
{type(obj)}"
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
+def test_enum_schema_evolution_compatible_reverse():
+ """Test enum schema evolution: deserialize OneEnumFieldStruct as
TwoEnumFieldStruct."""
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ # Deserialize OneEnumFieldStruct as TwoEnumFieldStruct
+ fory = pyfory.Fory(xlang=True, compatible=True)
+ fory.register_type(TestEnum, type_id=210)
+ fory.register_type(TwoEnumFieldStruct, type_id=211)
+
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized as TwoEnumFieldStruct: {obj}")
+ assert isinstance(obj, TwoEnumFieldStruct), f"Expected TwoEnumFieldStruct,
got {type(obj)}"
+ assert obj.f1 == TestEnum.VALUE_C, f"Expected f1=VALUE_C, got f1={obj.f1}"
+ # f2 should be None (missing field due to schema evolution)
+ f2_value = getattr(obj, "f2", None)
+ assert f2_value is None, f"Expected f2=None, got f2={f2_value}"
+
+ new_bytes = fory.serialize(obj)
+ with open(data_file, "wb") as f:
+ f.write(new_bytes)
+
+
if __name__ == "__main__":
"""
This file is executed by PythonXlangTest.java and other cross-language
tests.
diff --git a/rust/fory-derive/src/object/util.rs
b/rust/fory-derive/src/object/util.rs
index 0e9add7a3..1389dfeb9 100644
--- a/rust/fory-derive/src/object/util.rs
+++ b/rust/fory-derive/src/object/util.rs
@@ -994,7 +994,24 @@ fn to_snake_case(name: &str) -> String {
result
}
-pub(crate) fn compute_struct_version_hash(fields: &[&Field]) -> i32 {
+/// Computes the fingerprint string for a struct type used in schema
versioning.
+///
+/// **Fingerprint Format:**
+///
+/// Each field contributes: `<field_name>,<type_id>,<ref>,<nullable>;`
+///
+/// Fields are sorted by field name (snake_case) lexicographically.
+///
+/// **Field Components:**
+/// - `field_name`: snake_case field name (Rust doesn't support field tag IDs
yet)
+/// - `type_id`: Fory TypeId as decimal string (e.g., "4" for INT32)
+/// - `ref`: "1" if reference tracking enabled, "0" otherwise (currently
always "0" in Rust)
+/// - `nullable`: "1" if null flag is written, "0" otherwise
+///
+/// **Example fingerprint:** `"age,4,0,0;name,12,0,1;"`
+///
+/// This format is consistent across Go, Java, Rust, and C++ implementations.
+pub(crate) fn compute_struct_fingerprint(fields: &[&Field]) -> String {
let mut field_info_map: HashMap<String, (u32, bool)> =
HashMap::with_capacity(fields.len());
for field in fields {
let name = field.ident.as_ref().unwrap().to_string();
@@ -1006,11 +1023,19 @@ pub(crate) fn compute_struct_version_hash(fields:
&[&Field]) -> i32 {
field_info_map.insert(name, (type_id, nullable));
}
+ // Sort field names lexicographically for fingerprint computation
+ // This matches Java/Go behavior where fingerprint fields are sorted by
name,
+ // not by the type-category-based ordering used for serialization
+ let mut sorted_names: Vec<String> =
field_info_map.keys().cloned().collect();
+ sorted_names.sort();
+
let mut fingerprint = String::new();
- for name in get_sorted_field_names(fields).iter() {
+ for name in sorted_names.iter() {
let (type_id, nullable) = field_info_map
.get(name)
.expect("Field metadata missing during struct hash computation");
+ // Format: <field_name>,<type_id>,<ref>,<nullable>;
+ // Since Rust doesn't support field tag IDs yet, use snake_case field
name
fingerprint.push_str(&to_snake_case(name));
fingerprint.push(',');
let effective_type_id = if *type_id == TypeId::UNKNOWN as u32 {
@@ -1020,9 +1045,26 @@ pub(crate) fn compute_struct_version_hash(fields:
&[&Field]) -> i32 {
};
fingerprint.push_str(&effective_type_id.to_string());
fingerprint.push(',');
+ // ref flag: currently always 0 in Rust (no ref tracking support yet)
+ fingerprint.push('0');
+ fingerprint.push(',');
fingerprint.push_str(if *nullable { "1;" } else { "0;" });
}
+ fingerprint
+}
+
+/// Computes the struct version hash from field metadata.
+///
+/// Uses `compute_struct_fingerprint` to build the fingerprint string,
+/// then hashes it with MurmurHash3_x64_128 using seed 47, and takes
+/// the low 32 bits as signed i32.
+///
+/// This provides the cross-language struct version ID used by class
+/// version checking, consistent with Go, Java, and C++ implementations.
+pub(crate) fn compute_struct_version_hash(fields: &[&Field]) -> i32 {
+ let fingerprint = compute_struct_fingerprint(fields);
+
let seed: u64 = 47;
let (hash, _) =
fory_core::meta::murmurhash3_x64_128(fingerprint.as_bytes(), seed);
let version = (hash & 0xFFFF_FFFF) as u32;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
