This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 88a803855 fix(go): reference tracking fails when >127 objects
serialized (#3086)
88a803855 is described below
commit 88a80385592779fff335495faefe79df86dfef53
Author: Jonathan Yoder <[email protected]>
AuthorDate: Wed Dec 24 12:25:43 2025 -0500
fix(go): reference tracking fails when >127 objects serialized (#3086)
## What does this PR do?
Fixes reference tracking bug that causes deserialization to fail with
hash mismatch errors when more than 127 objects are serialized with
reference tracking enabled.
## Root Cause
The check `if int8(refID) < NotNullValueFlag` in multiple files causes
integer overflow when `refID >= 128`:
```go
// BEFORE (buggy):
if int8(refID) < NotNullValueFlag {
```
- `int8(128)` = `-128` (two's complement overflow)
- `-128 < -1` (NotNullValueFlag) = `TRUE` (incorrectly!)
This causes the code to incorrectly enter the "reference found" branch,
corrupting buffer positions and causing hash mismatches.
## Fix
Replace int8 cast with int32 comparison:
```go
// AFTER (fixed):
if refID < int32(NotNullValueFlag) {
```
Since `refID` is already `int32` (returned from `TryPreserveRefId`), and
the flag constants are small negative int8 values that correctly
sign-extend to int32, this comparison is type-safe and avoids
truncation.
## Files Changed
- `array.go` (1 occurrence)
- `map.go` (3 occurrences)
- `pointer.go` (2 occurrences)
- `reader.go` (7 occurrences)
- `serializer.go` (1 occurrence)
- `set.go` (4 occurrences)
- `slice.go` (1 occurrence)
- `slice_dyn.go` (4 occurrences)
- `struct.go` (2 occurrences)
## Testing
Added regression test `TestRefTrackingLargeCount` in
`ref_resolver_test.go` that verifies serialization/deserialization works
correctly with 127, 128, and 200 items.
All existing tests pass.
## Related Issues
Closes #3085
## Checklist
- [x] I have read the [contributing guidelines](../CONTRIBUTING.md)
- [x] I have added tests that prove my fix is effective
- [x] All new and existing tests pass locally with my changes
---
go/fory/array.go | 2 +-
go/fory/map.go | 6 ++---
go/fory/pointer.go | 4 +--
go/fory/reader.go | 14 +++++-----
go/fory/ref_resolver_test.go | 64 ++++++++++++++++++++++++++++++++++++++++++++
go/fory/serializer.go | 2 +-
go/fory/set.go | 8 +++---
go/fory/slice.go | 2 +-
go/fory/slice_dyn.go | 8 +++---
go/fory/struct.go | 4 +--
10 files changed, 89 insertions(+), 25 deletions(-)
diff --git a/go/fory/array.go b/go/fory/array.go
index 9845d0e61..ea222cd01 100644
--- a/go/fory/array.go
+++ b/go/fory/array.go
@@ -46,7 +46,7 @@ func readArrayRefAndType(ctx *ReadContext, refMode RefMode,
readType bool, value
ctx.SetError(FromError(refErr))
return false
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
value.Set(obj)
diff --git a/go/fory/map.go b/go/fory/map.go
index 450b71d30..f910146e6 100644
--- a/go/fory/map.go
+++ b/go/fory/map.go
@@ -82,7 +82,7 @@ func readMapRefAndType(ctx *ReadContext, refMode RefMode,
readType bool, value r
ctx.SetError(FromError(refErr))
return false
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
value.Set(obj)
@@ -375,7 +375,7 @@ func (s mapSerializer) ReadData(ctx *ReadContext, type_
reflect.Type, value refl
ctx.SetError(FromError(err))
return
}
- if int8(refID) <
NotNullValueFlag {
+ if refID <
int32(NotNullValueFlag) {
// Reference to
existing object
obj :=
refResolver.GetReadObject(refID)
if obj.IsValid() {
@@ -466,7 +466,7 @@ func (s mapSerializer) ReadData(ctx *ReadContext, type_
reflect.Type, value refl
ctx.SetError(FromError(err))
return
}
- if int8(refID) <
NotNullValueFlag {
+ if refID <
int32(NotNullValueFlag) {
// Reference to
existing object
obj :=
refResolver.GetReadObject(refID)
if obj.IsValid() {
diff --git a/go/fory/pointer.go b/go/fory/pointer.go
index 17c560965..b223c4e68 100644
--- a/go/fory/pointer.go
+++ b/go/fory/pointer.go
@@ -103,7 +103,7 @@ func (s *ptrToValueSerializer) Read(ctx *ReadContext,
refMode RefMode, readType
ctx.SetError(FromError(refErr))
return
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Reference found
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
@@ -210,7 +210,7 @@ func (s *ptrToInterfaceSerializer) Read(ctx *ReadContext,
refMode RefMode, readT
ctx.SetError(FromError(refErr))
return
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Reference found
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
diff --git a/go/fory/reader.go b/go/fory/reader.go
index 288eed1aa..8bd05a883 100644
--- a/go/fory/reader.go
+++ b/go/fory/reader.go
@@ -569,7 +569,7 @@ func (c *ReadContext) ReadValue(value reflect.Value) {
c.SetError(FromError(err))
return
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Reference found
obj := c.RefResolver().GetReadObject(refID)
if obj.IsValid() {
@@ -625,7 +625,7 @@ func (c *ReadContext) ReadValue(value reflect.Value) {
// For named structs, register the pointer BEFORE reading data
// This is critical for circular references to work correctly
- if isNamedStruct && int8(refID) >= NotNullValueFlag {
+ if isNamedStruct && refID >= int32(NotNullValueFlag) {
c.RefResolver().SetReadObject(refID, newValue)
}
@@ -643,7 +643,7 @@ func (c *ReadContext) ReadValue(value reflect.Value) {
}
// Register reference after reading data for non-struct types
- if !isNamedStruct && int8(refID) >= NotNullValueFlag {
+ if !isNamedStruct && refID >= int32(NotNullValueFlag) {
c.RefResolver().SetReadObject(refID, newValue)
}
@@ -701,7 +701,7 @@ func (c *ReadContext) ReadStruct(value reflect.Value) {
}
// Handle null
- if int8(refID) == NullFlag {
+ if refID == int32(NullFlag) {
if isPtr {
value.Set(reflect.Zero(valueType))
}
@@ -709,7 +709,7 @@ func (c *ReadContext) ReadStruct(value reflect.Value) {
}
// Handle reference to existing object
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
obj := c.RefResolver().GetReadObject(refID)
if obj.IsValid() {
value.Set(obj)
@@ -771,7 +771,7 @@ func (c *ReadContext) readArrayValue(target reflect.Value) {
c.SetError(FromError(err))
return
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Reference to existing object
obj := c.RefResolver().GetReadObject(refID)
if obj.IsValid() {
@@ -812,7 +812,7 @@ func (c *ReadContext) readArrayValue(target reflect.Value) {
reflect.Copy(target, tempSlice)
// Register for circular refs
- if int8(refID) >= NotNullValueFlag {
+ if refID >= int32(NotNullValueFlag) {
c.RefResolver().SetReadObject(refID, target)
}
}
diff --git a/go/fory/ref_resolver_test.go b/go/fory/ref_resolver_test.go
index 9c2773fe9..cfabb602f 100644
--- a/go/fory/ref_resolver_test.go
+++ b/go/fory/ref_resolver_test.go
@@ -18,6 +18,7 @@
package fory
import (
+ "fmt"
"github.com/stretchr/testify/require"
"reflect"
"testing"
@@ -119,3 +120,66 @@ func same(x, y interface{}) bool {
}
return unsafe.Pointer(reflect.ValueOf(x).Pointer()) ==
unsafe.Pointer(reflect.ValueOf(y).Pointer())
}
+
+// TestRefTrackingLargeCount tests that reference tracking works correctly
+// when the number of tracked objects exceeds 127 (the int8 overflow boundary).
+// This is a regression test for https://github.com/apache/fory/issues/3085
+func TestRefTrackingLargeCount(t *testing.T) {
+ type Inner struct {
+ Name string
+ Operator string
+ Version string
+ }
+
+ type Outer struct {
+ Id int32
+ Name string
+ Items []Inner
+ }
+
+ tests := []struct {
+ name string
+ count int
+ }{
+ {"127 items (boundary)", 127},
+ {"128 items (overflow boundary)", 128},
+ {"200 items (well over boundary)", 200},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ f := New(WithRefTracking(true))
+
+ err := f.RegisterByName(&Inner{},
fmt.Sprintf("RefTest_Inner_%d", tt.count))
+ require.NoError(t, err)
+ err = f.RegisterByName(&Outer{},
fmt.Sprintf("RefTest_Outer_%d", tt.count))
+ require.NoError(t, err)
+
+ original := make([]Outer, tt.count)
+ for i := 0; i < tt.count; i++ {
+ original[i] = Outer{
+ Id: int32(i),
+ Name: fmt.Sprintf("item%d", i),
+ Items: []Inner{
+ {Name: "dep1", Operator: ">=",
Version: "1.0.0"},
+ },
+ }
+ }
+
+ data, err := f.Marshal(original)
+ require.NoError(t, err)
+
+ var loaded []Outer
+ err = f.Unmarshal(data, &loaded)
+ require.NoError(t, err, "Unmarshal should succeed with
%d items", tt.count)
+ require.Equal(t, len(original), len(loaded))
+
+ // Verify data integrity
+ for i := 0; i < tt.count; i++ {
+ require.Equal(t, original[i].Id, loaded[i].Id)
+ require.Equal(t, original[i].Name,
loaded[i].Name)
+ require.Equal(t, len(original[i].Items),
len(loaded[i].Items))
+ }
+ })
+ }
+}
diff --git a/go/fory/serializer.go b/go/fory/serializer.go
index b9737a542..3d5ed82e9 100644
--- a/go/fory/serializer.go
+++ b/go/fory/serializer.go
@@ -214,7 +214,7 @@ func (s *extensionSerializerAdapter) Read(ctx *ReadContext,
refMode RefMode, rea
ctx.SetError(FromError(refErr))
return
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
value.Set(obj)
diff --git a/go/fory/set.go b/go/fory/set.go
index 5538927ad..eed5cd087 100644
--- a/go/fory/set.go
+++ b/go/fory/set.go
@@ -290,7 +290,7 @@ func (s setSerializer) readSameType(ctx *ReadContext, buf
*ByteBuffer, value ref
if trackRefs {
// Handle reference tracking if enabled
refID, _ = ctx.RefResolver().TryPreserveRefId(buf)
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Use existing reference if available
elem := ctx.RefResolver().GetReadObject(refID)
value.SetMapIndex(reflect.ValueOf(elem),
reflect.ValueOf(true))
@@ -328,11 +328,11 @@ func (s setSerializer) readDifferentTypes(ctx
*ReadContext, buf *ByteBuffer, val
ctx.SetError(FromError(refErr))
return
}
- if int8(refID) == NullFlag {
+ if refID == int32(NullFlag) {
// Null element - skip for sets
continue
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Use existing reference if available
elem := ctx.RefResolver().GetReadObject(refID)
value.SetMapIndex(elem, reflect.ValueOf(true))
@@ -412,7 +412,7 @@ func (s setSerializer) Read(ctx *ReadContext, refMode
RefMode, readType bool, ha
ctx.SetError(FromError(refErr))
return
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Reference found
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
diff --git a/go/fory/slice.go b/go/fory/slice.go
index 73606ad2f..e085c6acb 100644
--- a/go/fory/slice.go
+++ b/go/fory/slice.go
@@ -73,7 +73,7 @@ func readSliceRefAndType(ctx *ReadContext, refMode RefMode,
readType bool, value
ctx.SetError(FromError(refErr))
return true
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
value.Set(obj)
diff --git a/go/fory/slice_dyn.go b/go/fory/slice_dyn.go
index 4373b3fd1..64cd22ce3 100644
--- a/go/fory/slice_dyn.go
+++ b/go/fory/slice_dyn.go
@@ -319,11 +319,11 @@ func (s sliceDynSerializer) readSameType(ctx
*ReadContext, buf *ByteBuffer, valu
ctx.SetError(FromError(refErr))
return
}
- if int8(refID) == NullFlag {
+ if refID == int32(NullFlag) {
continue
}
// Handle RefFlag - element references a previously
read object
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
value.Index(i).Set(obj)
@@ -379,10 +379,10 @@ func (s sliceDynSerializer) readDifferentTypes(
ctx.SetError(FromError(refErr))
return
}
- if int8(refID) == NullFlag {
+ if refID == int32(NullFlag) {
continue
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Reference to existing object
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
diff --git a/go/fory/struct.go b/go/fory/struct.go
index 849a86ca6..eb91e0b17 100644
--- a/go/fory/struct.go
+++ b/go/fory/struct.go
@@ -551,7 +551,7 @@ func (s *structSerializer) Read(ctx *ReadContext, refMode
RefMode, readType bool
ctx.SetError(FromError(refErr))
return
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Reference found
obj := ctx.RefResolver().GetReadObject(refID)
if obj.IsValid() {
@@ -2080,7 +2080,7 @@ func (s *skipStructSerializer) Read(ctx *ReadContext,
refMode RefMode, readType
ctx.SetError(FromError(refErr))
return
}
- if int8(refID) < NotNullValueFlag {
+ if refID < int32(NotNullValueFlag) {
// Reference found, nothing to skip
return
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
