This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new cf996a122 fix(go): preserve primitive map iteration across chunks
(#3990)
cf996a122 is described below
commit cf996a12200940043b9cf642c7b0dddeaa33a710
Author: Shawn Yang <[email protected]>
AuthorDate: Sat Aug 29 08:51:52 2026 +0800
fix(go): preserve primitive map iteration across chunks (#3990)
## Why?
## What does this PR do?
## Related issues
## AI Contribution Checklist
- [ ] Substantial AI assistance was used in this PR: `yes` / `no`
- [ ] If `yes`, I included a completed [AI Contribution
Checklist](https://github.com/apache/fory/blob/main/AI_POLICY.md#9-contributor-checklist-for-ai-assisted-prs)
in this PR description and the required `AI Usage Disclosure`.
- [ ] If `yes`, my PR description includes the required `ai_review`
summary and screenshot evidence or equivalent persisted links of the
final clean AI review results from both fresh reviewers described in
`AI_POLICY.md`, the Fory-guided reviewer and the independent general
reviewer, on the current PR diff or current HEAD after the latest code
changes.
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
Co-authored-by: Release Validation Test <[email protected]>
---
go/fory/map_primitive.go | 395 ++++++++++++++++++++----------------------
go/fory/map_primitive_test.go | 82 +++++++++
2 files changed, 266 insertions(+), 211 deletions(-)
diff --git a/go/fory/map_primitive.go b/go/fory/map_primitive.go
index 0aded80b2..8ac772f45 100644
--- a/go/fory/map_primitive.go
+++ b/go/fory/map_primitive.go
@@ -46,6 +46,8 @@ var (
intIntMapMaxLength = maxGraphCount(intIntMapElemBytes)
)
+// All primitive map writers must keep one traversal across chunks. A new range
+// per chunk can repeat or omit entries because Go map order is not guaranteed.
// writeMapStringString writes map[string]string using chunk protocol
// When hasGenerics=true, element types are known so we set DECL_TYPE flags
and skip type info
func writeMapStringString(buf *ByteBuffer, m map[string]string, hasGenerics
bool) {
@@ -55,37 +57,32 @@ func writeMapStringString(buf *ByteBuffer, m
map[string]string, hasGenerics bool
return
}
- // WriteData chunks of entries
remaining := length
- for remaining > 0 {
- chunkSize := remaining
- if chunkSize > MAX_CHUNK_SIZE {
- chunkSize = MAX_CHUNK_SIZE
- }
-
- if hasGenerics {
- // Element types are known from generics, set DECL_TYPE
flags
- buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
- buf.WriteUint8(uint8(chunkSize))
- } else {
- // Write type info for generic reader compatibility
- buf.WriteUint8(0)
- buf.WriteUint8(uint8(chunkSize))
- buf.WriteUint8(uint8(STRING)) // key type
- buf.WriteUint8(uint8(STRING)) // value type
- }
+ chunkRemaining := 0
+ for k, v := range m {
+ if chunkRemaining == 0 {
+ chunkRemaining = remaining
+ if chunkRemaining > MAX_CHUNK_SIZE {
+ chunkRemaining = MAX_CHUNK_SIZE
+ }
- // WriteData chunk entries
- count := 0
- for k, v := range m {
- writeString(buf, k)
- writeString(buf, v)
- count++
- if count >= chunkSize {
- break
+ if hasGenerics {
+ // Element types are known from generics, set
DECL_TYPE flags
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(uint8(chunkRemaining))
+ } else {
+ // Write type info for generic reader
compatibility
+ buf.WriteUint8(0)
+ buf.WriteUint8(uint8(chunkRemaining))
+ buf.WriteUint8(uint8(STRING)) // key type
+ buf.WriteUint8(uint8(STRING)) // value type
}
}
- remaining -= chunkSize
+
+ writeString(buf, k)
+ writeString(buf, v)
+ remaining--
+ chunkRemaining--
}
}
@@ -172,32 +169,29 @@ func writeMapStringInt64(buf *ByteBuffer, m
map[string]int64, hasGenerics bool)
}
remaining := length
- for remaining > 0 {
- chunkSize := remaining
- if chunkSize > MAX_CHUNK_SIZE {
- chunkSize = MAX_CHUNK_SIZE
- }
-
- if hasGenerics {
- buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
- buf.WriteUint8(uint8(chunkSize))
- } else {
- buf.WriteUint8(0)
- buf.WriteUint8(uint8(chunkSize))
- buf.WriteUint8(uint8(STRING)) // key type
- buf.WriteUint8(uint8(VARINT64)) // value type
- }
+ chunkRemaining := 0
+ for k, v := range m {
+ if chunkRemaining == 0 {
+ chunkRemaining = remaining
+ if chunkRemaining > MAX_CHUNK_SIZE {
+ chunkRemaining = MAX_CHUNK_SIZE
+ }
- count := 0
- for k, v := range m {
- writeString(buf, k)
- buf.WriteVarint64(v)
- count++
- if count >= chunkSize {
- break
+ if hasGenerics {
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(uint8(chunkRemaining))
+ } else {
+ buf.WriteUint8(0)
+ buf.WriteUint8(uint8(chunkRemaining))
+ buf.WriteUint8(uint8(STRING)) // key type
+ buf.WriteUint8(uint8(VARINT64)) // value type
}
}
- remaining -= chunkSize
+
+ writeString(buf, k)
+ buf.WriteVarint64(v)
+ remaining--
+ chunkRemaining--
}
}
@@ -256,32 +250,29 @@ func writeMapStringInt32(buf *ByteBuffer, m
map[string]int32, hasGenerics bool)
}
remaining := length
- for remaining > 0 {
- chunkSize := remaining
- if chunkSize > MAX_CHUNK_SIZE {
- chunkSize = MAX_CHUNK_SIZE
- }
-
- if hasGenerics {
- buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
- buf.WriteUint8(uint8(chunkSize))
- } else {
- buf.WriteUint8(0)
- buf.WriteUint8(uint8(chunkSize))
- buf.WriteUint8(uint8(STRING)) // key type
- buf.WriteUint8(uint8(VARINT32)) // value type
- }
+ chunkRemaining := 0
+ for k, v := range m {
+ if chunkRemaining == 0 {
+ chunkRemaining = remaining
+ if chunkRemaining > MAX_CHUNK_SIZE {
+ chunkRemaining = MAX_CHUNK_SIZE
+ }
- count := 0
- for k, v := range m {
- writeString(buf, k)
- buf.WriteVarint32(v)
- count++
- if count >= chunkSize {
- break
+ if hasGenerics {
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(uint8(chunkRemaining))
+ } else {
+ buf.WriteUint8(0)
+ buf.WriteUint8(uint8(chunkRemaining))
+ buf.WriteUint8(uint8(STRING)) // key type
+ buf.WriteUint8(uint8(VARINT32)) // value type
}
}
- remaining -= chunkSize
+
+ writeString(buf, k)
+ buf.WriteVarint32(v)
+ remaining--
+ chunkRemaining--
}
}
@@ -340,32 +331,29 @@ func writeMapStringInt(buf *ByteBuffer, m map[string]int,
hasGenerics bool) {
}
remaining := length
- for remaining > 0 {
- chunkSize := remaining
- if chunkSize > MAX_CHUNK_SIZE {
- chunkSize = MAX_CHUNK_SIZE
- }
-
- if hasGenerics {
- buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
- buf.WriteUint8(uint8(chunkSize))
- } else {
- buf.WriteUint8(0)
- buf.WriteUint8(uint8(chunkSize))
- buf.WriteUint8(uint8(STRING)) // key type
- buf.WriteUint8(uint8(VARINT64)) // value type (int
serialized as varint64)
- }
+ chunkRemaining := 0
+ for k, v := range m {
+ if chunkRemaining == 0 {
+ chunkRemaining = remaining
+ if chunkRemaining > MAX_CHUNK_SIZE {
+ chunkRemaining = MAX_CHUNK_SIZE
+ }
- count := 0
- for k, v := range m {
- writeString(buf, k)
- buf.WriteVarint64(int64(v))
- count++
- if count >= chunkSize {
- break
+ if hasGenerics {
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(uint8(chunkRemaining))
+ } else {
+ buf.WriteUint8(0)
+ buf.WriteUint8(uint8(chunkRemaining))
+ buf.WriteUint8(uint8(STRING)) // key type
+ buf.WriteUint8(uint8(VARINT64)) // value type
(int serialized as varint64)
}
}
- remaining -= chunkSize
+
+ writeString(buf, k)
+ buf.WriteVarint64(int64(v))
+ remaining--
+ chunkRemaining--
}
}
@@ -424,32 +412,29 @@ func writeMapStringFloat64(buf *ByteBuffer, m
map[string]float64, hasGenerics bo
}
remaining := length
- for remaining > 0 {
- chunkSize := remaining
- if chunkSize > MAX_CHUNK_SIZE {
- chunkSize = MAX_CHUNK_SIZE
- }
-
- if hasGenerics {
- buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
- buf.WriteUint8(uint8(chunkSize))
- } else {
- buf.WriteUint8(0)
- buf.WriteUint8(uint8(chunkSize))
- buf.WriteUint8(uint8(STRING)) // key type
- buf.WriteUint8(uint8(FLOAT64)) // value type
- }
+ chunkRemaining := 0
+ for k, v := range m {
+ if chunkRemaining == 0 {
+ chunkRemaining = remaining
+ if chunkRemaining > MAX_CHUNK_SIZE {
+ chunkRemaining = MAX_CHUNK_SIZE
+ }
- count := 0
- for k, v := range m {
- writeString(buf, k)
- buf.WriteFloat64(v)
- count++
- if count >= chunkSize {
- break
+ if hasGenerics {
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(uint8(chunkRemaining))
+ } else {
+ buf.WriteUint8(0)
+ buf.WriteUint8(uint8(chunkRemaining))
+ buf.WriteUint8(uint8(STRING)) // key type
+ buf.WriteUint8(uint8(FLOAT64)) // value type
}
}
- remaining -= chunkSize
+
+ writeString(buf, k)
+ buf.WriteFloat64(v)
+ remaining--
+ chunkRemaining--
}
}
@@ -508,32 +493,29 @@ func writeMapStringBool(buf *ByteBuffer, m
map[string]bool, hasGenerics bool) {
}
remaining := length
- for remaining > 0 {
- chunkSize := remaining
- if chunkSize > MAX_CHUNK_SIZE {
- chunkSize = MAX_CHUNK_SIZE
- }
-
- if hasGenerics {
- buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
- buf.WriteUint8(uint8(chunkSize))
- } else {
- buf.WriteUint8(0)
- buf.WriteUint8(uint8(chunkSize))
- buf.WriteUint8(uint8(STRING)) // key type
- buf.WriteUint8(uint8(BOOL)) // value type
- }
+ chunkRemaining := 0
+ for k, v := range m {
+ if chunkRemaining == 0 {
+ chunkRemaining = remaining
+ if chunkRemaining > MAX_CHUNK_SIZE {
+ chunkRemaining = MAX_CHUNK_SIZE
+ }
- count := 0
- for k, v := range m {
- writeString(buf, k)
- buf.WriteBool(v)
- count++
- if count >= chunkSize {
- break
+ if hasGenerics {
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(uint8(chunkRemaining))
+ } else {
+ buf.WriteUint8(0)
+ buf.WriteUint8(uint8(chunkRemaining))
+ buf.WriteUint8(uint8(STRING)) // key type
+ buf.WriteUint8(uint8(BOOL)) // value type
}
}
- remaining -= chunkSize
+
+ writeString(buf, k)
+ buf.WriteBool(v)
+ remaining--
+ chunkRemaining--
}
}
@@ -596,32 +578,29 @@ func writeMapInt32Int32(buf *ByteBuffer, m
map[int32]int32, hasGenerics bool) {
}
remaining := length
- for remaining > 0 {
- chunkSize := remaining
- if chunkSize > MAX_CHUNK_SIZE {
- chunkSize = MAX_CHUNK_SIZE
- }
-
- if hasGenerics {
- buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
- buf.WriteUint8(uint8(chunkSize))
- } else {
- buf.WriteUint8(0)
- buf.WriteUint8(uint8(chunkSize))
- buf.WriteUint8(uint8(VARINT32)) // key type
- buf.WriteUint8(uint8(VARINT32)) // value type
- }
+ chunkRemaining := 0
+ for k, v := range m {
+ if chunkRemaining == 0 {
+ chunkRemaining = remaining
+ if chunkRemaining > MAX_CHUNK_SIZE {
+ chunkRemaining = MAX_CHUNK_SIZE
+ }
- count := 0
- for k, v := range m {
- buf.WriteVarint32(k)
- buf.WriteVarint32(v)
- count++
- if count >= chunkSize {
- break
+ if hasGenerics {
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(uint8(chunkRemaining))
+ } else {
+ buf.WriteUint8(0)
+ buf.WriteUint8(uint8(chunkRemaining))
+ buf.WriteUint8(uint8(VARINT32)) // key type
+ buf.WriteUint8(uint8(VARINT32)) // value type
}
}
- remaining -= chunkSize
+
+ buf.WriteVarint32(k)
+ buf.WriteVarint32(v)
+ remaining--
+ chunkRemaining--
}
}
@@ -680,32 +659,29 @@ func writeMapInt64Int64(buf *ByteBuffer, m
map[int64]int64, hasGenerics bool) {
}
remaining := length
- for remaining > 0 {
- chunkSize := remaining
- if chunkSize > MAX_CHUNK_SIZE {
- chunkSize = MAX_CHUNK_SIZE
- }
-
- if hasGenerics {
- buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
- buf.WriteUint8(uint8(chunkSize))
- } else {
- buf.WriteUint8(0)
- buf.WriteUint8(uint8(chunkSize))
- buf.WriteUint8(uint8(VARINT64)) // key type
- buf.WriteUint8(uint8(VARINT64)) // value type
- }
+ chunkRemaining := 0
+ for k, v := range m {
+ if chunkRemaining == 0 {
+ chunkRemaining = remaining
+ if chunkRemaining > MAX_CHUNK_SIZE {
+ chunkRemaining = MAX_CHUNK_SIZE
+ }
- count := 0
- for k, v := range m {
- buf.WriteVarint64(k)
- buf.WriteVarint64(v)
- count++
- if count >= chunkSize {
- break
+ if hasGenerics {
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(uint8(chunkRemaining))
+ } else {
+ buf.WriteUint8(0)
+ buf.WriteUint8(uint8(chunkRemaining))
+ buf.WriteUint8(uint8(VARINT64)) // key type
+ buf.WriteUint8(uint8(VARINT64)) // value type
}
}
- remaining -= chunkSize
+
+ buf.WriteVarint64(k)
+ buf.WriteVarint64(v)
+ remaining--
+ chunkRemaining--
}
}
@@ -764,32 +740,29 @@ func writeMapIntInt(buf *ByteBuffer, m map[int]int,
hasGenerics bool) {
}
remaining := length
- for remaining > 0 {
- chunkSize := remaining
- if chunkSize > MAX_CHUNK_SIZE {
- chunkSize = MAX_CHUNK_SIZE
- }
-
- if hasGenerics {
- buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
- buf.WriteUint8(uint8(chunkSize))
- } else {
- buf.WriteUint8(0)
- buf.WriteUint8(uint8(chunkSize))
- buf.WriteUint8(uint8(VARINT64)) // key type (int
serialized as varint64)
- buf.WriteUint8(uint8(VARINT64)) // value type
- }
+ chunkRemaining := 0
+ for k, v := range m {
+ if chunkRemaining == 0 {
+ chunkRemaining = remaining
+ if chunkRemaining > MAX_CHUNK_SIZE {
+ chunkRemaining = MAX_CHUNK_SIZE
+ }
- count := 0
- for k, v := range m {
- buf.WriteVarint64(int64(k))
- buf.WriteVarint64(int64(v))
- count++
- if count >= chunkSize {
- break
+ if hasGenerics {
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(uint8(chunkRemaining))
+ } else {
+ buf.WriteUint8(0)
+ buf.WriteUint8(uint8(chunkRemaining))
+ buf.WriteUint8(uint8(VARINT64)) // key type
(int serialized as varint64)
+ buf.WriteUint8(uint8(VARINT64)) // value type
}
}
- remaining -= chunkSize
+
+ buf.WriteVarint64(int64(k))
+ buf.WriteVarint64(int64(v))
+ remaining--
+ chunkRemaining--
}
}
diff --git a/go/fory/map_primitive_test.go b/go/fory/map_primitive_test.go
index c4eaebc27..58e9c5137 100644
--- a/go/fory/map_primitive_test.go
+++ b/go/fory/map_primitive_test.go
@@ -18,6 +18,7 @@
package fory
import (
+ "strconv"
"testing"
"github.com/stretchr/testify/require"
@@ -59,3 +60,84 @@ func TestPrimitiveMapReaderRejectsNullChunks(t *testing.T) {
_ = f.readCtx.ReadStringStringMap(RefModeNone, false)
require.Error(t, f.readCtx.CheckError())
}
+
+func buildPrimitiveMap[K comparable, V any](size int, entry func(int) (K, V))
map[K]V {
+ result := make(map[K]V, size)
+ for i := 0; i < size; i++ {
+ key, value := entry(i)
+ result[key] = value
+ }
+ return result
+}
+
+func requirePrimitiveMapRoundTrip[K comparable, V comparable](t *testing.T,
value map[K]V) {
+ t.Helper()
+ f := NewFory(WithXlang(false), WithCompatible(false))
+ data, err := Serialize(f, value)
+ require.NoError(t, err)
+
+ var result map[K]V
+ require.NoError(t, Deserialize(f, data, &result))
+ require.Equal(t, value, result)
+}
+
+func TestPrimitiveMapChunks(t *testing.T) {
+ for _, size := range []int{MAX_CHUNK_SIZE + 1, MAX_CHUNK_SIZE*2 + 1} {
+ t.Run(strconv.Itoa(size), func(t *testing.T) {
+ t.Run("string_string", func(t *testing.T) {
+ value := buildPrimitiveMap(size, func(i int)
(string, string) {
+ return strconv.Itoa(i),
strconv.Itoa(i*3 + 1)
+ })
+ requirePrimitiveMapRoundTrip(t, value)
+ })
+ t.Run("string_int64", func(t *testing.T) {
+ value := buildPrimitiveMap(size, func(i int)
(string, int64) {
+ return strconv.Itoa(i), int64(i*3 + 1)
+ })
+ requirePrimitiveMapRoundTrip(t, value)
+ })
+ t.Run("string_int32", func(t *testing.T) {
+ value := buildPrimitiveMap(size, func(i int)
(string, int32) {
+ return strconv.Itoa(i), int32(i*3 + 1)
+ })
+ requirePrimitiveMapRoundTrip(t, value)
+ })
+ t.Run("string_int", func(t *testing.T) {
+ value := buildPrimitiveMap(size, func(i int)
(string, int) {
+ return strconv.Itoa(i), i*3 + 1
+ })
+ requirePrimitiveMapRoundTrip(t, value)
+ })
+ t.Run("string_float64", func(t *testing.T) {
+ value := buildPrimitiveMap(size, func(i int)
(string, float64) {
+ return strconv.Itoa(i), float64(i) +
0.25
+ })
+ requirePrimitiveMapRoundTrip(t, value)
+ })
+ t.Run("string_bool", func(t *testing.T) {
+ value := buildPrimitiveMap(size, func(i int)
(string, bool) {
+ return strconv.Itoa(i), i%2 == 0
+ })
+ requirePrimitiveMapRoundTrip(t, value)
+ })
+ t.Run("int32_int32", func(t *testing.T) {
+ value := buildPrimitiveMap(size, func(i int)
(int32, int32) {
+ return int32(i), int32(i*3 + 1)
+ })
+ requirePrimitiveMapRoundTrip(t, value)
+ })
+ t.Run("int64_int64", func(t *testing.T) {
+ value := buildPrimitiveMap(size, func(i int)
(int64, int64) {
+ return int64(i), int64(i*3 + 1)
+ })
+ requirePrimitiveMapRoundTrip(t, value)
+ })
+ t.Run("int_int", func(t *testing.T) {
+ value := buildPrimitiveMap(size, func(i int)
(int, int) {
+ return i, i*3 + 1
+ })
+ requirePrimitiveMapRoundTrip(t, value)
+ })
+ })
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
