This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 9a48b8dda feat(compiler): add compatible mode for idl (#3245)
9a48b8dda is described below
commit 9a48b8ddab2785aea67f9e4e75887ff144001be6
Author: Shawn Yang <[email protected]>
AuthorDate: Fri Jan 30 19:11:01 2026 +0800
feat(compiler): add compatible mode for idl (#3245)
## Why?
- Enable IDL compatible mode across languages so struct/union metadata
and type-def diffs round-trip correctly.
## What does this PR do?
- Align union handling in fingerprints/type IDs across
Java/C++/Rust/Python/Go for compatibility.
- Fix compatible-mode read/write paths for struct/union fields and
dynamic type resolution, including named union typedef handling.
- Improve Go struct detection, primitive array skipping, and type-def
diff diagnostics; add debug output hooks in Java/Go/Python.
- Expand IDL roundtrip coverage for compatible mode and union scenarios.
- Minor QoL: union string representations.
## Related issues
#3099
#2906
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---
AGENTS.md | 1 +
cpp/fory/serialization/struct_serializer.h | 99 +++++++--------
cpp/fory/serialization/type_resolver.cc | 7 +-
cpp/fory/serialization/union_serializer.h | 1 -
go/fory/field_info.go | 9 +-
go/fory/skip.go | 24 ++++
go/fory/struct.go | 133 ++++++++++++++++----
go/fory/type_def.go | 121 ++++++++++++++----
go/fory/types.go | 1 +
integration_tests/idl_tests/cpp/main.cc | 53 +++++++-
.../idl_tests/go/idl_roundtrip_test.go | 22 +++-
.../apache/fory/idl_tests/IdlRoundTripTest.java | 138 +++++++++++++++++----
.../idl_tests/python/src/idl_tests/roundtrip.py | 23 +++-
.../idl_tests/rust/tests/idl_roundtrip.rs | 18 ++-
.../main/java/org/apache/fory/meta/ClassDef.java | 56 ++++++---
.../main/java/org/apache/fory/meta/FieldTypes.java | 37 ++++++
.../org/apache/fory/resolver/XtypeResolver.java | 16 ++-
.../org/apache/fory/serializer/FieldGroups.java | 7 ++
.../apache/fory/serializer/ObjectSerializer.java | 21 ++++
.../fory/serializer/SerializationBinding.java | 12 +-
.../apache/fory/serializer/UnionSerializer.java | 4 +
.../apache/fory/serializer/struct/Fingerprint.java | 5 +-
.../src/main/java/org/apache/fory/type/Types.java | 1 +
.../java/org/apache/fory/type/union/Union.java | 2 +-
python/pyfory/meta/typedef.py | 44 ++++---
python/pyfory/meta/typedef_encoder.py | 20 +--
python/pyfory/registry.py | 6 +-
python/pyfory/struct.py | 28 +++--
python/pyfory/types.py | 18 +++
python/pyfory/union.py | 3 +
rust/fory-core/src/meta/type_meta.rs | 8 +-
rust/fory-derive/src/object/util.rs | 8 +-
32 files changed, 732 insertions(+), 214 deletions(-)
diff --git a/AGENTS.md b/AGENTS.md
index f329427ec..c26a3995d 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -484,6 +484,7 @@ Fory python has two implementations for the protocol:
- **Python mode**: Pure python implementation based on `xlang serialization
format`, used for debugging and testing only. This mode can be enabled by
setting `ENABLE_FORY_CYTHON_SERIALIZATION=0` environment variable.
- **Cython mode**: Cython based implementation based on `xlang serialization
format`, which is used by default and has better performance than pure python.
This mode can be enabled by setting `ENABLE_FORY_CYTHON_SERIALIZATION=1`
environment variable.
- **Python mode** and **Cython mode** reused some code from each other to
reduce code duplication.
+- **Debug Struct Serialization**: set `ENABLE_FORY_PYTHON_JIT=0` when debug
struct fields serialization error, this mode is more easy to debug and add
logs. Even struct serialization itself has no bug, by enable this mode and
adding debug logs, we can narrow the bug scope more easily.
Code structure:
diff --git a/cpp/fory/serialization/struct_serializer.h
b/cpp/fory/serialization/struct_serializer.h
index a9103efdc..67471c493 100644
--- a/cpp/fory/serialization/struct_serializer.h
+++ b/cpp/fory/serialization/struct_serializer.h
@@ -552,6 +552,15 @@ template <typename T> struct CompileTimeFieldHelpers {
}
return field_is_nullable_v<RawFieldType>;
}
+ // Else if FORY_FIELD_CONFIG is defined, use nullable from config
+ else if constexpr (::fory::detail::has_field_config_v<T>) {
+ if constexpr (::fory::detail::GetFieldConfigEntry<T,
+ Index>::has_entry &&
+ ::fory::detail::GetFieldConfigEntry<T, Index>::nullable)
{
+ return true;
+ }
+ return field_is_nullable_v<RawFieldType>;
+ }
// For non-wrapped types, use xlang defaults:
// Only std::optional is nullable (field_is_nullable_v returns true for
// optional). For xlang consistency, shared_ptr/unique_ptr are NOT
@@ -620,6 +629,15 @@ template <typename T> struct CompileTimeFieldHelpers {
else if constexpr (::fory::detail::has_field_tags_v<T>) {
return ::fory::detail::GetFieldTagEntry<T, Index>::track_ref;
}
+ // Else if FORY_FIELD_CONFIG is defined, use ref from config
+ else if constexpr (::fory::detail::has_field_config_v<T>) {
+ if constexpr (::fory::detail::GetFieldConfigEntry<T,
+ Index>::has_entry &&
+ ::fory::detail::GetFieldConfigEntry<T, Index>::ref) {
+ return true;
+ }
+ return field_track_ref_v<RawFieldType>;
+ }
// Default: shared_ptr/SharedWeak track refs
else {
return field_track_ref_v<RawFieldType>;
@@ -646,6 +664,15 @@ template <typename T> struct CompileTimeFieldHelpers {
else if constexpr (::fory::detail::has_field_tags_v<T>) {
return ::fory::detail::GetFieldTagEntry<T, Index>::dynamic_value;
}
+ // Else if FORY_FIELD_CONFIG is defined, use dynamic_value from config
+ else if constexpr (::fory::detail::has_field_config_v<T>) {
+ constexpr int dynamic_value =
+ ::fory::detail::GetFieldConfigEntry<T, Index>::dynamic_value;
+ if constexpr (dynamic_value != -1) {
+ return dynamic_value;
+ }
+ return -1;
+ }
// Default: AUTO (use std::is_polymorphic to decide)
else {
return -1;
@@ -1802,7 +1829,7 @@ void write_single_field(const T &obj, WriteContext &ctx,
}
// Per Rust implementation: primitives are written directly without ref/type
- if constexpr (is_primitive_field && !field_type_is_nullable) {
+ if constexpr (is_primitive_field && !field_type_is_nullable && !is_nullable)
{
if constexpr (::fory::detail::has_field_config_v<T> &&
(std::is_same_v<FieldType, uint32_t> ||
std::is_same_v<FieldType, uint64_t> ||
@@ -1890,7 +1917,7 @@ void write_single_field(const T &obj, WriteContext &ctx,
// - TRUE (1): always write type info
// - FALSE (0): never write type info for this field
// - AUTO (-1): write type info if is_polymorphic (auto-detected)
- bool polymorphic_write_type =
+ constexpr bool polymorphic_write_type =
(dynamic_val == 1) || (dynamic_val == -1 && is_polymorphic);
bool write_type =
polymorphic_write_type || ((is_struct || is_ext) && ctx.is_compatible());
@@ -2123,23 +2150,15 @@ void read_single_field_by_index(T &obj, ReadContext
&ctx) {
// - TRUE (1): always read type info
// - FALSE (0): never read type info for this field
// - AUTO (-1): read type info if is_polymorphic_field (auto-detected)
- bool read_type =
- (dynamic_val == 1) || (dynamic_val == -1 && is_polymorphic_field);
+ // Struct/EXT fields need type info in compatible mode for TypeMeta.
+ bool read_type = (dynamic_val == 1) ||
+ (dynamic_val == -1 && is_polymorphic_field) ||
+ ((is_struct_field || is_ext_field) && ctx.is_compatible());
// get field metadata from fory::field<> or FORY_FIELD_TAGS or defaults
constexpr bool is_nullable = Helpers::template field_nullable<Index>();
constexpr bool track_ref = Helpers::template field_track_ref<Index>();
- // In compatible mode, nested struct fields always carry type metadata
- // (xtype_id + meta index). We must read this metadata so that
- // `Serializer<T>::read` can dispatch to `read_compatible` with the correct
- // remote TypeMeta instead of treating the bytes as part of the first field
- // value.
- if (!is_polymorphic_field && (is_struct_field || is_ext_field) &&
- ctx.is_compatible()) {
- read_type = true;
- }
-
// Per xlang spec, all non-primitive fields have ref flags.
// Primitive types: bool, int8-64, var_int32/64, sli_int64, float16/32/64
// Non-primitives include: string, list, set, map, struct, enum, etc.
@@ -2154,7 +2173,8 @@ void read_single_field_by_index(T &obj, ReadContext &ctx)
{
// shared_ptr) that don't need ref metadata, bypass Serializer<T>::read
// and use direct buffer reads with Error&.
constexpr bool is_raw_prim = is_raw_primitive_v<FieldType>;
- if constexpr (is_raw_prim && is_primitive_field && !field_type_is_nullable) {
+ if constexpr (is_raw_prim && is_primitive_field && !field_type_is_nullable &&
+ !is_nullable) {
auto read_value = [&ctx]() -> FieldType {
if constexpr (is_configurable_int_v<FieldType>) {
return read_configurable_int<FieldType, T, Index>(ctx);
@@ -2303,18 +2323,10 @@ void read_single_field_by_index_compatible(T &obj,
ReadContext &ctx,
// - TRUE (1): always read type info
// - FALSE (0): never read type info for this field
// - AUTO (-1): read type info if is_polymorphic_field (auto-detected)
- bool read_type =
- (dynamic_val == 1) || (dynamic_val == -1 && is_polymorphic_field);
-
- // In compatible mode, nested struct fields always carry type metadata
- // (xtype_id + meta index). We must read this metadata so that
- // `Serializer<T>::read` can dispatch to `read_compatible` with the correct
- // remote TypeMeta instead of treating the bytes as part of the first field
- // value.
- if (!is_polymorphic_field && (is_struct_field || is_ext_field) &&
- ctx.is_compatible()) {
- read_type = true;
- }
+ // Struct/EXT fields need type info in compatible mode for TypeMeta.
+ bool read_type = (dynamic_val == 1) ||
+ (dynamic_val == -1 && is_polymorphic_field) ||
+ ((is_struct_field || is_ext_field) && ctx.is_compatible());
// In compatible mode, trust the remote field metadata (remote_ref_mode)
// to tell us whether a ref/null flag was written before the value payload.
@@ -2937,41 +2949,22 @@ struct Serializer<T,
std::enable_if_t<is_fory_serializable_v<T>>> {
if (FORY_PREDICT_FALSE(ctx.has_error())) {
return T{};
}
-
- // Check LOCAL type to decide if we should read meta_index (matches
- // Rust logic)
- auto local_type_info_res =
- ctx.type_resolver().template get_type_info<T>();
- if (!local_type_info_res.ok()) {
- ctx.set_error(std::move(local_type_info_res).error());
- return T{};
- }
- const TypeInfo *local_type_info = local_type_info_res.value();
- uint32_t local_type_id = local_type_info->type_id;
- uint8_t local_type_id_low = local_type_id & 0xff;
-
- if (local_type_id_low ==
+ uint8_t remote_type_id_low = remote_type_id & 0xff;
+ const bool remote_has_meta =
+ remote_type_id_low ==
static_cast<uint8_t>(TypeId::COMPATIBLE_STRUCT) ||
- local_type_id_low ==
- static_cast<uint8_t>(TypeId::NAMED_COMPATIBLE_STRUCT)) {
+ remote_type_id_low ==
+ static_cast<uint8_t>(TypeId::NAMED_COMPATIBLE_STRUCT);
+ if (remote_has_meta) {
// Read TypeMeta inline using streaming protocol
auto remote_type_info_res = ctx.read_type_meta();
if (!remote_type_info_res.ok()) {
ctx.set_error(std::move(remote_type_info_res).error());
return T{};
}
-
return read_compatible(ctx, remote_type_info_res.value());
- } else {
- // Local type is not compatible struct - verify type match and read
- // data
- if (remote_type_id != local_type_id) {
- ctx.set_error(
- Error::type_mismatch(remote_type_id, local_type_id));
- return T{};
- }
- return read_data(ctx);
}
+ return read_data(ctx);
} else {
// read_type=false in compatible mode: same version, use sorted order
// (fast path)
diff --git a/cpp/fory/serialization/type_resolver.cc
b/cpp/fory/serialization/type_resolver.cc
index 2549073ef..2e0f28c20 100644
--- a/cpp/fory/serialization/type_resolver.cc
+++ b/cpp/fory/serialization/type_resolver.cc
@@ -857,6 +857,10 @@ void TypeMeta::assign_field_ids(const TypeMeta *local_type,
case TypeId::EXT:
case TypeId::NAMED_EXT:
return static_cast<uint32_t>(TypeId::EXT);
+ case TypeId::BINARY:
+ case TypeId::INT8_ARRAY:
+ case TypeId::UINT8_ARRAY:
+ return static_cast<uint32_t>(TypeId::BINARY);
default:
return tid;
}
@@ -1072,7 +1076,8 @@ std::string TypeMeta::compute_struct_fingerprint(
if (effective_type_id == static_cast<uint32_t>(TypeId::ENUM) ||
effective_type_id == static_cast<uint32_t>(TypeId::NAMED_ENUM) ||
effective_type_id == static_cast<uint32_t>(TypeId::STRUCT) ||
- effective_type_id == static_cast<uint32_t>(TypeId::NAMED_STRUCT)) {
+ effective_type_id == static_cast<uint32_t>(TypeId::NAMED_STRUCT) ||
+ effective_type_id == static_cast<uint32_t>(TypeId::UNION)) {
effective_type_id = static_cast<uint32_t>(TypeId::UNKNOWN);
}
fingerprint.append(std::to_string(effective_type_id));
diff --git a/cpp/fory/serialization/union_serializer.h
b/cpp/fory/serialization/union_serializer.h
index 734472996..6592b3bd2 100644
--- a/cpp/fory/serialization/union_serializer.h
+++ b/cpp/fory/serialization/union_serializer.h
@@ -547,7 +547,6 @@ struct Serializer<T,
std::enable_if_t<detail::is_union_type_v<T>>> {
if (FORY_PREDICT_FALSE(ctx.has_error())) {
return default_value();
}
-
T result{};
bool matched = detail::dispatch_union_case<T>(case_id, [&](auto tag) {
constexpr size_t index = decltype(tag)::value;
diff --git a/go/fory/field_info.go b/go/fory/field_info.go
index 5a778cf21..33466bc0f 100644
--- a/go/fory/field_info.go
+++ b/go/fory/field_info.go
@@ -481,13 +481,20 @@ func isStructField(t reflect.Type) bool {
if info, ok := getOptionalInfo(t); ok {
t = info.valueType
}
+ if t.Kind() == reflect.Ptr {
+ t = t.Elem()
+ }
if isUnionType(t) {
return false
}
+ // Date/Timestamp are built-in types with dedicated encodings, not user
structs.
+ if t == dateType || t == timestampType {
+ return false
+ }
if t.Kind() == reflect.Struct {
return true
}
- return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
+ return false
}
// isSetReflectType checks if a reflect.Type is a Set type (map[T]struct{})
diff --git a/go/fory/skip.go b/go/fory/skip.go
index 9606fafe5..2eea25d4c 100644
--- a/go/fory/skip.go
+++ b/go/fory/skip.go
@@ -604,6 +604,30 @@ func skipValue(ctx *ReadContext, fieldDef FieldDef,
readRefFlag bool, isField bo
return
}
_ = ctx.buffer.ReadBinary(int(length), err)
+ case BOOL_ARRAY, INT8_ARRAY, UINT8_ARRAY:
+ length := ctx.buffer.ReadLength(err)
+ if ctx.HasError() {
+ return
+ }
+ _ = ctx.buffer.ReadBinary(length, err)
+ case INT16_ARRAY, UINT16_ARRAY, FLOAT16_ARRAY:
+ length := ctx.buffer.ReadLength(err)
+ if ctx.HasError() {
+ return
+ }
+ _ = ctx.buffer.ReadBinary(length*2, err)
+ case INT32_ARRAY, UINT32_ARRAY, FLOAT32_ARRAY:
+ length := ctx.buffer.ReadLength(err)
+ if ctx.HasError() {
+ return
+ }
+ _ = ctx.buffer.ReadBinary(length*4, err)
+ case INT64_ARRAY, UINT64_ARRAY, FLOAT64_ARRAY:
+ length := ctx.buffer.ReadLength(err)
+ if ctx.HasError() {
+ return
+ }
+ _ = ctx.buffer.ReadBinary(length*8, err)
// Date/Time types
case DATE:
diff --git a/go/fory/struct.go b/go/fory/struct.go
index 66c0b9fab..828dd28cd 100644
--- a/go/fory/struct.go
+++ b/go/fory/struct.go
@@ -149,12 +149,43 @@ func computeLocalNullable(typeResolver *TypeResolver,
field reflect.StructField,
if foryTag.NullableSet {
nullableFlag = foryTag.Nullable
}
- if isNonNullablePrimitiveKind(fieldType.Kind()) && !isEnum {
+ if isNonNullablePrimitiveKind(fieldType.Kind()) && !isEnum &&
!isOptional {
nullableFlag = false
}
return nullableFlag
}
+func primitiveTypeIdMatchesKind(typeId TypeId, kind reflect.Kind) bool {
+ switch typeId {
+ case BOOL:
+ return kind == reflect.Bool
+ case INT8:
+ return kind == reflect.Int8
+ case INT16:
+ return kind == reflect.Int16
+ case INT32, VARINT32:
+ return kind == reflect.Int32 || kind == reflect.Int
+ case INT64, VARINT64, TAGGED_INT64:
+ return kind == reflect.Int64 || kind == reflect.Int
+ case UINT8:
+ return kind == reflect.Uint8
+ case UINT16:
+ return kind == reflect.Uint16
+ case UINT32, VAR_UINT32:
+ return kind == reflect.Uint32 || kind == reflect.Uint
+ case UINT64, VAR_UINT64, TAGGED_UINT64:
+ return kind == reflect.Uint64 || kind == reflect.Uint
+ case FLOAT32:
+ return kind == reflect.Float32
+ case FLOAT64:
+ return kind == reflect.Float64
+ case STRING:
+ return kind == reflect.String
+ default:
+ return false
+ }
+}
+
func applyNestedRefOverride(serializer Serializer, fieldType reflect.Type,
foryTag ForyTag) Serializer {
if !foryTag.NestedRefSet || !foryTag.NestedRefValid {
return serializer
@@ -697,6 +728,33 @@ func (s *structSerializer)
initFieldsFromTypeDef(typeResolver *TypeResolver) err
shouldRead = true
fieldType = localType // Use local type
for struct fields
}
+ } else if typeLookupFailed && (internalDefTypeId ==
UNION || internalDefTypeId == TYPED_UNION || internalDefTypeId == NAMED_UNION) {
+ // For union fields with failed type lookup
(named unions aren't in typeIDToTypeInfo),
+ // allow reading if the local type is a union.
+ if isUnionType(localType) {
+ shouldRead = true
+ fieldType = localType
+ }
+ } else if typeLookupFailed &&
isPrimitiveType(int16(internalDefTypeId)) {
+ baseLocal := localType
+ if optInfo, ok := getOptionalInfo(baseLocal);
ok {
+ baseLocal = optInfo.valueType
+ }
+ if baseLocal.Kind() == reflect.Ptr {
+ baseLocal = baseLocal.Elem()
+ }
+ if
primitiveTypeIdMatchesKind(internalDefTypeId, baseLocal.Kind()) {
+ shouldRead = true
+ fieldType = localType
+ }
+ } else if typeLookupFailed &&
isPrimitiveArrayType(int16(internalDefTypeId)) {
+ // Primitive arrays/slices use array type IDs
but may not be registered in typeIDToTypeInfo.
+ // Allow reading using the local slice/array
type when the type IDs match.
+ localTypeId := typeIdFromKind(localType)
+ if TypeId(localTypeId&0xFF) ==
internalDefTypeId {
+ shouldRead = true
+ fieldType = localType
+ }
} else if typeLookupFailed && defTypeId == LIST {
// For list fields with failed type lookup
(e.g., named struct element types),
// allow reading using the local slice type.
@@ -940,6 +998,10 @@ func (s *structSerializer)
initFieldsFromTypeDef(typeResolver *TypeResolver) err
for i, field := range fields {
if field.Meta.FieldIndex < 0 {
// Field exists in remote TypeDef but not locally
+ if DebugOutputEnabled && s.type_ != nil {
+ fmt.Printf("[Go][fory-debug] [%s]
typeDefDiffers: missing local field for remote def idx=%d name=%q tagID=%d
typeId=%d\n",
+ s.name, i, s.fieldDefs[i].name,
s.fieldDefs[i].tagID, s.fieldDefs[i].fieldType.TypeId())
+ }
s.typeDefDiffers = true
break
}
@@ -951,6 +1013,24 @@ func (s *structSerializer)
initFieldsFromTypeDef(typeResolver *TypeResolver) err
// Check if local Go field is nullable based on local
field definitions
localNullable :=
localNullableByIndex[field.Meta.FieldIndex]
if remoteNullable != localNullable {
+ if DebugOutputEnabled && s.type_ != nil {
+ fmt.Printf("[Go][fory-debug] [%s]
typeDefDiffers: nullable mismatch idx=%d name=%q tagID=%d remote=%v local=%v\n",
+ s.name, i, s.fieldDefs[i].name,
s.fieldDefs[i].tagID, remoteNullable, localNullable)
+ }
+ s.typeDefDiffers = true
+ break
+ }
+ remoteTypeId :=
TypeId(s.fieldDefs[i].fieldType.TypeId() & 0xFF)
+ localTypeId :=
typeResolver.getTypeIdByType(field.Meta.Type)
+ if localTypeId == 0 {
+ localTypeId = typeIdFromKind(field.Meta.Type)
+ }
+ localTypeId = TypeId(localTypeId & 0xFF)
+ if !typeIdEqualForDiff(remoteTypeId, localTypeId) {
+ if DebugOutputEnabled && s.type_ != nil {
+ fmt.Printf("[Go][fory-debug] [%s]
typeDefDiffers: type ID mismatch idx=%d name=%q tagID=%d remote=%d local=%d\n",
+ s.name, i, s.fieldDefs[i].name,
s.fieldDefs[i].tagID, remoteTypeId, localTypeId)
+ }
s.typeDefDiffers = true
break
}
@@ -964,6 +1044,24 @@ func (s *structSerializer)
initFieldsFromTypeDef(typeResolver *TypeResolver) err
return nil
}
+func typeIdEqualForDiff(remoteTypeId TypeId, localTypeId TypeId) bool {
+ if remoteTypeId == localTypeId {
+ return true
+ }
+ if remoteTypeId == UNION && (localTypeId == TYPED_UNION || localTypeId
== NAMED_UNION) {
+ return true
+ }
+ if localTypeId == UNION && (remoteTypeId == TYPED_UNION || remoteTypeId
== NAMED_UNION) {
+ return true
+ }
+ // Treat byte array encodings as compatible for diffing.
+ if (remoteTypeId == INT8_ARRAY || remoteTypeId == UINT8_ARRAY ||
remoteTypeId == BINARY) &&
+ (localTypeId == INT8_ARRAY || localTypeId == UINT8_ARRAY ||
localTypeId == BINARY) {
+ return true
+ }
+ return false
+}
+
func (s *structSerializer) computeHash() int32 {
// Build FieldFingerprintInfo for each field
fields := make([]FieldFingerprintInfo, 0, len(s.fields))
@@ -985,14 +1083,14 @@ func (s *structSerializer) computeHash() int32 {
}
}
// Unions use UNION type ID in fingerprints, regardless
of typed/named variants.
- internalId := TypeId(typeId & 0xFF)
+ internalId := typeId & 0xFF
if internalId == TYPED_UNION || internalId ==
NAMED_UNION || internalId == UNION {
typeId = UNION
}
// For user-defined types (struct, ext types), use
UNKNOWN in fingerprint
// This matches Java's behavior where user-defined
types return UNKNOWN
// to ensure consistent fingerprint computation across
languages
- if isUserDefinedType(int16(typeId)) {
+ if isUserDefinedType(typeId) {
typeId = UNKNOWN
}
fieldTypeForHash := field.Meta.Type
@@ -1379,6 +1477,10 @@ func (s *structSerializer) WriteData(ctx *WriteContext,
value reflect.Value) {
// writeRemainingField writes a non-primitive field (string, slice, map,
struct, enum)
func (s *structSerializer) writeRemainingField(ctx *WriteContext, ptr
unsafe.Pointer, field *FieldInfo, value reflect.Value) {
buf := ctx.Buffer()
+ if DebugOutputEnabled {
+ fmt.Printf("[fory-debug] write field %s: fieldInfo=%v typeId=%v
serializer=%T, buffer writerIndex=%d\n",
+ field.Meta.Name, field.Meta.FieldDef,
field.Meta.TypeId, field.Serializer, buf.writerIndex)
+ }
if field.Kind == FieldKindOptional {
if ptr != nil {
if writeOptionFast(ctx, field, unsafe.Add(ptr,
field.Offset)) {
@@ -2542,6 +2644,10 @@ func (s *structSerializer) ReadData(ctx *ReadContext,
value reflect.Value) {
// readRemainingField reads a non-primitive field (string, slice, map, struct,
enum)
func (s *structSerializer) readRemainingField(ctx *ReadContext, ptr
unsafe.Pointer, field *FieldInfo, value reflect.Value) {
buf := ctx.Buffer()
+ if DebugOutputEnabled {
+ fmt.Printf("[fory-debug] read remaining field %s: fieldInfo=%v
typeId=%v, buffer readerIndex=%d\n",
+ field.Meta.Name, field.Meta.FieldDef,
field.Meta.TypeId, buf.readerIndex)
+ }
ctxErr := ctx.Err()
if field.Kind == FieldKindOptional {
if ptr != nil {
@@ -2873,11 +2979,6 @@ func (s *structSerializer) readRemainingField(ctx
*ReadContext, ptr unsafe.Point
return
}
}
-
- if DebugOutputEnabled {
- fmt.Printf("[fory-debug] read normal field %s: fieldInfo=%v
typeId=%v serializer=%T, buffer readerIndex=%d\n",
- field.Meta.Name, field.Meta.FieldDef,
field.Meta.TypeId, field.Serializer, buf.readerIndex)
- }
// Slow path for RefModeTracking cases that break from the switch above
fieldValue := value.Field(field.Meta.FieldIndex)
if field.Serializer != nil {
@@ -3202,15 +3303,6 @@ func (s *structSerializer) readFieldsInOrder(ctx
*ReadContext, value reflect.Val
s.skipField(ctx, field)
return
}
- if field.Kind == FieldKindOptional {
- fieldValue := value.Field(field.Meta.FieldIndex)
- if field.Serializer != nil {
- field.Serializer.Read(ctx, field.RefMode,
field.Meta.WriteType, field.Meta.HasGenerics, fieldValue)
- } else {
- ctx.ReadValue(fieldValue, RefModeTracking, true)
- }
- return
- }
// Fast path for fixed-size primitive types (no ref flag from
remote schema)
if isFixedSizePrimitive(field.DispatchId) {
@@ -3371,7 +3463,7 @@ func (s *structSerializer) readFieldsInOrder(ctx
*ReadContext, value reflect.Val
// skipField skips a field that doesn't exist or is incompatible
// Uses context error state for deferred error checking.
func (s *structSerializer) skipField(ctx *ReadContext, field *FieldInfo) {
- if field.Meta.FieldDef.name != "" {
+ if field.Meta.FieldDef.name != "" || field.Meta.FieldDef.tagID >= 0 {
if DebugOutputEnabled {
fmt.Printf("[Go][fory-debug] skipField name=%s
typeId=%d fieldType=%s\n",
field.Meta.FieldDef.name,
@@ -3427,11 +3519,6 @@ func writeEnumField(ctx *WriteContext, field *FieldInfo,
fieldValue reflect.Valu
targetValue = fieldValue.Elem()
}
}
-
- if DebugOutputEnabled {
- fmt.Printf("[fory-debug] write field %s: fieldInfo=%v typeId=%v
serializer=%T, buffer writerIndex=%d\n",
- field.Meta.Name, field.Meta.FieldDef,
field.Meta.TypeId, field.Serializer, buf.writerIndex)
- }
// For pointer enum fields, the serializer is ptrToValueSerializer
wrapping enumSerializer.
// We need to call the inner enumSerializer directly with the
dereferenced value.
if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok {
diff --git a/go/fory/type_def.go b/go/fory/type_def.go
index c485b0259..6fa0a6509 100644
--- a/go/fory/type_def.go
+++ b/go/fory/type_def.go
@@ -18,10 +18,12 @@
package fory
import (
+ "bytes"
+ "compress/zlib"
"fmt"
- "strings"
-
+ "io"
"reflect"
+ "strings"
"github.com/apache/fory/go/fory/meta"
)
@@ -107,43 +109,43 @@ func (td *TypeDef) ComputeDiff(localDef *TypeDef) string {
// Build field maps for comparison
remoteFields := make(map[string]FieldDef)
for _, fd := range td.fieldDefs {
- remoteFields[fd.name] = fd
+ remoteFields[fieldKey(fd)] = fd
}
localFields := make(map[string]FieldDef)
for _, fd := range localDef.fieldDefs {
- localFields[fd.name] = fd
+ localFields[fieldKey(fd)] = fd
}
// Find fields only in remote
- for fieldName, fd := range remoteFields {
- if _, exists := localFields[fieldName]; !exists {
+ for fieldKey, fd := range remoteFields {
+ if _, exists := localFields[fieldKey]; !exists {
diff.WriteString(fmt.Sprintf(" field '%s': only in
remote, type=%s, nullable=%v\n",
- fieldName, fieldTypeToString(fd.fieldType),
fd.nullable))
+ fieldLabel(fd),
fieldTypeToString(fd.fieldType), fd.nullable))
}
}
// Find fields only in local
- for fieldName, fd := range localFields {
- if _, exists := remoteFields[fieldName]; !exists {
+ for fieldKey, fd := range localFields {
+ if _, exists := remoteFields[fieldKey]; !exists {
diff.WriteString(fmt.Sprintf(" field '%s': only in
local, type=%s, nullable=%v\n",
- fieldName, fieldTypeToString(fd.fieldType),
fd.nullable))
+ fieldLabel(fd),
fieldTypeToString(fd.fieldType), fd.nullable))
}
}
// Compare common fields
- for fieldName, remoteField := range remoteFields {
- if localField, exists := localFields[fieldName]; exists {
+ for fieldKey, remoteField := range remoteFields {
+ if localField, exists := localFields[fieldKey]; exists {
// Compare field types
remoteTypeStr :=
fieldTypeToString(remoteField.fieldType)
localTypeStr := fieldTypeToString(localField.fieldType)
if remoteTypeStr != localTypeStr {
diff.WriteString(fmt.Sprintf(" field '%s':
type mismatch, remote=%s, local=%s\n",
- fieldName, remoteTypeStr, localTypeStr))
+ fieldLabel(remoteField), remoteTypeStr,
localTypeStr))
}
// Compare nullable
if remoteField.nullable != localField.nullable {
diff.WriteString(fmt.Sprintf(" field '%s':
nullable mismatch, remote=%v, local=%v\n",
- fieldName, remoteField.nullable,
localField.nullable))
+ fieldLabel(remoteField),
remoteField.nullable, localField.nullable))
}
}
}
@@ -152,7 +154,7 @@ func (td *TypeDef) ComputeDiff(localDef *TypeDef) string {
if len(td.fieldDefs) == len(localDef.fieldDefs) {
orderDifferent := false
for i := range td.fieldDefs {
- if td.fieldDefs[i].name != localDef.fieldDefs[i].name {
+ if fieldKey(td.fieldDefs[i]) !=
fieldKey(localDef.fieldDefs[i]) {
orderDifferent = true
break
}
@@ -164,7 +166,7 @@ func (td *TypeDef) ComputeDiff(localDef *TypeDef) string {
if i > 0 {
diff.WriteString(", ")
}
- diff.WriteString(fd.name)
+ diff.WriteString(fieldLabel(fd))
}
diff.WriteString("]\n")
diff.WriteString(" local: [")
@@ -172,7 +174,7 @@ func (td *TypeDef) ComputeDiff(localDef *TypeDef) string {
if i > 0 {
diff.WriteString(", ")
}
- diff.WriteString(fd.name)
+ diff.WriteString(fieldLabel(fd))
}
diff.WriteString("]\n")
}
@@ -181,6 +183,23 @@ func (td *TypeDef) ComputeDiff(localDef *TypeDef) string {
return diff.String()
}
+func fieldKey(fd FieldDef) string {
+ if fd.tagID >= 0 {
+ return fmt.Sprintf("id:%d", fd.tagID)
+ }
+ return "name:" + fd.name
+}
+
+func fieldLabel(fd FieldDef) string {
+ if fd.tagID >= 0 {
+ if fd.name != "" {
+ return fmt.Sprintf("%s(id=%d)", fd.name, fd.tagID)
+ }
+ return fmt.Sprintf("id=%d", fd.tagID)
+ }
+ return fd.name
+}
+
func (td *TypeDef) writeTypeDef(buffer *ByteBuffer, err *Error) {
buffer.WriteBinary(td.encoded)
}
@@ -931,7 +950,7 @@ type SimpleFieldType struct {
func NewSimpleFieldType(typeId TypeId) *SimpleFieldType {
return &SimpleFieldType{
BaseFieldType: BaseFieldType{
- typeId: typeId,
+ typeId: typeId & 0xff,
},
}
}
@@ -997,6 +1016,9 @@ func buildFieldType(fory *Fory, fieldValue reflect.Value)
(FieldType, error) {
fieldType = info.valueType
fieldValue = reflect.Zero(fieldType)
}
+ if isUnionType(fieldType) {
+ return NewSimpleFieldType(UNION), nil
+ }
// Handle Interface type, we can't determine the actual type here, so
leave it as dynamic type
if fieldType.Kind() == reflect.Interface {
return NewDynamicFieldType(UNKNOWN), nil
@@ -1016,12 +1038,20 @@ func buildFieldType(fory *Fory, fieldValue
reflect.Value) (FieldType, error) {
return NewSimpleFieldType(BOOL_ARRAY), nil
case reflect.Int8:
return NewSimpleFieldType(INT8_ARRAY), nil
+ case reflect.Uint8:
+ return NewSimpleFieldType(BINARY), nil
case reflect.Int16:
return NewSimpleFieldType(INT16_ARRAY), nil
+ case reflect.Uint16:
+ return NewSimpleFieldType(UINT16_ARRAY), nil
case reflect.Int32:
return NewSimpleFieldType(INT32_ARRAY), nil
+ case reflect.Uint32:
+ return NewSimpleFieldType(UINT32_ARRAY), nil
case reflect.Int64, reflect.Int:
return NewSimpleFieldType(INT64_ARRAY), nil
+ case reflect.Uint64, reflect.Uint:
+ return NewSimpleFieldType(UINT64_ARRAY), nil
case reflect.Float32:
return NewSimpleFieldType(FLOAT32_ARRAY), nil
case reflect.Float64:
@@ -1079,6 +1109,11 @@ func buildFieldType(fory *Fory, fieldValue
reflect.Value) (FieldType, error) {
typeId = TypeId(typeInfo.TypeID)
if isUserDefinedType(typeId) {
+ internalTypeId := TypeId(typeId & 0xFF)
+ switch internalTypeId {
+ case UNION, TYPED_UNION, NAMED_UNION, ENUM, NAMED_ENUM:
+ return NewSimpleFieldType(typeId), nil
+ }
return NewDynamicFieldType(typeId), nil
}
@@ -1391,20 +1426,28 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer,
header int64) (*TypeDef, erro
globalHeader := header
hasFieldsMeta := (globalHeader & HAS_FIELDS_META_FLAG) != 0
isCompressed := (globalHeader & COMPRESS_META_FLAG) != 0
- metaSize := int(globalHeader & META_SIZE_MASK)
- if metaSize == META_SIZE_MASK {
- metaSize += int(buffer.ReadVarUint32(&bufErr))
+ metaSizeBits := int(globalHeader & META_SIZE_MASK)
+ metaSize := metaSizeBits
+ extraMetaSize := 0
+ if metaSizeBits == META_SIZE_MASK {
+ extraMetaSize = int(buffer.ReadVarUint32(&bufErr))
+ metaSize += extraMetaSize
}
// Store the encoded bytes for the TypeDef (including meta header and
metadata)
- // todo: handle compression if is_compressed is true
- if isCompressed {
- }
- encoded := buffer.ReadBinary(metaSize, &bufErr)
+ encodedMeta := buffer.ReadBinary(metaSize, &bufErr)
if bufErr.HasError() {
return nil, bufErr.TakeError()
}
- metaBuffer := NewByteBuffer(encoded)
+ decodedMeta := encodedMeta
+ if isCompressed {
+ decodedMetaBytes, err := decompressMeta(encodedMeta)
+ if err != nil {
+ return nil, err
+ }
+ decodedMeta = decodedMetaBytes
+ }
+ metaBuffer := NewByteBuffer(decodedMeta)
var metaErr Error
// ReadData 1-byte meta header
@@ -1546,6 +1589,8 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header
int64) (*TypeDef, erro
}
}
+ encoded := buildTypeDefEncoded(globalHeader, metaSizeBits,
extraMetaSize, encodedMeta)
+
// Create TypeDef
typeDef := NewTypeDef(typeId, nsBytes, nameBytes, registeredByName,
isCompressed, fieldInfos)
typeDef.encoded = encoded
@@ -1570,6 +1615,30 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer,
header int64) (*TypeDef, erro
return typeDef, nil
}
+func buildTypeDefEncoded(header int64, metaSizeBits, extraMetaSize int,
metaBytes []byte) []byte {
+ capacity := 8 + len(metaBytes) + 5
+ buffer := NewByteBuffer(make([]byte, 0, capacity))
+ buffer.WriteInt64(header)
+ if metaSizeBits == META_SIZE_MASK {
+ buffer.WriteVarUint32(uint32(extraMetaSize))
+ }
+ buffer.WriteBinary(metaBytes)
+ return buffer.Bytes()
+}
+
+func decompressMeta(encoded []byte) ([]byte, error) {
+ reader, err := zlib.NewReader(bytes.NewReader(encoded))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create meta decompressor:
%w", err)
+ }
+ defer reader.Close()
+ decoded, err := io.ReadAll(reader)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decompress meta: %w", err)
+ }
+ return decoded, nil
+}
+
/*
readFieldDef reads a single field's definition from the buffer
field def layout as following:
diff --git a/go/fory/types.go b/go/fory/types.go
index 6555222ea..df9112451 100644
--- a/go/fory/types.go
+++ b/go/fory/types.go
@@ -275,6 +275,7 @@ func isUserDefinedType(typeID int16) bool {
id == NAMED_EXT ||
id == ENUM ||
id == NAMED_ENUM ||
+ id == UNION ||
id == TYPED_UNION ||
id == NAMED_UNION
}
diff --git a/integration_tests/idl_tests/cpp/main.cc
b/integration_tests/idl_tests/cpp/main.cc
index dc43c3027..d8ef4ead5 100644
--- a/integration_tests/idl_tests/cpp/main.cc
+++ b/integration_tests/idl_tests/cpp/main.cc
@@ -18,6 +18,7 @@
*/
#include <any>
+#include <cctype>
#include <chrono>
#include <cstdlib>
#include <fstream>
@@ -158,10 +159,11 @@ fory::Result<void, fory::Error> ValidateGraph(const
graph::Graph &graph_value) {
using StringMap = std::map<std::string, std::string>;
-fory::Result<void, fory::Error> RunRoundTrip() {
+fory::Result<void, fory::Error> RunRoundTrip(bool compatible) {
auto fory = fory::serialization::Fory::builder()
.xlang(true)
- .check_struct_version(true)
+ .compatible(compatible)
+ .check_struct_version(!compatible)
.track_ref(false)
.build();
@@ -507,7 +509,8 @@ fory::Result<void, fory::Error> RunRoundTrip() {
auto ref_fory = fory::serialization::Fory::builder()
.xlang(true)
- .check_struct_version(true)
+ .compatible(compatible)
+ .check_struct_version(!compatible)
.track_ref(true)
.build();
tree::register_types(ref_fory);
@@ -548,10 +551,52 @@ fory::Result<void, fory::Error> RunRoundTrip() {
return fory::Result<void, fory::Error>();
}
+bool ParseCompatibleMode(const char *value, bool *compatible) {
+ if (value == nullptr || value[0] == '\0') {
+ return false;
+ }
+ std::string normalized(value);
+ for (char &ch : normalized) {
+ ch = static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
+ }
+ if (normalized == "1" || normalized == "true" || normalized == "yes") {
+ *compatible = true;
+ return true;
+ }
+ if (normalized == "0" || normalized == "false" || normalized == "no") {
+ *compatible = false;
+ return true;
+ }
+ return false;
+}
+
} // namespace
int main() {
- auto result = RunRoundTrip();
+ const char *compat_env = std::getenv("IDL_COMPATIBLE");
+ bool compatible = false;
+ if (compat_env != nullptr && compat_env[0] != '\0') {
+ if (!ParseCompatibleMode(compat_env, &compatible)) {
+ std::cerr << "Unsupported IDL_COMPATIBLE value: " << compat_env
+ << std::endl;
+ return 1;
+ }
+ auto result = RunRoundTrip(compatible);
+ if (!result.ok()) {
+ std::cerr << "IDL roundtrip failed: " << result.error().message()
+ << std::endl;
+ return 1;
+ }
+ return 0;
+ }
+
+ auto result = RunRoundTrip(false);
+ if (!result.ok()) {
+ std::cerr << "IDL roundtrip failed: " << result.error().message()
+ << std::endl;
+ return 1;
+ }
+ result = RunRoundTrip(true);
if (!result.ok()) {
std::cerr << "IDL roundtrip failed: " << result.error().message()
<< std::endl;
diff --git a/integration_tests/idl_tests/go/idl_roundtrip_test.go
b/integration_tests/idl_tests/go/idl_roundtrip_test.go
index b5fb182b7..abe5d31fe 100644
--- a/integration_tests/idl_tests/go/idl_roundtrip_test.go
+++ b/integration_tests/idl_tests/go/idl_roundtrip_test.go
@@ -71,8 +71,20 @@ func buildAddressBook() addressbook.AddressBook {
}
}
-func TestAddressBookRoundTrip(t *testing.T) {
- f := fory.NewFory(fory.WithXlang(true), fory.WithRefTracking(false))
+func TestAddressBookRoundTripCompatible(t *testing.T) {
+ runAddressBookRoundTrip(t, true)
+}
+
+func TestAddressBookRoundTripSchemaConsistent(t *testing.T) {
+ runAddressBookRoundTrip(t, false)
+}
+
+func runAddressBookRoundTrip(t *testing.T, compatible bool) {
+ f := fory.NewFory(
+ fory.WithXlang(true),
+ fory.WithRefTracking(false),
+ fory.WithCompatible(compatible),
+ )
if err := addressbook.RegisterTypes(f); err != nil {
t.Fatalf("register types: %v", err)
}
@@ -115,7 +127,11 @@ func TestAddressBookRoundTrip(t *testing.T) {
anyHolder := buildAnyHolder()
runLocalAnyRoundTrip(t, f, anyHolder)
- refFory := fory.NewFory(fory.WithXlang(true),
fory.WithRefTracking(true))
+ refFory := fory.NewFory(
+ fory.WithXlang(true),
+ fory.WithRefTracking(true),
+ fory.WithCompatible(compatible),
+ )
if err := treepkg.RegisterTypes(refFory); err != nil {
t.Fatalf("register tree types: %v", err)
}
diff --git
a/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java
b/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java
index e97e6ca1f..ff64b0977 100644
---
a/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java
+++
b/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java
@@ -71,6 +71,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.fory.Fory;
+import org.apache.fory.config.CompatibleMode;
import org.apache.fory.config.Language;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -78,8 +79,17 @@ import org.testng.annotations.Test;
public class IdlRoundTripTest {
@Test
- public void testAddressBookRoundTrip() throws Exception {
- Fory fory = Fory.builder().withLanguage(Language.XLANG).build();
+ public void testAddressBookRoundTripCompatible() throws Exception {
+ runAddressBookRoundTrip(true);
+ }
+
+ @Test
+ public void testAddressBookRoundTripSchemaConsistent() throws Exception {
+ runAddressBookRoundTrip(false);
+ }
+
+ private void runAddressBookRoundTrip(boolean compatible) throws Exception {
+ Fory fory = buildFory(compatible);
AddressbookForyRegistration.register(fory);
AddressBook book = buildAddressBook();
@@ -96,7 +106,7 @@ public class IdlRoundTripTest {
Map<String, String> env = new HashMap<>();
env.put("DATA_FILE", dataFile.toAbsolutePath().toString());
- PeerCommand command = buildPeerCommand(peer, env);
+ PeerCommand command = buildPeerCommand(peer, env, compatible);
runPeer(command, peer);
byte[] peerBytes = Files.readAllBytes(dataFile);
@@ -156,8 +166,17 @@ public class IdlRoundTripTest {
}
@Test
- public void testPrimitiveTypesRoundTrip() throws Exception {
- Fory fory = Fory.builder().withLanguage(Language.XLANG).build();
+ public void testPrimitiveTypesRoundTripCompatible() throws Exception {
+ runPrimitiveTypesRoundTrip(true);
+ }
+
+ @Test
+ public void testPrimitiveTypesRoundTripSchemaConsistent() throws Exception {
+ runPrimitiveTypesRoundTrip(false);
+ }
+
+ private void runPrimitiveTypesRoundTrip(boolean compatible) throws Exception
{
+ Fory fory = buildFory(compatible);
AddressbookForyRegistration.register(fory);
ComplexPbForyRegistration.register(fory);
@@ -175,7 +194,7 @@ public class IdlRoundTripTest {
Map<String, String> env = new HashMap<>();
env.put("DATA_FILE_PRIMITIVES", dataFile.toAbsolutePath().toString());
- PeerCommand command = buildPeerCommand(peer, env);
+ PeerCommand command = buildPeerCommand(peer, env, compatible);
runPeer(command, peer);
byte[] peerBytes = Files.readAllBytes(dataFile);
@@ -186,8 +205,17 @@ public class IdlRoundTripTest {
}
@Test
- public void testOptionalTypesRoundTrip() throws Exception {
- Fory fory = Fory.builder().withLanguage(Language.XLANG).build();
+ public void testOptionalTypesRoundTripCompatible() throws Exception {
+ runOptionalTypesRoundTrip(true);
+ }
+
+ @Test
+ public void testOptionalTypesRoundTripSchemaConsistent() throws Exception {
+ runOptionalTypesRoundTrip(false);
+ }
+
+ private void runOptionalTypesRoundTrip(boolean compatible) throws Exception {
+ Fory fory = buildFory(compatible);
OptionalTypesForyRegistration.register(fory);
OptionalHolder holder = buildOptionalHolder();
@@ -204,7 +232,7 @@ public class IdlRoundTripTest {
Map<String, String> env = new HashMap<>();
env.put("DATA_FILE_OPTIONAL_TYPES",
dataFile.toAbsolutePath().toString());
- PeerCommand command = buildPeerCommand(peer, env);
+ PeerCommand command = buildPeerCommand(peer, env, compatible);
runPeer(command, peer);
byte[] peerBytes = Files.readAllBytes(dataFile);
@@ -215,8 +243,17 @@ public class IdlRoundTripTest {
}
@Test
- public void testAnyRoundTrip() {
- Fory fory = Fory.builder().withLanguage(Language.XLANG).build();
+ public void testAnyRoundTripCompatible() {
+ runAnyRoundTrip(true);
+ }
+
+ @Test
+ public void testAnyRoundTripSchemaConsistent() {
+ runAnyRoundTrip(false);
+ }
+
+ private void runAnyRoundTrip(boolean compatible) {
+ Fory fory = buildFory(compatible);
AnyExampleForyRegistration.register(fory);
AnyHolder holder = buildAnyHolder();
@@ -228,8 +265,17 @@ public class IdlRoundTripTest {
}
@Test
- public void testTreeRoundTrip() throws Exception {
- Fory fory =
Fory.builder().withLanguage(Language.XLANG).withRefTracking(true).build();
+ public void testTreeRoundTripCompatible() throws Exception {
+ runTreeRoundTrip(true);
+ }
+
+ @Test
+ public void testTreeRoundTripSchemaConsistent() throws Exception {
+ runTreeRoundTrip(false);
+ }
+
+ private void runTreeRoundTrip(boolean compatible) throws Exception {
+ Fory fory = buildRefFory(compatible);
TreeForyRegistration.register(fory);
TreeNode tree = buildTree();
@@ -247,7 +293,7 @@ public class IdlRoundTripTest {
Map<String, String> env = new HashMap<>();
env.put("DATA_FILE_TREE", dataFile.toAbsolutePath().toString());
- PeerCommand command = buildPeerCommand(peer, env);
+ PeerCommand command = buildPeerCommand(peer, env, compatible);
runPeer(command, peer);
byte[] peerBytes = Files.readAllBytes(dataFile);
@@ -258,8 +304,17 @@ public class IdlRoundTripTest {
}
@Test
- public void testGraphRoundTrip() throws Exception {
- Fory fory =
Fory.builder().withLanguage(Language.XLANG).withRefTracking(true).build();
+ public void testGraphRoundTripCompatible() throws Exception {
+ runGraphRoundTrip(true);
+ }
+
+ @Test
+ public void testGraphRoundTripSchemaConsistent() throws Exception {
+ runGraphRoundTrip(false);
+ }
+
+ private void runGraphRoundTrip(boolean compatible) throws Exception {
+ Fory fory = buildRefFory(compatible);
GraphForyRegistration.register(fory);
Graph graph = buildGraph();
@@ -277,7 +332,7 @@ public class IdlRoundTripTest {
Map<String, String> env = new HashMap<>();
env.put("DATA_FILE_GRAPH", dataFile.toAbsolutePath().toString());
- PeerCommand command = buildPeerCommand(peer, env);
+ PeerCommand command = buildPeerCommand(peer, env, compatible);
runPeer(command, peer);
byte[] peerBytes = Files.readAllBytes(dataFile);
@@ -288,8 +343,17 @@ public class IdlRoundTripTest {
}
@Test
- public void testFlatbuffersRoundTrip() throws Exception {
- Fory fory = Fory.builder().withLanguage(Language.XLANG).build();
+ public void testFlatbuffersRoundTripCompatible() throws Exception {
+ runFlatbuffersRoundTrip(true);
+ }
+
+ @Test
+ public void testFlatbuffersRoundTripSchemaConsistent() throws Exception {
+ runFlatbuffersRoundTrip(false);
+ }
+
+ private void runFlatbuffersRoundTrip(boolean compatible) throws Exception {
+ Fory fory = buildFory(compatible);
MonsterForyRegistration.register(fory);
ComplexFbsForyRegistration.register(fory);
@@ -319,7 +383,7 @@ public class IdlRoundTripTest {
Map<String, String> env = new HashMap<>();
env.put("DATA_FILE_FLATBUFFERS_MONSTER",
monsterFile.toAbsolutePath().toString());
env.put("DATA_FILE_FLATBUFFERS_TEST2",
containerFile.toAbsolutePath().toString());
- PeerCommand command = buildPeerCommand(peer, env);
+ PeerCommand command = buildPeerCommand(peer, env, compatible);
runPeer(command, peer);
byte[] peerMonsterBytes = Files.readAllBytes(monsterFile);
@@ -334,6 +398,23 @@ public class IdlRoundTripTest {
}
}
+ private Fory buildFory(boolean compatible) {
+ return Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCompatibleMode(
+ compatible ? CompatibleMode.COMPATIBLE :
CompatibleMode.SCHEMA_CONSISTENT)
+ .build();
+ }
+
+ private Fory buildRefFory(boolean compatible) {
+ return Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCompatibleMode(
+ compatible ? CompatibleMode.COMPATIBLE :
CompatibleMode.SCHEMA_CONSISTENT)
+ .withRefTracking(true)
+ .build();
+ }
+
private List<String> resolvePeers() {
String peerEnv = System.getenv("IDL_PEER_LANG");
if (peerEnv == null || peerEnv.trim().isEmpty()) {
@@ -350,13 +431,15 @@ public class IdlRoundTripTest {
return peers;
}
- private PeerCommand buildPeerCommand(String peer, Map<String, String>
environment) {
+ private PeerCommand buildPeerCommand(
+ String peer, Map<String, String> environment, boolean compatible) {
Path repoRoot = repoRoot();
Path idlRoot = repoRoot.resolve("integration_tests").resolve("idl_tests");
Path workDir = idlRoot;
List<String> command;
PeerCommand peerCommand = new PeerCommand();
peerCommand.environment.putAll(environment);
+ peerCommand.environment.put("IDL_COMPATIBLE",
Boolean.toString(compatible));
switch (peer) {
case "python":
@@ -374,11 +457,19 @@ public class IdlRoundTripTest {
break;
case "go":
workDir = idlRoot.resolve("go");
- command = Arrays.asList("go", "test", "-run",
"TestAddressBookRoundTrip", "-v");
+ String goTest =
+ compatible
+ ? "TestAddressBookRoundTripCompatible"
+ : "TestAddressBookRoundTripSchemaConsistent";
+ command = Arrays.asList("go", "test", "-run", goTest, "-v");
break;
case "rust":
workDir = idlRoot.resolve("rust");
- command = Arrays.asList("cargo", "test", "--test", "idl_roundtrip");
+ String rustTest =
+ compatible
+ ? "test_address_book_roundtrip_compatible"
+ : "test_address_book_roundtrip_schema_consistent";
+ command = Arrays.asList("cargo", "test", "--test", "idl_roundtrip",
rustTest);
break;
case "cpp":
command = Collections.singletonList("./cpp/run.sh");
@@ -394,6 +485,7 @@ public class IdlRoundTripTest {
private void runPeer(PeerCommand command, String peer) throws IOException,
InterruptedException {
ProcessBuilder builder = new ProcessBuilder(command.command);
+ builder.inheritIO();
builder.directory(command.workDir.toFile());
builder.environment().putAll(command.environment);
diff --git a/integration_tests/idl_tests/python/src/idl_tests/roundtrip.py
b/integration_tests/idl_tests/python/src/idl_tests/roundtrip.py
index e0085e652..7cc573880 100644
--- a/integration_tests/idl_tests/python/src/idl_tests/roundtrip.py
+++ b/integration_tests/idl_tests/python/src/idl_tests/roundtrip.py
@@ -500,8 +500,8 @@ def file_roundtrip_graph(fory: pyfory.Fory, graph_value:
"graph.Graph") -> None:
Path(data_file).write_bytes(fory.serialize(decoded))
-def main() -> int:
- fory = pyfory.Fory(xlang=True)
+def run_roundtrip(compatible: bool) -> None:
+ fory = pyfory.Fory(xlang=True, compatible=compatible)
complex_pb.register_complex_pb_types(fory)
addressbook.register_addressbook_types(fory)
monster.register_monster_types(fory)
@@ -534,7 +534,7 @@ def main() -> int:
any_holder = build_any_holder()
local_roundtrip_any(fory, any_holder)
- ref_fory = pyfory.Fory(xlang=True, ref=True)
+ ref_fory = pyfory.Fory(xlang=True, ref=True, compatible=compatible)
tree.register_tree_types(ref_fory)
graph.register_graph_types(ref_fory)
tree_root = build_tree()
@@ -543,6 +543,23 @@ def main() -> int:
graph_value = build_graph()
local_roundtrip_graph(ref_fory, graph_value)
file_roundtrip_graph(ref_fory, graph_value)
+
+
+def resolve_compatible_modes() -> list[bool]:
+ value = os.environ.get("IDL_COMPATIBLE")
+ if value is None or value.strip() == "":
+ return [False, True]
+ normalized = value.strip().lower()
+ if normalized in ("1", "true", "yes"):
+ return [True]
+ if normalized in ("0", "false", "no"):
+ return [False]
+ raise ValueError(f"Unsupported IDL_COMPATIBLE value: {value}")
+
+
+def main() -> int:
+ for compatible in resolve_compatible_modes():
+ run_roundtrip(compatible)
return 0
diff --git a/integration_tests/idl_tests/rust/tests/idl_roundtrip.rs
b/integration_tests/idl_tests/rust/tests/idl_roundtrip.rs
index 069d716af..ec95b5e87 100644
--- a/integration_tests/idl_tests/rust/tests/idl_roundtrip.rs
+++ b/integration_tests/idl_tests/rust/tests/idl_roundtrip.rs
@@ -436,8 +436,17 @@ fn assert_graph(value: &graph::Graph) {
}
#[test]
-fn test_address_book_roundtrip() {
- let mut fory = Fory::default().xlang(true);
+fn test_address_book_roundtrip_compatible() {
+ run_address_book_roundtrip(true);
+}
+
+#[test]
+fn test_address_book_roundtrip_schema_consistent() {
+ run_address_book_roundtrip(false);
+}
+
+fn run_address_book_roundtrip(compatible: bool) {
+ let mut fory = Fory::default().xlang(true).compatible(compatible);
complex_pb::register_types(&mut fory).expect("register complex pb types");
addressbook::register_types(&mut fory).expect("register types");
monster::register_types(&mut fory).expect("register monster types");
@@ -543,7 +552,10 @@ fn test_address_book_roundtrip() {
let result: Result<AnyHolder, _> = fory.deserialize(&bytes);
assert!(result.is_err());
- let mut ref_fory = Fory::default().xlang(true).track_ref(true);
+ let mut ref_fory = Fory::default()
+ .xlang(true)
+ .compatible(compatible)
+ .track_ref(true);
tree::register_types(&mut ref_fory).expect("register tree types");
graph::register_types(&mut ref_fory).expect("register graph types");
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
index 75e901e28..821d19c13 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
@@ -240,43 +240,45 @@ public class ClassDef implements Serializable {
// Build field maps for comparison
Map<String, FieldInfo> remoteFields = new HashMap<>();
for (FieldInfo fi : this.fieldsInfo) {
- remoteFields.put(fi.getFieldName(), fi);
+ remoteFields.put(fieldKey(fi), fi);
}
Map<String, FieldInfo> localFields = new HashMap<>();
for (FieldInfo fi : localDef.fieldsInfo) {
- localFields.put(fi.getFieldName(), fi);
+ localFields.put(fieldKey(fi), fi);
}
// Find fields only in remote
- for (String fieldName : remoteFields.keySet()) {
- if (!localFields.containsKey(fieldName)) {
+ for (String fieldKey : remoteFields.keySet()) {
+ if (!localFields.containsKey(fieldKey)) {
+ FieldInfo remoteField = remoteFields.get(fieldKey);
diff.append(" field '")
- .append(fieldName)
+ .append(fieldLabel(remoteField))
.append("': only in remote, type=")
- .append(remoteFields.get(fieldName).getFieldType())
+ .append(remoteField.getFieldType())
.append("\n");
}
}
// Find fields only in local
- for (String fieldName : localFields.keySet()) {
- if (!remoteFields.containsKey(fieldName)) {
+ for (String fieldKey : localFields.keySet()) {
+ if (!remoteFields.containsKey(fieldKey)) {
+ FieldInfo localField = localFields.get(fieldKey);
diff.append(" field '")
- .append(fieldName)
+ .append(fieldLabel(localField))
.append("': only in local, type=")
- .append(localFields.get(fieldName).getFieldType())
+ .append(localField.getFieldType())
.append("\n");
}
}
// Compare common fields
- for (String fieldName : remoteFields.keySet()) {
- if (localFields.containsKey(fieldName)) {
- FieldInfo remoteField = remoteFields.get(fieldName);
- FieldInfo localField = localFields.get(fieldName);
+ for (String fieldKey : remoteFields.keySet()) {
+ if (localFields.containsKey(fieldKey)) {
+ FieldInfo remoteField = remoteFields.get(fieldKey);
+ FieldInfo localField = localFields.get(fieldKey);
if (!Objects.equals(remoteField.getFieldType(),
localField.getFieldType())) {
diff.append(" field '")
- .append(fieldName)
+ .append(fieldLabel(remoteField))
.append("': type mismatch, remote=")
.append(remoteField.getFieldType())
.append(", local=")
@@ -291,7 +293,7 @@ public class ClassDef implements Serializable {
boolean orderDifferent = false;
for (int i = 0; i < this.fieldsInfo.size(); i++) {
if (!Objects.equals(
- this.fieldsInfo.get(i).getFieldName(),
localDef.fieldsInfo.get(i).getFieldName())) {
+ fieldKey(this.fieldsInfo.get(i)),
fieldKey(localDef.fieldsInfo.get(i)))) {
orderDifferent = true;
break;
}
@@ -303,7 +305,7 @@ public class ClassDef implements Serializable {
if (i > 0) {
diff.append(", ");
}
- diff.append(this.fieldsInfo.get(i).getFieldName());
+ diff.append(fieldLabel(this.fieldsInfo.get(i)));
}
diff.append("]\n");
diff.append(" local: [");
@@ -311,7 +313,7 @@ public class ClassDef implements Serializable {
if (i > 0) {
diff.append(", ");
}
- diff.append(localDef.fieldsInfo.get(i).getFieldName());
+ diff.append(fieldLabel(localDef.fieldsInfo.get(i)));
}
diff.append("]\n");
}
@@ -320,6 +322,24 @@ public class ClassDef implements Serializable {
return diff.length() > 0 ? diff.toString() : null;
}
+ private static String fieldKey(FieldInfo fieldInfo) {
+ if (fieldInfo.hasFieldId()) {
+ return "id:" + fieldInfo.getFieldId();
+ }
+ return "name:" + fieldInfo.getFieldName();
+ }
+
+ private static String fieldLabel(FieldInfo fieldInfo) {
+ if (fieldInfo.hasFieldId()) {
+ String name = fieldInfo.getFieldName();
+ if (name == null || name.startsWith("$tag")) {
+ return "id=" + fieldInfo.getFieldId();
+ }
+ return name + "(id=" + fieldInfo.getFieldId() + ")";
+ }
+ return fieldInfo.getFieldName();
+ }
+
/** Write class definition to buffer. */
public void writeClassDef(MemoryBuffer buffer) {
buffer.writeBytes(encoded, 0, encoded.length);
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java
b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java
index a84f97d40..3f653b378 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java
@@ -478,6 +478,18 @@ public class FieldTypes {
}
return TypeRef.of(cls, new TypeExtMeta(typeId, nullable, trackingRef));
}
+ if (Types.isPrimitiveArray(internalTypeId)) {
+ if (declared != null) {
+ Class<?> declaredRaw = declared.getRawType();
+ if (declaredRaw.isArray()) {
+ return TypeRef.of(declaredRaw, new TypeExtMeta(typeId, nullable,
trackingRef));
+ }
+ }
+ cls = getPrimitiveArrayClass(internalTypeId);
+ if (cls != null) {
+ return TypeRef.of(cls, new TypeExtMeta(typeId, nullable,
trackingRef));
+ }
+ }
if (resolver instanceof XtypeResolver) {
ClassInfo xtypeInfo = ((XtypeResolver) resolver).getXtypeInfo(typeId);
Preconditions.checkNotNull(xtypeInfo);
@@ -543,6 +555,31 @@ public class FieldTypes {
}
}
+ private static Class<?> getPrimitiveArrayClass(int typeId) {
+ switch (typeId) {
+ case Types.BOOL_ARRAY:
+ return boolean[].class;
+ case Types.INT8_ARRAY:
+ case Types.UINT8_ARRAY:
+ return byte[].class;
+ case Types.INT16_ARRAY:
+ case Types.UINT16_ARRAY:
+ return short[].class;
+ case Types.INT32_ARRAY:
+ case Types.UINT32_ARRAY:
+ return int[].class;
+ case Types.INT64_ARRAY:
+ case Types.UINT64_ARRAY:
+ return long[].class;
+ case Types.FLOAT32_ARRAY:
+ return float[].class;
+ case Types.FLOAT64_ARRAY:
+ return double[].class;
+ default:
+ return null;
+ }
+ }
+
/**
* Class for collection field type, which store collection element type
information. Nested
* collection/map generics example:
diff --git
a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
index 3e7694361..4f8cd30bb 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
@@ -83,6 +83,7 @@ import org.apache.fory.serializer.DeferedLazySerializer;
import
org.apache.fory.serializer.DeferedLazySerializer.DeferredLazyObjectSerializer;
import org.apache.fory.serializer.EnumSerializer;
import org.apache.fory.serializer.NonexistentClass;
+import org.apache.fory.serializer.NonexistentClass.NonexistentEnum;
import org.apache.fory.serializer.NonexistentClass.NonexistentMetaShared;
import org.apache.fory.serializer.NonexistentClassSerializers;
import
org.apache.fory.serializer.NonexistentClassSerializers.NonexistentClassSerializer;
@@ -111,6 +112,7 @@ import org.apache.fory.type.GenericType;
import org.apache.fory.type.Generics;
import org.apache.fory.type.TypeUtils;
import org.apache.fory.type.Types;
+import org.apache.fory.type.union.Union;
import org.apache.fory.type.unsigned.Uint16;
import org.apache.fory.type.unsigned.Uint32;
import org.apache.fory.type.unsigned.Uint8;
@@ -520,6 +522,7 @@ public class XtypeResolver extends TypeResolver {
switch (typeId) {
case Types.NAMED_COMPATIBLE_STRUCT:
case Types.NAMED_ENUM:
+ case Types.NAMED_UNION:
case Types.NAMED_STRUCT:
case Types.NAMED_EXT:
return false;
@@ -538,6 +541,7 @@ public class XtypeResolver extends TypeResolver {
switch (typeId) {
case Types.NAMED_COMPATIBLE_STRUCT:
case Types.NAMED_ENUM:
+ case Types.NAMED_UNION:
case Types.NAMED_STRUCT:
case Types.NAMED_EXT:
return true;
@@ -563,6 +567,9 @@ public class XtypeResolver extends TypeResolver {
if (rawType.isEnum()) {
return true;
}
+ if (Union.class.isAssignableFrom(rawType)) {
+ return true;
+ }
if (rawType == NonexistentMetaShared.class) {
return true;
}
@@ -585,9 +592,12 @@ public class XtypeResolver extends TypeResolver {
if (clz.isArray()) {
return true;
}
- if (clz == NonexistentMetaShared.class) {
+ if (clz == NonexistentEnum.class) {
return true;
}
+ if (clz == NonexistentMetaShared.class) {
+ return false;
+ }
ClassInfo classInfo = getClassInfo(clz, false);
if (classInfo != null) {
Serializer<?> s = classInfo.serializer;
@@ -609,8 +619,8 @@ public class XtypeResolver extends TypeResolver {
public boolean isBuildIn(Descriptor descriptor) {
Class<?> rawType = descriptor.getRawType();
byte typeIdByte = getInternalTypeId(descriptor);
- if (rawType == NonexistentMetaShared.class) {
- return true;
+ if (NonexistentClass.class.isAssignableFrom(rawType)) {
+ return false;
}
return !Types.isUserDefinedType(typeIdByte) && typeIdByte != Types.UNKNOWN;
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java
index 0e28c8561..ea20bdb83 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java
@@ -212,6 +212,13 @@ public class FieldGroups {
}
}
+ public String getName() {
+ if (fieldAccessor != null) {
+ return fieldAccessor.getField().getName();
+ }
+ return qualifiedFieldName;
+ }
+
@Override
public String toString() {
String[] rsplit = StringUtils.rsplit(qualifiedFieldName, ".", 1);
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
index 6091d6931..d935bca91 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
@@ -155,6 +155,13 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
private void writeOtherFields(MemoryBuffer buffer, T value) {
for (SerializationFieldInfo fieldInfo : otherFields) {
+ if (Utils.DEBUG_OUTPUT_ENABLED) {
+ LOG.info(
+ "[Java] write field {} of type {}, writer index {}",
+ fieldInfo.descriptor.getName(),
+ fieldInfo.typeRef,
+ buffer.writerIndex());
+ }
FieldAccessor fieldAccessor = fieldInfo.fieldAccessor;
Object fieldValue = fieldAccessor.getObject(value);
binding.writeField(fieldInfo, buffer, fieldValue);
@@ -163,6 +170,13 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
private void writeBuildInFields(MemoryBuffer buffer, T value, Fory fory) {
for (SerializationFieldInfo fieldInfo : this.buildInFields) {
+ if (Utils.DEBUG_OUTPUT_ENABLED) {
+ LOG.info(
+ "[Java] write field {} of type {}, writer index {}",
+ fieldInfo.descriptor.getName(),
+ fieldInfo.typeRef,
+ buffer.writerIndex());
+ }
AbstractObjectSerializer.writeBuildInField(binding, fieldInfo, buffer,
value);
}
}
@@ -171,6 +185,13 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver) {
Generics generics = fory.getGenerics();
for (SerializationFieldInfo fieldInfo : containerFields) {
+ if (Utils.DEBUG_OUTPUT_ENABLED) {
+ LOG.info(
+ "[Java] write field {} of type {}, writer index {}",
+ fieldInfo.descriptor.getName(),
+ fieldInfo.typeRef,
+ buffer.writerIndex());
+ }
FieldAccessor fieldAccessor = fieldInfo.fieldAccessor;
Object fieldValue = fieldAccessor.getObject(value);
writeContainerFieldValue(binding, refResolver, generics, fieldInfo,
buffer, fieldValue);
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java
index 0fcd14c9f..c621fbeca 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java
@@ -610,19 +610,15 @@ abstract class SerializationBinding {
@Override
Object readField(SerializationFieldInfo fieldInfo, RefMode refMode,
MemoryBuffer buffer) {
if (fieldInfo.useDeclaredTypeInfo) {
- if (refMode == RefMode.TRACKING) {
- return fory.xreadRef(buffer, fieldInfo.classInfo);
- } else {
- if (refMode != RefMode.NULL_ONLY || buffer.readByte() !=
Fory.NULL_FLAG) {
- return fory.xreadNonRef(buffer, fieldInfo.classInfo);
- }
- }
+ return fieldInfo.classInfo.getSerializer().xread(buffer, refMode);
} else {
if (refMode == RefMode.TRACKING) {
return fory.xreadRef(buffer, fieldInfo.classInfoHolder);
} else {
if (refMode != RefMode.NULL_ONLY || buffer.readByte() !=
Fory.NULL_FLAG) {
- return fory.xreadNonRef(buffer, fieldInfo.classInfoHolder);
+ ClassInfo classInfo = xtypeResolver.readClassInfo(buffer,
fieldInfo.classInfoHolder);
+ Serializer<?> serializer = classInfo.getSerializer();
+ return serializer.xread(buffer, refMode);
}
}
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/UnionSerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/UnionSerializer.java
index f11c8a717..da8c38297 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/UnionSerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/UnionSerializer.java
@@ -23,6 +23,8 @@ import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.util.function.BiFunction;
import org.apache.fory.Fory;
+import org.apache.fory.logging.Logger;
+import org.apache.fory.logging.LoggerFactory;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.resolver.ClassInfo;
import org.apache.fory.resolver.RefResolver;
@@ -52,6 +54,8 @@ import org.apache.fory.type.union.Union6;
* union types in other languages like C++'s std::variant, Rust's enum, or
Python's typing.Union.
*/
public class UnionSerializer extends Serializer<Union> {
+ private static final Logger LOG =
LoggerFactory.getLogger(UnionSerializer.class);
+
/** Array of factories for creating Union instances by type tag. */
@SuppressWarnings("unchecked")
private static final BiFunction<Integer, Object, Union>[] FACTORIES =
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java
index 0d7be7179..75d2c9e11 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/struct/Fingerprint.java
@@ -161,9 +161,8 @@ public class Fingerprint {
}
int typeId = Types.getDescriptorTypeId(fory, descriptor);
int internalTypeId = typeId & 0xff;
- if (Types.isUnionType(internalTypeId)) {
- return Types.UNION;
- }
+ // union must also be set to `UNKNOWN`, we can't know a type is union at
compile-time for some
+ // languages.
if (Types.isUserDefinedType((byte) internalTypeId)) {
return Types.UNKNOWN;
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Types.java
b/java/fory-core/src/main/java/org/apache/fory/type/Types.java
index e27640e74..568ee5d86 100644
--- a/java/fory-core/src/main/java/org/apache/fory/type/Types.java
+++ b/java/fory-core/src/main/java/org/apache/fory/type/Types.java
@@ -253,6 +253,7 @@ public class Types {
return isStructType(typeId)
|| isExtType(typeId)
|| isEnumType(typeId)
+ || typeId == UNION
|| typeId == TYPED_UNION
|| typeId == NAMED_UNION;
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/type/union/Union.java
b/java/fory-core/src/main/java/org/apache/fory/type/union/Union.java
index 196f2d24a..56ba4bddb 100644
--- a/java/fory-core/src/main/java/org/apache/fory/type/union/Union.java
+++ b/java/fory-core/src/main/java/org/apache/fory/type/union/Union.java
@@ -151,6 +151,6 @@ public class Union {
@Override
public String toString() {
- return "Union{index=" + index + ", value=" + value + "}";
+ return getClass().getSimpleName() + "{index=" + index + ", value=" + value
+ "}";
}
}
diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py
index a986c1da3..811116ae9 100644
--- a/python/pyfory/meta/typedef.py
+++ b/python/pyfory/meta/typedef.py
@@ -18,7 +18,7 @@
import enum
import typing
from typing import List
-from pyfory.types import TypeId, is_primitive_type, is_polymorphic_type
+from pyfory.types import TypeId, is_primitive_type, is_polymorphic_type,
is_union_type
from pyfory.buffer import Buffer
from pyfory.type_util import infer_field
from pyfory.meta.metastring import Encoding
@@ -141,6 +141,8 @@ class TypeDef:
from pyfory.serializer import NonExistEnumSerializer
return NonExistEnumSerializer(resolver.fory)
+ if self.type_id & 0xFF == TypeId.NAMED_UNION:
+ return resolver.get_typeinfo_by_name(self.namespace,
self.typename).serializer
from pyfory.struct import DataClassSerializer
@@ -262,6 +264,13 @@ class FieldType:
# Handle list wrapper
if isinstance(type_, list):
type_ = type_[0]
+ if is_union_type(self.type_id):
+ if type_ is None:
+ return None
+ try:
+ return resolver.get_typeinfo(cls=type_).serializer
+ except Exception:
+ return None
# Types that need to be handled dynamically during deserialization
# For these types, we don't know the concrete type at compile time
if self.type_id & 0xFF in [
@@ -271,9 +280,6 @@ class FieldType:
TypeId.NAMED_STRUCT,
TypeId.COMPATIBLE_STRUCT,
TypeId.NAMED_COMPATIBLE_STRUCT,
- TypeId.UNION,
- TypeId.TYPED_UNION,
- TypeId.NAMED_UNION,
TypeId.UNKNOWN,
]:
return None
@@ -372,9 +378,16 @@ class DynamicFieldType(FieldType):
super().__init__(type_id, is_monomorphic, is_nullable, is_tracking_ref)
def create_serializer(self, resolver, type_):
- # For dynamic field types (UNKNOWN, STRUCT, etc.), always return None
- # This ensures type info is written/read at runtime, which is required
- # for cross-language compatibility (Java always writes type info for
struct fields)
+ # For dynamic field types (UNKNOWN, STRUCT, etc.), default to None so
+ # type info is written/read at runtime for cross-language
compatibility.
+ # Exception: union fields are declared, so we should use the union
serializer
+ # to write/read the union payload correctly.
+ if isinstance(type_, list):
+ type_ = type_[0]
+ assert not is_union_type(self.type_id), (
+ "Union fields don't write field type info, \
+ they are not dynamic field types"
+ )
return None
def __repr__(self):
@@ -456,7 +469,13 @@ def build_field_infos(type_resolver, cls):
# Get just the field names for sorting
current_field_names = [fi.name for fi in field_infos]
- sorted_field_names, serializers = _sort_fields(type_resolver,
current_field_names, serializers, nullable_map)
+ sorted_field_names, serializers = _sort_fields(
+ type_resolver,
+ current_field_names,
+ serializers,
+ nullable_map,
+ field_infos,
+ )
field_infos_map = {field_info.name: field_info for field_info in
field_infos}
new_field_infos = []
for field_name in sorted_field_names:
@@ -572,14 +591,12 @@ def build_field_type_from_type_ids_with_ref(
TypeId.NAMED_STRUCT,
TypeId.COMPATIBLE_STRUCT,
TypeId.NAMED_COMPATIBLE_STRUCT,
- TypeId.UNION,
- TypeId.TYPED_UNION,
- TypeId.NAMED_UNION,
]:
return DynamicFieldType(type_id, False, is_nullable, is_tracking_ref)
else:
if type_id <= 0 or type_id >= TypeId.BOUND:
raise TypeError(f"Unknown type: {type_id} for field: {field_name}")
+ # union/enum go here too
return FieldType(type_id, morphic, is_nullable, is_tracking_ref)
@@ -614,10 +631,7 @@ def build_field_type_from_type_ids(type_resolver,
field_name: str, type_ids, vis
TypeId.NAMED_STRUCT,
TypeId.COMPATIBLE_STRUCT,
TypeId.NAMED_COMPATIBLE_STRUCT,
- TypeId.UNION,
- TypeId.TYPED_UNION,
- TypeId.NAMED_UNION,
- ]:
+ ] or is_union_type(type_id):
return DynamicFieldType(type_id, False, is_nullable, tracking_ref)
else:
if type_id <= 0 or type_id >= TypeId.BOUND:
diff --git a/python/pyfory/meta/typedef_encoder.py
b/python/pyfory/meta/typedef_encoder.py
index 0de4f2fa2..fc2d7fdb0 100644
--- a/python/pyfory/meta/typedef_encoder.py
+++ b/python/pyfory/meta/typedef_encoder.py
@@ -47,7 +47,7 @@ TYPENAME_ENCODER = MetaStringEncoder("$", "_")
FIELD_NAME_ENCODER = MetaStringEncoder("$", "_")
-def encode_typedef(type_resolver, cls):
+def encode_typedef(type_resolver, cls, include_fields: bool = True):
"""
Encode the typedef of the type for xlang serialization.
@@ -58,14 +58,16 @@ def encode_typedef(type_resolver, cls):
Returns:
The encoded TypeDef.
"""
- field_infos = build_field_infos(type_resolver, cls)
-
- # Check for duplicate field names
- field_names = [field_info.name for field_info in field_infos]
- duplicate_field_names = [name for name, count in
Counter(field_names).items() if count > 1]
- if duplicate_field_names:
- # TODO: handle duplicate field names for inheritance in future
- raise ValueError(f"Duplicate field names: {duplicate_field_names}")
+ if include_fields:
+ field_infos = build_field_infos(type_resolver, cls)
+ # Check for duplicate field names
+ field_names = [field_info.name for field_info in field_infos]
+ duplicate_field_names = [name for name, count in
Counter(field_names).items() if count > 1]
+ if duplicate_field_names:
+ # TODO: handle duplicate field names for inheritance in future
+ raise ValueError(f"Duplicate field names: {duplicate_field_names}")
+ else:
+ field_infos = []
buffer = Buffer.allocate(64)
diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py
index 8a078ba7e..d6dbc71aa 100644
--- a/python/pyfory/registry.py
+++ b/python/pyfory/registry.py
@@ -535,11 +535,11 @@ class TypeResolver:
if type_id not in self._type_id_to_typeinfo or not internal:
self._type_id_to_typeinfo[type_id] = typeinfo
self._types_info[cls] = typeinfo
- # Create TypeDef for NAMED_ENUM and NAMED_EXT when meta_share is
enabled
+ # Create TypeDef for named non-struct types when meta_share is enabled
if self.meta_share and type_id is not None:
base_type_id = type_id & 0xFF
- if base_type_id in (TypeId.NAMED_ENUM, TypeId.NAMED_EXT):
- type_def = encode_typedef(self, cls)
+ if base_type_id in (TypeId.NAMED_ENUM, TypeId.NAMED_EXT,
TypeId.NAMED_UNION):
+ type_def = encode_typedef(self, cls,
include_fields=is_struct_type(base_type_id))
if type_def is not None:
typeinfo.type_def = type_def
return typeinfo
diff --git a/python/pyfory/struct.py b/python/pyfory/struct.py
index f63e542c5..d90faeb98 100644
--- a/python/pyfory/struct.py
+++ b/python/pyfory/struct.py
@@ -52,6 +52,7 @@ from pyfory.types import (
get_primitive_type_size,
is_polymorphic_type,
is_primitive_type,
+ is_union_type,
)
from pyfory.type_util import (
TypeVisitor,
@@ -298,6 +299,10 @@ _ENABLE_FORY_PYTHON_JIT =
os.environ.get("ENABLE_FORY_PYTHON_JIT", "True").lower
"true",
"1",
)
+_ENABLE_FORY_DEBUG_OUTPUT = os.environ.get("ENABLE_FORY_DEBUG_OUTPUT",
"False").lower() in (
+ "true",
+ "1",
+)
class DataClassSerializer(Serializer):
@@ -1032,6 +1037,11 @@ class DataClassSerializer(Serializer):
serializer = self._serializers[index]
is_nullable = self._nullable_fields.get(field_name, False)
is_dynamic = self._dynamic_fields.get(field_name, False)
+ if _ENABLE_FORY_DEBUG_OUTPUT:
+ print(
+ f"xwrite field '{field_name}': {field_value!r},
writer_index={buffer.get_writer_index()}, "
+ f"nullable={is_nullable}, dynamic={is_dynamic},
serializer={serializer}"
+ )
if is_nullable:
if field_value is None:
buffer.write_int8(-3)
@@ -1069,6 +1079,11 @@ class DataClassSerializer(Serializer):
serializer = self._serializers[index]
is_nullable = self._nullable_fields.get(field_name, False)
is_dynamic = self._dynamic_fields.get(field_name, False)
+ if _ENABLE_FORY_DEBUG_OUTPUT:
+ print(
+ f"xread field '{field_name}':
reader_index={buffer.get_reader_index()}, "
+ f"nullable={is_nullable}, dynamic={is_dynamic},
serializer={serializer}"
+ )
if is_nullable:
ref_id = buffer.read_int8()
if ref_id == -3:
@@ -1261,7 +1276,7 @@ def group_fields(type_resolver, field_names, serializers,
nullable_map=None, fie
)
)
for type_id, serializer, field_name, sort_key in type_ids:
- if type_id in {TypeId.TYPED_UNION, TypeId.NAMED_UNION}:
+ if is_union_type(type_id):
type_id = TypeId.UNION
is_nullable = nullable_map.get(field_name, False)
if is_primitive_type(type_id):
@@ -1272,11 +1287,7 @@ def group_fields(type_resolver, field_names,
serializers, nullable_map=None, fie
container = collection_types
elif is_map_type(serializer.type_):
container = map_types
- elif is_polymorphic_type(type_id) or type_id in {
- TypeId.ENUM,
- TypeId.NAMED_ENUM,
- TypeId.UNION,
- }:
+ elif is_polymorphic_type(type_id) or type_id in {TypeId.ENUM,
TypeId.NAMED_ENUM} or is_union_type(type_id):
container = other_types
elif type_id >= TypeId.BOUND:
# Native mode user-registered types have type_id >= BOUND
@@ -1361,8 +1372,9 @@ def compute_struct_fingerprint(type_resolver,
field_names, serializers, nullable
nullable_flag = "1" if nullable_map.get(field_name, False) else "0"
else:
type_id = type_resolver.get_typeinfo(serializer.type_).type_id &
0xFF
- if type_id in {TypeId.TYPED_UNION, TypeId.NAMED_UNION}:
- type_id = TypeId.UNION
+ if is_union_type(type_id):
+ # customized types can't be detected at compile time for some
languages
+ type_id = TypeId.UNKNOWN
is_nullable = nullable_map.get(field_name, False)
# For polymorphic or enum types, set type_id to UNKNOWN but
preserve nullable from map
diff --git a/python/pyfory/types.py b/python/pyfory/types.py
index 50ff34777..5c12d4af7 100644
--- a/python/pyfory/types.py
+++ b/python/pyfory/types.py
@@ -414,6 +414,12 @@ _struct_type_ids = {
TypeId.NAMED_COMPATIBLE_STRUCT,
}
+_union_type_ids = {
+ TypeId.UNION,
+ TypeId.TYPED_UNION,
+ TypeId.NAMED_UNION,
+}
+
def is_polymorphic_type(type_id: int) -> bool:
return type_id in _polymorphic_type_ids
@@ -421,3 +427,15 @@ def is_polymorphic_type(type_id: int) -> bool:
def is_struct_type(type_id: int) -> bool:
return type_id in _struct_type_ids
+
+
+def is_union_type(type_or_id) -> bool:
+ if type_or_id is None:
+ return False
+ if isinstance(type_or_id, int):
+ type_id = type_or_id
+ else:
+ type_id = getattr(type_or_id, "type_id", None)
+ if type_id is None or not isinstance(type_id, int):
+ return False
+ return (type_id & 0xFF) in _union_type_ids
diff --git a/python/pyfory/union.py b/python/pyfory/union.py
index 1f9134d7a..e36a5e288 100644
--- a/python/pyfory/union.py
+++ b/python/pyfory/union.py
@@ -39,6 +39,9 @@ class Union:
def value(self) -> object:
return self._value
+ def __repr__(self) -> str:
+ return f"{self.__class__.__name__}(case_id={self._case_id},
value={self._value})"
+
class UnionSerializer(Serializer):
"""
diff --git a/rust/fory-core/src/meta/type_meta.rs
b/rust/fory-core/src/meta/type_meta.rs
index 04496175c..31e356778 100644
--- a/rust/fory-core/src/meta/type_meta.rs
+++ b/rust/fory-core/src/meta/type_meta.rs
@@ -23,8 +23,8 @@ use crate::meta::{
};
use crate::resolver::type_resolver::{TypeInfo, TypeResolver};
use crate::types::{
- TypeId, COMPATIBLE_STRUCT, ENUM, EXT, NAMED_COMPATIBLE_STRUCT, NAMED_ENUM,
NAMED_EXT,
- NAMED_STRUCT, PRIMITIVE_TYPES, STRUCT, UNKNOWN,
+ TypeId, BINARY, COMPATIBLE_STRUCT, ENUM, EXT, INT8_ARRAY,
NAMED_COMPATIBLE_STRUCT, NAMED_ENUM,
+ NAMED_EXT, NAMED_STRUCT, PRIMITIVE_TYPES, STRUCT, UINT8_ARRAY, UNKNOWN,
};
use crate::util::to_snake_case;
@@ -32,7 +32,7 @@ use crate::util::to_snake_case;
/// This treats all struct variants (STRUCT, COMPATIBLE_STRUCT, NAMED_STRUCT,
/// NAMED_COMPATIBLE_STRUCT) and UNKNOWN as equivalent to STRUCT.
/// UNKNOWN (0) is used for polymorphic types (interfaces) in cross-language
serialization.
-/// Similarly for ENUM and EXT variants.
+/// Similarly for ENUM and EXT variants, and byte array encodings.
fn normalize_type_id_for_eq(type_id: u32) -> u32 {
let low = type_id & 0xff;
match low {
@@ -49,6 +49,8 @@ fn normalize_type_id_for_eq(type_id: u32) -> u32 {
_ if low == ENUM || low == NAMED_ENUM => ENUM,
// All ext variants normalize to EXT
_ if low == EXT || low == NAMED_EXT => EXT,
+ // Byte array encodings normalize to BINARY
+ _ if low == BINARY || low == INT8_ARRAY || low == UINT8_ARRAY =>
BINARY,
// Everything else stays the same
_ => type_id,
}
diff --git a/rust/fory-derive/src/object/util.rs
b/rust/fory-derive/src/object/util.rs
index cf56dca40..291fdd99e 100644
--- a/rust/fory-derive/src/object/util.rs
+++ b/rust/fory-derive/src/object/util.rs
@@ -1435,8 +1435,12 @@ fn compute_struct_fingerprint(fields: &[&Field]) ->
String {
};
let nullable_flag = if nullable { "1" } else { "0" };
- // User-defined types (UNKNOWN) use 0 in fingerprint, matching Java
behavior
- let effective_type_id = if info.type_id == TypeId::UNKNOWN as u32 {
+ // User-defined types (UNKNOWN) and unions use 0 in fingerprint,
matching Java behavior
+ let effective_type_id = if info.type_id == TypeId::UNKNOWN as u32
+ || info.type_id == TypeId::UNION as u32
+ || info.type_id == TypeId::TYPED_UNION as u32
+ || info.type_id == TypeId::NAMED_UNION as u32
+ {
0
} else {
info.type_id
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
