This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/main by this push:
new 513d5585 feat(java): support meta shared serialization for xlang in
java (#2276)
513d5585 is described below
commit 513d5585df2e2cbe15311aa4fa73de23f984a33c
Author: Shawn Yang <shawn.ck.y...@gmail.com>
AuthorDate: Mon Jun 2 15:49:32 2025 +0800
feat(java): support meta shared serialization for xlang in java (#2276)
## What does this PR do?
This PR supports meta shared serialization for xlang in java.
Multi-dimensional array field is not supported in this PR. We track it
in #2197
## Related issues
#2197
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
.../src/main/java/org/apache/fory/Fory.java | 24 +++++--
.../fory/builder/MetaSharedCodecBuilder.java | 6 +-
.../apache/fory/builder/ObjectCodecBuilder.java | 4 +-
.../java/org/apache/fory/config/ForyBuilder.java | 3 +
.../main/java/org/apache/fory/meta/ClassDef.java | 49 +++++++++-----
.../fory/serializer/MetaSharedSerializer.java | 20 +++++-
.../serializer/NonexistentClassSerializers.java | 6 +-
.../apache/fory/serializer/ObjectSerializer.java | 2 +-
.../apache/fory/serializer/SerializationUtils.java | 2 +-
.../org/apache/fory/serializer/Serializer.java | 3 +-
.../org/apache/fory/xlang/MetaSharedXlangTest.java | 79 ++++++++++++++++++++++
11 files changed, 168 insertions(+), 30 deletions(-)
diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java
b/java/fory-core/src/main/java/org/apache/fory/Fory.java
index c26c01bb..9b097b16 100644
--- a/java/fory-core/src/main/java/org/apache/fory/Fory.java
+++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java
@@ -326,8 +326,7 @@ public final class Fory implements BaseFory {
if (language == Language.JAVA) {
write(buffer, obj);
} else {
- buffer.writeByte((byte) Language.JAVA.ordinal());
- xwriteRef(buffer, obj);
+ xwrite(buffer, obj);
}
return buffer;
} catch (StackOverflowError t) {
@@ -412,6 +411,21 @@ public final class Fory implements BaseFory {
}
}
+ private void xwrite(MemoryBuffer buffer, Object obj) {
+ buffer.writeByte((byte) Language.JAVA.ordinal());
+ int startOffset = buffer.writerIndex();
+ boolean shareMeta = config.isMetaShareEnabled();
+ if (shareMeta) {
+ buffer.writeInt32(-1); // preserve 4-byte for meta start offsets.
+ }
+ xwriteRef(buffer, obj);
+ MetaContext metaContext = serializationContext.getMetaContext();
+ if (shareMeta && metaContext != null &&
!metaContext.writingClassDefs.isEmpty()) {
+ buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4);
+ classResolver.writeClassDefs(buffer);
+ }
+ }
+
/** Serialize a nullable referencable object to <code>buffer</code>. */
public void writeRef(MemoryBuffer buffer, Object obj) {
if (!refResolver.writeRefOrNull(buffer, obj)) {
@@ -861,13 +875,13 @@ public final class Fory implements BaseFory {
"outOfBandBuffers should be null when the serialized stream is "
+ "produced with bufferCallback null.");
}
+ if (shareMeta) {
+ readClassDefs(buffer);
+ }
Object obj;
if (isTargetXLang) {
obj = xreadRef(buffer);
} else {
- if (shareMeta) {
- readClassDefs(buffer);
- }
obj = readRef(buffer);
}
return obj;
diff --git
a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java
b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java
index f9308ee5..099b0081 100644
---
a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java
+++
b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java
@@ -76,7 +76,11 @@ public class MetaSharedCodecBuilder extends
ObjectCodecBuilder {
this.classDef = classDef;
Collection<Descriptor> descriptors =
fory(
- f -> MetaSharedSerializer.consolidateFields(f.getClassResolver(),
beanClass, classDef));
+ f ->
+ MetaSharedSerializer.consolidateFields(
+ f.isCrossLanguage() ? f.getXtypeResolver() :
f.getClassResolver(),
+ beanClass,
+ classDef));
DescriptorGrouper grouper =
fory.getClassResolver().createDescriptorGrouper(descriptors, false);
objectCodecOptimizer =
new ObjectCodecOptimizer(beanClass, grouper,
!fory.isBasicTypesRefIgnored(), ctx);
diff --git
a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java
b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java
index ed07302e..b4e525d7 100644
---
a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java
+++
b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java
@@ -88,13 +88,15 @@ public class ObjectCodecBuilder extends
BaseObjectCodecBuilder {
super(TypeRef.of(beanClass), fory,
Generated.GeneratedObjectSerializer.class);
Collection<Descriptor> descriptors;
boolean shareMeta = fory.getConfig().isMetaShareEnabled();
+ boolean xlang = fory.isCrossLanguage();
if (shareMeta) {
descriptors =
fory(
f ->
f.getClassResolver()
.getClassDef(beanClass, true)
- .getDescriptors(f.getClassResolver(), beanClass));
+ .getDescriptors(
+ xlang ? f.getXtypeResolver() : f.getClassResolver(),
beanClass));
} else {
descriptors = fory.getClassResolver().getFieldDescriptors(beanClass,
true);
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
index 7edcf021..cd3955a1 100644
--- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
+++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
@@ -95,6 +95,9 @@ public final class ForyBuilder {
*/
public ForyBuilder withLanguage(Language language) {
this.language = language;
+ if (language != Language.JAVA) {
+ codeGenEnabled = false;
+ }
return this;
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
index cef444b6..f5c9e41f 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
@@ -239,9 +239,10 @@ public class ClassDef implements Serializable {
*
* @param cls class load in current process.
*/
- public List<Descriptor> getDescriptors(ClassResolver resolver, Class<?> cls)
{
+ public List<Descriptor> getDescriptors(TypeResolver resolver, Class<?> cls) {
if (descriptors == null) {
- SortedMap<Member, Descriptor> allDescriptorsMap =
resolver.getAllDescriptorsMap(cls, true);
+ SortedMap<Member, Descriptor> allDescriptorsMap =
+ resolver.getFory().getClassResolver().getAllDescriptorsMap(cls,
true);
Map<String, Descriptor> descriptorsMap = new HashMap<>();
for (Map.Entry<Member, Descriptor> e : allDescriptorsMap.entrySet()) {
if (descriptorsMap.put(
@@ -333,9 +334,16 @@ public class ClassDef implements Serializable {
* null. Don't invoke this method if class does have
<code>fieldName</code> field. In such case,
* reflection should be used to get the descriptor.
*/
- Descriptor toDescriptor(ClassResolver classResolver, Descriptor
descriptor) {
+ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) {
TypeRef<?> declared = descriptor != null ? descriptor.getTypeRef() :
null;
- TypeRef<?> typeRef = fieldType.toTypeToken(classResolver, declared);
+ TypeRef<?> typeRef = fieldType.toTypeToken(resolver, declared);
+ if (descriptor != null) {
+ if (typeRef.equals(declared)) {
+ return descriptor;
+ } else {
+ descriptor.copyWithTypeName(typeRef.getType().getTypeName());
+ }
+ }
// This field doesn't exist in peer class, so any legal modifier will be
OK.
int stubModifiers = ReflectionUtils.getField(getClass(),
"fieldName").getModifiers();
return new Descriptor(typeRef, fieldName, stubModifiers, definedClass);
@@ -407,7 +415,7 @@ public class ClassDef implements Serializable {
*
* @see FinalObjectTypeStub
*/
- public abstract TypeRef<?> toTypeToken(ClassResolver classResolver,
TypeRef<?> declared);
+ public abstract TypeRef<?> toTypeToken(TypeResolver classResolver,
TypeRef<?> declared);
@Override
public boolean equals(Object o) {
@@ -555,10 +563,6 @@ public class ClassDef implements Serializable {
ClassInfo classInfo = resolver.getXtypeInfo(xtypeId);
Preconditions.checkNotNull(classInfo);
Class<?> cls = classInfo.getCls();
- if (Types.isPrimitiveArray(xtypeId)) {
- FieldType type = buildFieldType(resolver,
resolver.buildGenericType(cls));
- return new ArrayFieldType(xtypeId, true, nullable,
trackingRef, type, 1);
- }
return new RegisteredFieldType(
resolver.isMonomorphic(cls), nullable, trackingRef, xtypeId);
} else {
@@ -584,8 +588,21 @@ public class ClassDef implements Serializable {
}
@Override
- public TypeRef<?> toTypeToken(ClassResolver classResolver, TypeRef<?>
declared) {
- Class<?> cls = classResolver.getRegisteredClass(classId);
+ public TypeRef<?> toTypeToken(TypeResolver resolver, TypeRef<?> declared) {
+ Class<?> cls;
+ if (resolver instanceof XtypeResolver) {
+ cls = ((XtypeResolver) resolver).getXtypeInfo(classId).getCls();
+ if (Types.isPrimitiveType(classId)) {
+ if (declared.isPrimitive() && !nullable) {
+ cls = TypeUtils.unwrap(cls);
+ }
+ if (nullable && !declared.isPrimitive()) {
+ cls = TypeUtils.wrap(cls);
+ }
+ }
+ } else {
+ cls = ((ClassResolver) resolver).getRegisteredClass(classId);
+ }
if (cls == null) {
LOG.warn("Class {} not registered, take it as Struct type for
deserialization.", classId);
cls = NonexistentClass.NonexistentMetaShared.class;
@@ -652,7 +669,7 @@ public class ClassDef implements Serializable {
}
@Override
- public TypeRef<?> toTypeToken(ClassResolver classResolver, TypeRef<?>
declared) {
+ public TypeRef<?> toTypeToken(TypeResolver classResolver, TypeRef<?>
declared) {
// TODO support preserve element TypeExtMeta
TypeRef<? extends Collection<?>> collectionTypeRef =
collectionOf(
@@ -749,7 +766,7 @@ public class ClassDef implements Serializable {
}
@Override
- public TypeRef<?> toTypeToken(ClassResolver classResolver, TypeRef<?>
declared) {
+ public TypeRef<?> toTypeToken(TypeResolver classResolver, TypeRef<?>
declared) {
// TODO support preserve element TypeExtMeta, it will be lost when
building other TypeRef
return mapOf(
keyType.toTypeToken(classResolver, declared),
@@ -798,7 +815,7 @@ public class ClassDef implements Serializable {
}
@Override
- public TypeRef<?> toTypeToken(ClassResolver classResolver, TypeRef<?>
declared) {
+ public TypeRef<?> toTypeToken(TypeResolver classResolver, TypeRef<?>
declared) {
return TypeRef.of(NonexistentClass.NonexistentEnum.class);
}
}
@@ -825,7 +842,7 @@ public class ClassDef implements Serializable {
}
@Override
- public TypeRef<?> toTypeToken(ClassResolver classResolver, TypeRef<?>
declared) {
+ public TypeRef<?> toTypeToken(TypeResolver classResolver, TypeRef<?>
declared) {
TypeRef<?> componentTypeRef = componentType.toTypeToken(classResolver,
declared);
Class<?> componentRawType = componentTypeRef.getRawType();
if (NonexistentClass.class.isAssignableFrom(componentRawType)) {
@@ -893,7 +910,7 @@ public class ClassDef implements Serializable {
}
@Override
- public TypeRef<?> toTypeToken(ClassResolver classResolver, TypeRef<?>
declared) {
+ public TypeRef<?> toTypeToken(TypeResolver classResolver, TypeRef<?>
declared) {
return isMonomorphic()
? TypeRef.of(FinalObjectTypeStub.class, new TypeExtMeta(nullable,
trackingRef))
: TypeRef.of(Object.class, new TypeExtMeta(nullable, trackingRef));
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java
index 8ac9c022..4d49e257 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java
@@ -36,6 +36,7 @@ import org.apache.fory.reflect.FieldAccessor;
import org.apache.fory.resolver.ClassInfoHolder;
import org.apache.fory.resolver.ClassResolver;
import org.apache.fory.resolver.RefResolver;
+import org.apache.fory.resolver.TypeResolver;
import org.apache.fory.type.Descriptor;
import org.apache.fory.type.DescriptorGrouper;
import org.apache.fory.type.Generics;
@@ -84,7 +85,10 @@ public class MetaSharedSerializer<T> extends
AbstractObjectSerializer<T> {
"Class version check should be disabled when compatible mode is
enabled.");
Preconditions.checkArgument(
fory.getConfig().isMetaShareEnabled(), "Meta share must be enabled.");
- Collection<Descriptor> descriptors = consolidateFields(this.classResolver,
type, classDef);
+ boolean xlang = fory.isCrossLanguage();
+ Collection<Descriptor> descriptors =
+ consolidateFields(
+ xlang ? fory.getXtypeResolver() : fory.getClassResolver(), type,
classDef);
DescriptorGrouper descriptorGrouper =
classResolver.createDescriptorGrouper(descriptors, false);
// d.getField() may be null if not exists in this class when meta share
enabled.
Tuple3<
@@ -118,6 +122,11 @@ public class MetaSharedSerializer<T> extends
AbstractObjectSerializer<T> {
serializer.write(buffer, value);
}
+ @Override
+ public void xwrite(MemoryBuffer buffer, T value) {
+ write(buffer, value);
+ }
+
@Override
public T read(MemoryBuffer buffer) {
if (isRecord) {
@@ -192,6 +201,11 @@ public class MetaSharedSerializer<T> extends
AbstractObjectSerializer<T> {
return obj;
}
+ @Override
+ public T xread(MemoryBuffer buffer) {
+ return read(buffer);
+ }
+
private void readFields(MemoryBuffer buffer, Object[] fields) {
int counter = 0;
Fory fory = this.fory;
@@ -280,7 +294,7 @@ public class MetaSharedSerializer<T> extends
AbstractObjectSerializer<T> {
}
public static Collection<Descriptor> consolidateFields(
- ClassResolver classResolver, Class<?> cls, ClassDef classDef) {
- return classDef.getDescriptors(classResolver, cls);
+ TypeResolver resolver, Class<?> cls, ClassDef classDef) {
+ return classDef.getDescriptors(resolver, cls);
}
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java
index 4c48e6c3..70d4d1ca 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java
@@ -19,6 +19,8 @@
package org.apache.fory.serializer;
+import static org.apache.fory.serializer.SerializationUtils.getTypeResolver;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -36,6 +38,7 @@ import org.apache.fory.resolver.ClassResolver;
import org.apache.fory.resolver.MetaContext;
import org.apache.fory.resolver.MetaStringResolver;
import org.apache.fory.resolver.RefResolver;
+import org.apache.fory.resolver.TypeResolver;
import org.apache.fory.serializer.NonexistentClass.NonexistentEnum;
import org.apache.fory.type.Descriptor;
import org.apache.fory.type.DescriptorGrouper;
@@ -155,11 +158,12 @@ public final class NonexistentClassSerializers {
private ClassFieldsInfo getClassFieldsInfo(ClassDef classDef) {
ClassFieldsInfo fieldsInfo = fieldsInfoMap.get(classDef.getId());
+ TypeResolver resolver = getTypeResolver(fory);
if (fieldsInfo == null) {
// Use `NonexistentSkipClass` since it doesn't have any field.
Collection<Descriptor> descriptors =
MetaSharedSerializer.consolidateFields(
- fory.getClassResolver(),
NonexistentClass.NonexistentSkip.class, classDef);
+ resolver, NonexistentClass.NonexistentSkip.class, classDef);
DescriptorGrouper descriptorGrouper =
fory.getClassResolver().createDescriptorGrouper(descriptors,
false);
Tuple3<
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
index c3adf224..741491a0 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java
@@ -96,7 +96,7 @@ public final class ObjectSerializer<T> extends
AbstractObjectSerializer<T> {
boolean shareMeta = fory.getConfig().isMetaShareEnabled();
if (shareMeta) {
ClassDef classDef = classResolver.getClassDef(cls, resolveParent);
- descriptors = classDef.getDescriptors(classResolver, cls);
+ descriptors = classDef.getDescriptors(typeResolver, cls);
} else {
descriptors = fory.getClassResolver().getFieldDescriptors(cls,
resolveParent);
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java
index c88910db..51760f73 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java
@@ -26,7 +26,7 @@ import org.apache.fory.resolver.TypeResolver;
@Internal
class SerializationUtils {
- static TypeResolver getTypeResolver(Fory fory) {
+ public static TypeResolver getTypeResolver(Fory fory) {
return fory.isCrossLanguage() ? fory.getXtypeResolver() :
fory.getClassResolver();
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java
index ef10c460..076d5ae6 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java
@@ -67,7 +67,8 @@ public abstract class Serializer<T> {
}
public T xread(MemoryBuffer buffer) {
- throw new UnsupportedOperationException();
+ throw new UnsupportedOperationException(
+ getClass() + " doesn't support xlang serialization for " + type);
}
public Serializer(Fory fory, Class<T> type) {
diff --git
a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java
b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java
new file mode 100644
index 00000000..0528c834
--- /dev/null
+++
b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fory.xlang;
+
+import lombok.Data;
+import org.apache.fory.CrossLanguageTest.Bar;
+import org.apache.fory.CrossLanguageTest.Foo;
+import org.apache.fory.Fory;
+import org.apache.fory.ForyTestBase;
+import org.apache.fory.config.CompatibleMode;
+import org.apache.fory.config.Language;
+import org.apache.fory.test.bean.BeanB;
+import org.testng.annotations.Test;
+
+public class MetaSharedXlangTest extends ForyTestBase {
+
+ @Test
+ public void testMetaSharedBasic() {
+ Fory fory =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCodegen(false)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .build();
+ fory.register(Foo.class, "example.foo");
+ fory.register(Bar.class, "example.bar");
+ serDeCheck(fory, Bar.create());
+ serDeCheck(fory, Foo.create());
+ }
+
+ @Test
+ public void testMetaSharedComplex1() {
+ Fory fory =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCodegen(false)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .build();
+ fory.register(BeanB.class, "example.b");
+ serDeCheck(fory, BeanB.createBeanB(2));
+ }
+
+ @Data
+ static class MDArrayFieldStruct {
+ int[][] arr;
+ }
+
+ // @Test
+ public void testMDArrayField() {
+ Fory fory =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCodegen(false)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .build();
+ // TODO support multi-dimensional array serialization
+ fory.register(MDArrayFieldStruct.class, "example.a");
+ MDArrayFieldStruct s = new MDArrayFieldStruct();
+ s.arr = new int[][] {{1, 2}, {3, 4}};
+ serDeCheck(fory, s);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org
For additional commands, e-mail: commits-h...@fury.apache.org
