This is an automated email from the ASF dual-hosted git repository. chaokunyang pushed a commit to branch releases-0.12 in repository https://gitbox.apache.org/repos/asf/fory.git
commit 9659d8debdab7ff5839d54b8c8c17ec782273290 Author: Shawn Yang <[email protected]> AuthorDate: Thu Aug 14 01:58:09 2025 +0800 feat(java): support object stream serialization for graalvm (#2464) ## What does this PR do? <!-- Describe the purpose of this PR. --> ## Related issues Closes #2460 ## Does this PR introduce any user-facing change? <!-- If any user-facing interface changes, please [open an issue](https://github.com/apache/fory/issues/new/choose) describing the need to do so and update the document if necessary. --> - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark <!-- When the PR has an impact on performance (if you don't know whether the PR will have an impact on performance, you can submit the PR first, and if it will have impact on performance, the code reviewer will explain it), be sure to attach a benchmark data here. --> --- docs/guide/graalvm_guide.md | 4 ++ .../main/java/org/apache/fory/graalvm/Main.java | 1 + .../apache/fory/graalvm/ObjectStreamExample.java | 58 ++++++++++++++++++++ .../graalvm_tests/native-image.properties | 1 + .../org/apache/fory/AbstractThreadSafeFory.java | 9 +++ .../src/main/java/org/apache/fory/BaseFory.java | 9 +++ .../src/main/java/org/apache/fory/Fory.java | 5 ++ .../java/org/apache/fory/builder/CodecUtils.java | 64 ++++++++++++++++++---- .../org/apache/fory/resolver/ClassResolver.java | 35 ++++++++++++ .../fory-core/native-image.properties | 4 ++ 10 files changed, 180 insertions(+), 10 deletions(-) diff --git a/docs/guide/graalvm_guide.md b/docs/guide/graalvm_guide.md index e9bef16a8..7a1dc7998 100644 --- a/docs/guide/graalvm_guide.md +++ b/docs/guide/graalvm_guide.md @@ -70,6 +70,8 @@ public class Example { fory = Fory.builder().build(); // register and generate serializer code. fory.register(Record.class, true); + // ensure lazy initialized serializers being compiled by fory. + fory.ensureSerializersCompiled(); } public static void main(String[] args) { @@ -115,6 +117,8 @@ public class ThreadSafeExample { Fory f = Fory.builder().build(); // register and generate serializer code. f.register(Foo.class, true); + // ensure lazy initialized serializers being compiled by fory. + fory.ensureSerializersCompiled(); return f; }); } diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java index e042682a3..48469e9e7 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java @@ -35,6 +35,7 @@ public class Main { ThreadSafeExample.main(args); CompatibleThreadSafeExample.main(args); ProxyExample.main(args); + ObjectStreamExample.main(args); Benchmark.main(args); CollectionExample.main(args); } diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java new file mode 100644 index 000000000..a73c90bdf --- /dev/null +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.fory.graalvm; + +import org.apache.fory.Fory; + +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class ObjectStreamExample extends AbstractMap<Integer, Integer> { + private static final Fory FORY = Fory.builder() + .withName(ObjectStreamExample.class.getName()) + .registerGuavaTypes(false) + .build(); + + static { + FORY.register(ObjectStreamExample.class, true); + FORY.ensureSerializersCompiled(); + } + + final int[] ints = new int[10]; + + public static void main(String[] args) { + FORY.reset(); + byte[] bytes = FORY.serialize(new ObjectStreamExample()); + FORY.reset(); + ObjectStreamExample o = (ObjectStreamExample) FORY.deserialize(bytes); + System.out.println(Arrays.toString(o.ints)); + } + + @Override + public Set<Entry<Integer, Integer>> entrySet() { + HashSet<Entry<Integer, Integer>> set = new HashSet<>(); + for (int i = 0; i < ints.length; i++) { + set.add(new AbstractMap.SimpleEntry<>(i, ints[i])); + } + return set; + } +} \ No newline at end of file diff --git a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties index 27cd0385f..35d26127a 100644 --- a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties +++ b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties @@ -27,5 +27,6 @@ Args=-H:+ReportExceptionStackTraces \ org.apache.fory.graalvm.ThreadSafeExample,\ org.apache.fory.graalvm.CompatibleThreadSafeExample,\ org.apache.fory.graalvm.ProxyExample,\ + org.apache.fory.graalvm.ObjectStreamExample,\ org.apache.fory.graalvm.CollectionExample,\ org.apache.fory.graalvm.Benchmark diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index 8c8405ca3..a438ec5cb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -82,6 +82,15 @@ public abstract class AbstractThreadSafeFory implements ThreadSafeFory { registerCallback(fory -> fory.getClassResolver().setClassChecker(classChecker)); } + @Override + public void ensureSerializersCompiled() { + execute( + fory -> { + fory.ensureSerializersCompiled(); + return null; + }); + } + @Internal public abstract void registerCallback(Consumer<Fory> callback); } diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index 3f27ecdad..ff0b68c71 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -105,6 +105,15 @@ public interface BaseFory { void setSerializerFactory(SerializerFactory serializerFactory); + /** + * Ensure all compilation for serializers and accessors even for lazy initialized serializers. + * This method will block until all compilation is done. + * + * <p>This method is mainly used for graalvm native image build time and trigger compilation ahead + * for online service ahead to avoid cold start. + */ + void ensureSerializersCompiled(); + /** Return serialized <code>obj</code> as a byte array. */ byte[] serialize(Object obj); diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 99eed4bbf..5bacf507f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -1620,6 +1620,11 @@ public final class Fory implements BaseFory { method)); } + @Override + public void ensureSerializersCompiled() { + classResolver.ensureSerializersCompiled(); + } + public JITContext getJITContext() { return jitContext; } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java index 9f96a889d..c23cec203 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java @@ -20,19 +20,26 @@ package org.apache.fory.builder; import java.util.Collections; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import org.apache.fory.Fory; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.CompileUnit; +import org.apache.fory.collection.Tuple2; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.FieldResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.util.ClassLoaderUtils; +import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.Preconditions; /** Codec util to create and load jit serializer class. */ +@SuppressWarnings("rawtypes") public class CodecUtils { + private static ConcurrentHashMap<Tuple2<String, Class<?>>, Class> graalvmSerializers = + new ConcurrentHashMap<>(); // TODO(chaokunyang) how to uninstall org.apache.fory.codegen/builder classes for graalvm build // time @@ -40,30 +47,47 @@ public class CodecUtils { public static <T> Class<? extends Serializer<T>> loadOrGenObjectCodecClass( Class<T> cls, Fory fory) { Preconditions.checkNotNull(fory); - BaseObjectCodecBuilder codecBuilder = new ObjectCodecBuilder(cls, fory); - return loadOrGenCodecClass(cls, fory, codecBuilder); + return loadSerializer( + "loadOrGenObjectCodecClass", + cls, + () -> loadOrGenCodecClass(cls, fory, new ObjectCodecBuilder(cls, fory))); } public static <T> Class<? extends Serializer<T>> loadOrGenMetaSharedCodecClass( Fory fory, Class<T> cls, ClassDef classDef) { Preconditions.checkNotNull(fory); - MetaSharedCodecBuilder codecBuilder = - new MetaSharedCodecBuilder(TypeRef.of(cls), fory, classDef); - return loadOrGenCodecClass(cls, fory, codecBuilder); + return loadSerializer( + "loadOrGenMetaSharedCodecClass", + cls, + () -> + loadOrGenCodecClass( + cls, fory, new MetaSharedCodecBuilder(TypeRef.of(cls), fory, classDef))); } public static <T> Class<? extends Serializer<T>> loadOrGenCompatibleCodecClass( Class<T> cls, Fory fory) { - FieldResolver resolver = FieldResolver.of(fory, cls, true, false); - return loadOrGenCompatibleCodecClass(cls, fory, resolver, Generated.GeneratedSerializer.class); + return loadSerializer( + "loadOrGenCompatibleCodecClass", + cls, + () -> { + FieldResolver resolver = FieldResolver.of(fory, cls, true, false); + return loadOrGenCompatibleCodecClass( + cls, fory, resolver, Generated.GeneratedSerializer.class); + }); } public static <T> Class<? extends Serializer<T>> loadOrGenCompatibleCodecClass( Class<T> cls, Fory fory, FieldResolver fieldResolver, Class<?> parentSerializerClass) { Preconditions.checkNotNull(fory); - BaseObjectCodecBuilder codecBuilder = - new CompatibleCodecBuilder(TypeRef.of(cls), fory, fieldResolver, parentSerializerClass); - return loadOrGenCodecClass(cls, fory, codecBuilder); + return loadSerializer( + "loadOrGenCompatibleCodecClass", + cls, + () -> { + BaseObjectCodecBuilder codecBuilder = + new CompatibleCodecBuilder( + TypeRef.of(cls), fory, fieldResolver, parentSerializerClass); + return loadOrGenCodecClass(cls, fory, codecBuilder); + }); } @SuppressWarnings("unchecked") @@ -126,4 +150,24 @@ public class CodecUtils { } return codeGenerator; } + + private static <T> Class<? extends Serializer<T>> loadSerializer( + String name, Class<?> cls, Callable<Class<? extends Serializer<T>>> func) { + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + Tuple2<String, Class<?>> key = Tuple2.of(name, cls); + Class serializerClass = graalvmSerializers.get(key); + if (serializerClass != null) { + return serializerClass; + } + } + try { + Class serializerClass = func.call(); + if (GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE) { + graalvmSerializers.putIfAbsent(Tuple2.of(name, cls), serializerClass); + } + return serializerClass; + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 0e62c54d4..54895c592 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -274,6 +274,7 @@ public class ClassResolver implements TypeResolver { private final IdentityMap<Type, GenericType> genericTypes = new IdentityMap<>(); private final Map<Class, Map<String, GenericType>> classGenericTypes = new HashMap<>(); private final Map<List<ClassLoader>, CodeGenerator> codeGeneratorMap = new HashMap<>(); + private final Set<ClassInfo> initialClassInfos = new HashSet<>(); } public ClassResolver(Fory fory) { @@ -331,6 +332,14 @@ public class ClassResolver implements TypeResolver { addDefaultSerializers(); shimDispatcher.initialize(); innerEndClassId = extRegistry.classIdGenerator; + if (GraalvmSupport.isGraalBuildtime()) { + classInfoMap.forEach( + (cls, classInfo) -> { + if (classInfo.serializer != null) { + extRegistry.initialClassInfos.add(classInfo); + } + }); + } } private void addDefaultSerializers() { @@ -2197,6 +2206,32 @@ public class ClassResolver implements TypeResolver { return fory; } + /** + * Ensure all compilation for serializers and accessors even for lazy initialized serializers. + * This method will block until all compilation is done. + */ + public void ensureSerializersCompiled() { + try { + classInfoMap.forEach( + (cls, classInfo) -> { + if (classInfo.serializer == null) { + getSerializer(classInfo.cls, isSerializable(classInfo.cls)); + } + }); + if (GraalvmSupport.isGraalBuildtime()) { + classInfoMap.forEach( + (cls, classInfo) -> { + if (classInfo.serializer != null + && !extRegistry.initialClassInfos.contains(classInfo)) { + classInfo.serializer = null; + } + }); + } + } finally { + fory.getJITContext().unlock(); + } + } + private static final ConcurrentMap<Integer, GraalvmClassRegistry> GRAALVM_REGISTRY = new ConcurrentHashMap<>(); diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 319193bfa..50f599329 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -193,6 +193,7 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.builder.AccessorHelper,\ org.apache.fory.builder.JITContext,\ org.apache.fory.builder.ObjectCodecBuilder,\ + org.apache.fory.builder.CodecUtils,\ org.apache.fory.codegen.CodeGenerator$DefineState,\ org.apache.fory.codegen.CodeGenerator,\ org.apache.fory.codegen.CodegenContext,\ @@ -300,6 +301,7 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.collection.SubListSerializers,\ org.apache.fory.serializer.collection.SubListSerializers$SubListViewSerializer,\ org.apache.fory.serializer.collection.SubListSerializers$SubListSerializer,\ + org.apache.fory.serializer.collection.MapSerializers$DefaultJavaMapSerializer,\ org.apache.fory.serializer.JavaSerializer$1,\ org.apache.fory.serializer.JavaSerializer$2,\ org.apache.fory.serializer.JavaSerializer$3,\ @@ -312,6 +314,7 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.LocaleSerializer,\ org.apache.fory.serializer.LazySerializer,\ org.apache.fory.serializer.LazySerializer$LazyObjectSerializer,\ + org.apache.fory.serializer.CodegenSerializer$LazyInitBeanSerializer,\ org.apache.fory.serializer.NoneSerializer,\ org.apache.fory.serializer.NonexistentClassSerializers$ClassFieldsInfo,\ org.apache.fory.serializer.NonexistentClassSerializers$NonexistentClassSerializer,\ @@ -332,6 +335,7 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.ReplaceResolveSerializer$1,\ org.apache.fory.serializer.ReplaceResolveSerializer$ReplaceStub,\ org.apache.fory.serializer.ReplaceResolveSerializer,\ + org.apache.fory.serializer.ReplaceResolveSerializer$ReplaceResolveInfo,\ org.apache.fory.serializer.Serializers$AtomicBooleanSerializer,\ org.apache.fory.serializer.Serializers$AtomicIntegerSerializer,\ org.apache.fory.serializer.Serializers$AtomicLongSerializer,\ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
