This is an automated email from the ASF dual-hosted git repository. chaokunyang pushed a commit to branch releases-0.12 in repository https://gitbox.apache.org/repos/asf/fory.git
commit da0146ce5cf9efd00758ecaeeebaa6c21adf8e2d Author: Emre Şafak <[email protected]> AuthorDate: Tue Sep 2 10:58:53 2025 -0400 fix(ci): Exit with subprocess return code in run_ci.py (#2560) CI was not reflecting failures in shell script fallbacks (we are migrating to python) * Use sys.exit() to propagate the return code from subprocess.call(). * This ensures that the CI pipeline correctly reflects the success or failure of the executed scripts. You might want to merge #2561 first, to reduce the number of errors. --------- Co-authored-by: chaokunyang <[email protected]> --- .github/pull_request_template.md | 6 ++ .github/workflows/ci.yml | 17 ++++-- ci/run_ci.py | 7 +-- ci/run_ci.sh | 4 +- ci/tasks/java.py | 61 +++++++++++-------- integration_tests/graalvm_tests/pom.xml | 1 - .../apache/fory/graalvm/ObjectStreamExample.java | 40 ++++++------ .../java/org/apache/fory/graalvm/ProxyExample.java | 1 + .../graalvm_tests/proxy-config.json | 10 +++ .../META-INF/native-image/proxy-config.json | 7 --- integration_tests/jdk_compatibility_tests/pom.xml | 15 +++++ .../java/org/apache/fory/builder/CodecBuilder.java | 4 ++ .../java/org/apache/fory/codegen/Expression.java | 3 +- .../org/apache/fory/resolver/AllowListChecker.java | 2 +- .../org/apache/fory/resolver/ClassResolver.java | 54 ++++++++++++---- .../apache/fory/serializer/JdkProxySerializer.java | 22 +++++-- .../apache/fory/serializer/LambdaSerializer.java | 71 ++++++++++++++++------ .../fory/serializer/ObjectStreamSerializer.java | 4 +- .../org/apache/fory/serializer/Serializers.java | 27 ++++++++ .../main/java/org/apache/fory/type/TypeUtils.java | 4 +- .../java/org/apache/fory/util/GraalvmSupport.java | 17 ++++++ .../fory-core/native-image.properties | 9 +++ .../fory-core/reflection-config.json | 9 +++ .../fory-core/serialization-config.json | 11 ++++ .../fory/format/encoder/RowEncoderBuilder.java | 3 - .../format/encoder/ImplementInterfaceTest.java | 1 - python/pyfory/_fory.py | 39 ++++-------- python/pyfory/_registry.py | 11 +++- python/pyfory/tests/test_function.py | 8 +-- python/pyfory/tests/test_serializer.py | 16 ++++- python/setup.py | 6 +- 31 files changed, 342 insertions(+), 148 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 61ea3b195..79e0d94aa 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,6 +10,12 @@ Contribution Checklist - Fory has a strong focus on performance. If the PR you submit will have an impact on performance, please benchmark it first and provide the benchmark result here. --> +## Why? + +## What does this PR do? + +<!-- Describe the purpose of this PR. --> + ## What does this PR do? <!-- Describe the purpose of this PR. --> diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c79022757..e77e71013 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,8 @@ jobs: run: python ./ci/run_ci.py cpp --install-deps-only - name: Install python dependencies run: pip install pyarrow==15.0.0 Cython wheel pytest setuptools -U + - name: Install pyfory for xlang tests + run: pip install -e python/ - name: Run CI with Maven run: python ./ci/run_ci.py java --version ${{ matrix.java-version }} - name: Upload Test Report @@ -93,6 +95,8 @@ jobs: python-version: 3.8 - name: Install bazel run: python ./ci/run_ci.py cpp --install-deps-only + - name: Install pyfory for xlang tests + run: pip install -e python/ - name: Install python dependencies run: pip install pyarrow==15.0.0 Cython wheel pytest setuptools -U - name: Run CI with Maven @@ -190,10 +194,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - name: Set up JDK8 + - name: Set up JDK11 uses: actions/setup-java@v4 with: - java-version: 8 + java-version: 11 distribution: "temurin" - name: Set up Python 3.8 uses: actions/setup-python@v5 @@ -272,9 +276,14 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install bazel + - name: Install bazel (Unix) + if: runner.os != 'Windows' shell: bash run: python ./ci/run_ci.py cpp --install-deps-only + - name: Install bazel (Windows) + if: runner.os == 'Windows' + shell: bash + run: ./ci/run_ci.sh install_bazel_windows - name: Run Python CI shell: bash run: python ./ci/run_ci.py python @@ -301,7 +310,7 @@ jobs: run: python ./ci/run_ci.py cpp --install-deps-only - name: Install python dependencies run: pip install pyarrow==15.0.0 cython wheel pytest setuptools -U - - name: Install pyfory + - name: Install pyfory for xlang tests run: pip install -e python/ - name: Run Golang CI run: python ./ci/run_ci.py go diff --git a/ci/run_ci.py b/ci/run_ci.py index e637cc94f..ed1a3f59b 100644 --- a/ci/run_ci.py +++ b/ci/run_ci.py @@ -84,7 +84,7 @@ def run_shell_script(command, *args): cmd = [bash_path, script_path, command] cmd.extend(args) logging.info(f"Falling back to shell script with bash: {' '.join(cmd)}") - return subprocess.call(cmd) + sys.exit(subprocess.call(cmd)) else: logging.error( "Bash is not available on this Windows system. Cannot run shell script." @@ -101,7 +101,7 @@ def run_shell_script(command, *args): cmd = [script_path, command] cmd.extend(args) logging.info(f"Falling back to shell script: {' '.join(cmd)}") - return subprocess.call(cmd) + sys.exit(subprocess.call(cmd)) def parse_args(): @@ -237,14 +237,13 @@ def parse_args(): if USE_PYTHON_JAVA: func(**arg_dict) else: - if not arg_dict.get("version"): func(**arg_dict) return # Map Python version argument to shell script command version = arg_dict.get("version", "17") release = arg_dict.get("release", False) - + if release: logging.info("Release mode requested - using Python implementation") func(**arg_dict) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 63a1330d1..789f78cda 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -141,7 +141,7 @@ install_jdks() { graalvm_test() { cd "$ROOT"/java - mvn -T10 -B --no-transfer-progress clean install -DskipTests + mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite' echo "Start to build graalvm native image" cd "$ROOT"/integration_tests/graalvm_tests mvn -DskipTests=true --no-transfer-progress -Pnative package @@ -231,7 +231,7 @@ case $1 in echo "Executing fory java tests" cd "$ROOT/java" set +e - mvn -T16 --batch-mode --no-transfer-progress test + mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format,!:fory-testsuite' testcode=$? if [[ $testcode -ne 0 ]]; then exit $testcode diff --git a/ci/tasks/java.py b/ci/tasks/java.py index 7bb73e50a..6fa0754ed 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -17,7 +17,6 @@ import logging import os -import sys import subprocess import re from . import common @@ -26,7 +25,7 @@ from . import common def get_jdk_major_version(): try: # Run the 'java -version' command - result = subprocess.run(['java', '-version'], capture_output=True, text=True) + result = subprocess.run(["java", "-version"], capture_output=True, text=True) output = result.stderr # java -version outputs to stderr # Use regex to find the version string @@ -37,8 +36,8 @@ def get_jdk_major_version(): version_string = match.group(1) # Parse the version string - version_parts = version_string.split('.') - if version_parts[0] == '1': + version_parts = version_string.split(".") + if version_parts[0] == "1": # Java 8 or earlier return int(version_parts[1]) else: @@ -81,54 +80,56 @@ def create_toolchains_xml(jdk_mappings): import os import xml.etree.ElementTree as ET from xml.dom import minidom - + # Create ~/.m2 directory if it doesn't exist m2_dir = os.path.expanduser("~/.m2") os.makedirs(m2_dir, exist_ok=True) - + # Create the root element toolchains = ET.Element("toolchains") - + for version, jdk_name in jdk_mappings.items(): toolchain = ET.SubElement(toolchains, "toolchain") - + # Set type type_elem = ET.SubElement(toolchain, "type") type_elem.text = "jdk" - + # Set provides provides = ET.SubElement(toolchain, "provides") version_elem = ET.SubElement(provides, "version") version_elem.text = version vendor_elem = ET.SubElement(provides, "vendor") vendor_elem.text = "azul" - + # Set configuration configuration = ET.SubElement(toolchain, "configuration") jdk_home = ET.SubElement(configuration, "jdkHome") jdk_home.text = os.path.abspath(os.path.join(common.PROJECT_ROOT_DIR, jdk_name)) - + # Create pretty XML string - rough_string = ET.tostring(toolchains, 'unicode') + rough_string = ET.tostring(toolchains, "unicode") reparsed = minidom.parseString(rough_string) pretty_xml = reparsed.toprettyxml(indent=" ") - + # Add proper XML header with encoding xml_header = '<?xml version="1.0" encoding="UTF8"?>\n' - pretty_xml = xml_header + pretty_xml.split('\n', 1)[1] # Remove the default header and add our custom one - + pretty_xml = ( + xml_header + pretty_xml.split("\n", 1)[1] + ) # Remove the default header and add our custom one + # Write to ~/.m2/toolchains.xml toolchains_path = os.path.join(m2_dir, "toolchains.xml") - with open(toolchains_path, 'w', encoding='utf-8') as f: + with open(toolchains_path, "w", encoding="utf-8") as f: f.write(pretty_xml) - + logging.info(f"Created toolchains.xml at {toolchains_path}") logging.info("Toolchains configuration:") for version, jdk_name in jdk_mappings.items(): jdk_path = os.path.join(common.PROJECT_ROOT_DIR, jdk_name) logging.info(f" JDK {version}: {jdk_path}") # print toolchains.xml - with open(toolchains_path, 'r', encoding='utf-8') as f: + with open(toolchains_path, "r", encoding="utf-8") as f: logging.info(f.read()) @@ -149,7 +150,9 @@ def run_java8(): logging.info("Executing fory java tests with Java 8") install_jdks() common.cd_project_subdir("java") - common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test -pl '!fory-format'") + common.exec_cmd( + "mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format,!:fory-testsuite'" + ) logging.info("Executing fory java tests succeeds") @@ -208,7 +211,9 @@ def run_integration_tests(): logging.info("Executing fory integration tests") common.cd_project_subdir("java") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests") + common.exec_cmd( + "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'" + ) logging.info("benchmark tests") common.cd_project_subdir("java/benchmark") @@ -270,7 +275,9 @@ def run_graalvm_test(): logging.info("Start GraalVM tests") common.cd_project_subdir("java") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests") + common.exec_cmd( + "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'" + ) logging.info("Start to build graalvm native image") common.cd_project_subdir("integration_tests/graalvm_tests") @@ -285,17 +292,19 @@ def run_graalvm_test(): def run_release(): """Release to Maven Central.""" - logging.info(f"Starting release to Maven Central with Java") + logging.info("Starting release to Maven Central with Java") common.cd_project_subdir("java") - + # Clean and install without tests first logging.info("Cleaning and installing dependencies") common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests") - + # Deploy to Maven Central logging.info("Deploying to Maven Central") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean deploy -Dgpg.skip -DskipTests -Papache-release") - + common.exec_cmd( + "mvn -T10 -B --no-transfer-progress clean deploy -Dgpg.skip -DskipTests -Papache-release" + ) + logging.info("Release to Maven Central completed successfully") diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index 236bcf625..5f73cae6d 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -173,7 +173,6 @@ <!-- for fast build --> <!-- <buildArg>-Ob</buildArg> --> <buildArg>-H:+UnlockExperimentalVMOptions</buildArg> - <buildArg>-H:DynamicProxyConfigurationFiles=src/main/resources/META-INF/native-image/proxy-config.json</buildArg> </buildArgs> </configuration> </plugin> diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java index a73c90bdf..227ce126d 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java @@ -17,42 +17,42 @@ * under the License. */ - package org.apache.fory.graalvm; - -import org.apache.fory.Fory; +package org.apache.fory.graalvm; import java.util.AbstractMap; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import org.apache.fory.Fory; public class ObjectStreamExample extends AbstractMap<Integer, Integer> { - private static final Fory FORY = Fory.builder() - .withName(ObjectStreamExample.class.getName()) - .registerGuavaTypes(false) - .build(); + private static final Fory FORY = + Fory.builder() + .withName(ObjectStreamExample.class.getName()) + .registerGuavaTypes(false) + .build(); static { - FORY.register(ObjectStreamExample.class, true); - FORY.ensureSerializersCompiled(); + FORY.register(ObjectStreamExample.class, true); + FORY.ensureSerializersCompiled(); } final int[] ints = new int[10]; public static void main(String[] args) { - FORY.reset(); - byte[] bytes = FORY.serialize(new ObjectStreamExample()); - FORY.reset(); - ObjectStreamExample o = (ObjectStreamExample) FORY.deserialize(bytes); - System.out.println(Arrays.toString(o.ints)); + FORY.reset(); + byte[] bytes = FORY.serialize(new ObjectStreamExample()); + FORY.reset(); + ObjectStreamExample o = (ObjectStreamExample) FORY.deserialize(bytes); + System.out.println(Arrays.toString(o.ints)); } @Override public Set<Entry<Integer, Integer>> entrySet() { - HashSet<Entry<Integer, Integer>> set = new HashSet<>(); - for (int i = 0; i < ints.length; i++) { - set.add(new AbstractMap.SimpleEntry<>(i, ints[i])); - } - return set; + HashSet<Entry<Integer, Integer>> set = new HashSet<>(); + for (int i = 0; i < ints.length; i++) { + set.add(new AbstractMap.SimpleEntry<>(i, ints[i])); + } + return set; } -} \ No newline at end of file +} diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java index 990faaa2b..2fd195675 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java @@ -50,6 +50,7 @@ public class ProxyExample { .build(); // register and generate serializer code. fory.register(TestInvocationHandler.class, true); + fory.ensureSerializersCompiled(); return fory; } diff --git a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json new file mode 100644 index 000000000..549b812dd --- /dev/null +++ b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json @@ -0,0 +1,10 @@ +[ + { + "interfaces": [ + "java.util.function.Function" + ], + "condition": { + "typeReachable": "org.apache.fory.graalvm.ProxyExample$TestInvocationHandler" + } + } +] \ No newline at end of file diff --git a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json deleted file mode 100644 index 305ed56fe..000000000 --- a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "interfaces": [ - "java.util.function.Function" - ] - } -] diff --git a/integration_tests/jdk_compatibility_tests/pom.xml b/integration_tests/jdk_compatibility_tests/pom.xml index 891782a14..d53fc9e6a 100644 --- a/integration_tests/jdk_compatibility_tests/pom.xml +++ b/integration_tests/jdk_compatibility_tests/pom.xml @@ -38,11 +38,26 @@ </properties> <dependencies> + <dependency> + <groupId>org.apache.fory</groupId> + <artifactId>fory-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.fory</groupId> + <artifactId>fory-format</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.apache.fory</groupId> <artifactId>benchmark</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index cf37d8ff7..eefb7212b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -257,6 +257,10 @@ public abstract class CodecBuilder { if (ref == null) { Class<?> funcInterface = methodInfo.f0; TypeRef<?> getterType = TypeRef.of(funcInterface); + if (GraalvmSupport.isGraalBuildtime()) { + // generate getter ahead at native image build time. + Functions.makeGetterFunction(beanClass, fieldName); + } Expression getter = new StaticInvoke( Functions.class, diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index f5135bcb6..a6a751e8c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -2432,7 +2432,8 @@ public interface Expression { action.apply( new Reference(i), new Reference(leftElemValue, leftElemType, true), - // elemValue nullability check uses isNullAt inside action, so elemValueRef's nullable is false. + // elemValue nullability check uses isNullAt inside action, so elemValueRef's nullable + // is false. new Reference(rightElemValue, rightElemType, false)); ExprCode elementExprCode = elemExpr.genCode(ctx); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java index b49ded2e4..9b280cce5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java @@ -239,7 +239,7 @@ public class AllowListChecker implements ClassChecker { lock.writeLock().lock(); listeners.put(classResolver, true); } finally { - lock.writeLock().unlock(); + lock.writeLock().unlock(); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 54895c592..3f2f339cd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -37,6 +37,7 @@ import com.google.common.collect.HashBiMap; import java.io.Externalizable; import java.io.IOException; import java.io.Serializable; +import java.lang.invoke.SerializedLambda; import java.lang.reflect.Field; import java.lang.reflect.Member; import java.lang.reflect.Type; @@ -87,7 +88,8 @@ import org.apache.fory.ForyCopyable; import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.CodecUtils; -import org.apache.fory.builder.Generated; +import org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer; +import org.apache.fory.builder.Generated.GeneratedObjectSerializer; import org.apache.fory.builder.JITContext; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; @@ -163,6 +165,7 @@ import org.apache.fory.type.ScalaTypes; import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; import org.apache.fory.util.GraalvmSupport; +import org.apache.fory.util.GraalvmSupport.GraalvmSerializerHolder; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; import org.apache.fory.util.function.Functions; @@ -420,6 +423,7 @@ public class ClassResolver implements TypeResolver { register(AtomicReference.class); register(EnumSet.allOf(Language.class).getClass()); register(EnumSet.of(Language.JAVA).getClass()); + register(SerializedLambda.class); register(Throwable.class, StackTraceElement.class, Exception.class, RuntimeException.class); register(NullPointerException.class); register(IOException.class); @@ -968,14 +972,14 @@ public class ClassResolver implements TypeResolver { return CollectionSerializers.EnumSetSerializer.class; } else if (Charset.class.isAssignableFrom(cls)) { return Serializers.CharsetSerializer.class; - } else if (Functions.isLambda(cls)) { - return LambdaSerializer.class; } else if (ReflectionUtils.isJdkProxy(cls)) { if (JavaSerializer.getWriteReplaceMethod(cls) != null) { return ReplaceResolveSerializer.class; } else { return JdkProxySerializer.class; } + } else if (Functions.isLambda(cls)) { + return LambdaSerializer.class; } else if (Calendar.class.isAssignableFrom(cls)) { return TimeSerializers.CalendarSerializer.class; } else if (ZoneId.class.isAssignableFrom(cls)) { @@ -1423,7 +1427,7 @@ public class ClassResolver implements TypeResolver { if (deserializationClassInfo != null && GraalvmSupport.isGraalBuildtime()) { getGraalvmClassRegistry() .deserializerClassMap - .put(classDef.getId(), deserializationClassInfo.serializer.getClass()); + .put(classDef.getId(), getGraalvmSerializerClass(deserializationClassInfo.serializer)); Tuple2<ClassDef, ClassInfo> classDefTuple = extRegistry.classIdToDef.get(classDef.getId()); // empty serializer for graalvm build time classDefTuple.f1.serializer = null; @@ -1432,7 +1436,9 @@ public class ClassResolver implements TypeResolver { } if (GraalvmSupport.isGraalBuildtime()) { // Instance for generated class should be hold at graalvm runtime only. - getGraalvmClassRegistry().serializerClassMap.put(cls, classInfo.serializer.getClass()); + getGraalvmClassRegistry() + .serializerClassMap + .put(cls, getGraalvmSerializerClass(classInfo.serializer)); classInfo.serializer = null; } } @@ -1560,13 +1566,21 @@ public class ClassResolver implements TypeResolver { } boolean needToWriteClassDef(Serializer serializer) { - return fory.getConfig().getCompatibleMode() == CompatibleMode.COMPATIBLE - && (serializer instanceof Generated.GeneratedObjectSerializer - // May already switched to MetaSharedSerializer when update class info cache. - || serializer instanceof Generated.GeneratedMetaSharedSerializer - || serializer instanceof LazyInitBeanSerializer - || serializer instanceof ObjectSerializer - || serializer instanceof MetaSharedSerializer); + if (fory.getConfig().getCompatibleMode() != CompatibleMode.COMPATIBLE) { + return false; + } + if (GraalvmSupport.isGraalBuildtime() && serializer instanceof GraalvmSerializerHolder) { + Class<? extends Serializer> serializerClass = + ((GraalvmSerializerHolder) serializer).getSerializerClass(); + return GeneratedObjectSerializer.class.isAssignableFrom(serializerClass) + || GeneratedMetaSharedSerializer.class.isAssignableFrom(serializerClass); + } + return (serializer instanceof GeneratedObjectSerializer + // May already switched to MetaSharedSerializer when update class info cache. + || serializer instanceof GeneratedMetaSharedSerializer + || serializer instanceof LazyInitBeanSerializer + || serializer instanceof ObjectSerializer + || serializer instanceof MetaSharedSerializer); } private ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer, MetaContext metaContext) { @@ -2212,6 +2226,10 @@ public class ClassResolver implements TypeResolver { */ public void ensureSerializersCompiled() { try { + fory.getJITContext().lock(); + Serializers.newSerializer(fory, LambdaSerializer.STUB_LAMBDA_CLASS, LambdaSerializer.class); + Serializers.newSerializer( + fory, JdkProxySerializer.SUBT_PROXY.getClass(), JdkProxySerializer.class); classInfoMap.forEach( (cls, classInfo) -> { if (classInfo.serializer == null) { @@ -2262,6 +2280,13 @@ public class ClassResolver implements TypeResolver { fory.getConfig().getConfigHash(), k -> new GraalvmClassRegistry()); } + private Class<? extends Serializer> getGraalvmSerializerClass(Serializer serializer) { + if (serializer instanceof GraalvmSerializerHolder) { + return ((GraalvmSerializerHolder) serializer).getSerializerClass(); + } + return serializer.getClass(); + } + private Class<? extends Serializer> getSerializerClassFromGraalvmRegistry(Class<?> cls) { GraalvmClassRegistry registry = getGraalvmClassRegistry(); List<ClassResolver> classResolvers = registry.resolvers; @@ -2307,7 +2332,10 @@ public class ClassResolver implements TypeResolver { if (Functions.isLambda(cls) || ReflectionUtils.isJdkProxy(cls)) { return null; } - throw new RuntimeException(String.format("Class %s is not registered", cls)); + throw new RuntimeException( + String.format( + "Class %s is not registered, registered classes: %s", + cls, registry.deserializerClassMap)); } return null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java index ac1dd80c5..9c2bc7418 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java @@ -21,6 +21,7 @@ package org.apache.fory.serializer; import java.lang.reflect.Field; import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; import java.lang.reflect.Proxy; import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; @@ -32,7 +33,6 @@ import org.apache.fory.util.Preconditions; /** Serializer for jdk {@link Proxy}. */ @SuppressWarnings({"rawtypes", "unchecked"}) public class JdkProxySerializer extends Serializer { - // Make offset compatible with graalvm native image. private static final Field FIELD; private static final long PROXY_HANDLER_FIELD_OFFSET; @@ -42,10 +42,22 @@ public class JdkProxySerializer extends Serializer { PROXY_HANDLER_FIELD_OFFSET = Platform.objectFieldOffset(FIELD); } - private static final InvocationHandler STUB_HANDLER = - (proxy, method, args) -> { - throw new IllegalStateException("Deserialization stub handler still active"); - }; + private static class StubInvocationHandler implements InvocationHandler { + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + throw new IllegalStateException("Deserialization stub handler still active"); + } + } + + private static final InvocationHandler STUB_HANDLER = new StubInvocationHandler(); + + private interface StubInterface { + int apply(); + } + + public static Object SUBT_PROXY = + Proxy.newProxyInstance( + Serializer.class.getClassLoader(), new Class[] {StubInterface.class}, STUB_HANDLER); public JdkProxySerializer(Fory fory, Class cls) { super(fory, cls); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java index 2edfafeac..efa80b772 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java @@ -21,13 +21,18 @@ package org.apache.fory.serializer; import java.io.ObjectStreamClass; import java.io.Serializable; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; import java.lang.invoke.SerializedLambda; -import java.lang.reflect.Method; import java.util.Objects; import org.apache.fory.Fory; +import org.apache.fory.collection.ClassValueCache; +import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.util.Preconditions; +import org.apache.fory.util.function.SerializableFunction; +import org.apache.fory.util.unsafe._JDKAccess; /** * Serializer for java serializable lambda. Use fory to serialize java lambda instead of JDK @@ -36,20 +41,34 @@ import org.apache.fory.util.Preconditions; */ @SuppressWarnings({"unchecked", "rawtypes"}) public class LambdaSerializer extends Serializer { + public static Class<?> STUB_LAMBDA_CLASS = + ((SerializableFunction<Integer, Integer>) (x -> x * 2)).getClass(); private static final Class<SerializedLambda> SERIALIZED_LAMBDA = SerializedLambda.class; - private static final Method READ_RESOLVE_METHOD = - (Method) - Objects.requireNonNull( - ReflectionUtils.getObjectFieldValue( - ObjectStreamClass.lookup(SERIALIZED_LAMBDA), "readResolveMethod")); + private static final MethodHandle READ_RESOLVE_HANDLE; private static final boolean SERIALIZED_LAMBDA_HAS_JDK_WRITE = JavaSerializer.getWriteObjectMethod(SERIALIZED_LAMBDA) != null; private static final boolean SERIALIZED_LAMBDA_HAS_JDK_READ = JavaSerializer.getReadObjectMethod(SERIALIZED_LAMBDA) != null; - private final Method writeReplaceMethod; + private static final ClassValueCache<MethodHandle> writeReplaceMethodCache = + ClassValueCache.newClassKeySoftCache(32); + + private final MethodHandle writeReplaceHandle; private Serializer dataSerializer; - public LambdaSerializer(Fory fory, Class cls) { + static { + try { + // Initialize READ_RESOLVE_HANDLE + MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(SERIALIZED_LAMBDA); + Object readResolveMethod = + ReflectionUtils.getObjectFieldValue( + ObjectStreamClass.lookup(SERIALIZED_LAMBDA), "readResolveMethod"); + READ_RESOLVE_HANDLE = lookup.unreflect((java.lang.reflect.Method) readResolveMethod); + } catch (IllegalAccessException e) { + throw new ForyException(e); + } + } + + public LambdaSerializer(Fory fory, Class<?> cls) { super(fory, cls); if (cls != ReplaceStub.class) { if (!Serializable.class.isAssignableFrom(cls)) { @@ -59,12 +78,28 @@ public class LambdaSerializer extends Serializer { cls, Serializable.class.getName()); throw new UnsupportedOperationException(msg); } - writeReplaceMethod = - (Method) + MethodHandle methodHandle = writeReplaceMethodCache.getIfPresent(cls); + if (methodHandle == null) { + try { + MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(cls); + Object writeReplaceMethod = ReflectionUtils.getObjectFieldValue( ObjectStreamClass.lookup(cls), "writeReplaceMethod"); + methodHandle = + lookup.unreflect( + (java.lang.reflect.Method) Objects.requireNonNull(writeReplaceMethod)); + writeReplaceMethodCache.put(cls, methodHandle); + } catch (Throwable e) { + throw new RuntimeException( + String.format("Failed to create writeReplace MethodHandle for %s", cls), e); + } + } + writeReplaceHandle = methodHandle; } else { - writeReplaceMethod = null; + writeReplaceHandle = null; + } + if (cls == STUB_LAMBDA_CLASS) { + getDataSerializer(); } } @@ -72,10 +107,10 @@ public class LambdaSerializer extends Serializer { public void write(MemoryBuffer buffer, Object value) { assert value.getClass() != ReplaceStub.class; try { - Object replacement = writeReplaceMethod.invoke(value); + Object replacement = writeReplaceHandle.invoke(value); Preconditions.checkArgument(SERIALIZED_LAMBDA.isInstance(replacement)); getDataSerializer().write(buffer, replacement); - } catch (Exception e) { + } catch (Throwable e) { throw new RuntimeException("Can't serialize lambda " + value, e); } } @@ -83,10 +118,10 @@ public class LambdaSerializer extends Serializer { @Override public Object copy(Object value) { try { - Object replacement = writeReplaceMethod.invoke(value); + Object replacement = writeReplaceHandle.invoke(value); Object newReplacement = getDataSerializer().copy(replacement); - return READ_RESOLVE_METHOD.invoke(newReplacement); - } catch (Exception e) { + return READ_RESOLVE_HANDLE.invoke(newReplacement); + } catch (Throwable e) { throw new RuntimeException("Can't copy lambda " + value, e); } } @@ -95,8 +130,8 @@ public class LambdaSerializer extends Serializer { public Object read(MemoryBuffer buffer) { try { Object replacement = getDataSerializer().read(buffer); - return READ_RESOLVE_METHOD.invoke(replacement); - } catch (Exception e) { + return READ_RESOLVE_HANDLE.invoke(replacement); + } catch (Throwable e) { throw new RuntimeException("Can't deserialize lambda", e); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index b976046fc..ef672d9b7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -61,6 +61,7 @@ import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.FieldResolver; import org.apache.fory.resolver.FieldResolver.ClassField; import org.apache.fory.util.ExceptionUtils; +import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.Preconditions; import org.apache.fory.util.unsafe._JDKAccess; @@ -354,7 +355,8 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer { this.slotsSerializer = (CompatibleSerializerBase) Serializers.newSerializer(fory, type, c)); } - if (sc == CompatibleSerializer.class) { + if (sc == CompatibleSerializer.class || GraalvmSupport.isGraalBuildtime()) { + // skip init generated serializer at graalvm build time this.slotsSerializer = new CompatibleSerializer(fory, type, fieldResolver); } else { this.slotsSerializer = (CompatibleSerializerBase) Serializers.newSerializer(fory, type, sc); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index ce936d740..c1719b64b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -43,6 +43,7 @@ import java.util.function.Function; import java.util.function.ToIntFunction; import java.util.regex.Pattern; import org.apache.fory.Fory; +import org.apache.fory.builder.Generated; import org.apache.fory.collection.Tuple2; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; @@ -50,6 +51,7 @@ import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.GraalvmSupport; +import org.apache.fory.util.GraalvmSupport.GraalvmSerializerHolder; import org.apache.fory.util.StringUtils; import org.apache.fory.util.unsafe._JDKAccess; @@ -88,6 +90,11 @@ public class Serializers { } Tuple2<MethodType, MethodHandle> ctrInfo = CTR_MAP.getIfPresent(serializerClass); if (ctrInfo != null) { + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } MethodType sig = ctrInfo.f0; MethodHandle handle = ctrInfo.f1; if (sig.equals(SIG1)) { @@ -125,6 +132,11 @@ public class Serializers { try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG1); CTR_MAP.put(serializerClass, Tuple2.of(SIG1, ctr)); + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } return (Serializer<T>) ctr.invoke(fory, type); } catch (NoSuchMethodException e) { ExceptionUtils.ignore(e); @@ -132,6 +144,11 @@ public class Serializers { try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG2); CTR_MAP.put(serializerClass, Tuple2.of(SIG2, ctr)); + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } return (Serializer<T>) ctr.invoke(fory); } catch (NoSuchMethodException e) { ExceptionUtils.ignore(e); @@ -139,10 +156,20 @@ public class Serializers { try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG3); CTR_MAP.put(serializerClass, Tuple2.of(SIG3, ctr)); + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } return (Serializer<T>) ctr.invoke(type); } catch (NoSuchMethodException e) { MethodHandle ctr = ReflectionUtils.getCtrHandle(serializerClass); CTR_MAP.put(serializerClass, Tuple2.of(SIG4, ctr)); + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } return (Serializer<T>) ctr.invoke(); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index afee63b3d..dde3fca6a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -657,8 +657,8 @@ public class TypeUtils { } /** - * Check if a class is one of {@link Optional), {@link OptionalInt}, - * {@link OptionaLong}, or {@link OptionalDouble}. + * Check if a class is one of {@link Optional}, {@link OptionalInt}, {@link OptionalLong}, or + * {@link OptionalDouble}. */ public static boolean isOptionalType(Class<?> type) { return type == Optional.class diff --git a/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java b/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java index 55d023981..edf00db99 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java @@ -19,6 +19,10 @@ package org.apache.fory.util; +import java.util.Objects; +import org.apache.fory.Fory; +import org.apache.fory.serializer.Serializer; + /** A helper for Graalvm native image support. */ public class GraalvmSupport { // https://github.com/oracle/graal/blob/master/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/ImageInfo.java @@ -46,4 +50,17 @@ public class GraalvmSupport { return IN_GRAALVM_NATIVE_IMAGE && GRAAL_IMAGE_RUNTIME.equals(System.getProperty(GRAAL_IMAGE_CODE_KEY)); } + + public static class GraalvmSerializerHolder extends Serializer { + private final Class serializerClass; + + public GraalvmSerializerHolder(Fory fory, Class<?> type, Class<?> serializerClass) { + super(fory, type); + this.serializerClass = Objects.requireNonNull(serializerClass); + } + + public Class<? extends Serializer> getSerializerClass() { + return serializerClass; + } + } } diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 50f599329..c9dc1b45d 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -226,6 +226,8 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.logging.LoggerFactory,\ org.apache.fory.memory.BoundsChecking,\ org.apache.fory.memory.MemoryBuffer,\ + org.apache.fory.memory.MemoryAllocator,\ + org.apache.fory.memory.MemoryBuffer$DefaultMemoryAllocator,\ org.apache.fory.memory.MemoryUtils,\ org.apache.fory.memory.Platform,\ org.apache.fory.meta.ClassDef$ArrayFieldType,\ @@ -308,6 +310,8 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.JavaSerializer$4,\ org.apache.fory.serializer.JavaSerializer,\ org.apache.fory.serializer.JdkProxySerializer$ReplaceStub,\ + org.apache.fory.serializer.JdkProxySerializer$StubInterface,\ + org.apache.fory.serializer.JdkProxySerializer$StubInvocationHandler,\ org.apache.fory.serializer.JdkProxySerializer,\ org.apache.fory.serializer.LambdaSerializer$ReplaceStub,\ org.apache.fory.serializer.LambdaSerializer,\ @@ -433,6 +437,11 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.LazySerializer$LazyObjectSerializer,\ org.apache.fory.serializer.shim.ShimDispatcher,\ org.apache.fory.serializer.shim.ProtobufDispatcher,\ + org.apache.fory.serializer.ObjectStreamSerializer,\ + org.apache.fory.serializer.ObjectStreamSerializer$1,\ + org.apache.fory.serializer.ObjectStreamSerializer$StreamClassInfo,\ + org.apache.fory.serializer.collection.ChildContainerSerializers$ChildCollectionSerializer,\ + org.apache.fory.serializer.collection.ChildContainerSerializers$ChildMapSerializer,\ org.apache.fory.shaded.org.codehaus.janino.IClass$1,\ org.apache.fory.type.Descriptor$1,\ org.apache.fory.type.Descriptor,\ diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json new file mode 100644 index 000000000..505934bf9 --- /dev/null +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json @@ -0,0 +1,9 @@ +[ + { + "name": "java.lang.reflect.Proxy", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true + } +] diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json new file mode 100644 index 000000000..154ba0da2 --- /dev/null +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json @@ -0,0 +1,11 @@ +[ + { + "name": "java.lang.reflect.Proxy", + "fields": [], + "methods": [ + { "name": "<init>" }, + { "name": "writeReplace" } + ], + "customTargetConstructorClass": "java.lang.Object" + } +] diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java index 49ec2e134..f24da862d 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java @@ -27,9 +27,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.OptionalDouble; -import java.util.OptionalInt; -import java.util.OptionalLong; import java.util.SortedMap; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java index b942d5dc0..a31b11799 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java @@ -26,7 +26,6 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.TreeSet; - import lombok.Data; import org.apache.arrow.vector.types.pojo.Field; import org.apache.fory.annotation.ForyField; diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 35ab78d36..6bc87ba1a 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -133,9 +133,7 @@ class Fory: """ self.language = language self.is_py = language == Language.PYTHON - self.require_type_registration = ( - _ENABLE_TYPE_REGISTRATION_FORCIBLY or require_type_registration - ) + self.require_type_registration = _ENABLE_TYPE_REGISTRATION_FORCIBLY or require_type_registration self.ref_tracking = ref_tracking if self.ref_tracking: self.ref_resolver = MapRefResolver() @@ -151,8 +149,7 @@ class Fory: self.buffer = Buffer.allocate(32) if not require_type_registration: warnings.warn( - "Type registration is disabled, unknown types can be deserialized " - "which may be insecure.", + "Type registration is disabled, unknown types can be deserialized which may be insecure.", RuntimeWarning, stacklevel=2, ) @@ -166,7 +163,7 @@ class Fory: self._unsupported_callback = None self._unsupported_objects = None self._peer_language = None - + def register( self, cls: Union[type, TypeVar], @@ -345,7 +342,7 @@ class Fory: buffers: Iterable = None, unsupported_objects: Iterable = None, ): - if type(buffer) == bytes: + if isinstance(buffer, bytes): buffer = Buffer(buffer) if unsupported_objects is not None: self._unsupported_objects = iter(unsupported_objects) @@ -360,10 +357,7 @@ class Fory: if get_bit(buffer, reader_index, 0): return None is_little_endian_ = get_bit(buffer, reader_index, 1) - assert is_little_endian_, ( - "Big endian is not supported for now, " - "please ensure peer machine is little endian." - ) + assert is_little_endian_, "Big endian is not supported for now, please ensure peer machine is little endian." is_target_x_lang = get_bit(buffer, reader_index, 2) if is_target_x_lang: self._peer_language = Language(buffer.read_int8()) @@ -371,16 +365,10 @@ class Fory: self._peer_language = Language.PYTHON is_out_of_band_serialization_enabled = get_bit(buffer, reader_index, 3) if is_out_of_band_serialization_enabled: - assert buffers is not None, ( - "buffers shouldn't be null when the serialized stream is " - "produced with buffer_callback not null." - ) + assert buffers is not None, "buffers shouldn't be null when the serialized stream is produced with buffer_callback not null." self._buffers = iter(buffers) else: - assert buffers is None, ( - "buffers should be null when the serialized stream is " - "produced with buffer_callback null." - ) + assert buffers is None, "buffers should be null when the serialized stream is produced with buffer_callback null." if is_target_x_lang: obj = self.xdeserialize_ref(buffer) else: @@ -532,9 +520,7 @@ class SerializationContext: self.objects.clear() -_ENABLE_TYPE_REGISTRATION_FORCIBLY = os.getenv( - "ENABLE_TYPE_REGISTRATION_FORCIBLY", "0" -) in { +_ENABLE_TYPE_REGISTRATION_FORCIBLY = os.getenv("ENABLE_TYPE_REGISTRATION_FORCIBLY", "0") in { "1", "true", } @@ -544,8 +530,8 @@ class _PicklerStub: def dump(self, o): raise ValueError( f"Type {type(o)} is not registered, " - f"pickle is not allowed when type registration enabled, Please register" - f"the type or pass unsupported_callback" + f"pickle is not allowed when type registration enabled, " + f"Please register the type or pass unsupported_callback" ) def clear_memo(self): @@ -554,7 +540,4 @@ class _PicklerStub: class _UnpicklerStub: def load(self): - raise ValueError( - "pickle is not allowed when type registration enabled, Please register" - "the type or pass unsupported_callback" - ) + raise ValueError("pickle is not allowed when type registration enabled, Please register the type or pass unsupported_callback") diff --git a/python/pyfory/_registry.py b/python/pyfory/_registry.py index 1fe214366..f72baac63 100644 --- a/python/pyfory/_registry.py +++ b/python/pyfory/_registry.py @@ -376,6 +376,9 @@ class TypeResolver: serializer: Serializer = None, internal: bool = False, ): + # Set default type_id when None, similar to _register_xtype + if type_id is None and typename is not None: + type_id = self._next_type_id() return self.__register_type( cls, type_id=type_id, @@ -395,7 +398,7 @@ class TypeResolver: serializer: Serializer = None, internal: bool = False, ): - dynamic_type = type_id < 0 + dynamic_type = type_id is not None and type_id < 0 if not internal and serializer is None: serializer = self._create_serializer(cls) if typename is None: @@ -468,6 +471,8 @@ class TypeResolver: type_id = TypeId.NAMED_ENUM elif type(serializer) is PickleSerializer: type_id = PickleSerializer.PICKLE_TYPE_ID + elif isinstance(serializer, FunctionSerializer): + type_id = TypeId.NAMED_EXT elif isinstance(serializer, (ObjectSerializer, StatefulSerializer, ReduceSerializer)): type_id = TypeId.NAMED_EXT if not self.require_registration: @@ -491,8 +496,8 @@ class TypeResolver: break else: if cls is types.FunctionType: - # Use PickleSerializer for function types (including lambdas) - serializer = PickleSerializer(self.fory, cls) + # Use FunctionSerializer for function types (including lambdas) + serializer = FunctionSerializer(self.fory, cls) elif dataclasses.is_dataclass(cls): serializer = DataClassSerializer(self.fory, cls) elif issubclass(cls, enum.Enum): diff --git a/python/pyfory/tests/test_function.py b/python/pyfory/tests/test_function.py index ad2e03c06..738ef69d6 100644 --- a/python/pyfory/tests/test_function.py +++ b/python/pyfory/tests/test_function.py @@ -26,7 +26,7 @@ def test_lambda_functions_serialization(): # Register the necessary types fory.register_type(tuple) fory.register_type(list) - fory.register_type(dict) + # dict is already registered by default with MapSerializer # Simple lambda simple_lambda = lambda x: x * 2 # noqa: E731 @@ -64,7 +64,7 @@ def test_regular_functions_serialization(): # Register the necessary types for complex functions fory.register_type(tuple) fory.register_type(list) - fory.register_type(dict) + # dict is already registered by default with MapSerializer # Test complex function serialized = fory.serialize(complex_function) @@ -79,7 +79,7 @@ def test_nested_functions_serialization(): # Register the necessary types fory.register_type(tuple) fory.register_type(list) - fory.register_type(dict) + # dict is already registered by default with MapSerializer def outer_function(x): def inner_function(y): @@ -104,7 +104,7 @@ def test_local_class_serialization(): # Register the necessary types fory.register_type(tuple) fory.register_type(list) - fory.register_type(dict) + # dict is already registered by default with MapSerializer def create_local_class(): from dataclasses import dataclass diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 3f6cbcb49..6df8875b4 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -469,6 +469,8 @@ def test_pickle_fallback(): def test_unsupported_callback(): fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False) + # Test with functions that now have proper serialization support + # Functions should no longer be treated as unsupported def f1(x): return x @@ -478,10 +480,18 @@ def test_unsupported_callback(): obj1 = [1, True, f1, f2, {1: 2}] unsupported_objects = [] binary1 = fory.serialize(obj1, unsupported_callback=unsupported_objects.append) - assert len(unsupported_objects) == 2 - assert unsupported_objects == [f1, f2] + # Functions are now properly supported, so unsupported_objects should be empty + assert len(unsupported_objects) == 0 new_obj1 = fory.deserialize(binary1, unsupported_objects=unsupported_objects) - assert new_obj1 == obj1 + # Functions should roundtrip correctly + assert len(new_obj1) == len(obj1) + assert new_obj1[0] == obj1[0] # 1 + assert new_obj1[1] == obj1[1] # True + assert new_obj1[2](5) == f1(5) # Test f1 functionality + assert new_obj1[3](5) == f2(5) # Test f2 functionality + assert new_obj1[4] == obj1[4] # {1: 2} + # Don't check full equality since functions are new objects after deserialization + # The functionality test above already confirmed they work correctly def test_slice(): diff --git a/python/setup.py b/python/setup.py index 87123efa8..d31f5544f 100644 --- a/python/setup.py +++ b/python/setup.py @@ -37,8 +37,10 @@ project_dir = abspath(pjoin(setup_dir, os.pardir)) fory_cpp_src_dir = abspath(pjoin(setup_dir, "../src/")) print(f"setup_dir: {setup_dir}") +print(f"project_dir: {project_dir}") print(f"fory_cpp_src_dir: {fory_cpp_src_dir}") + class BinaryDistribution(Distribution): def __init__(self, attrs=None): super().__init__(attrs=attrs) @@ -50,7 +52,9 @@ class BinaryDistribution(Distribution): elif arch in ("aarch64", "arm64"): bazel_args += ["--copt=-fsigned-char"] bazel_args += ["//:cp_fory_so"] - subprocess.check_call(bazel_args) + # Ensure Windows path compatibility + cwd_path = os.path.normpath(project_dir) + subprocess.check_call(bazel_args, cwd=cwd_path) def has_ext_modules(self): return True --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
