This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new bb430b03e fix(ci): Exit with subprocess return code in run_ci.py
(#2560)
bb430b03e is described below
commit bb430b03e58dbd339a49e78d4bd3bd4d49e11f5c
Author: Emre Şafak <[email protected]>
AuthorDate: Tue Sep 2 10:58:53 2025 -0400
fix(ci): Exit with subprocess return code in run_ci.py (#2560)
## Why?
CI was not reflecting failures in shell script fallbacks (we are
migrating to python)
## What does this PR do?
* Use sys.exit() to propagate the return code from subprocess.call().
* This ensures that the CI pipeline correctly reflects the success or
failure of the executed scripts.
## Related issues
#2555
## Notes
You might want to merge #2561 first, to reduce the number of errors.
---------
Co-authored-by: chaokunyang <[email protected]>
---
.github/pull_request_template.md | 1 +
.github/workflows/ci.yml | 17 ++++--
ci/run_ci.py | 7 +--
ci/run_ci.sh | 4 +-
ci/tasks/java.py | 61 +++++++++++--------
integration_tests/graalvm_tests/pom.xml | 1 -
.../apache/fory/graalvm/ObjectStreamExample.java | 40 ++++++------
.../java/org/apache/fory/graalvm/ProxyExample.java | 1 +
.../graalvm_tests/proxy-config.json | 10 +++
.../META-INF/native-image/proxy-config.json | 7 ---
integration_tests/jdk_compatibility_tests/pom.xml | 15 +++++
.../java/org/apache/fory/builder/CodecBuilder.java | 4 ++
.../java/org/apache/fory/codegen/Expression.java | 3 +-
.../org/apache/fory/resolver/AllowListChecker.java | 2 +-
.../org/apache/fory/resolver/ClassResolver.java | 54 ++++++++++++----
.../apache/fory/serializer/JdkProxySerializer.java | 22 +++++--
.../apache/fory/serializer/LambdaSerializer.java | 71 ++++++++++++++++------
.../fory/serializer/ObjectStreamSerializer.java | 4 +-
.../org/apache/fory/serializer/Serializers.java | 27 ++++++++
.../main/java/org/apache/fory/type/TypeUtils.java | 4 +-
.../java/org/apache/fory/util/GraalvmSupport.java | 17 ++++++
.../fory-core/native-image.properties | 9 +++
.../fory-core/reflection-config.json | 9 +++
.../fory-core/serialization-config.json | 11 ++++
.../fory/format/encoder/RowEncoderBuilder.java | 3 -
.../format/encoder/ImplementInterfaceTest.java | 1 -
python/pyfory/_fory.py | 39 ++++--------
python/pyfory/_registry.py | 11 +++-
python/pyfory/tests/test_function.py | 8 +--
python/pyfory/tests/test_serializer.py | 16 ++++-
python/setup.py | 6 +-
31 files changed, 337 insertions(+), 148 deletions(-)
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 2b4a80c57..10c620545 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -9,6 +9,7 @@ Contribution Checklist
- Fory has a strong focus on performance. If the PR you submit will have
an impact on performance, please benchmark it first and provide the benchmark
result here.
-->
+
## Why?
<!-- Describe the purpose of this PR. -->
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b6f47cf79..eeb6ea48a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -62,6 +62,8 @@ jobs:
run: python ./ci/run_ci.py cpp --install-deps-only
- name: Install python dependencies
run: pip install pyarrow==15.0.0 Cython wheel pytest setuptools -U
+ - name: Install pyfory for xlang tests
+ run: pip install -e python/
- name: Run CI with Maven
run: python ./ci/run_ci.py java --version ${{ matrix.java-version }}
- name: Upload Test Report
@@ -93,6 +95,8 @@ jobs:
python-version: 3.8
- name: Install bazel
run: python ./ci/run_ci.py cpp --install-deps-only
+ - name: Install pyfory for xlang tests
+ run: pip install -e python/
- name: Install python dependencies
run: pip install pyarrow==15.0.0 Cython wheel pytest setuptools -U
- name: Run CI with Maven
@@ -190,10 +194,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- - name: Set up JDK8
+ - name: Set up JDK11
uses: actions/setup-java@v4
with:
- java-version: 8
+ java-version: 11
distribution: "temurin"
- name: Set up Python 3.8
uses: actions/setup-python@v5
@@ -274,9 +278,14 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- - name: Install bazel
+ - name: Install bazel (Unix)
+ if: runner.os != 'Windows'
shell: bash
run: python ./ci/run_ci.py cpp --install-deps-only
+ - name: Install bazel (Windows)
+ if: runner.os == 'Windows'
+ shell: bash
+ run: ./ci/run_ci.sh install_bazel_windows
- name: Run Python CI
shell: bash
run: python ./ci/run_ci.py python
@@ -303,7 +312,7 @@ jobs:
run: python ./ci/run_ci.py cpp --install-deps-only
- name: Install python dependencies
run: pip install pyarrow==15.0.0 cython wheel pytest setuptools -U
- - name: Install pyfory
+ - name: Install pyfory for xlang tests
run: pip install -e python/
- name: Run Golang CI
run: python ./ci/run_ci.py go
diff --git a/ci/run_ci.py b/ci/run_ci.py
index e637cc94f..ed1a3f59b 100644
--- a/ci/run_ci.py
+++ b/ci/run_ci.py
@@ -84,7 +84,7 @@ def run_shell_script(command, *args):
cmd = [bash_path, script_path, command]
cmd.extend(args)
logging.info(f"Falling back to shell script with bash: {'
'.join(cmd)}")
- return subprocess.call(cmd)
+ sys.exit(subprocess.call(cmd))
else:
logging.error(
"Bash is not available on this Windows system. Cannot run
shell script."
@@ -101,7 +101,7 @@ def run_shell_script(command, *args):
cmd = [script_path, command]
cmd.extend(args)
logging.info(f"Falling back to shell script: {' '.join(cmd)}")
- return subprocess.call(cmd)
+ sys.exit(subprocess.call(cmd))
def parse_args():
@@ -237,14 +237,13 @@ def parse_args():
if USE_PYTHON_JAVA:
func(**arg_dict)
else:
-
if not arg_dict.get("version"):
func(**arg_dict)
return
# Map Python version argument to shell script command
version = arg_dict.get("version", "17")
release = arg_dict.get("release", False)
-
+
if release:
logging.info("Release mode requested - using Python
implementation")
func(**arg_dict)
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
index 63a1330d1..789f78cda 100755
--- a/ci/run_ci.sh
+++ b/ci/run_ci.sh
@@ -141,7 +141,7 @@ install_jdks() {
graalvm_test() {
cd "$ROOT"/java
- mvn -T10 -B --no-transfer-progress clean install -DskipTests
+ mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl
'!:fory-format,!:fory-testsuite'
echo "Start to build graalvm native image"
cd "$ROOT"/integration_tests/graalvm_tests
mvn -DskipTests=true --no-transfer-progress -Pnative package
@@ -231,7 +231,7 @@ case $1 in
echo "Executing fory java tests"
cd "$ROOT/java"
set +e
- mvn -T16 --batch-mode --no-transfer-progress test
+ mvn -T16 --batch-mode --no-transfer-progress test -pl
'!:fory-format,!:fory-testsuite'
testcode=$?
if [[ $testcode -ne 0 ]]; then
exit $testcode
diff --git a/ci/tasks/java.py b/ci/tasks/java.py
index 7bb73e50a..6fa0754ed 100644
--- a/ci/tasks/java.py
+++ b/ci/tasks/java.py
@@ -17,7 +17,6 @@
import logging
import os
-import sys
import subprocess
import re
from . import common
@@ -26,7 +25,7 @@ from . import common
def get_jdk_major_version():
try:
# Run the 'java -version' command
- result = subprocess.run(['java', '-version'], capture_output=True,
text=True)
+ result = subprocess.run(["java", "-version"], capture_output=True,
text=True)
output = result.stderr # java -version outputs to stderr
# Use regex to find the version string
@@ -37,8 +36,8 @@ def get_jdk_major_version():
version_string = match.group(1)
# Parse the version string
- version_parts = version_string.split('.')
- if version_parts[0] == '1':
+ version_parts = version_string.split(".")
+ if version_parts[0] == "1":
# Java 8 or earlier
return int(version_parts[1])
else:
@@ -81,54 +80,56 @@ def create_toolchains_xml(jdk_mappings):
import os
import xml.etree.ElementTree as ET
from xml.dom import minidom
-
+
# Create ~/.m2 directory if it doesn't exist
m2_dir = os.path.expanduser("~/.m2")
os.makedirs(m2_dir, exist_ok=True)
-
+
# Create the root element
toolchains = ET.Element("toolchains")
-
+
for version, jdk_name in jdk_mappings.items():
toolchain = ET.SubElement(toolchains, "toolchain")
-
+
# Set type
type_elem = ET.SubElement(toolchain, "type")
type_elem.text = "jdk"
-
+
# Set provides
provides = ET.SubElement(toolchain, "provides")
version_elem = ET.SubElement(provides, "version")
version_elem.text = version
vendor_elem = ET.SubElement(provides, "vendor")
vendor_elem.text = "azul"
-
+
# Set configuration
configuration = ET.SubElement(toolchain, "configuration")
jdk_home = ET.SubElement(configuration, "jdkHome")
jdk_home.text = os.path.abspath(os.path.join(common.PROJECT_ROOT_DIR,
jdk_name))
-
+
# Create pretty XML string
- rough_string = ET.tostring(toolchains, 'unicode')
+ rough_string = ET.tostring(toolchains, "unicode")
reparsed = minidom.parseString(rough_string)
pretty_xml = reparsed.toprettyxml(indent=" ")
-
+
# Add proper XML header with encoding
xml_header = '<?xml version="1.0" encoding="UTF8"?>\n'
- pretty_xml = xml_header + pretty_xml.split('\n', 1)[1] # Remove the
default header and add our custom one
-
+ pretty_xml = (
+ xml_header + pretty_xml.split("\n", 1)[1]
+ ) # Remove the default header and add our custom one
+
# Write to ~/.m2/toolchains.xml
toolchains_path = os.path.join(m2_dir, "toolchains.xml")
- with open(toolchains_path, 'w', encoding='utf-8') as f:
+ with open(toolchains_path, "w", encoding="utf-8") as f:
f.write(pretty_xml)
-
+
logging.info(f"Created toolchains.xml at {toolchains_path}")
logging.info("Toolchains configuration:")
for version, jdk_name in jdk_mappings.items():
jdk_path = os.path.join(common.PROJECT_ROOT_DIR, jdk_name)
logging.info(f" JDK {version}: {jdk_path}")
# print toolchains.xml
- with open(toolchains_path, 'r', encoding='utf-8') as f:
+ with open(toolchains_path, "r", encoding="utf-8") as f:
logging.info(f.read())
@@ -149,7 +150,9 @@ def run_java8():
logging.info("Executing fory java tests with Java 8")
install_jdks()
common.cd_project_subdir("java")
- common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test -pl
'!fory-format'")
+ common.exec_cmd(
+ "mvn -T16 --batch-mode --no-transfer-progress test -pl
'!:fory-format,!:fory-testsuite'"
+ )
logging.info("Executing fory java tests succeeds")
@@ -208,7 +211,9 @@ def run_integration_tests():
logging.info("Executing fory integration tests")
common.cd_project_subdir("java")
- common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install
-DskipTests")
+ common.exec_cmd(
+ "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl
'!:fory-format,!:fory-testsuite'"
+ )
logging.info("benchmark tests")
common.cd_project_subdir("java/benchmark")
@@ -270,7 +275,9 @@ def run_graalvm_test():
logging.info("Start GraalVM tests")
common.cd_project_subdir("java")
- common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install
-DskipTests")
+ common.exec_cmd(
+ "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl
'!:fory-format,!:fory-testsuite'"
+ )
logging.info("Start to build graalvm native image")
common.cd_project_subdir("integration_tests/graalvm_tests")
@@ -285,17 +292,19 @@ def run_graalvm_test():
def run_release():
"""Release to Maven Central."""
- logging.info(f"Starting release to Maven Central with Java")
+ logging.info("Starting release to Maven Central with Java")
common.cd_project_subdir("java")
-
+
# Clean and install without tests first
logging.info("Cleaning and installing dependencies")
common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install
-DskipTests")
-
+
# Deploy to Maven Central
logging.info("Deploying to Maven Central")
- common.exec_cmd("mvn -T10 -B --no-transfer-progress clean deploy
-Dgpg.skip -DskipTests -Papache-release")
-
+ common.exec_cmd(
+ "mvn -T10 -B --no-transfer-progress clean deploy -Dgpg.skip
-DskipTests -Papache-release"
+ )
+
logging.info("Release to Maven Central completed successfully")
diff --git a/integration_tests/graalvm_tests/pom.xml
b/integration_tests/graalvm_tests/pom.xml
index 84da032e4..94a0d83d3 100644
--- a/integration_tests/graalvm_tests/pom.xml
+++ b/integration_tests/graalvm_tests/pom.xml
@@ -173,7 +173,6 @@
<!-- for fast build -->
<!-- <buildArg>-Ob</buildArg> -->
<buildArg>-H:+UnlockExperimentalVMOptions</buildArg>
-
<buildArg>-H:DynamicProxyConfigurationFiles=src/main/resources/META-INF/native-image/proxy-config.json</buildArg>
</buildArgs>
</configuration>
</plugin>
diff --git
a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java
b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java
index a73c90bdf..227ce126d 100644
---
a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java
+++
b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java
@@ -17,42 +17,42 @@
* under the License.
*/
- package org.apache.fory.graalvm;
-
-import org.apache.fory.Fory;
+package org.apache.fory.graalvm;
import java.util.AbstractMap;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import org.apache.fory.Fory;
public class ObjectStreamExample extends AbstractMap<Integer, Integer> {
- private static final Fory FORY = Fory.builder()
- .withName(ObjectStreamExample.class.getName())
- .registerGuavaTypes(false)
- .build();
+ private static final Fory FORY =
+ Fory.builder()
+ .withName(ObjectStreamExample.class.getName())
+ .registerGuavaTypes(false)
+ .build();
static {
- FORY.register(ObjectStreamExample.class, true);
- FORY.ensureSerializersCompiled();
+ FORY.register(ObjectStreamExample.class, true);
+ FORY.ensureSerializersCompiled();
}
final int[] ints = new int[10];
public static void main(String[] args) {
- FORY.reset();
- byte[] bytes = FORY.serialize(new ObjectStreamExample());
- FORY.reset();
- ObjectStreamExample o = (ObjectStreamExample) FORY.deserialize(bytes);
- System.out.println(Arrays.toString(o.ints));
+ FORY.reset();
+ byte[] bytes = FORY.serialize(new ObjectStreamExample());
+ FORY.reset();
+ ObjectStreamExample o = (ObjectStreamExample) FORY.deserialize(bytes);
+ System.out.println(Arrays.toString(o.ints));
}
@Override
public Set<Entry<Integer, Integer>> entrySet() {
- HashSet<Entry<Integer, Integer>> set = new HashSet<>();
- for (int i = 0; i < ints.length; i++) {
- set.add(new AbstractMap.SimpleEntry<>(i, ints[i]));
- }
- return set;
+ HashSet<Entry<Integer, Integer>> set = new HashSet<>();
+ for (int i = 0; i < ints.length; i++) {
+ set.add(new AbstractMap.SimpleEntry<>(i, ints[i]));
+ }
+ return set;
}
-}
\ No newline at end of file
+}
diff --git
a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java
b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java
index 990faaa2b..2fd195675 100644
---
a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java
+++
b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java
@@ -50,6 +50,7 @@ public class ProxyExample {
.build();
// register and generate serializer code.
fory.register(TestInvocationHandler.class, true);
+ fory.ensureSerializersCompiled();
return fory;
}
diff --git
a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json
b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json
new file mode 100644
index 000000000..549b812dd
--- /dev/null
+++
b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json
@@ -0,0 +1,10 @@
+[
+ {
+ "interfaces": [
+ "java.util.function.Function"
+ ],
+ "condition": {
+ "typeReachable":
"org.apache.fory.graalvm.ProxyExample$TestInvocationHandler"
+ }
+ }
+]
\ No newline at end of file
diff --git
a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json
b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json
deleted file mode 100644
index 305ed56fe..000000000
---
a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json
+++ /dev/null
@@ -1,7 +0,0 @@
-[
- {
- "interfaces": [
- "java.util.function.Function"
- ]
- }
-]
diff --git a/integration_tests/jdk_compatibility_tests/pom.xml
b/integration_tests/jdk_compatibility_tests/pom.xml
index a2eb0f6b8..2464b080e 100644
--- a/integration_tests/jdk_compatibility_tests/pom.xml
+++ b/integration_tests/jdk_compatibility_tests/pom.xml
@@ -38,11 +38,26 @@
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.fory</groupId>
+ <artifactId>fory-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.fory</groupId>
+ <artifactId>fory-format</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.fory</groupId>
<artifactId>benchmark</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java
b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java
index cf37d8ff7..eefb7212b 100644
--- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java
+++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java
@@ -257,6 +257,10 @@ public abstract class CodecBuilder {
if (ref == null) {
Class<?> funcInterface = methodInfo.f0;
TypeRef<?> getterType = TypeRef.of(funcInterface);
+ if (GraalvmSupport.isGraalBuildtime()) {
+ // generate getter ahead at native image build time.
+ Functions.makeGetterFunction(beanClass, fieldName);
+ }
Expression getter =
new StaticInvoke(
Functions.class,
diff --git
a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java
b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java
index f5135bcb6..a6a751e8c 100644
--- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java
+++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java
@@ -2432,7 +2432,8 @@ public interface Expression {
action.apply(
new Reference(i),
new Reference(leftElemValue, leftElemType, true),
- // elemValue nullability check uses isNullAt inside action, so
elemValueRef's nullable is false.
+ // elemValue nullability check uses isNullAt inside action, so
elemValueRef's nullable
+ // is false.
new Reference(rightElemValue, rightElemType, false));
ExprCode elementExprCode = elemExpr.genCode(ctx);
diff --git
a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java
b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java
index b49ded2e4..9b280cce5 100644
---
a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java
+++
b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java
@@ -239,7 +239,7 @@ public class AllowListChecker implements ClassChecker {
lock.writeLock().lock();
listeners.put(classResolver, true);
} finally {
- lock.writeLock().unlock();
+ lock.writeLock().unlock();
}
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
index 54895c592..3f2f339cd 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
@@ -37,6 +37,7 @@ import com.google.common.collect.HashBiMap;
import java.io.Externalizable;
import java.io.IOException;
import java.io.Serializable;
+import java.lang.invoke.SerializedLambda;
import java.lang.reflect.Field;
import java.lang.reflect.Member;
import java.lang.reflect.Type;
@@ -87,7 +88,8 @@ import org.apache.fory.ForyCopyable;
import org.apache.fory.annotation.CodegenInvoke;
import org.apache.fory.annotation.Internal;
import org.apache.fory.builder.CodecUtils;
-import org.apache.fory.builder.Generated;
+import org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer;
+import org.apache.fory.builder.Generated.GeneratedObjectSerializer;
import org.apache.fory.builder.JITContext;
import org.apache.fory.codegen.CodeGenerator;
import org.apache.fory.codegen.Expression;
@@ -163,6 +165,7 @@ import org.apache.fory.type.ScalaTypes;
import org.apache.fory.type.TypeUtils;
import org.apache.fory.type.Types;
import org.apache.fory.util.GraalvmSupport;
+import org.apache.fory.util.GraalvmSupport.GraalvmSerializerHolder;
import org.apache.fory.util.Preconditions;
import org.apache.fory.util.StringUtils;
import org.apache.fory.util.function.Functions;
@@ -420,6 +423,7 @@ public class ClassResolver implements TypeResolver {
register(AtomicReference.class);
register(EnumSet.allOf(Language.class).getClass());
register(EnumSet.of(Language.JAVA).getClass());
+ register(SerializedLambda.class);
register(Throwable.class, StackTraceElement.class, Exception.class,
RuntimeException.class);
register(NullPointerException.class);
register(IOException.class);
@@ -968,14 +972,14 @@ public class ClassResolver implements TypeResolver {
return CollectionSerializers.EnumSetSerializer.class;
} else if (Charset.class.isAssignableFrom(cls)) {
return Serializers.CharsetSerializer.class;
- } else if (Functions.isLambda(cls)) {
- return LambdaSerializer.class;
} else if (ReflectionUtils.isJdkProxy(cls)) {
if (JavaSerializer.getWriteReplaceMethod(cls) != null) {
return ReplaceResolveSerializer.class;
} else {
return JdkProxySerializer.class;
}
+ } else if (Functions.isLambda(cls)) {
+ return LambdaSerializer.class;
} else if (Calendar.class.isAssignableFrom(cls)) {
return TimeSerializers.CalendarSerializer.class;
} else if (ZoneId.class.isAssignableFrom(cls)) {
@@ -1423,7 +1427,7 @@ public class ClassResolver implements TypeResolver {
if (deserializationClassInfo != null &&
GraalvmSupport.isGraalBuildtime()) {
getGraalvmClassRegistry()
.deserializerClassMap
- .put(classDef.getId(),
deserializationClassInfo.serializer.getClass());
+ .put(classDef.getId(),
getGraalvmSerializerClass(deserializationClassInfo.serializer));
Tuple2<ClassDef, ClassInfo> classDefTuple =
extRegistry.classIdToDef.get(classDef.getId());
// empty serializer for graalvm build time
classDefTuple.f1.serializer = null;
@@ -1432,7 +1436,9 @@ public class ClassResolver implements TypeResolver {
}
if (GraalvmSupport.isGraalBuildtime()) {
// Instance for generated class should be hold at graalvm runtime only.
- getGraalvmClassRegistry().serializerClassMap.put(cls,
classInfo.serializer.getClass());
+ getGraalvmClassRegistry()
+ .serializerClassMap
+ .put(cls, getGraalvmSerializerClass(classInfo.serializer));
classInfo.serializer = null;
}
}
@@ -1560,13 +1566,21 @@ public class ClassResolver implements TypeResolver {
}
boolean needToWriteClassDef(Serializer serializer) {
- return fory.getConfig().getCompatibleMode() == CompatibleMode.COMPATIBLE
- && (serializer instanceof Generated.GeneratedObjectSerializer
- // May already switched to MetaSharedSerializer when update class
info cache.
- || serializer instanceof Generated.GeneratedMetaSharedSerializer
- || serializer instanceof LazyInitBeanSerializer
- || serializer instanceof ObjectSerializer
- || serializer instanceof MetaSharedSerializer);
+ if (fory.getConfig().getCompatibleMode() != CompatibleMode.COMPATIBLE) {
+ return false;
+ }
+ if (GraalvmSupport.isGraalBuildtime() && serializer instanceof
GraalvmSerializerHolder) {
+ Class<? extends Serializer> serializerClass =
+ ((GraalvmSerializerHolder) serializer).getSerializerClass();
+ return GeneratedObjectSerializer.class.isAssignableFrom(serializerClass)
+ ||
GeneratedMetaSharedSerializer.class.isAssignableFrom(serializerClass);
+ }
+ return (serializer instanceof GeneratedObjectSerializer
+ // May already switched to MetaSharedSerializer when update class info
cache.
+ || serializer instanceof GeneratedMetaSharedSerializer
+ || serializer instanceof LazyInitBeanSerializer
+ || serializer instanceof ObjectSerializer
+ || serializer instanceof MetaSharedSerializer);
}
private ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer,
MetaContext metaContext) {
@@ -2212,6 +2226,10 @@ public class ClassResolver implements TypeResolver {
*/
public void ensureSerializersCompiled() {
try {
+ fory.getJITContext().lock();
+ Serializers.newSerializer(fory, LambdaSerializer.STUB_LAMBDA_CLASS,
LambdaSerializer.class);
+ Serializers.newSerializer(
+ fory, JdkProxySerializer.SUBT_PROXY.getClass(),
JdkProxySerializer.class);
classInfoMap.forEach(
(cls, classInfo) -> {
if (classInfo.serializer == null) {
@@ -2262,6 +2280,13 @@ public class ClassResolver implements TypeResolver {
fory.getConfig().getConfigHash(), k -> new GraalvmClassRegistry());
}
+ private Class<? extends Serializer> getGraalvmSerializerClass(Serializer
serializer) {
+ if (serializer instanceof GraalvmSerializerHolder) {
+ return ((GraalvmSerializerHolder) serializer).getSerializerClass();
+ }
+ return serializer.getClass();
+ }
+
private Class<? extends Serializer>
getSerializerClassFromGraalvmRegistry(Class<?> cls) {
GraalvmClassRegistry registry = getGraalvmClassRegistry();
List<ClassResolver> classResolvers = registry.resolvers;
@@ -2307,7 +2332,10 @@ public class ClassResolver implements TypeResolver {
if (Functions.isLambda(cls) || ReflectionUtils.isJdkProxy(cls)) {
return null;
}
- throw new RuntimeException(String.format("Class %s is not registered",
cls));
+ throw new RuntimeException(
+ String.format(
+ "Class %s is not registered, registered classes: %s",
+ cls, registry.deserializerClassMap));
}
return null;
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java
index ac1dd80c5..9c2bc7418 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java
@@ -21,6 +21,7 @@ package org.apache.fory.serializer;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import org.apache.fory.Fory;
import org.apache.fory.memory.MemoryBuffer;
@@ -32,7 +33,6 @@ import org.apache.fory.util.Preconditions;
/** Serializer for jdk {@link Proxy}. */
@SuppressWarnings({"rawtypes", "unchecked"})
public class JdkProxySerializer extends Serializer {
-
// Make offset compatible with graalvm native image.
private static final Field FIELD;
private static final long PROXY_HANDLER_FIELD_OFFSET;
@@ -42,10 +42,22 @@ public class JdkProxySerializer extends Serializer {
PROXY_HANDLER_FIELD_OFFSET = Platform.objectFieldOffset(FIELD);
}
- private static final InvocationHandler STUB_HANDLER =
- (proxy, method, args) -> {
- throw new IllegalStateException("Deserialization stub handler still
active");
- };
+ private static class StubInvocationHandler implements InvocationHandler {
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws
Throwable {
+ throw new IllegalStateException("Deserialization stub handler still
active");
+ }
+ }
+
+ private static final InvocationHandler STUB_HANDLER = new
StubInvocationHandler();
+
+ private interface StubInterface {
+ int apply();
+ }
+
+ public static Object SUBT_PROXY =
+ Proxy.newProxyInstance(
+ Serializer.class.getClassLoader(), new Class[]
{StubInterface.class}, STUB_HANDLER);
public JdkProxySerializer(Fory fory, Class cls) {
super(fory, cls);
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java
index 2edfafeac..efa80b772 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java
@@ -21,13 +21,18 @@ package org.apache.fory.serializer;
import java.io.ObjectStreamClass;
import java.io.Serializable;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
import java.lang.invoke.SerializedLambda;
-import java.lang.reflect.Method;
import java.util.Objects;
import org.apache.fory.Fory;
+import org.apache.fory.collection.ClassValueCache;
+import org.apache.fory.exception.ForyException;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.reflect.ReflectionUtils;
import org.apache.fory.util.Preconditions;
+import org.apache.fory.util.function.SerializableFunction;
+import org.apache.fory.util.unsafe._JDKAccess;
/**
* Serializer for java serializable lambda. Use fory to serialize java lambda
instead of JDK
@@ -36,20 +41,34 @@ import org.apache.fory.util.Preconditions;
*/
@SuppressWarnings({"unchecked", "rawtypes"})
public class LambdaSerializer extends Serializer {
+ public static Class<?> STUB_LAMBDA_CLASS =
+ ((SerializableFunction<Integer, Integer>) (x -> x * 2)).getClass();
private static final Class<SerializedLambda> SERIALIZED_LAMBDA =
SerializedLambda.class;
- private static final Method READ_RESOLVE_METHOD =
- (Method)
- Objects.requireNonNull(
- ReflectionUtils.getObjectFieldValue(
- ObjectStreamClass.lookup(SERIALIZED_LAMBDA),
"readResolveMethod"));
+ private static final MethodHandle READ_RESOLVE_HANDLE;
private static final boolean SERIALIZED_LAMBDA_HAS_JDK_WRITE =
JavaSerializer.getWriteObjectMethod(SERIALIZED_LAMBDA) != null;
private static final boolean SERIALIZED_LAMBDA_HAS_JDK_READ =
JavaSerializer.getReadObjectMethod(SERIALIZED_LAMBDA) != null;
- private final Method writeReplaceMethod;
+ private static final ClassValueCache<MethodHandle> writeReplaceMethodCache =
+ ClassValueCache.newClassKeySoftCache(32);
+
+ private final MethodHandle writeReplaceHandle;
private Serializer dataSerializer;
- public LambdaSerializer(Fory fory, Class cls) {
+ static {
+ try {
+ // Initialize READ_RESOLVE_HANDLE
+ MethodHandles.Lookup lookup =
_JDKAccess._trustedLookup(SERIALIZED_LAMBDA);
+ Object readResolveMethod =
+ ReflectionUtils.getObjectFieldValue(
+ ObjectStreamClass.lookup(SERIALIZED_LAMBDA),
"readResolveMethod");
+ READ_RESOLVE_HANDLE = lookup.unreflect((java.lang.reflect.Method)
readResolveMethod);
+ } catch (IllegalAccessException e) {
+ throw new ForyException(e);
+ }
+ }
+
+ public LambdaSerializer(Fory fory, Class<?> cls) {
super(fory, cls);
if (cls != ReplaceStub.class) {
if (!Serializable.class.isAssignableFrom(cls)) {
@@ -59,12 +78,28 @@ public class LambdaSerializer extends Serializer {
cls, Serializable.class.getName());
throw new UnsupportedOperationException(msg);
}
- writeReplaceMethod =
- (Method)
+ MethodHandle methodHandle = writeReplaceMethodCache.getIfPresent(cls);
+ if (methodHandle == null) {
+ try {
+ MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(cls);
+ Object writeReplaceMethod =
ReflectionUtils.getObjectFieldValue(
ObjectStreamClass.lookup(cls), "writeReplaceMethod");
+ methodHandle =
+ lookup.unreflect(
+ (java.lang.reflect.Method)
Objects.requireNonNull(writeReplaceMethod));
+ writeReplaceMethodCache.put(cls, methodHandle);
+ } catch (Throwable e) {
+ throw new RuntimeException(
+ String.format("Failed to create writeReplace MethodHandle for
%s", cls), e);
+ }
+ }
+ writeReplaceHandle = methodHandle;
} else {
- writeReplaceMethod = null;
+ writeReplaceHandle = null;
+ }
+ if (cls == STUB_LAMBDA_CLASS) {
+ getDataSerializer();
}
}
@@ -72,10 +107,10 @@ public class LambdaSerializer extends Serializer {
public void write(MemoryBuffer buffer, Object value) {
assert value.getClass() != ReplaceStub.class;
try {
- Object replacement = writeReplaceMethod.invoke(value);
+ Object replacement = writeReplaceHandle.invoke(value);
Preconditions.checkArgument(SERIALIZED_LAMBDA.isInstance(replacement));
getDataSerializer().write(buffer, replacement);
- } catch (Exception e) {
+ } catch (Throwable e) {
throw new RuntimeException("Can't serialize lambda " + value, e);
}
}
@@ -83,10 +118,10 @@ public class LambdaSerializer extends Serializer {
@Override
public Object copy(Object value) {
try {
- Object replacement = writeReplaceMethod.invoke(value);
+ Object replacement = writeReplaceHandle.invoke(value);
Object newReplacement = getDataSerializer().copy(replacement);
- return READ_RESOLVE_METHOD.invoke(newReplacement);
- } catch (Exception e) {
+ return READ_RESOLVE_HANDLE.invoke(newReplacement);
+ } catch (Throwable e) {
throw new RuntimeException("Can't copy lambda " + value, e);
}
}
@@ -95,8 +130,8 @@ public class LambdaSerializer extends Serializer {
public Object read(MemoryBuffer buffer) {
try {
Object replacement = getDataSerializer().read(buffer);
- return READ_RESOLVE_METHOD.invoke(replacement);
- } catch (Exception e) {
+ return READ_RESOLVE_HANDLE.invoke(replacement);
+ } catch (Throwable e) {
throw new RuntimeException("Can't deserialize lambda", e);
}
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java
index b976046fc..ef672d9b7 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java
@@ -61,6 +61,7 @@ import org.apache.fory.resolver.ClassInfo;
import org.apache.fory.resolver.FieldResolver;
import org.apache.fory.resolver.FieldResolver.ClassField;
import org.apache.fory.util.ExceptionUtils;
+import org.apache.fory.util.GraalvmSupport;
import org.apache.fory.util.Preconditions;
import org.apache.fory.util.unsafe._JDKAccess;
@@ -354,7 +355,8 @@ public class ObjectStreamSerializer extends
AbstractObjectSerializer {
this.slotsSerializer =
(CompatibleSerializerBase)
Serializers.newSerializer(fory, type, c));
}
- if (sc == CompatibleSerializer.class) {
+ if (sc == CompatibleSerializer.class ||
GraalvmSupport.isGraalBuildtime()) {
+ // skip init generated serializer at graalvm build time
this.slotsSerializer = new CompatibleSerializer(fory, type,
fieldResolver);
} else {
this.slotsSerializer = (CompatibleSerializerBase)
Serializers.newSerializer(fory, type, sc);
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java
index ce936d740..c1719b64b 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java
@@ -43,6 +43,7 @@ import java.util.function.Function;
import java.util.function.ToIntFunction;
import java.util.regex.Pattern;
import org.apache.fory.Fory;
+import org.apache.fory.builder.Generated;
import org.apache.fory.collection.Tuple2;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.memory.Platform;
@@ -50,6 +51,7 @@ import org.apache.fory.reflect.ReflectionUtils;
import org.apache.fory.resolver.ClassResolver;
import org.apache.fory.util.ExceptionUtils;
import org.apache.fory.util.GraalvmSupport;
+import org.apache.fory.util.GraalvmSupport.GraalvmSerializerHolder;
import org.apache.fory.util.StringUtils;
import org.apache.fory.util.unsafe._JDKAccess;
@@ -88,6 +90,11 @@ public class Serializers {
}
Tuple2<MethodType, MethodHandle> ctrInfo =
CTR_MAP.getIfPresent(serializerClass);
if (ctrInfo != null) {
+ if (GraalvmSupport.isGraalBuildtime()) {
+ if (Generated.class.isAssignableFrom(serializerClass)) {
+ return new GraalvmSerializerHolder(fory, type, serializerClass);
+ }
+ }
MethodType sig = ctrInfo.f0;
MethodHandle handle = ctrInfo.f1;
if (sig.equals(SIG1)) {
@@ -125,6 +132,11 @@ public class Serializers {
try {
MethodHandle ctr = lookup.findConstructor(serializerClass, SIG1);
CTR_MAP.put(serializerClass, Tuple2.of(SIG1, ctr));
+ if (GraalvmSupport.isGraalBuildtime()) {
+ if (Generated.class.isAssignableFrom(serializerClass)) {
+ return new GraalvmSerializerHolder(fory, type, serializerClass);
+ }
+ }
return (Serializer<T>) ctr.invoke(fory, type);
} catch (NoSuchMethodException e) {
ExceptionUtils.ignore(e);
@@ -132,6 +144,11 @@ public class Serializers {
try {
MethodHandle ctr = lookup.findConstructor(serializerClass, SIG2);
CTR_MAP.put(serializerClass, Tuple2.of(SIG2, ctr));
+ if (GraalvmSupport.isGraalBuildtime()) {
+ if (Generated.class.isAssignableFrom(serializerClass)) {
+ return new GraalvmSerializerHolder(fory, type, serializerClass);
+ }
+ }
return (Serializer<T>) ctr.invoke(fory);
} catch (NoSuchMethodException e) {
ExceptionUtils.ignore(e);
@@ -139,10 +156,20 @@ public class Serializers {
try {
MethodHandle ctr = lookup.findConstructor(serializerClass, SIG3);
CTR_MAP.put(serializerClass, Tuple2.of(SIG3, ctr));
+ if (GraalvmSupport.isGraalBuildtime()) {
+ if (Generated.class.isAssignableFrom(serializerClass)) {
+ return new GraalvmSerializerHolder(fory, type, serializerClass);
+ }
+ }
return (Serializer<T>) ctr.invoke(type);
} catch (NoSuchMethodException e) {
MethodHandle ctr = ReflectionUtils.getCtrHandle(serializerClass);
CTR_MAP.put(serializerClass, Tuple2.of(SIG4, ctr));
+ if (GraalvmSupport.isGraalBuildtime()) {
+ if (Generated.class.isAssignableFrom(serializerClass)) {
+ return new GraalvmSerializerHolder(fory, type, serializerClass);
+ }
+ }
return (Serializer<T>) ctr.invoke();
}
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java
b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java
index afee63b3d..dde3fca6a 100644
--- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java
+++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java
@@ -657,8 +657,8 @@ public class TypeUtils {
}
/**
- * Check if a class is one of {@link Optional), {@link OptionalInt},
- * {@link OptionaLong}, or {@link OptionalDouble}.
+ * Check if a class is one of {@link Optional}, {@link OptionalInt}, {@link
OptionalLong}, or
+ * {@link OptionalDouble}.
*/
public static boolean isOptionalType(Class<?> type) {
return type == Optional.class
diff --git
a/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java
b/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java
index 55d023981..edf00db99 100644
--- a/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java
+++ b/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java
@@ -19,6 +19,10 @@
package org.apache.fory.util;
+import java.util.Objects;
+import org.apache.fory.Fory;
+import org.apache.fory.serializer.Serializer;
+
/** A helper for Graalvm native image support. */
public class GraalvmSupport {
//
https://github.com/oracle/graal/blob/master/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/ImageInfo.java
@@ -46,4 +50,17 @@ public class GraalvmSupport {
return IN_GRAALVM_NATIVE_IMAGE
&&
GRAAL_IMAGE_RUNTIME.equals(System.getProperty(GRAAL_IMAGE_CODE_KEY));
}
+
+ public static class GraalvmSerializerHolder extends Serializer {
+ private final Class serializerClass;
+
+ public GraalvmSerializerHolder(Fory fory, Class<?> type, Class<?>
serializerClass) {
+ super(fory, type);
+ this.serializerClass = Objects.requireNonNull(serializerClass);
+ }
+
+ public Class<? extends Serializer> getSerializerClass() {
+ return serializerClass;
+ }
+ }
}
diff --git
a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties
b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties
index 50f599329..c9dc1b45d 100644
---
a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties
+++
b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties
@@ -226,6 +226,8 @@
Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\
org.apache.fory.logging.LoggerFactory,\
org.apache.fory.memory.BoundsChecking,\
org.apache.fory.memory.MemoryBuffer,\
+ org.apache.fory.memory.MemoryAllocator,\
+ org.apache.fory.memory.MemoryBuffer$DefaultMemoryAllocator,\
org.apache.fory.memory.MemoryUtils,\
org.apache.fory.memory.Platform,\
org.apache.fory.meta.ClassDef$ArrayFieldType,\
@@ -308,6 +310,8 @@
Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\
org.apache.fory.serializer.JavaSerializer$4,\
org.apache.fory.serializer.JavaSerializer,\
org.apache.fory.serializer.JdkProxySerializer$ReplaceStub,\
+ org.apache.fory.serializer.JdkProxySerializer$StubInterface,\
+ org.apache.fory.serializer.JdkProxySerializer$StubInvocationHandler,\
org.apache.fory.serializer.JdkProxySerializer,\
org.apache.fory.serializer.LambdaSerializer$ReplaceStub,\
org.apache.fory.serializer.LambdaSerializer,\
@@ -433,6 +437,11 @@
Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\
org.apache.fory.serializer.LazySerializer$LazyObjectSerializer,\
org.apache.fory.serializer.shim.ShimDispatcher,\
org.apache.fory.serializer.shim.ProtobufDispatcher,\
+ org.apache.fory.serializer.ObjectStreamSerializer,\
+ org.apache.fory.serializer.ObjectStreamSerializer$1,\
+ org.apache.fory.serializer.ObjectStreamSerializer$StreamClassInfo,\
+
org.apache.fory.serializer.collection.ChildContainerSerializers$ChildCollectionSerializer,\
+
org.apache.fory.serializer.collection.ChildContainerSerializers$ChildMapSerializer,\
org.apache.fory.shaded.org.codehaus.janino.IClass$1,\
org.apache.fory.type.Descriptor$1,\
org.apache.fory.type.Descriptor,\
diff --git
a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json
b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json
new file mode 100644
index 000000000..505934bf9
--- /dev/null
+++
b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json
@@ -0,0 +1,9 @@
+[
+ {
+ "name": "java.lang.reflect.Proxy",
+ "allDeclaredConstructors": true,
+ "allPublicConstructors": true,
+ "allDeclaredMethods": true,
+ "allPublicMethods": true
+ }
+]
diff --git
a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json
b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json
new file mode 100644
index 000000000..154ba0da2
--- /dev/null
+++
b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json
@@ -0,0 +1,11 @@
+[
+ {
+ "name": "java.lang.reflect.Proxy",
+ "fields": [],
+ "methods": [
+ { "name": "<init>" },
+ { "name": "writeReplace" }
+ ],
+ "customTargetConstructorClass": "java.lang.Object"
+ }
+]
diff --git
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java
index 49ec2e134..f24da862d 100644
---
a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java
+++
b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java
@@ -27,9 +27,6 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
-import java.util.OptionalDouble;
-import java.util.OptionalInt;
-import java.util.OptionalLong;
import java.util.SortedMap;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
diff --git
a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java
b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java
index 724acb773..c054edeab 100644
---
a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java
+++
b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java
@@ -26,7 +26,6 @@ import java.util.OptionalDouble;
import java.util.OptionalInt;
import java.util.OptionalLong;
import java.util.TreeSet;
-
import lombok.Data;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.fory.annotation.ForyField;
diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py
index 35ab78d36..6bc87ba1a 100644
--- a/python/pyfory/_fory.py
+++ b/python/pyfory/_fory.py
@@ -133,9 +133,7 @@ class Fory:
"""
self.language = language
self.is_py = language == Language.PYTHON
- self.require_type_registration = (
- _ENABLE_TYPE_REGISTRATION_FORCIBLY or require_type_registration
- )
+ self.require_type_registration = _ENABLE_TYPE_REGISTRATION_FORCIBLY or
require_type_registration
self.ref_tracking = ref_tracking
if self.ref_tracking:
self.ref_resolver = MapRefResolver()
@@ -151,8 +149,7 @@ class Fory:
self.buffer = Buffer.allocate(32)
if not require_type_registration:
warnings.warn(
- "Type registration is disabled, unknown types can be
deserialized "
- "which may be insecure.",
+ "Type registration is disabled, unknown types can be
deserialized which may be insecure.",
RuntimeWarning,
stacklevel=2,
)
@@ -166,7 +163,7 @@ class Fory:
self._unsupported_callback = None
self._unsupported_objects = None
self._peer_language = None
-
+
def register(
self,
cls: Union[type, TypeVar],
@@ -345,7 +342,7 @@ class Fory:
buffers: Iterable = None,
unsupported_objects: Iterable = None,
):
- if type(buffer) == bytes:
+ if isinstance(buffer, bytes):
buffer = Buffer(buffer)
if unsupported_objects is not None:
self._unsupported_objects = iter(unsupported_objects)
@@ -360,10 +357,7 @@ class Fory:
if get_bit(buffer, reader_index, 0):
return None
is_little_endian_ = get_bit(buffer, reader_index, 1)
- assert is_little_endian_, (
- "Big endian is not supported for now, "
- "please ensure peer machine is little endian."
- )
+ assert is_little_endian_, "Big endian is not supported for now, please
ensure peer machine is little endian."
is_target_x_lang = get_bit(buffer, reader_index, 2)
if is_target_x_lang:
self._peer_language = Language(buffer.read_int8())
@@ -371,16 +365,10 @@ class Fory:
self._peer_language = Language.PYTHON
is_out_of_band_serialization_enabled = get_bit(buffer, reader_index, 3)
if is_out_of_band_serialization_enabled:
- assert buffers is not None, (
- "buffers shouldn't be null when the serialized stream is "
- "produced with buffer_callback not null."
- )
+ assert buffers is not None, "buffers shouldn't be null when the
serialized stream is produced with buffer_callback not null."
self._buffers = iter(buffers)
else:
- assert buffers is None, (
- "buffers should be null when the serialized stream is "
- "produced with buffer_callback null."
- )
+ assert buffers is None, "buffers should be null when the
serialized stream is produced with buffer_callback null."
if is_target_x_lang:
obj = self.xdeserialize_ref(buffer)
else:
@@ -532,9 +520,7 @@ class SerializationContext:
self.objects.clear()
-_ENABLE_TYPE_REGISTRATION_FORCIBLY = os.getenv(
- "ENABLE_TYPE_REGISTRATION_FORCIBLY", "0"
-) in {
+_ENABLE_TYPE_REGISTRATION_FORCIBLY =
os.getenv("ENABLE_TYPE_REGISTRATION_FORCIBLY", "0") in {
"1",
"true",
}
@@ -544,8 +530,8 @@ class _PicklerStub:
def dump(self, o):
raise ValueError(
f"Type {type(o)} is not registered, "
- f"pickle is not allowed when type registration enabled, Please
register"
- f"the type or pass unsupported_callback"
+ f"pickle is not allowed when type registration enabled, "
+ f"Please register the type or pass unsupported_callback"
)
def clear_memo(self):
@@ -554,7 +540,4 @@ class _PicklerStub:
class _UnpicklerStub:
def load(self):
- raise ValueError(
- "pickle is not allowed when type registration enabled, Please
register"
- "the type or pass unsupported_callback"
- )
+ raise ValueError("pickle is not allowed when type registration
enabled, Please register the type or pass unsupported_callback")
diff --git a/python/pyfory/_registry.py b/python/pyfory/_registry.py
index 05715a579..2333ff782 100644
--- a/python/pyfory/_registry.py
+++ b/python/pyfory/_registry.py
@@ -379,6 +379,9 @@ class TypeResolver:
serializer: Serializer = None,
internal: bool = False,
):
+ # Set default type_id when None, similar to _register_xtype
+ if type_id is None and typename is not None:
+ type_id = self._next_type_id()
return self.__register_type(
cls,
type_id=type_id,
@@ -398,7 +401,7 @@ class TypeResolver:
serializer: Serializer = None,
internal: bool = False,
):
- dynamic_type = type_id < 0
+ dynamic_type = type_id is not None and type_id < 0
if not internal and serializer is None:
serializer = self._create_serializer(cls)
if typename is None:
@@ -471,6 +474,8 @@ class TypeResolver:
type_id = TypeId.NAMED_ENUM
elif type(serializer) is PickleSerializer:
type_id = PickleSerializer.PICKLE_TYPE_ID
+ elif isinstance(serializer, FunctionSerializer):
+ type_id = TypeId.NAMED_EXT
elif isinstance(serializer, (ObjectSerializer, StatefulSerializer,
ReduceSerializer)):
type_id = TypeId.NAMED_EXT
if not self.require_registration:
@@ -494,8 +499,8 @@ class TypeResolver:
break
else:
if cls is types.FunctionType:
- # Use PickleSerializer for function types (including lambdas)
- serializer = PickleSerializer(self.fory, cls)
+ # Use FunctionSerializer for function types (including lambdas)
+ serializer = FunctionSerializer(self.fory, cls)
elif dataclasses.is_dataclass(cls):
serializer = DataClassSerializer(self.fory, cls)
elif issubclass(cls, enum.Enum):
diff --git a/python/pyfory/tests/test_function.py
b/python/pyfory/tests/test_function.py
index ad2e03c06..738ef69d6 100644
--- a/python/pyfory/tests/test_function.py
+++ b/python/pyfory/tests/test_function.py
@@ -26,7 +26,7 @@ def test_lambda_functions_serialization():
# Register the necessary types
fory.register_type(tuple)
fory.register_type(list)
- fory.register_type(dict)
+ # dict is already registered by default with MapSerializer
# Simple lambda
simple_lambda = lambda x: x * 2 # noqa: E731
@@ -64,7 +64,7 @@ def test_regular_functions_serialization():
# Register the necessary types for complex functions
fory.register_type(tuple)
fory.register_type(list)
- fory.register_type(dict)
+ # dict is already registered by default with MapSerializer
# Test complex function
serialized = fory.serialize(complex_function)
@@ -79,7 +79,7 @@ def test_nested_functions_serialization():
# Register the necessary types
fory.register_type(tuple)
fory.register_type(list)
- fory.register_type(dict)
+ # dict is already registered by default with MapSerializer
def outer_function(x):
def inner_function(y):
@@ -104,7 +104,7 @@ def test_local_class_serialization():
# Register the necessary types
fory.register_type(tuple)
fory.register_type(list)
- fory.register_type(dict)
+ # dict is already registered by default with MapSerializer
def create_local_class():
from dataclasses import dataclass
diff --git a/python/pyfory/tests/test_serializer.py
b/python/pyfory/tests/test_serializer.py
index 3f6cbcb49..6df8875b4 100644
--- a/python/pyfory/tests/test_serializer.py
+++ b/python/pyfory/tests/test_serializer.py
@@ -469,6 +469,8 @@ def test_pickle_fallback():
def test_unsupported_callback():
fory = Fory(language=Language.PYTHON, ref_tracking=True,
require_type_registration=False)
+ # Test with functions that now have proper serialization support
+ # Functions should no longer be treated as unsupported
def f1(x):
return x
@@ -478,10 +480,18 @@ def test_unsupported_callback():
obj1 = [1, True, f1, f2, {1: 2}]
unsupported_objects = []
binary1 = fory.serialize(obj1,
unsupported_callback=unsupported_objects.append)
- assert len(unsupported_objects) == 2
- assert unsupported_objects == [f1, f2]
+ # Functions are now properly supported, so unsupported_objects should be
empty
+ assert len(unsupported_objects) == 0
new_obj1 = fory.deserialize(binary1,
unsupported_objects=unsupported_objects)
- assert new_obj1 == obj1
+ # Functions should roundtrip correctly
+ assert len(new_obj1) == len(obj1)
+ assert new_obj1[0] == obj1[0] # 1
+ assert new_obj1[1] == obj1[1] # True
+ assert new_obj1[2](5) == f1(5) # Test f1 functionality
+ assert new_obj1[3](5) == f2(5) # Test f2 functionality
+ assert new_obj1[4] == obj1[4] # {1: 2}
+ # Don't check full equality since functions are new objects after
deserialization
+ # The functionality test above already confirmed they work correctly
def test_slice():
diff --git a/python/setup.py b/python/setup.py
index 87123efa8..d31f5544f 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -37,8 +37,10 @@ project_dir = abspath(pjoin(setup_dir, os.pardir))
fory_cpp_src_dir = abspath(pjoin(setup_dir, "../src/"))
print(f"setup_dir: {setup_dir}")
+print(f"project_dir: {project_dir}")
print(f"fory_cpp_src_dir: {fory_cpp_src_dir}")
+
class BinaryDistribution(Distribution):
def __init__(self, attrs=None):
super().__init__(attrs=attrs)
@@ -50,7 +52,9 @@ class BinaryDistribution(Distribution):
elif arch in ("aarch64", "arm64"):
bazel_args += ["--copt=-fsigned-char"]
bazel_args += ["//:cp_fory_so"]
- subprocess.check_call(bazel_args)
+ # Ensure Windows path compatibility
+ cwd_path = os.path.normpath(project_dir)
+ subprocess.check_call(bazel_args, cwd=cwd_path)
def has_ext_modules(self):
return True
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
