This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 96a657a6a fix(java): stabilize GraalVM field offsets (#3853)
96a657a6a is described below
commit 96a657a6aa3853ce8fba081c1bc65f7afafff6af
Author: Shawn Yang <[email protected]>
AuthorDate: Wed Jul 15 17:19:56 2026 +0530
fix(java): stabilize GraalVM field offsets (#3853)
## Why?
## What does this PR do?
## Related issues
## AI Contribution Checklist
- [ ] Substantial AI assistance was used in this PR: `yes` / `no`
- [ ] If `yes`, I included a completed [AI Contribution
Checklist](https://github.com/apache/fory/blob/main/AI_POLICY.md#9-contributor-checklist-for-ai-assisted-prs)
in this PR description and the required `AI Usage Disclosure`.
- [ ] If `yes`, my PR description includes the required `ai_review`
summary and screenshot evidence or equivalent persisted links of the
final clean AI review results from both fresh reviewers described in
`AI_POLICY.md`, the Fory-guided reviewer and the independent general
reviewer, on the current PR diff or current HEAD after the latest code
changes.
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---
.github/workflows/ci.yml | 39 ++++--
ci/run_ci.py | 3 +
ci/run_ci.sh | 41 +++---
ci/tasks/java.py | 37 +++--
integration_tests/graalvm_tests/pom.xml | 2 +
.../main/java/org/apache/fory/graalvm/Main.java | 1 -
.../java/org/apache/fory/memory/MemoryBuffer.java | 25 ++--
.../fory/serializer/PlatformStringUtils.java | 149 ++++++++-------------
8 files changed, 155 insertions(+), 142 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fe1ada049..bb974a78e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -127,7 +127,7 @@ jobs:
echo "python=false" >> "$GITHUB_OUTPUT"
fi
- if grep -Eq '^(java/|integration_tests/graalvm_tests/)' <<<
"$changed_files"; then
+ if grep -Eq
'^(\.github/workflows/ci\.yml$|ci/run_ci\.(py|sh)$|ci/tasks/(common|java)\.py$|java/|integration_tests/graalvm_tests/)'
<<< "$changed_files"; then
echo "graalvm=true" >> "$GITHUB_OUTPUT"
else
echo "graalvm=false" >> "$GITHUB_OUTPUT"
@@ -576,7 +576,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- java-version: ["17", "21", "25"]
+ java-version: ["17", "25"]
steps:
- uses: actions/checkout@v5
- uses: graalvm/setup-graalvm@6f3fa030c4b8f77c1f554a860f593a654538fa38 #
1.5.6
@@ -592,13 +592,36 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- - name: Set up Python3.8
- uses: actions/setup-python@v5
+ - name: Build JPMS native image and run
+ shell: bash
+ run: ./ci/run_ci.sh graalvm_test
+
+ graalvm_json:
+ name: GraalVM JSON CI
+ needs: changes
+ if: needs.changes.outputs.graalvm == 'true'
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ java-version: ["17", "25"]
+ steps:
+ - uses: actions/checkout@v5
+ - uses: graalvm/setup-graalvm@6f3fa030c4b8f77c1f554a860f593a654538fa38 #
1.5.6
with:
- python-version: 3.8
- - name: Build native image and run
+ java-version: ${{ matrix.java-version }}
+ distribution: "graalvm"
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ native-image-job-reports: "true"
+ - name: Cache Maven local repository
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-maven-
+ - name: Build JSON JPMS native image and run
shell: bash
- run: python ./ci/run_ci.py java --version graalvm
+ run: ./ci/run_ci.sh graalvm_json_tests
kotlin:
name: Kotlin CI
@@ -917,7 +940,7 @@ jobs:
python-version: 3.11
cache: "pip"
- name: Set up Dart
- uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c #
v1.7.1
+ uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c #
v1.7.1
with:
sdk: stable
- name: Cache Maven local repository
diff --git a/ci/run_ci.py b/ci/run_ci.py
index 12192dcbd..f0a84c1b6 100644
--- a/ci/run_ci.py
+++ b/ci/run_ci.py
@@ -319,6 +319,7 @@ def parse_args():
"windows_java21",
"integration_tests",
"graalvm",
+ "graalvm_json_tests",
],
default=None,
help="Java version to use for testing",
@@ -423,6 +424,8 @@ def parse_args():
run_shell_script("windows_java21")
elif version == "graalvm":
run_shell_script("graalvm_test")
+ elif version == "graalvm_json_tests":
+ run_shell_script("graalvm_json_tests")
else:
run_shell_script(f"java{version}")
elif command == "cpp":
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
index 94360bb65..0196ce043 100755
--- a/ci/run_ci.sh
+++ b/ci/run_ci.sh
@@ -79,7 +79,10 @@ install_jdks() {
done
}
-graalvm_test() {
+run_graalvm_tests() {
+ local main_class="$1"
+ local java_version
+ local java_major
java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2; exit}')
if [[ "$java_version" == 1.* ]]; then
java_major=$(echo "$java_version" | cut -d. -f2)
@@ -87,26 +90,34 @@ graalvm_test() {
java_major=$(echo "$java_version" | cut -d. -f1)
fi
if [[ "$java_major" -ge 25 ]]; then
- export JDK_JAVA_OPTIONS="$(jdk25_runtime_options "ALL-UNNAMED")
$(jdk25_javac_options)"
+ export JDK_JAVA_OPTIONS="$(jdk25_javac_options)"
else
unset JDK_JAVA_OPTIONS
fi
cd "$ROOT"/java
- mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl
'!:fory-testsuite'
- echo "Start to build graalvm native image"
+ # GraalVM jobs consume production jars only; Java CI owns test/source jar
verification.
+ # Run the install goal directly after package so verify is not repeated in
every native job.
+ mvn -T10 -B --no-transfer-progress clean package install:install \
+ -pl .,fory-test-core,fory-core,fory-json \
+ -Dmaven.test.skip=true \
+ -Dmaven.source.skip=true \
+ -Dmaven.javadoc.skip=true
+ echo "Start to build GraalVM JPMS native image for $main_class"
cd "$ROOT"/integration_tests/graalvm_tests
- mvn -DskipTests=true --no-transfer-progress -Pnative clean package
- echo "Built GraalVM classpath native image"
- echo "Start to run GraalVM classpath native image"
- ./target/main
- if [[ "$java_major" -ge 25 ]]; then
- export JDK_JAVA_OPTIONS="$(jdk25_javac_options)"
- fi
- mvn -DskipTests=true --no-transfer-progress -Pnative-module clean package
- echo "Built GraalVM module-path native image"
- echo "Start to run GraalVM module-path native image"
+ mvn -DmainClass="$main_class" -DskipTests=true -Dassembly.skipAssembly=true \
+ --no-transfer-progress -Pnative-module clean package
+ echo "Built GraalVM JPMS native image"
+ echo "Start to run GraalVM JPMS native image"
./target/main-module
- echo "Execute graalvm tests succeed!"
+ echo "Execute GraalVM tests for $main_class succeed!"
+}
+
+graalvm_test() {
+ run_graalvm_tests org.apache.fory.graalvm.Main
+}
+
+graalvm_json_tests() {
+ run_graalvm_tests org.apache.fory.graalvm.ForyJsonExample
}
jdk25_access_options() {
diff --git a/ci/tasks/java.py b/ci/tasks/java.py
index 8e6ecc082..7af581443 100644
--- a/ci/tasks/java.py
+++ b/ci/tasks/java.py
@@ -352,9 +352,9 @@ def run_integration_tests():
logging.info("Executing fory integration tests succeeds")
-def run_graalvm_test():
- """Run GraalVM tests."""
- logging.info("Start GraalVM tests")
+def run_graalvm_tests(main_class):
+ """Build and run JPMS GraalVM tests."""
+ logging.info(f"Start GraalVM tests for {main_class}")
java_major = get_jdk_major_version()
if java_major is not None and java_major >= 25:
os.environ["JDK_JAVA_OPTIONS"] = " ".join(jdk25_javac_options())
@@ -362,19 +362,34 @@ def run_graalvm_test():
os.environ.pop("JDK_JAVA_OPTIONS", None)
common.cd_project_subdir("java")
+ # Java CI owns test/source jar verification. GraalVM jobs install only the
+ # production multi-release jars consumed by native-image.
common.exec_cmd(
- "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl
'!:fory-testsuite'"
+ "mvn -T10 -B --no-transfer-progress clean package install:install "
+ "-pl .,fory-test-core,fory-core,fory-json -Dmaven.test.skip=true "
+ "-Dmaven.source.skip=true -Dmaven.javadoc.skip=true"
)
- logging.info("Start to build graalvm native image")
+ logging.info(f"Start to build GraalVM JPMS native image for {main_class}")
common.cd_project_subdir("integration_tests/graalvm_tests")
- common.exec_cmd("mvn -DskipTests=true --no-transfer-progress -Pnative
package")
+ common.exec_cmd(
+ f"mvn -DmainClass={main_class} -DskipTests=true "
+ "-Dassembly.skipAssembly=true --no-transfer-progress "
+ "-Pnative-module clean package"
+ )
+
+ logging.info("Built GraalVM JPMS native image")
+ logging.info("Start to run GraalVM JPMS native image")
+ common.exec_cmd("./target/main-module")
+ logging.info(f"Execute GraalVM tests for {main_class} succeed!")
+
+
+def run_graalvm_test():
+ run_graalvm_tests("org.apache.fory.graalvm.Main")
- logging.info("Built graalvm native image")
- logging.info("Start to run graalvm native image")
- common.exec_cmd("./target/main")
- logging.info("Execute graalvm tests succeed!")
+def run_graalvm_json_tests():
+ run_graalvm_tests("org.apache.fory.graalvm.ForyJsonExample")
def run_release():
@@ -435,3 +450,5 @@ def run(version=None, release=False, install_jdks=False,
install_fory=False):
run_integration_tests()
elif version == "graalvm":
run_graalvm_test()
+ elif version == "graalvm_json_tests":
+ run_graalvm_json_tests()
diff --git a/integration_tests/graalvm_tests/pom.xml
b/integration_tests/graalvm_tests/pom.xml
index 13acb89b9..f2b872df0 100644
--- a/integration_tests/graalvm_tests/pom.xml
+++ b/integration_tests/graalvm_tests/pom.xml
@@ -281,6 +281,8 @@
<executable>native-image</executable>
<arguments>
<argument>--no-fallback</argument>
+ <!-- These images verify native-image compatibility, not
runtime performance. -->
+ <argument>-O1</argument>
<argument>-H:+UnlockExperimentalVMOptions</argument>
<argument>-o</argument>
<argument>${project.build.directory}/${moduleImageName}</argument>
diff --git
a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java
b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java
index 14e290f31..f84eeb50b 100644
---
a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java
+++
b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/Main.java
@@ -46,6 +46,5 @@ public class Main {
ExceptionExample.main(args);
AbstractClassExample.main(args);
FeatureTestExample.main(args);
- ForyJsonExample.main(args);
}
}
diff --git
a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java
b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java
index c0f002ca9..8cd0929bd 100644
--- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java
+++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java
@@ -77,6 +77,7 @@ public final class MemoryBuffer {
private static final int LONG_ARRAY_OFFSET;
private static final int FLOAT_ARRAY_OFFSET;
private static final int DOUBLE_ARRAY_OFFSET;
+ private static final long BUFFER_ADDRESS_FIELD_OFFSET;
// GraalVM native-image recognizes arrayBaseOffset only when the call stores
directly into the
// target static field. Keep these assignments in this shape so native
images recompute heap array
@@ -91,6 +92,7 @@ public final class MemoryBuffer {
LONG_ARRAY_OFFSET = 0;
FLOAT_ARRAY_OFFSET = 0;
DOUBLE_ARRAY_OFFSET = 0;
+ BUFFER_ADDRESS_FIELD_OFFSET = -1;
} else {
BOOLEAN_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(boolean[].class);
BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
@@ -100,29 +102,24 @@ public final class MemoryBuffer {
LONG_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(long[].class);
FLOAT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(float[].class);
DOUBLE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(double[].class);
+ try {
+ Field addressField = Buffer.class.getDeclaredField("address");
+ // GraalVM native-image only recomputes a hosted object field offset
when this call stores
+ // directly into its final static owner. Do not route it through a
helper or local offset.
+ BUFFER_ADDRESS_FIELD_OFFSET = UNSAFE.objectFieldOffset(addressField);
+ checkArgument(BUFFER_ADDRESS_FIELD_OFFSET != 0);
+ } catch (NoSuchFieldException e) {
+ throw new IllegalStateException(e);
+ }
}
}
/** Limits each raw Unsafe copy to let large copies hit safepoint polls
between chunks. */
private static final long UNSAFE_COPY_THRESHOLD = 1024L * 1024L;
- private static final long BUFFER_ADDRESS_FIELD_OFFSET =
- AndroidSupport.IS_ANDROID ? -1 : bufferAddressFieldOffset();
-
// Global allocator instance that can be customized
private static volatile MemoryAllocator globalAllocator = new
DefaultMemoryAllocator();
- private static long bufferAddressFieldOffset() {
- try {
- Field addressField = Buffer.class.getDeclaredField("address");
- long offset = UNSAFE.objectFieldOffset(addressField);
- checkArgument(offset != 0);
- return offset;
- } catch (NoSuchFieldException e) {
- throw new IllegalStateException(e);
- }
- }
-
private static boolean unaligned() {
String arch = System.getProperty("os.arch", "");
if ("ppc64le".equals(arch) || "ppc64".equals(arch) ||
"s390x".equals(arch)) {
diff --git
a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java
b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java
index a5f3747b3..978d11955 100644
---
a/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java
+++
b/java/fory-core/src/main/java/org/apache/fory/serializer/PlatformStringUtils.java
@@ -34,6 +34,14 @@ final class PlatformStringUtils {
private static final Unsafe UNSAFE = AndroidSupport.IS_ANDROID ? null :
_UnsafeUtils.UNSAFE;
private static final int BYTE_ARRAY_OFFSET;
private static final int CHAR_ARRAY_OFFSET;
+ static final boolean JDK_STRING_FIELD_ACCESS;
+ static final boolean STRING_VALUE_FIELD_IS_CHARS;
+ static final boolean STRING_VALUE_FIELD_IS_BYTES;
+ static final boolean STRING_HAS_COUNT_OFFSET;
+ private static final long STRING_VALUE_FIELD_OFFSET;
+ private static final long STRING_CODER_FIELD_OFFSET;
+ private static final long STRING_COUNT_FIELD_OFFSET;
+ private static final long STRING_OFFSET_FIELD_OFFSET;
// GraalVM native-image needs arrayBaseOffset calls to store directly into
their static fields so
// it can recompute the offsets for the image runtime.
@@ -45,65 +53,60 @@ final class PlatformStringUtils {
BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
CHAR_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(char[].class);
}
- }
-
- private static final StringFields STRING_FIELDS = stringFields();
-
- static final boolean JDK_STRING_FIELD_ACCESS = STRING_FIELDS.fieldAccess;
- static final boolean STRING_VALUE_FIELD_IS_CHARS =
- JDK_STRING_FIELD_ACCESS && STRING_FIELDS.valueFieldIsChars;
- static final boolean STRING_VALUE_FIELD_IS_BYTES =
- JDK_STRING_FIELD_ACCESS && STRING_FIELDS.valueFieldIsBytes;
- static final boolean STRING_HAS_COUNT_OFFSET = JDK_STRING_FIELD_ACCESS &&
STRING_FIELDS.counted;
-
- private static final long STRING_VALUE_FIELD_OFFSET =
STRING_FIELDS.valueOffset;
- private static final long STRING_CODER_FIELD_OFFSET =
STRING_FIELDS.coderOffset;
- private static final long STRING_COUNT_FIELD_OFFSET =
STRING_FIELDS.countOffset;
- private static final long STRING_OFFSET_FIELD_OFFSET =
STRING_FIELDS.offsetOffset;
-
- private PlatformStringUtils() {}
-
- private static StringFields stringFields() {
if (AndroidSupport.IS_ANDROID
|| GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE
|| !_JDKAccess.JDK_INTERNAL_FIELD_ACCESS) {
- return StringFields.noAccess();
- }
- try {
- Field valueField = String.class.getDeclaredField("value");
- boolean valueFieldIsChars = valueField.getType() == char[].class;
- boolean valueFieldIsBytes = valueField.getType() == byte[].class;
- long valueOffset = UNSAFE.objectFieldOffset(valueField);
- Field countField = getStringFieldNullable("count");
- Field offsetField = getStringFieldNullable("offset");
- boolean counted = false;
- long countOffset = -1;
- long offsetOffset = -1;
- if (countField != null || offsetField != null) {
- Preconditions.checkArgument(
- countField != null && offsetField != null, "Current jdk not
supported");
- Preconditions.checkArgument(
- countField.getType() == int.class && offsetField.getType() ==
int.class,
- "Current jdk not supported");
- counted = true;
- countOffset = UNSAFE.objectFieldOffset(countField);
- offsetOffset = UNSAFE.objectFieldOffset(offsetField);
+ JDK_STRING_FIELD_ACCESS = false;
+ STRING_VALUE_FIELD_IS_CHARS = false;
+ STRING_VALUE_FIELD_IS_BYTES = false;
+ STRING_HAS_COUNT_OFFSET = false;
+ STRING_VALUE_FIELD_OFFSET = -1;
+ STRING_CODER_FIELD_OFFSET = -1;
+ STRING_COUNT_FIELD_OFFSET = -1;
+ STRING_OFFSET_FIELD_OFFSET = -1;
+ } else {
+ try {
+ Field valueField = String.class.getDeclaredField("value");
+ boolean valueFieldIsChars = valueField.getType() == char[].class;
+ boolean valueFieldIsBytes = valueField.getType() == byte[].class;
+ JDK_STRING_FIELD_ACCESS = true;
+ STRING_VALUE_FIELD_IS_CHARS = valueFieldIsChars;
+ STRING_VALUE_FIELD_IS_BYTES = valueFieldIsBytes;
+ // GraalVM native-image only recomputes hosted object field offsets
when each call stores
+ // directly into its final static owner. Do not route these through
locals or holder
+ // objects.
+ STRING_VALUE_FIELD_OFFSET = UNSAFE.objectFieldOffset(valueField);
+
+ Field countField = getStringFieldNullable("count");
+ Field offsetField = getStringFieldNullable("offset");
+ if (countField != null || offsetField != null) {
+ Preconditions.checkArgument(
+ countField != null && offsetField != null, "Current jdk not
supported");
+ Preconditions.checkArgument(
+ countField.getType() == int.class && offsetField.getType() ==
int.class,
+ "Current jdk not supported");
+ STRING_HAS_COUNT_OFFSET = true;
+ STRING_COUNT_FIELD_OFFSET = UNSAFE.objectFieldOffset(countField);
+ STRING_OFFSET_FIELD_OFFSET = UNSAFE.objectFieldOffset(offsetField);
+ } else {
+ STRING_HAS_COUNT_OFFSET = false;
+ STRING_COUNT_FIELD_OFFSET = -1;
+ STRING_OFFSET_FIELD_OFFSET = -1;
+ }
+ if (valueFieldIsBytes) {
+ STRING_CODER_FIELD_OFFSET =
+ UNSAFE.objectFieldOffset(String.class.getDeclaredField("coder"));
+ } else {
+ STRING_CODER_FIELD_OFFSET = -1;
+ }
+ } catch (NoSuchFieldException e) {
+ throw new RuntimeException(e);
}
- long coderOffset = valueFieldIsBytes ? stringCoderFieldOffset() : -1;
- return new StringFields(
- true,
- valueFieldIsChars,
- valueFieldIsBytes,
- counted,
- valueOffset,
- coderOffset,
- countOffset,
- offsetOffset);
- } catch (NoSuchFieldException e) {
- throw new RuntimeException(e);
}
}
+ private PlatformStringUtils() {}
+
private static Field getStringFieldNullable(String fieldName) {
try {
return String.class.getDeclaredField(fieldName);
@@ -112,48 +115,6 @@ final class PlatformStringUtils {
}
}
- private static long stringCoderFieldOffset() {
- try {
- return UNSAFE.objectFieldOffset(String.class.getDeclaredField("coder"));
- } catch (NoSuchFieldException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static final class StringFields {
- private final boolean fieldAccess;
- private final boolean valueFieldIsChars;
- private final boolean valueFieldIsBytes;
- private final boolean counted;
- private final long valueOffset;
- private final long coderOffset;
- private final long countOffset;
- private final long offsetOffset;
-
- private StringFields(
- boolean fieldAccess,
- boolean valueFieldIsChars,
- boolean valueFieldIsBytes,
- boolean counted,
- long valueOffset,
- long coderOffset,
- long countOffset,
- long offsetOffset) {
- this.fieldAccess = fieldAccess;
- this.valueFieldIsChars = valueFieldIsChars;
- this.valueFieldIsBytes = valueFieldIsBytes;
- this.counted = counted;
- this.valueOffset = valueOffset;
- this.coderOffset = coderOffset;
- this.countOffset = countOffset;
- this.offsetOffset = offsetOffset;
- }
-
- private static StringFields noAccess() {
- return new StringFields(false, false, false, false, -1, -1, -1, -1);
- }
- }
-
static Object getStringValue(String value) {
return UNSAFE.getObject(value, STRING_VALUE_FIELD_OFFSET);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
