This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/main by this push:
new c8698b65 fix(java): fix async compilation switch for non-public nested
class (#1883)
c8698b65 is described below
commit c8698b65f976987edc189a537ca66bf42b9cfcdc
Author: Shawn Yang <[email protected]>
AuthorDate: Sun Oct 13 11:18:46 2024 +0800
fix(java): fix async compilation switch for non-public nested class (#1883)
## What does this PR do?
- fix async compilation switch for non-public nested class
- fix install sbt for scala
- fix install python
## Related issues
Closes #1879
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fury/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
.github/workflows/ci.yml | 26 +++++++++-
ci/run_ci.sh | 1 +
.../fury/builder/BaseObjectCodecBuilder.java | 13 +++--
.../org/apache/fury/builder/JITContextTest.java | 60 ++++++++++++++++++++++
4 files changed, 95 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 212fcf10..fa2d3e92 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -54,6 +54,10 @@ jobs:
with:
java-version: ${{ matrix.java-version }}
distribution: "temurin"
+ - name: Set up Python3.8
+ uses: actions/setup-python@v5
+ with:
+ python-version: 3.8
- name: Install bazel
run: ./ci/run_ci.sh install_bazel
- name: Install python
@@ -78,6 +82,10 @@ jobs:
with:
java-version: ${{ matrix.java-version }}
distribution: "adopt-openj9"
+ - name: Set up Python3.8
+ uses: actions/setup-python@v5
+ with:
+ python-version: 3.8
- name: Install bazel
run: ./ci/run_ci.sh install_bazel
- name: Install python
@@ -100,6 +108,10 @@ jobs:
distribution: "graalvm"
github-token: ${{ secrets.GITHUB_TOKEN }}
native-image-job-reports: "true"
+ - name: Set up Python3.8
+ uses: actions/setup-python@v5
+ with:
+ python-version: 3.8
- name: Build native image and run
run: ./ci/run_ci.sh graalvm_test
@@ -111,12 +123,20 @@ jobs:
- name: Set up JDK8
uses: actions/setup-java@v4
with:
- java-version: 8
+ java-version: 11
distribution: "temurin"
- name: Install fury java
run: cd java && mvn -T10 --no-transfer-progress clean install
-DskipTests && cd -
- name: Test
run: |
+ sudo apt-get update
+ sudo apt-get install apt-transport-https curl gnupg -yqq
+ echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" |
sudo tee /etc/apt/sources.list.d/sbt.list
+ echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee
/etc/apt/sources.list.d/sbt_old.list
+ curl -sL
"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823";
| sudo -H gpg --no-default-keyring --keyring
gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --import
+ sudo chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+ sudo apt-get update
+ sudo apt-get install sbt
# Avoid sbt download jackson-databind error
rm -rf /home/runner/.m2/repository/com/fasterxml/jackson/
cd scala && sbt +test && cd -
@@ -224,6 +244,10 @@ jobs:
go-version: ${{ matrix.go-version }}
- name: Display Go version
run: go version
+ - name: Set up Python3.8
+ uses: actions/setup-python@v5
+ with:
+ python-version: 3.8
- name: Install bazel
run: ./ci/run_ci.sh install_bazel
- name: Install python
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
index 2883254b..14664e9b 100755
--- a/ci/run_ci.sh
+++ b/ci/run_ci.sh
@@ -24,6 +24,7 @@ ROOT="$(git rev-parse --show-toplevel)"
echo "Root path: $ROOT, home path: $HOME"
cd "$ROOT"
+
install_python() {
wget -q
https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.2-0-Linux-x86_64.sh -O
Miniconda3.sh
bash Miniconda3.sh -b -p $HOME/miniconda && rm -f miniconda.*
diff --git
a/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
b/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
index b1b43d36..7e50919b 100644
---
a/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
+++
b/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
@@ -60,6 +60,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.function.Supplier;
import org.apache.fury.Fury;
+import org.apache.fury.codegen.Code;
import org.apache.fury.codegen.CodeGenerator;
import org.apache.fury.codegen.CodegenContext;
import org.apache.fury.codegen.Expression;
@@ -127,7 +128,7 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
private final Map<Class<?>, Reference> serializerMap = new HashMap<>();
private final Map<String, Object> sharedFieldMap = new HashMap<>();
protected final Class<?> parentSerializerClass;
- private final Map<String, String> jitCallbackUpdateFields;
+ private final Map<String, Expression> jitCallbackUpdateFields;
protected LinkedList<String> walkPath = new LinkedList<>();
public BaseObjectCodecBuilder(TypeRef<?> beanType, Fury fury, Class<?>
parentSerializerClass) {
@@ -266,9 +267,13 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
// build encode/decode expr before add constructor to fill up
jitCallbackUpdateFields.
if (!jitCallbackUpdateFields.isEmpty()) {
StringJoiner stringJoiner = new StringJoiner(", ",
"registerJITNotifyCallback(this,", ");\n");
- for (Map.Entry<String, String> entry :
jitCallbackUpdateFields.entrySet()) {
+ for (Map.Entry<String, Expression> entry :
jitCallbackUpdateFields.entrySet()) {
+ Code.ExprCode exprCode = entry.getValue().genCode(ctx);
+ if (StringUtils.isNotBlank(exprCode.code())) {
+ stringJoiner.add(exprCode.code());
+ }
stringJoiner.add("\"" + entry.getKey() + "\"");
- stringJoiner.add(entry.getValue());
+ stringJoiner.add(exprCode.value().toString());
}
// add this code after field serialization initialization to avoid
// it overrides field updates by this callback.
@@ -526,7 +531,7 @@ public abstract class BaseObjectCodecBuilder extends
CodecBuilder {
// `serializerClass` is already jit generated class.
boolean hasJITResult = fury.getJITContext().hasJITResult(cls);
if (hasJITResult) {
- jitCallbackUpdateFields.put(name, ctx.type(cls) + ".class");
+ jitCallbackUpdateFields.put(name, getClassExpr(cls));
ctx.addField(
false, ctx.type(Serializer.class), name, new
Cast(newSerializerExpr, SERIALIZER_TYPE));
serializerRef = new Reference(name, SERIALIZER_TYPE, false);
diff --git
a/java/fury-core/src/test/java/org/apache/fury/builder/JITContextTest.java
b/java/fury-core/src/test/java/org/apache/fury/builder/JITContextTest.java
index 33a23ce2..86143fe9 100644
--- a/java/fury-core/src/test/java/org/apache/fury/builder/JITContextTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/builder/JITContextTest.java
@@ -20,16 +20,19 @@
package org.apache.fury.builder;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import java.util.List;
+import lombok.Data;
import org.apache.fury.Fury;
import org.apache.fury.FuryTestBase;
import org.apache.fury.config.CompatibleMode;
import org.apache.fury.config.Language;
import org.apache.fury.logging.Logger;
import org.apache.fury.logging.LoggerFactory;
+import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.resolver.MetaContext;
import org.apache.fury.serializer.Serializer;
import org.apache.fury.test.bean.BeanA;
@@ -131,4 +134,61 @@ public class JITContextTest extends FuryTestBase {
if (!scopedMetaShare)
fury.getSerializationContext().setMetaContext(context);
assertEquals(fury.deserialize(bytes2), beanA);
}
+
+ @Test(timeOut = 60000)
+ public void testAsyncCompilationSwitch() throws InterruptedException {
+ final Fury fury =
+ Fury.builder()
+ .withLanguage(Language.JAVA)
+ .requireClassRegistration(false)
+ .withRefTracking(true)
+ .withAsyncCompilation(true)
+ .build();
+
+ TestAccessLevel o = new TestAccessLevel(new PkgAccessLevel(1), new
PrivateAccessLevel(2));
+ serDeCheck(fury, o);
+ Class<?>[] classes = {PkgAccessLevel.class, PrivateAccessLevel.class};
+ for (Class<?> cls : classes) {
+ while (!(fury.getClassResolver().getSerializer(cls) instanceof
Generated)) {
+ Thread.sleep(1000);
+ LOG.warn("Wait async compilation finish for {}", cls);
+ }
+ }
+ Serializer<TestAccessLevel> serializer =
+ fury.getClassResolver().getSerializer(TestAccessLevel.class);
+ assertTrue(ReflectionUtils.getObjectFieldValue(serializer, "serializer")
instanceof Generated);
+ assertTrue(ReflectionUtils.getObjectFieldValue(serializer, "serializer1")
instanceof Generated);
+ serDeCheck(fury, o);
+ }
+
+ @Data
+ public static final class TestAccessLevel {
+ PkgAccessLevel f1;
+ PrivateAccessLevel f2;
+
+ public TestAccessLevel(PkgAccessLevel f1, PrivateAccessLevel f2) {
+ this.f1 = f1;
+ this.f2 = f2;
+ }
+ }
+
+ // test pkg level class
+ @Data
+ private static final class PkgAccessLevel {
+ private final int f1;
+
+ public PkgAccessLevel(int f1) {
+ this.f1 = f1;
+ }
+ }
+
+ // test private class, class should be final for switch
+ @Data
+ private static final class PrivateAccessLevel {
+ private final int f1;
+
+ public PrivateAccessLevel(int f1) {
+ this.f1 = f1;
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
