This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 06a74a10 fix(java): fix fastutil Object2ObjectOpenHashMap
serialization (#1618)
06a74a10 is described below
commit 06a74a102c9191f40d4c78247b4e6f8c2e4e23cc
Author: Shawn Yang <[email protected]>
AuthorDate: Thu May 9 22:49:17 2024 +0800
fix(java): fix fastutil Object2ObjectOpenHashMap serialization (#1618)
## What does this PR do?
fix fastutil Object2ObjectOpenHashMap serialization
## Related issues
Closes #1615
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/incubator-fury/issues/new/choose)
describing the need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
.../org/apache/fury/resolver/FieldResolver.java | 22 ++++----
.../fury/serializer/CompatibleSerializer.java | 13 +++--
java/fury-testsuite/pom.xml | 6 ++
.../src/test/java/org/apache/fury/TestBase.java | 65 ++++++++++++++++++++++
.../fury/test/Object2ObjectOpenHashMapTest.java | 46 +++++++++++++++
5 files changed, 136 insertions(+), 16 deletions(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/FieldResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/FieldResolver.java
index 047a2af1..17d6b2e8 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/FieldResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/FieldResolver.java
@@ -50,8 +50,8 @@ import org.apache.fury.reflect.FieldAccessor;
import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.reflect.TypeRef;
import org.apache.fury.serializer.PrimitiveSerializers;
-import org.apache.fury.serializer.collection.CollectionSerializer;
-import org.apache.fury.serializer.collection.MapSerializer;
+import org.apache.fury.serializer.collection.AbstractCollectionSerializer;
+import org.apache.fury.serializer.collection.AbstractMapSerializer;
import org.apache.fury.type.Descriptor;
import org.apache.fury.type.TypeUtils;
import org.apache.fury.util.MurmurHash3;
@@ -567,7 +567,8 @@ public class FieldResolver {
if (fieldType == FieldTypes.COLLECTION_ELEMENT_FINAL) {
ClassInfo elementClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
- CollectionSerializer collectionSerializer = (CollectionSerializer)
classInfo.getSerializer();
+ AbstractCollectionSerializer collectionSerializer =
+ (AbstractCollectionSerializer) classInfo.getSerializer();
try {
collectionSerializer.setElementSerializer(elementClassInfo.getSerializer());
o = collectionSerializer.read(buffer);
@@ -580,7 +581,7 @@ public class FieldResolver {
ClassInfo keyClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo valueClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
- MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
+ AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
@@ -594,7 +595,7 @@ public class FieldResolver {
} else if (fieldType == FieldTypes.MAP_KEY_FINAL) {
ClassInfo keyClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
- MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
+ AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
o = mapSerializer.read(buffer);
@@ -607,7 +608,7 @@ public class FieldResolver {
Preconditions.checkArgument(fieldType == FieldTypes.MAP_VALUE_FINAL);
ClassInfo valueClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
- MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
+ AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
o = mapSerializer.read(buffer);
@@ -625,7 +626,8 @@ public class FieldResolver {
if (fieldType == FieldTypes.COLLECTION_ELEMENT_FINAL) {
ClassInfo elementClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer,
fieldInfo.getClassInfoHolder());
- CollectionSerializer collectionSerializer = (CollectionSerializer)
classInfo.getSerializer();
+ AbstractCollectionSerializer collectionSerializer =
+ (AbstractCollectionSerializer) classInfo.getSerializer();
try {
collectionSerializer.setElementSerializer(elementClassInfo.getSerializer());
o = collectionSerializer.read(buffer);
@@ -638,7 +640,7 @@ public class FieldResolver {
ClassInfo keyClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo valueClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer,
fieldInfo.getClassInfoHolder());
- MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
+ AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
@@ -652,7 +654,7 @@ public class FieldResolver {
} else if (fieldType == FieldTypes.MAP_KEY_FINAL) {
ClassInfo keyClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer,
fieldInfo.getClassInfoHolder());
- MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
+ AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
o = mapSerializer.read(buffer);
@@ -665,7 +667,7 @@ public class FieldResolver {
Preconditions.checkArgument(fieldType == FieldTypes.MAP_VALUE_FINAL);
ClassInfo valueClassInfo = classResolver.readClassInfo(buffer,
classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer,
fieldInfo.getClassInfoHolder());
- MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
+ AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
o = mapSerializer.read(buffer);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/CompatibleSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/CompatibleSerializer.java
index b48e8993..f9cf8a8a 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/CompatibleSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/CompatibleSerializer.java
@@ -34,8 +34,8 @@ import org.apache.fury.resolver.ClassInfo;
import org.apache.fury.resolver.ClassResolver;
import org.apache.fury.resolver.FieldResolver;
import org.apache.fury.resolver.RefResolver;
-import org.apache.fury.serializer.collection.CollectionSerializer;
-import org.apache.fury.serializer.collection.MapSerializer;
+import org.apache.fury.serializer.collection.AbstractCollectionSerializer;
+import org.apache.fury.serializer.collection.AbstractMapSerializer;
import org.apache.fury.util.Preconditions;
import org.apache.fury.util.record.RecordInfo;
import org.apache.fury.util.record.RecordUtils;
@@ -241,7 +241,8 @@ public final class CompatibleSerializer<T> extends
CompatibleSerializerBase<T> {
// following write is consistent with
`BaseSeqCodecBuilder.serializeForCollection`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
classResolver.writeClass(buffer, classInfo);
- CollectionSerializer collectionSerializer = (CollectionSerializer)
classInfo.getSerializer();
+ AbstractCollectionSerializer collectionSerializer =
+ (AbstractCollectionSerializer) classInfo.getSerializer();
try {
collectionSerializer.setElementSerializer(elementClassInfo.getSerializer());
collectionSerializer.write(buffer, fieldValue);
@@ -261,7 +262,7 @@ public final class CompatibleSerializer<T> extends
CompatibleSerializerBase<T> {
// following write is consistent with `BaseSeqCodecBuilder.serializeForMap`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
classResolver.writeClass(buffer, classInfo);
- MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
+ AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
@@ -281,7 +282,7 @@ public final class CompatibleSerializer<T> extends
CompatibleSerializerBase<T> {
// following write is consistent with `BaseSeqCodecBuilder.serializeForMap`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
classResolver.writeClass(buffer, classInfo);
- MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
+ AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
mapSerializer.write(buffer, fieldValue);
@@ -299,7 +300,7 @@ public final class CompatibleSerializer<T> extends
CompatibleSerializerBase<T> {
// following write is consistent with `BaseSeqCodecBuilder.serializeForMap`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
classResolver.writeClass(buffer, classInfo);
- MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
+ AbstractMapSerializer mapSerializer = (AbstractMapSerializer)
classInfo.getSerializer();
try {
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
mapSerializer.write(buffer, fieldValue);
diff --git a/java/fury-testsuite/pom.xml b/java/fury-testsuite/pom.xml
index c5d91898..165e17db 100644
--- a/java/fury-testsuite/pom.xml
+++ b/java/fury-testsuite/pom.xml
@@ -93,6 +93,12 @@
<version>2.57</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
+ <version>8.5.6</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/java/fury-testsuite/src/test/java/org/apache/fury/TestBase.java
b/java/fury-testsuite/src/test/java/org/apache/fury/TestBase.java
new file mode 100644
index 00000000..4fe88d46
--- /dev/null
+++ b/java/fury-testsuite/src/test/java/org/apache/fury/TestBase.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fury;
+
+import org.apache.fury.config.FuryBuilder;
+import org.apache.fury.config.Language;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+
+/** Fury unit test base class. */
+@SuppressWarnings("unchecked")
+public abstract class TestBase {
+
+ public static FuryBuilder builder() {
+ return
Fury.builder().withLanguage(Language.JAVA).requireClassRegistration(false);
+ }
+
+ @DataProvider
+ public static Object[][] trackingRef() {
+ return new Object[][] {{false}, {true}};
+ }
+
+ @DataProvider
+ public static Object[][] enableCodegen() {
+ return new Object[][] {{false}, {true}};
+ }
+
+ @DataProvider
+ public static Object[][] compressNumber() {
+ return new Object[][] {{false}, {true}};
+ }
+
+ public static Object serDeCheck(Fury fury, Object obj) {
+ Object o = serDe(fury, obj);
+ Assert.assertEquals(o, obj);
+ return o;
+ }
+
+ public static <T> T serDe(Fury fury, T obj) {
+ try {
+ byte[] bytes = fury.serialize(obj);
+ return (T) (fury.deserialize(bytes));
+ } catch (Throwable t) {
+ // Catch for add breakpoint for debugging.
+ throw t;
+ }
+ }
+}
diff --git
a/java/fury-testsuite/src/test/java/org/apache/fury/test/Object2ObjectOpenHashMapTest.java
b/java/fury-testsuite/src/test/java/org/apache/fury/test/Object2ObjectOpenHashMapTest.java
new file mode 100644
index 00000000..2d78a51b
--- /dev/null
+++
b/java/fury-testsuite/src/test/java/org/apache/fury/test/Object2ObjectOpenHashMapTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fury.test;
+
+import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
+import java.util.Map;
+import lombok.Data;
+import org.apache.fury.Fury;
+import org.apache.fury.TestBase;
+import org.apache.fury.config.CompatibleMode;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class Object2ObjectOpenHashMapTest extends TestBase {
+ @Data
+ public static class TestObject2ObjectOpenHashMap {
+ Map<String, String> ext = new Object2ObjectOpenHashMap<>();
+ }
+
+ @Test(dataProvider = "enableCodegen")
+ public void testObject2ObjectOpenHashMap(boolean enableCodegen) {
+ Fury fury =
+
builder().withCompatibleMode(CompatibleMode.COMPATIBLE).withCodegen(enableCodegen).build();
+
+ TestObject2ObjectOpenHashMap o = new TestObject2ObjectOpenHashMap();
+ byte[] bytes = fury.serializeJavaObject(o);
+ Assert.assertEquals(fury.deserializeJavaObject(bytes,
TestObject2ObjectOpenHashMap.class), o);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
