This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new d59c57c3 feat(java): support deserialize unexisted array/enum classes
(#1569)
d59c57c3 is described below
commit d59c57c33e5d4b2f5fc1c1af466113abbb1a8457
Author: Shawn Yang <[email protected]>
AuthorDate: Fri Apr 26 16:48:48 2024 +0800
feat(java): support deserialize unexisted array/enum classes (#1569)
## What does this PR do?
This PR support sdeserialize unexisted array/enum object.
For type compatible serialization, deserialization process may not have
the class in serialization peer. In such cases, we should support skip
object data of unexisted array/enum
## Related issues
Closes #1564
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/incubator-fury/issues/new/choose)
describing the need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
.../java/org/apache/fury/builder/CodecBuilder.java | 4 +-
.../java/org/apache/fury/codegen/JaninoUtils.java | 10 ++
.../org/apache/fury/meta/MetaStringEncoder.java | 5 +
.../java/org/apache/fury/resolver/ClassInfo.java | 39 ++++--
.../org/apache/fury/resolver/ClassResolver.java | 136 +++++++++++++-----
.../org/apache/fury/resolver/MetaStringBytes.java | 6 +-
.../apache/fury/serializer/ArraySerializers.java | 152 ++++++++++++++++++++-
.../apache/fury/serializer/CodegenSerializer.java | 7 +-
.../fury/serializer/UnexistedClassSerializers.java | 18 +++
.../main/java/org/apache/fury/type/TypeUtils.java | 8 ++
.../java/org/apache/fury/util/ReflectionUtils.java | 30 +++-
.../java/org/apache/fury/util/StringUtils.java | 8 ++
.../serializer/UnexistedClassSerializersTest.java | 130 ++++++++++++++++++
13 files changed, 505 insertions(+), 48 deletions(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/builder/CodecBuilder.java
b/java/fury-core/src/main/java/org/apache/fury/builder/CodecBuilder.java
index d8ebc77c..a9753fec 100644
--- a/java/fury-core/src/main/java/org/apache/fury/builder/CodecBuilder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/builder/CodecBuilder.java
@@ -345,7 +345,9 @@ public abstract class CodecBuilder {
if (duplicatedFields.contains(fieldName) ||
!sourcePublicAccessible(beanClass)) {
return unsafeSetField(bean, d, value);
}
- if (!d.isFinalField() && Modifier.isPublic(d.getModifiers())) {
+ if (!d.isFinalField()
+ && Modifier.isPublic(d.getModifiers())
+ && Modifier.isPublic(d.getRawType().getModifiers())) {
return new Expression.SetField(bean, fieldName, value);
} else if (d.getWriteMethod() != null &&
Modifier.isPublic(d.getWriteMethod().getModifiers())) {
return new Invoke(bean, d.getWriteMethod().getName(), value);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/codegen/JaninoUtils.java
b/java/fury-core/src/main/java/org/apache/fury/codegen/JaninoUtils.java
index 90b8cc3c..bddd900a 100644
--- a/java/fury-core/src/main/java/org/apache/fury/codegen/JaninoUtils.java
+++ b/java/fury-core/src/main/java/org/apache/fury/codegen/JaninoUtils.java
@@ -51,6 +51,16 @@ import org.codehaus.janino.util.ClassFile;
public class JaninoUtils {
private static final Logger LOG = LoggerFactory.getLogger(JaninoUtils.class);
+ public static Class<?> compileClass(
+ ClassLoader loader, String pkg, String className, String code) {
+ ByteArrayClassLoader classLoader = compile(loader, new CompileUnit(pkg,
className, code));
+ try {
+ return classLoader.loadClass(StringUtils.isBlank(pkg) ? className : pkg
+ "." + className);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
public static ByteArrayClassLoader compile(
ClassLoader parentClassLoader, CompileUnit... compileUnits) {
final Map<String, byte[]> classes = toBytecode(parentClassLoader,
compileUnits);
diff --git
a/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
b/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
index fe6796a7..6c8d69c2 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
@@ -25,6 +25,11 @@ import org.apache.fury.util.Preconditions;
/** Encodes plain text strings into MetaString objects with specified encoding
mechanisms. */
public class MetaStringEncoder {
+ public static final MetaStringEncoder PACKAGE_ENCODER = new
MetaStringEncoder('.', '_');
+ public static final MetaStringDecoder PACKAGE_DECODER = new
MetaStringDecoder('.', '_');
+ public static final MetaStringEncoder TYPE_NAME_ENCODER = new
MetaStringEncoder('$', '_');
+ public static final MetaStringDecoder TYPE_NAME_DECODER = new
MetaStringDecoder('$', '_');
+
private final char specialChar1;
private final char specialChar2;
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
index 25f67022..bc45d355 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
@@ -19,12 +19,17 @@
package org.apache.fury.resolver;
+import static org.apache.fury.meta.MetaStringEncoder.PACKAGE_ENCODER;
+import static org.apache.fury.meta.MetaStringEncoder.TYPE_NAME_ENCODER;
+
+import org.apache.fury.collection.Tuple2;
import org.apache.fury.config.Language;
import org.apache.fury.meta.MetaString.Encoding;
-import org.apache.fury.meta.MetaStringEncoder;
import org.apache.fury.serializer.Serializer;
+import org.apache.fury.type.TypeUtils;
import org.apache.fury.util.Preconditions;
import org.apache.fury.util.ReflectionUtils;
+import org.apache.fury.util.StringUtils;
import org.apache.fury.util.function.Functions;
/**
@@ -32,6 +37,9 @@ import org.apache.fury.util.function.Functions;
* serialization.
*/
public class ClassInfo {
+ static final String ARRAY_PREFIX = "1";
+ static final String ENUM_PREFIX = "2";
+
final Class<?> cls;
final MetaStringBytes fullClassNameBytes;
final MetaStringBytes packageNameBytes;
@@ -77,7 +85,7 @@ public class ClassInfo {
if (cls != null && classResolver.getFury().getLanguage() != Language.JAVA)
{
this.fullClassNameBytes =
metaStringResolver.getOrCreateMetaStringBytes(
- new MetaStringEncoder('.', '_').encode(cls.getName(),
Encoding.UTF_8));
+ PACKAGE_ENCODER.encode(cls.getName(), Encoding.UTF_8));
} else {
this.fullClassNameBytes = null;
}
@@ -85,13 +93,28 @@ public class ClassInfo {
&& (classId == ClassResolver.NO_CLASS_ID || classId ==
ClassResolver.REPLACE_STUB_ID)) {
// REPLACE_STUB_ID for write replace class in `ClassSerializer`.
String packageName = ReflectionUtils.getPackage(cls);
+ String className = ReflectionUtils.getClassNameWithoutPackage(cls);
+ if (cls.isArray()) {
+ Tuple2<Class<?>, Integer> componentInfo =
TypeUtils.getArrayComponentInfo(cls);
+ Class<?> ctype = componentInfo.f0;
+ if (!ctype.isPrimitive()) { // primitive array has special format like
[[[III.
+ String componentName = ctype.getName();
+ packageName = ReflectionUtils.getPackage(componentName);
+ String componentSimpleName =
ReflectionUtils.getClassNameWithoutPackage(componentName);
+ String prefix = StringUtils.repeat(ARRAY_PREFIX, componentInfo.f1);
+ if (ctype.isEnum()) {
+ className = prefix + ENUM_PREFIX + componentSimpleName;
+ } else {
+ className = prefix + componentSimpleName;
+ }
+ }
+ } else if (cls.isEnum()) {
+ className = ENUM_PREFIX + className;
+ }
this.packageNameBytes =
- metaStringResolver.getOrCreateMetaStringBytes(
- new MetaStringEncoder('.', '_').encode(packageName));
+
metaStringResolver.getOrCreateMetaStringBytes(PACKAGE_ENCODER.encode(packageName));
this.classNameBytes =
- metaStringResolver.getOrCreateMetaStringBytes(
- new MetaStringEncoder('_', '$')
- .encode(ReflectionUtils.getClassNameWithoutPackage(cls)));
+
metaStringResolver.getOrCreateMetaStringBytes(TYPE_NAME_ENCODER.encode(className));
} else {
this.packageNameBytes = null;
this.classNameBytes = null;
@@ -99,7 +122,7 @@ public class ClassInfo {
if (tag != null) {
this.typeTagBytes =
metaStringResolver.getOrCreateMetaStringBytes(
- new MetaStringEncoder('.', '_').encode(tag, Encoding.UTF_8));
+ PACKAGE_ENCODER.encode(tag, Encoding.UTF_8));
} else {
this.typeTagBytes = null;
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
index de5ce261..1e4c3703 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
@@ -19,6 +19,9 @@
package org.apache.fury.resolver;
+import static org.apache.fury.meta.MetaStringEncoder.PACKAGE_DECODER;
+import static org.apache.fury.meta.MetaStringEncoder.PACKAGE_ENCODER;
+import static org.apache.fury.meta.MetaStringEncoder.TYPE_NAME_DECODER;
import static
org.apache.fury.serializer.CodegenSerializer.loadCodegenSerializer;
import static
org.apache.fury.serializer.CodegenSerializer.loadCompatibleCodegenSerializer;
import static
org.apache.fury.serializer.CodegenSerializer.supportCodegenForJavaSerialization;
@@ -94,7 +97,10 @@ import org.apache.fury.logging.Logger;
import org.apache.fury.logging.LoggerFactory;
import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.meta.ClassDef;
+import org.apache.fury.meta.MetaString;
import org.apache.fury.serializer.ArraySerializers;
+import
org.apache.fury.serializer.ArraySerializers.UnexistedArrayClassSerializer;
+import
org.apache.fury.serializer.ArraySerializers.UnexistedEnumArrayClassSerializer;
import org.apache.fury.serializer.BufferSerializers;
import org.apache.fury.serializer.CodegenSerializer.LazyInitBeanSerializer;
import org.apache.fury.serializer.CompatibleSerializer;
@@ -114,9 +120,13 @@ import org.apache.fury.serializer.Serializers;
import org.apache.fury.serializer.StringSerializer;
import org.apache.fury.serializer.StructSerializer;
import org.apache.fury.serializer.TimeSerializers;
+import
org.apache.fury.serializer.UnexistedClassSerializers.UnexistedArrayClass;
import
org.apache.fury.serializer.UnexistedClassSerializers.UnexistedClassSerializer;
+import
org.apache.fury.serializer.UnexistedClassSerializers.UnexistedEnumArrayClass;
+import
org.apache.fury.serializer.UnexistedClassSerializers.UnexistedEnumClassSerializer;
import
org.apache.fury.serializer.UnexistedClassSerializers.UnexistedMetaSharedClass;
import org.apache.fury.serializer.UnexistedClassSerializers.UnexistedSkipClass;
+import
org.apache.fury.serializer.UnexistedClassSerializers.UnexistedSkipEnumClass;
import org.apache.fury.serializer.collection.ChildContainerSerializers;
import org.apache.fury.serializer.collection.CollectionSerializer;
import org.apache.fury.serializer.collection.CollectionSerializers;
@@ -209,7 +219,7 @@ public class ClassResolver {
private final ObjectMap<MetaStringBytes, Class<?>> classNameBytes2Class =
new ObjectMap<>(16, furyMapLoadFactor);
// Every deserialization for unregistered class will query it, performance
is important.
- private final ObjectMap<ClassNameBytes, Class<?>>
compositeClassNameBytes2Class =
+ private final ObjectMap<ClassNameBytes, ClassInfo>
compositeClassNameBytes2ClassInfo =
new ObjectMap<>(16, furyMapLoadFactor);
private final HashMap<Short, Class<?>> typeIdToClassXLangMap = new
HashMap<>(8, loadFactor);
private final HashMap<String, Class<?>> typeTagToClassXLangMap = new
HashMap<>(8, loadFactor);
@@ -1469,21 +1479,21 @@ public class ClassResolver {
*/
public Class<?> readClassInternal(MemoryBuffer buffer) {
int header = buffer.readVarUint32Small14();
+ final ClassInfo classInfo;
if ((header & 0b1) != 0) {
if (metaContextShareEnabled) {
return readClassWithMetaShare(buffer);
}
MetaStringBytes packageBytes =
metaStringResolver.readMetaStringBytesWithFlag(buffer, header);
MetaStringBytes simpleClassNameBytes =
metaStringResolver.readMetaStringBytes(buffer);
- final Class<?> cls = loadBytesToClass(packageBytes,
simpleClassNameBytes);
- currentReadClass = cls;
- return cls;
+ classInfo = loadBytesToClassInfo(packageBytes, simpleClassNameBytes);
+
} else {
- ClassInfo classInfo = registeredId2ClassInfo[(short) (header >> 1)];
- final Class<?> cls = classInfo.cls;
- currentReadClass = cls;
- return cls;
+ classInfo = registeredId2ClassInfo[(short) (header >> 1)];
}
+ final Class<?> cls = classInfo.cls;
+ currentReadClass = cls;
+ return cls;
}
/**
@@ -1556,46 +1566,100 @@ public class ClassResolver {
private ClassInfo readClassInfoFromBytes(
MemoryBuffer buffer, ClassInfo classInfoCache, int header) {
MetaStringBytes simpleClassNameBytesCache = classInfoCache.classNameBytes;
+ MetaStringBytes packageBytes;
+ MetaStringBytes simpleClassNameBytes;
if (simpleClassNameBytesCache != null) {
MetaStringBytes packageNameBytesCache = classInfoCache.packageNameBytes;
- MetaStringBytes packageBytes =
+ packageBytes =
metaStringResolver.readMetaStringBytesWithFlag(buffer,
packageNameBytesCache, header);
assert packageNameBytesCache != null;
- MetaStringBytes simpleClassNameBytes =
+ simpleClassNameBytes =
metaStringResolver.readMetaStringBytes(buffer,
simpleClassNameBytesCache);
if (simpleClassNameBytesCache.hashCode == simpleClassNameBytes.hashCode
&& packageNameBytesCache.hashCode == packageBytes.hashCode) {
return classInfoCache;
- } else {
- Class<?> cls = loadBytesToClass(packageBytes, simpleClassNameBytes);
- return getClassInfo(cls);
}
} else {
- MetaStringBytes packageBytes =
metaStringResolver.readMetaStringBytesWithFlag(buffer, header);
- MetaStringBytes simpleClassNameBytes =
metaStringResolver.readMetaStringBytes(buffer);
- Class<?> cls = loadBytesToClass(packageBytes, simpleClassNameBytes);
- return getClassInfo(cls);
+ packageBytes = metaStringResolver.readMetaStringBytesWithFlag(buffer,
header);
+ simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer);
+ }
+ ClassInfo classInfo = loadBytesToClassInfo(packageBytes,
simpleClassNameBytes);
+ if (classInfo.serializer == null) {
+ return getClassInfo(classInfo.cls);
}
+ return classInfo;
}
- private Class<?> loadBytesToClass(
+ private ClassInfo loadBytesToClassInfo(
MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) {
ClassNameBytes classNameBytes =
new ClassNameBytes(packageBytes.hashCode,
simpleClassNameBytes.hashCode);
- Class<?> cls = compositeClassNameBytes2Class.get(classNameBytes);
- if (cls == null) {
- String packageName = packageBytes.decode('.', '_');
- String className = simpleClassNameBytes.decode('.', '$');
- String entireClassName;
- if (StringUtils.isBlank(packageName)) {
- entireClassName = className;
+ ClassInfo classInfo =
compositeClassNameBytes2ClassInfo.get(classNameBytes);
+ if (classInfo == null) {
+ classInfo = populateBytesToClassInfo(classNameBytes, packageBytes,
simpleClassNameBytes);
+ }
+ return classInfo;
+ }
+
+ private ClassInfo populateBytesToClassInfo(
+ ClassNameBytes classNameBytes,
+ MetaStringBytes packageBytes,
+ MetaStringBytes simpleClassNameBytes) {
+ String packageName = packageBytes.decode(PACKAGE_DECODER);
+ String rawPkg = packageName;
+ String className = simpleClassNameBytes.decode(TYPE_NAME_DECODER);
+ boolean isArray = className.startsWith(ClassInfo.ARRAY_PREFIX);
+ if (isArray) {
+ int dimension = 0;
+ while (className.charAt(dimension) == ClassInfo.ARRAY_PREFIX.charAt(0)) {
+ dimension++;
+ }
+ packageName = StringUtils.repeat("[", dimension) + "L" + packageName;
+ className = className.substring(dimension) + ";";
+ }
+ boolean isEnum = className.startsWith(ClassInfo.ENUM_PREFIX);
+ if (isEnum) {
+ className = className.substring(1);
+ }
+ String entireClassName;
+ if (StringUtils.isBlank(rawPkg)) {
+ if (isArray) {
+ entireClassName = packageName + className;
} else {
- entireClassName = packageName + "." + className;
+ entireClassName = className;
}
- cls = loadClass(entireClassName);
- compositeClassNameBytes2Class.put(classNameBytes, cls);
+ } else {
+ entireClassName = packageName + "." + className;
}
- return cls;
+ MetaStringBytes fullClassNameBytes =
+ metaStringResolver.getOrCreateMetaStringBytes(
+ PACKAGE_ENCODER.encode(entireClassName,
MetaString.Encoding.UTF_8));
+ Class<?> cls = loadClass(entireClassName, isArray, isEnum);
+ ClassInfo classInfo =
+ new ClassInfo(
+ cls,
+ fullClassNameBytes,
+ packageBytes,
+ simpleClassNameBytes,
+ false,
+ null,
+ null,
+ NO_CLASS_ID);
+ if (cls == UnexistedSkipEnumClass.class) {
+ classInfo.serializer = new UnexistedEnumClassSerializer(fury);
+ } else if (cls == UnexistedArrayClass.class) {
+ classInfo.serializer = new UnexistedArrayClassSerializer(fury,
entireClassName);
+ } else if (cls == UnexistedEnumArrayClass.class) {
+ classInfo.serializer = new UnexistedEnumArrayClassSerializer(fury,
entireClassName);
+ } else {
+ // don't create serializer here, if the class is an interface,
+ // there won't be serializer since interface has no instance.
+ if (!classInfoMap.containsKey(cls)) {
+ classInfoMap.put(cls, classInfo);
+ }
+ }
+ compositeClassNameBytes2ClassInfo.put(classNameBytes, classInfo);
+ return classInfo;
}
public void xwriteClass(MemoryBuffer buffer, Class<?> cls) {
@@ -1628,6 +1692,10 @@ public class ClassResolver {
}
private Class<?> loadClass(String className) {
+ return loadClass(className, false, false);
+ }
+
+ private Class<?> loadClass(String className, boolean isArray, boolean
isEnum) {
extRegistry.classChecker.checkClass(this, className);
try {
return Class.forName(className, false, fury.getClassLoader());
@@ -1641,8 +1709,14 @@ public class ClassResolver {
className, fury.getClassLoader(),
Thread.currentThread().getContextClassLoader());
if (fury.getConfig().deserializeUnexistedClass()) {
LOG.warn(msg);
- // FIXME create a subclass dynamically may be better?
- return UnexistedSkipClass.class;
+ if (isArray) {
+ return isEnum ? UnexistedEnumArrayClass.class :
UnexistedArrayClass.class;
+ } else if (isEnum) {
+ return UnexistedSkipEnumClass.class;
+ } else {
+ // FIXME create a subclass dynamically may be better?
+ return UnexistedSkipClass.class;
+ }
}
throw new IllegalStateException(msg, ex);
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringBytes.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringBytes.java
index f58f406d..24f84bdb 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringBytes.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringBytes.java
@@ -60,10 +60,14 @@ final class MetaStringBytes {
}
public String decode(char specialChar1, char specialChar2) {
+ return decode(new MetaStringDecoder(specialChar1, specialChar2));
+ }
+
+ public String decode(MetaStringDecoder decoder) {
int header = (int) (hashCode & 0xff);
int encodingFlags = header & 0b111;
MetaString.Encoding encoding = MetaString.Encoding.values()[encodingFlags];
- return new MetaStringDecoder(specialChar1, specialChar2).decode(bytes,
encoding);
+ return decoder.decode(bytes, encoding);
}
@Override
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java
index 5e193e8c..4f9ead51 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java
@@ -44,7 +44,6 @@ public class ArraySerializers {
private final Class<T> innerType;
private final Serializer componentTypeSerializer;
private final ClassInfoHolder classInfoHolder;
- private final int dimension;
private final int[] stubDims;
public ObjectArraySerializer(Fury fury, Class<T[]> cls) {
@@ -60,7 +59,6 @@ public class ArraySerializers {
innerType = t;
}
}
- this.dimension = dimension;
this.innerType = (Class<T>) innerType;
Class<?> componentType = cls.getComponentType();
if (ReflectionUtils.isMonomorphic(componentType)) {
@@ -81,9 +79,10 @@ public class ArraySerializers {
@Override
public void write(MemoryBuffer buffer, T[] arr) {
int len = arr.length;
- buffer.writeVarUint32Small7(len);
RefResolver refResolver = fury.getRefResolver();
Serializer componentSerializer = this.componentTypeSerializer;
+ int header = componentSerializer != null ? 0b1 : 0b0;
+ buffer.writeVarUint32Small7(len << 1 | header);
if (componentSerializer != null) {
for (T t : arr) {
if (!refResolver.writeRefOrNull(buffer, t)) {
@@ -122,11 +121,13 @@ public class ArraySerializers {
public T[] read(MemoryBuffer buffer) {
// Some jdk8 will crash if use varint, why?
int numElements = buffer.readVarUint32Small7();
+ boolean isFinal = (numElements & 0b1) != 0;
+ numElements >>>= 1;
Object[] value = newArray(numElements);
RefResolver refResolver = fury.getRefResolver();
refResolver.reference(value);
- final Serializer componentTypeSerializer = this.componentTypeSerializer;
- if (componentTypeSerializer != null) {
+ if (isFinal) {
+ final Serializer componentTypeSerializer =
this.componentTypeSerializer;
for (int i = 0; i < numElements; i++) {
Object elem;
int nextReadRefId = refResolver.tryPreserveRefId(buffer);
@@ -695,4 +696,145 @@ public class ArraySerializers {
double.class,
new int[] {Platform.DOUBLE_ARRAY_OFFSET, 8,
Type.FURY_PRIMITIVE_DOUBLE_ARRAY.getId()});
}
+
+ public abstract static class AbstractedUnexistedArrayClassSerializer extends
Serializer {
+ private final String className;
+ private final int dims;
+
+ public AbstractedUnexistedArrayClassSerializer(
+ Fury fury, String className, Class<?> stubClass) {
+ super(fury, stubClass);
+ this.className = className;
+ this.dims = TypeUtils.getArrayDimensions(className);
+ }
+
+ @Override
+ public Object[] read(MemoryBuffer buffer) {
+ switch (dims) {
+ case 1:
+ return read1DArray(buffer);
+ case 2:
+ return read2DArray(buffer);
+ case 3:
+ return read3DArray(buffer);
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Unsupported array dimension %s for class %s",
dims, className));
+ }
+ }
+
+ protected abstract Object readInnerElement(MemoryBuffer buffer);
+
+ private Object[] read1DArray(MemoryBuffer buffer) {
+ int numElements = buffer.readVarUint32Small7();
+ boolean isFinal = (numElements & 0b1) != 0;
+ numElements >>>= 1;
+ RefResolver refResolver = fury.getRefResolver();
+ Object[] value = new Object[numElements];
+ refResolver.reference(value);
+ if (isFinal) {
+ for (int i = 0; i < numElements; i++) {
+ Object elem;
+ int nextReadRefId = refResolver.tryPreserveRefId(buffer);
+ if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) {
+ elem = readInnerElement(buffer);
+ refResolver.setReadObject(nextReadRefId, elem);
+ } else {
+ elem = refResolver.getReadObject();
+ }
+ value[i] = elem;
+ }
+ } else {
+ for (int i = 0; i < numElements; i++) {
+ value[i] = fury.readRef(buffer);
+ }
+ }
+ return value;
+ }
+
+ private Object[][] read2DArray(MemoryBuffer buffer) {
+ int numElements = buffer.readVarUint32Small7();
+ boolean isFinal = (numElements & 0b1) != 0;
+ numElements >>>= 1;
+ RefResolver refResolver = fury.getRefResolver();
+ Object[][] value = new Object[numElements][];
+ refResolver.reference(value);
+ if (isFinal) {
+ for (int i = 0; i < numElements; i++) {
+ Object[] elem;
+ int nextReadRefId = refResolver.tryPreserveRefId(buffer);
+ if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) {
+ elem = read1DArray(buffer);
+ refResolver.setReadObject(nextReadRefId, elem);
+ } else {
+ elem = (Object[]) refResolver.getReadObject();
+ }
+ value[i] = elem;
+ }
+ } else {
+ for (int i = 0; i < numElements; i++) {
+ value[i] = (Object[]) fury.readRef(buffer);
+ }
+ }
+ return value;
+ }
+
+ private Object[] read3DArray(MemoryBuffer buffer) {
+ int numElements = buffer.readVarUint32Small7();
+ boolean isFinal = (numElements & 0b1) != 0;
+ numElements >>>= 1;
+ RefResolver refResolver = fury.getRefResolver();
+ Object[][][] value = new Object[numElements][][];
+ refResolver.reference(value);
+ if (isFinal) {
+ for (int i = 0; i < numElements; i++) {
+ Object[][] elem;
+ int nextReadRefId = refResolver.tryPreserveRefId(buffer);
+ if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) {
+ elem = read2DArray(buffer);
+ refResolver.setReadObject(nextReadRefId, elem);
+ } else {
+ elem = (Object[][]) refResolver.getReadObject();
+ }
+ value[i] = elem;
+ }
+ } else {
+ for (int i = 0; i < numElements; i++) {
+ value[i] = (Object[][]) fury.readRef(buffer);
+ }
+ }
+ return value;
+ }
+ }
+
+ public static final class UnexistedEnumArrayClassSerializer
+ extends AbstractedUnexistedArrayClassSerializer {
+ public UnexistedEnumArrayClassSerializer(Fury fury, String className) {
+ super(fury, className,
UnexistedClassSerializers.UnexistedEnumArrayClass.class);
+ }
+
+ @Override
+ protected Object readInnerElement(MemoryBuffer buffer) {
+ return buffer.readVarUint32Small7();
+ }
+ }
+
+ public static final class UnexistedArrayClassSerializer
+ extends AbstractedUnexistedArrayClassSerializer {
+
+ private final
CompatibleSerializer<UnexistedClassSerializers.UnexistedSkipClass>
+ componentSerializer;
+
+ public UnexistedArrayClassSerializer(Fury fury, String className) {
+ super(fury, className,
UnexistedClassSerializers.UnexistedArrayClass.class);
+ // TODO(chaokunyang) meta share mode not supported currently.
+ componentSerializer =
+ new CompatibleSerializer<>(fury,
UnexistedClassSerializers.UnexistedSkipClass.class);
+ }
+
+ @Override
+ protected Object readInnerElement(MemoryBuffer buffer) {
+ return componentSerializer.read(buffer);
+ }
+ }
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/CodegenSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/CodegenSerializer.java
index 9f7b8b66..c5711a85 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/CodegenSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/CodegenSerializer.java
@@ -35,7 +35,12 @@ public final class CodegenSerializer {
// bean class can be static nested class, but can't be a non-static inner
class
// If a class is a static class, the enclosing class must not be null.
// If enclosing class is null, it must not be a static class.
- return cls.getEnclosingClass() == null ||
Modifier.isStatic(cls.getModifiers());
+ try {
+ return cls.getEnclosingClass() == null ||
Modifier.isStatic(cls.getModifiers());
+
+ } catch (Throwable t) {
+ throw new RuntimeException(t);
+ }
}
@SuppressWarnings("unchecked")
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/UnexistedClassSerializers.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/UnexistedClassSerializers.java
index 09d22f78..44864e2d 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/UnexistedClassSerializers.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/UnexistedClassSerializers.java
@@ -56,6 +56,12 @@ public final class UnexistedClassSerializers {
/** Ensure no fields here to avoid conflicts with peer class fields. */
public static class UnexistedSkipClass implements UnexistedClass {}
+ public static class UnexistedArrayClass implements UnexistedClass {}
+
+ public static class UnexistedEnumArrayClass implements UnexistedClass {}
+
+ public static class UnexistedSkipEnumClass implements UnexistedClass {}
+
public static class UnexistedMetaSharedClass extends LazyMap implements
UnexistedClass {
private final ClassDef classDef;
@@ -239,4 +245,16 @@ public final class UnexistedClassSerializers {
return obj;
}
}
+
+ public static final class UnexistedEnumClassSerializer extends Serializer {
+
+ public UnexistedEnumClassSerializer(Fury fury) {
+ super(fury, UnexistedSkipEnumClass.class);
+ }
+
+ @Override
+ public Object read(MemoryBuffer buffer) {
+ return buffer.readVarUint32Small7();
+ }
+ }
}
diff --git a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
index 459ab9e8..f1191ca4 100644
--- a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
+++ b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java
@@ -364,6 +364,14 @@ public class TypeUtils {
return getArrayComponentInfo(type).f1;
}
+ public static int getArrayDimensions(String className) {
+ int dimension = 0;
+ while (className.charAt(dimension) == '[') {
+ dimension++;
+ }
+ return dimension;
+ }
+
public static Class<?> getArrayComponent(Class<?> type) {
return getArrayComponentInfo(type).f0;
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/util/ReflectionUtils.java
b/java/fury-core/src/main/java/org/apache/fury/util/ReflectionUtils.java
index 8144a0ba..a7f04a96 100644
--- a/java/fury-core/src/main/java/org/apache/fury/util/ReflectionUtils.java
+++ b/java/fury-core/src/main/java/org/apache/fury/util/ReflectionUtils.java
@@ -397,6 +397,8 @@ public class ReflectionUtils {
}
public static void setObjectFieldValue(Object obj, Field field, Object
value) {
+ Preconditions.checkArgument(
+ !field.getType().isPrimitive(), "Field %s is primitive type", field);
Platform.putObject(obj, Platform.objectFieldOffset(field), value);
}
@@ -426,6 +428,10 @@ public class ReflectionUtils {
return null;
}
+ /**
+ * Get classname with package name stripped. Note that this is different
from {@link
+ * Class#getSimpleName()} since it return className without enclosing
classname for inner classes.
+ */
public static String getClassNameWithoutPackage(Class<?> clz) {
String className = clz.getName();
int index = className.lastIndexOf(".");
@@ -436,6 +442,19 @@ public class ReflectionUtils {
}
}
+ /**
+ * Get classname with package name stripped. Note that this is different
from {@link
+ * Class#getSimpleName()} since it return className without enclosing
classname for inner classes.
+ */
+ public static String getClassNameWithoutPackage(String className) {
+ int index = className.lastIndexOf(".");
+ if (index != -1) {
+ return className.substring(index + 1);
+ } else {
+ return className;
+ }
+ }
+
public static boolean isPublic(TypeToken<?> targetType) {
return Modifier.isPublic(getRawType(targetType).getModifiers());
}
@@ -508,6 +527,15 @@ public class ReflectionUtils {
return pkg;
}
+ public static String getPackage(String className) {
+ int index = className.lastIndexOf(".");
+ if (index != -1) {
+ return className.substring(0, index);
+ } else {
+ return "";
+ }
+ }
+
/**
* Returns the canonical name of the underlying class as defined by
<cite>The Java Language
* Specification</cite>. Throw {@link IllegalArgumentException} if the
underlying class does not
@@ -534,7 +562,7 @@ public class ReflectionUtils {
// jdk class are loaded by bootstrap class loader, which will return
null.
classLoader = Thread.currentThread().getContextClassLoader();
}
- return classLoader.loadClass(className);
+ return Class.forName(className, false, classLoader);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
diff --git a/java/fury-core/src/main/java/org/apache/fury/util/StringUtils.java
b/java/fury-core/src/main/java/org/apache/fury/util/StringUtils.java
index 43362d7c..26031448 100644
--- a/java/fury-core/src/main/java/org/apache/fury/util/StringUtils.java
+++ b/java/fury-core/src/main/java/org/apache/fury/util/StringUtils.java
@@ -184,4 +184,12 @@ public class StringUtils {
sb.setCharAt(i, tmp);
}
}
+
+ public static String repeat(String str, int numRepeat) {
+ StringBuilder builder = new StringBuilder(numRepeat * str.length());
+ for (int i = 0; i < numRepeat; i++) {
+ builder.append(str);
+ }
+ return builder.toString();
+ }
}
diff --git
a/java/fury-core/src/test/java/org/apache/fury/serializer/UnexistedClassSerializersTest.java
b/java/fury-core/src/test/java/org/apache/fury/serializer/UnexistedClassSerializersTest.java
index 1fb65633..f753a834 100644
---
a/java/fury-core/src/test/java/org/apache/fury/serializer/UnexistedClassSerializersTest.java
+++
b/java/fury-core/src/test/java/org/apache/fury/serializer/UnexistedClassSerializersTest.java
@@ -23,14 +23,19 @@ import static org.testng.Assert.assertEquals;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
+import java.lang.reflect.Array;
import java.util.List;
import org.apache.fury.Fury;
import org.apache.fury.FuryTestBase;
+import org.apache.fury.codegen.CompileUnit;
+import org.apache.fury.codegen.JaninoUtils;
import org.apache.fury.config.CompatibleMode;
import org.apache.fury.config.FuryBuilder;
import org.apache.fury.config.Language;
import org.apache.fury.resolver.MetaContext;
import org.apache.fury.test.bean.Struct;
+import org.apache.fury.util.ReflectionUtils;
+import org.codehaus.commons.compiler.util.reflect.ByteArrayClassLoader;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -96,6 +101,131 @@ public class UnexistedClassSerializersTest extends
FuryTestBase {
}
}
+ @Test
+ public void testSkipUnexistedEnum() {
+ Fury fury1 = furyBuilder().withDeserializeUnexistedClass(true).build();
+ String enumCode = ("enum TestEnum {" + " A, B" + "}");
+
+ Class<?> cls = JaninoUtils.compileClass(getClass().getClassLoader(), "",
"TestEnum", enumCode);
+ Object c = cls.getEnumConstants()[1];
+ assertEquals(c.toString(), "B");
+ byte[] bytes = fury1.serialize(c);
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ Fury fury2 = furyBuilder().withDeserializeUnexistedClass(true).build();
+ Object o = fury2.deserialize(bytes);
+ assertEquals(o, 1);
+ }
+
+ @Test
+ public void testSkipUnexistedEnumAndArrayField() throws Exception {
+ String enumStructCode1 =
+ ("public class TestEnumStruct {\n"
+ + " public enum TestEnum {\n"
+ + " A, B\n"
+ + " }\n"
+ + " public String f1;\n"
+ + " public TestEnum f2;\n"
+ + " public TestEnum[] f3;\n"
+ + " public TestEnum[][] f4;\n"
+ + "}");
+ Class<?> cls1 =
+ JaninoUtils.compileClass(
+ getClass().getClassLoader(), "", "TestEnumStruct",
enumStructCode1);
+ Class<?> enumClass = cls1.getDeclaredClasses()[0];
+ Object o = cls1.newInstance();
+ ReflectionUtils.setObjectFieldValue(o, "f1", "str");
+ ReflectionUtils.setObjectFieldValue(o, "f2",
enumClass.getEnumConstants()[1]);
+ Object[] enumArray = (Object[]) Array.newInstance(enumClass, 2);
+ enumArray[0] = enumClass.getEnumConstants()[0];
+ enumArray[1] = enumClass.getEnumConstants()[1];
+ ReflectionUtils.setObjectFieldValue(o, "f3", enumArray);
+ Object[] enumArray2 = (Object[]) Array.newInstance(enumClass, 2, 2);
+ enumArray2[0] = enumArray;
+ enumArray2[1] = enumArray;
+ ReflectionUtils.setObjectFieldValue(o, "f4", enumArray2);
+ Fury fury1 =
+ furyBuilder()
+ .withDeserializeUnexistedClass(true)
+ .withClassLoader(cls1.getClassLoader())
+ .build();
+ byte[] bytes = fury1.serialize(o);
+ {
+ Object o1 = fury1.deserialize(bytes);
+ assertEquals(ReflectionUtils.getObjectFieldValue(o1, "f2"),
enumClass.getEnumConstants()[1]);
+ assertEquals(ReflectionUtils.getObjectFieldValue(o1, "f3"), enumArray);
+ }
+ ByteArrayClassLoader classLoader =
+ JaninoUtils.compile(
+ getClass().getClassLoader(),
+ new CompileUnit(
+ "",
+ "TestEnumStruct",
+ ("public class TestEnumStruct {" + " public String f1;" +
"}")));
+ Fury fury2 =
+
furyBuilder().withDeserializeUnexistedClass(true).withClassLoader(classLoader).build();
+ Object o1 = fury2.deserialize(bytes);
+ Assert.assertEquals(ReflectionUtils.getObjectFieldValue(o1, "f1"), "str");
+ }
+
+ @DataProvider
+ public Object[][] componentFinal() {
+ return new Object[][] {{false}, {true}};
+ }
+
+ @Test(dataProvider = "componentFinal")
+ public void testSkipUnexistedObjectArrayField(boolean componentFinal) throws
Exception {
+ String enumStructCode1 =
+ ("public class TestArrayStruct {\n"
+ + " public static "
+ + (componentFinal ? " final " : "")
+ + "class TestClass {\n"
+ + " }\n"
+ + " public String f1;\n"
+ + " public TestClass f2;\n"
+ + " public TestClass[] f3;\n"
+ + " public TestClass[][] f4;\n"
+ + "}");
+ Class<?> cls1 =
+ JaninoUtils.compile(
+ getClass().getClassLoader(),
+ new CompileUnit("", "TestArrayStruct", enumStructCode1))
+ .loadClass("TestArrayStruct");
+ Class<?> testClass = cls1.getDeclaredClasses()[0];
+ Object o = cls1.newInstance();
+ ReflectionUtils.setObjectFieldValue(o, "f1", "str");
+ ReflectionUtils.setObjectFieldValue(o, "f2", testClass.newInstance());
+ Object[] arr = (Object[]) Array.newInstance(testClass, 2);
+ arr[0] = testClass.newInstance();
+ arr[1] = testClass.newInstance();
+ ReflectionUtils.setObjectFieldValue(o, "f3", arr);
+ Object[] arr2D = (Object[]) Array.newInstance(testClass, 2, 2);
+ arr2D[0] = arr;
+ arr2D[1] = arr;
+ ReflectionUtils.setObjectFieldValue(o, "f4", arr2D);
+ Fury fury1 =
+ furyBuilder()
+ .withDeserializeUnexistedClass(true)
+ .withClassLoader(cls1.getClassLoader())
+ .build();
+ byte[] bytes = fury1.serialize(o);
+ {
+ Object o1 = fury1.deserialize(bytes);
+ assertEquals(ReflectionUtils.getObjectFieldValue(o1, "f2").getClass(),
testClass);
+ assertEquals(ReflectionUtils.getObjectFieldValue(o1, "f3").getClass(),
arr.getClass());
+ }
+ ByteArrayClassLoader classLoader =
+ JaninoUtils.compile(
+ getClass().getClassLoader(),
+ new CompileUnit(
+ "",
+ "TestArrayStruct",
+ ("public class TestArrayStruct {" + " public String f1;" +
"}")));
+ Fury fury2 =
+
furyBuilder().withDeserializeUnexistedClass(true).withClassLoader(classLoader).build();
+ Object o1 = fury2.deserialize(bytes);
+ Assert.assertEquals(ReflectionUtils.getObjectFieldValue(o1, "f1"), "str");
+ }
+
@Test(dataProvider = "metaShareConfig")
public void testDeserializeUnexistedNewFury(
boolean referenceTracking,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
