This is an automated email from the ASF dual-hosted git repository. chaokunyang pushed a commit to branch releases-0.12 in repository https://gitbox.apache.org/repos/asf/fory.git
commit 55bddbb1fbb6c2d514bb4b1c3cafa21c46ecf4bf Author: Shawn Yang <[email protected]> AuthorDate: Fri Sep 5 00:17:33 2025 +0800 feat(java): support limit deserialization depth (#2578) ## Why? <!-- Describe the purpose of this PR. --> ## What does this PR do? <!-- Describe the details of this PR. --> ## Related issues <!-- Is there any related issue? If this PR closes them you say say fix/closes: - #xxxx0 - #xxxx1 - Fixes #xxxx2 --> ## Does this PR introduce any user-facing change? <!-- If any user-facing interface changes, please [open an issue](https://github.com/apache/fory/issues/new/choose) describing the need to do so and update the document if necessary. Delete section if not applicable. --> - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark <!-- When the PR has an impact on performance (if you don't know whether the PR will have an impact on performance, you can submit the PR first, and if it will have impact on performance, the code reviewer will explain it), be sure to attach a benchmark data here. Delete section if not applicable. --> --- docs/guide/java_serialization_guide.md | 56 ++++++----- .../src/main/java/org/apache/fory/Fory.java | 56 ++++++----- .../fory/builder/BaseObjectCodecBuilder.java | 22 +++-- .../java/org/apache/fory/builder/CodecBuilder.java | 2 +- .../apache/fory/builder/ObjectCodecOptimizer.java | 2 +- .../main/java/org/apache/fory/config/Config.java | 7 ++ .../java/org/apache/fory/config/ForyBuilder.java | 12 +++ .../org/apache/fory/resolver/XtypeResolver.java | 8 +- .../fory/serializer/AbstractObjectSerializer.java | 12 ++- .../fory/serializer/SerializationBinding.java | 108 +++++++++++---------- .../collection/CollectionLikeSerializer.java | 4 +- .../serializer/collection/MapLikeSerializer.java | 24 +++-- .../main/java/org/apache/fory/type/TypeUtils.java | 75 ++++++++++++++ .../src/test/java/org/apache/fory/ForyTest.java | 39 ++++++++ .../test/java/org/apache/fory/ForyTestBase.java | 29 +----- .../java/org/apache/fory/type/TypeUtilsTest.java | 31 ++++++ 16 files changed, 334 insertions(+), 153 deletions(-) diff --git a/docs/guide/java_serialization_guide.md b/docs/guide/java_serialization_guide.md index 7f1f7a2af..059668176 100644 --- a/docs/guide/java_serialization_guide.md +++ b/docs/guide/java_serialization_guide.md @@ -108,28 +108,30 @@ public class Example { ## ForyBuilder options -| Option Name | Description [...] -| ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [...] -| `timeRefIgnored` | Whether to ignore reference tracking of all time types registered in `TimeSerializers` and subclasses of those types when ref tracking is enabled. If ignored, ref tracking of every time type can be enabled by invoking `Fory#registerSerializer(Class, Serializer)`. For example, `fory.registerSerializer(Date.class, new DateSerializer(fory, true))`. Note that enabling ref tracking should happen before serializer codegen of any types which contain time [...] -| `compressInt` | Enables or disables int compression for smaller size. [...] -| `compressLong` | Enables or disables long compression for smaller size. [...] -| `compressString` | Enables or disables string compression for smaller size. [...] -| `classLoader` | The classloader should not be updated; Fory caches class metadata. Use `LoaderBinding` or `ThreadSafeFory` for classloader updates. [...] -| `compatibleMode` | Type forward/backward compatibility config. Also Related to `checkClassVersion` config. `SCHEMA_CONSISTENT`: Class schema must be consistent between serialization peer and deserialization peer. `COMPATIBLE`: Class schema can be different between serialization peer and deserialization peer. They can add/delete fields independently. [See more](#class-inconsistency-and-class-version-check). [...] -| `checkClassVersion` | Determines whether to check the consistency of the class schema. If enabled, Fory checks, writes, and checks consistency using the `classVersionHash`. It will be automatically disabled when `CompatibleMode#COMPATIBLE` is enabled. Disabling is not recommended unless you can ensure the class won't evolve. [...] -| `checkJdkClassSerializable` | Enables or disables checking of `Serializable` interface for classes under `java.*`. If a class under `java.*` is not `Serializable`, Fory will throw an `UnsupportedOperationException`. [...] -| `registerGuavaTypes` | Whether to pre-register Guava types such as `RegularImmutableMap`/`RegularImmutableList`. These types are not public API, but seem pretty stable. [...] -| `requireClassRegistration` | Disabling may allow unknown classes to be deserialized, potentially causing security risks. [...] -| `suppressClassRegistrationWarnings` | Whether to suppress class registration warnings. The warnings can be used for security audit, but may be annoying, this suppression will be enabled by default. [...] -| `metaShareEnabled` | Enables or disables meta share mode. [...] -| `scopedMetaShareEnabled` | Scoped meta share focuses on a single serialization process. Metadata created or identified during this process is exclusive to it and is not shared with by other serializations. [...] -| `metaCompressor` | Set a compressor for meta compression. Note that the passed MetaCompressor should be thread-safe. By default, a `Deflater` based compressor `DeflaterMetaCompressor` will be used. Users can pass other compressor such as `zstd` for better compression rate. [...] -| `deserializeNonexistentClass` | Enables or disables deserialization/skipping of data for non-existent classes. [...] -| `codeGenEnabled` | Disabling may result in faster initial serialization but slower subsequent serializations. [...] -| `asyncCompilationEnabled` | If enabled, serialization uses interpreter mode first and switches to JIT serialization after async serializer JIT for a class is finished. [...] -| `scalaOptimizationEnabled` | Enables or disables Scala-specific serialization optimization. [...] -| `copyRef` | When disabled, the copy performance will be better. But fory deep copy will ignore circular and shared reference. Same reference of an object graph will be copied into different objects in one `Fory#copy`. [...] -| `serializeEnumByName` | When Enabled, fory serialize enum by name instead of ordinal. [...] +| Option Name | Description [...] +| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [...] +| `timeRefIgnored` | Whether to ignore reference tracking of all time types registered in `TimeSerializers` and subclasses of those types when ref tracking is enabled. If ignored, ref tracking of every time type can be enabled by invoking `Fory#registerSerializer(Class, Serializer)`. For example, `fory.registerSerializer(Date.class, new DateSerializer(fory, true))`. Note that enabling ref tracking should happen before serializer codegen of any types which contain time fields. [...] +| `compressInt` | Enables or disables int compression for smaller size. [...] +| `compressLong` | Enables or disables long compression for smaller size. [...] +| `compressString` | Enables or disables string compression for smaller size. [...] +| `classLoader` | The classloader should not be updated; Fory caches class metadata. Use `LoaderBinding` or `ThreadSafeFory` for classloader updates. [...] +| `compatibleMode` | Type forward/backward compatibility config. Also Related to `checkClassVersion` config. `SCHEMA_CONSISTENT`: Class schema must be consistent between serialization peer and deserialization peer. `COMPATIBLE`: Class schema can be different between serialization peer and deserialization peer. They can add/delete fields independently. [See more](#class-inconsistency-and-class-version-check). [...] +| `checkClassVersion` | Determines whether to check the consistency of the class schema. If enabled, Fory checks, writes, and checks consistency using the `classVersionHash`. It will be automatically disabled when `CompatibleMode#COMPATIBLE` is enabled. Disabling is not recommended unless you can ensure the class won't evolve. [...] +| `checkJdkClassSerializable` | Enables or disables checking of `Serializable` interface for classes under `java.*`. If a class under `java.*` is not `Serializable`, Fory will throw an `UnsupportedOperationException`. [...] +| `registerGuavaTypes` | Whether to pre-register Guava types such as `RegularImmutableMap`/`RegularImmutableList`. These types are not public API, but seem pretty stable. [...] +| `requireClassRegistration` | Disabling may allow unknown classes to be deserialized, potentially causing security risks. [...] +| `requireClassRegistration` | Set max depth for deserialization, when depth exceeds, an exception will be thrown. This can be used to refuse deserialization DDOS attack. [...] + +| `suppressClassRegistrationWarnings` | Whether to suppress class registration warnings. The warnings can be used for security audit, but may be annoying, this suppression will be enabled by default. | `true` | +| `metaShareEnabled` | Enables or disables meta share mode. | `true` if `CompatibleMode.Compatible` is set, otherwise false. | +| `scopedMetaShareEnabled` | Scoped meta share focuses on a single serialization process. Metadata created or identified during this process is exclusive to it and is not shared with by other serializations. | `true` if `CompatibleMode.Compatible` is set, otherwise false. | +| `metaCompressor` | Set a compressor for meta compression. Note that the passed MetaCompressor should be thread-safe. By default, a `Deflater` based compressor `DeflaterMetaCompressor` will be used. Users can pass other compressor such as `zstd` for better compression rate. | `DeflaterMetaCompressor` | +| `deserializeNonexistentClass` | Enables or disables deserialization/skipping of data for non-existent classes. | `true` if `CompatibleMode.Compatible` is set, otherwise false. | +| `codeGenEnabled` | Disabling may result in faster initial serialization but slower subsequent serializations. | `true` | +| `asyncCompilationEnabled` | If enabled, serialization uses interpreter mode first and switches to JIT serialization after async serializer JIT for a class is finished. | `false` | +| `scalaOptimizationEnabled` | Enables or disables Scala-specific serialization optimization. | `false` | +| `copyRef` | When disabled, the copy performance will be better. But fory deep copy will ignore circular and shared reference. Same reference of an object graph will be copied into different objects in one `Fory#copy`. | `true` | +| `serializeEnumByName` | When Enabled, fory serialize enum by name instead of ordinal. | `false` | ## Advanced Usage @@ -1167,7 +1169,9 @@ Custom memory allocators are useful for: - **Debugging**: Add logging or tracking to monitor memory usage - **Off-heap Memory**: Integrate with off-heap memory management systems -### Security & Class Registration +### Security + +#### Class Registration `ForyBuilder#requireClassRegistration` can be used to disable class registration, this will allow to deserialize objects unknown types, @@ -1217,6 +1221,12 @@ simplify the customization of class check mechanism. You can use this checker or implement more sophisticated checker by yourself. +#### Limit max deserization depth + +Fory also provides a `ForyBuilder#withMaxDepth` to limit max deserialization depth. The default max depth is 50. + +If max depth is reached, Fory will throw `ForyException`. This can be used to prevent malicious data from causing stack overflow or other issues. + ### Register class by name Register class by id will have better performance and smaller space overhead. But in some cases, management for a bunch diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 1786f8c95..e01a7994d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -40,6 +40,7 @@ import org.apache.fory.config.LongEncoding; import org.apache.fory.exception.CopyException; import org.apache.fory.exception.DeserializationException; import org.apache.fory.exception.ForyException; +import org.apache.fory.exception.InsecureException; import org.apache.fory.exception.SerializationException; import org.apache.fory.io.ForyInputStream; import org.apache.fory.io.ForyReadableChannel; @@ -126,6 +127,7 @@ public final class Fory implements BaseFory { private Iterator<MemoryBuffer> outOfBandBuffers; private boolean peerOutOfBandEnabled; private int depth; + private final int maxDepth; private int copyDepth; private final boolean copyRefTracking; private final IdentityMap<Object, Object> originToCopyMap; @@ -141,6 +143,7 @@ public final class Fory implements BaseFory { this.shareMeta = config.isMetaShareEnabled(); compressInt = config.compressInt(); longEncoding = config.longEncoding(); + maxDepth = config.maxDepth(); if (refTracking) { this.refResolver = new MapRefResolver(); } else { @@ -653,17 +656,6 @@ public final class Fory implements BaseFory { case ClassResolver.STRING_CLASS_ID: stringSerializer.writeJavaString(buffer, (String) obj); break; - case ClassResolver.ARRAYLIST_CLASS_ID: - depth++; - arrayListSerializer.write(buffer, (ArrayList) obj); - depth--; - break; - case ClassResolver.HASHMAP_CLASS_ID: - depth++; - hashMapSerializer.write(buffer, (HashMap) obj); - depth--; - break; - // TODO(add fastpath for other types) default: depth++; classInfo.getSerializer().write(buffer, obj); @@ -1024,7 +1016,7 @@ public final class Fory implements BaseFory { /** Class should be read already. */ public Object readData(MemoryBuffer buffer, ClassInfo classInfo) { - depth++; + incReadDepth(); Serializer<?> serializer = classInfo.getSerializer(); Object read = serializer.read(buffer); depth--; @@ -1055,19 +1047,8 @@ public final class Fory implements BaseFory { return buffer.readFloat64(); case ClassResolver.STRING_CLASS_ID: return stringSerializer.readJavaString(buffer); - case ClassResolver.ARRAYLIST_CLASS_ID: - depth++; - Object list = arrayListSerializer.read(buffer); - depth--; - return list; - case ClassResolver.HASHMAP_CLASS_ID: - depth++; - Object map = hashMapSerializer.read(buffer); - depth--; - return map; - // TODO(add fastpath for other types) default: - depth++; + incReadDepth(); Object read = classInfo.getSerializer().read(buffer); depth--; return read; @@ -1112,7 +1093,7 @@ public final class Fory implements BaseFory { } public Object xreadNonRef(MemoryBuffer buffer, Serializer<?> serializer) { - depth++; + incReadDepth(); Object o = serializer.xread(buffer); depth--; return o; @@ -1142,7 +1123,7 @@ public final class Fory implements BaseFory { return buffer.readFloat64(); // TODO(add fastpath for other types) default: - depth++; + incReadDepth(); Object o = classInfo.getSerializer().xread(buffer); depth--; return o; @@ -1682,6 +1663,29 @@ public final class Fory implements BaseFory { this.depth += diff; } + public void incDepth() { + this.depth += 1; + } + + public void decDepth() { + this.depth -= 1; + } + + public void incReadDepth() { + if ((this.depth += 1) > maxDepth) { + throwReadDepthExceedException(); + } + } + + private void throwReadDepthExceedException() { + throw new InsecureException( + String.format( + "Read depth exceed max depth %s, " + + "the deserialization data may be malicious. If it's not malicious, " + + "please increase max read depth by ForyBuilder#withMaxDepth(largerDepth)", + maxDepth)); + } + public void incCopyDepth(int diff) { this.copyDepth += diff; } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 7460664c2..0b129581b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -1636,13 +1636,13 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder { obj = deserializeForMap(buffer, typeRef, serializer, invokeHint); } else { if (serializer != null) { - return new Invoke(serializer, "read", OBJECT_TYPE, buffer); + return read(serializer, buffer, OBJECT_TYPE); } if (isMonomorphic(cls)) { serializer = getOrCreateSerializer(cls); Class<?> returnType = ReflectionUtils.getReturnType(getRawType(serializer.type()), "read"); - obj = new Invoke(serializer, "read", TypeRef.of(returnType), buffer); + obj = read(serializer, buffer, TypeRef.of(returnType)); } else { obj = readForNotNullNonFinal(buffer, typeRef, serializer); } @@ -1651,13 +1651,24 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder { } } + protected Expression read(Expression serializer, Expression buffer, TypeRef<?> returnType) { + Class<?> type = returnType.getRawType(); + Expression read = new Invoke(serializer, "read", returnType, buffer); + if (ReflectionUtils.isMonomorphic(type) && !TypeUtils.hasExpandableLeafs(type)) { + return read; + } + read = uninline(read); + return new ListExpression( + new Invoke(foryRef, "incReadDepth"), read, new Invoke(foryRef, "decDepth"), read); + } + protected Expression readForNotNullNonFinal( Expression buffer, TypeRef<?> typeRef, Expression serializer) { if (serializer == null) { Expression classInfo = readClassInfo(getRawType(typeRef), buffer); serializer = inlineInvoke(classInfo, "getSerializer", SERIALIZER_TYPE); } - return new Invoke(serializer, "read", OBJECT_TYPE, buffer); + return read(serializer, buffer, OBJECT_TYPE); } /** @@ -1693,7 +1704,7 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder { new If( supportHook, new ListExpression(collection, hookRead), - new Invoke(serializer, "read", OBJECT_TYPE, buffer), + read(serializer, buffer, OBJECT_TYPE), false); if (invokeHint != null && invokeHint.genNewMethod) { invokeHint.add(buffer); @@ -1969,8 +1980,7 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder { expressions.add(chunksLoop, newMap); // first newMap to create map, last newMap as expr value Expression map = inlineInvoke(serializer, "onMapRead", OBJECT_TYPE, expressions); - Expression action = - new If(supportHook, map, new Invoke(serializer, "read", OBJECT_TYPE, buffer), false); + Expression action = new If(supportHook, map, read(serializer, buffer, OBJECT_TYPE), false); if (invokeHint != null && invokeHint.genNewMethod) { invokeHint.add(buffer); invokeHint.add(serializer); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index b9d0b44bb..ca155eb8c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -95,7 +95,7 @@ public abstract class CodecBuilder { protected final boolean isRecord; protected final boolean isInterface; private final Set<String> duplicatedFields; - protected Reference foryRef = new Reference(FORY_NAME, TypeRef.of(Fory.class)); + protected Reference foryRef = Reference.fieldRef(FORY_NAME, TypeRef.of(Fory.class)); public static final Reference recordComponentDefaultValues = new Reference("recordComponentDefaultValues", OBJECT_ARRAY_TYPE); protected final Map<String, Reference> fieldMap = new HashMap<>(); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java index 85749ffb8..992370f75 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java @@ -121,7 +121,7 @@ public class ObjectCodecOptimizer extends ExpressionOptimizer { MutableTuple3.of( new ArrayList<>(descriptorGrouper.getFinalDescriptors()), 5, finalReadGroups), MutableTuple3.of( - new ArrayList<>(descriptorGrouper.getOtherDescriptors()), 5, otherReadGroups), + new ArrayList<>(descriptorGrouper.getOtherDescriptors()), 4, otherReadGroups), MutableTuple3.of( new ArrayList<>(descriptorGrouper.getOtherDescriptors()), 9, otherWriteGroups)); for (MutableTuple3<List<Descriptor>, Integer, List<List<Descriptor>>> decs : groups) { diff --git a/java/fory-core/src/main/java/org/apache/fory/config/Config.java b/java/fory-core/src/main/java/org/apache/fory/config/Config.java index 251f0223c..904aa92ea 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/Config.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/Config.java @@ -63,6 +63,7 @@ public class Config implements Serializable { private final boolean deserializeNonexistentEnumValueAsNull; private final boolean serializeEnumByName; private final int bufferSizeLimitBytes; + private final int maxDepth; public Config(ForyBuilder builder) { name = builder.name; @@ -101,6 +102,7 @@ public class Config implements Serializable { deserializeNonexistentEnumValueAsNull = builder.deserializeNonexistentEnumValueAsNull; serializeEnumByName = builder.serializeEnumByName; bufferSizeLimitBytes = builder.bufferSizeLimitBytes; + maxDepth = builder.maxDepth; } /** Returns the name for Fory serialization. */ @@ -357,4 +359,9 @@ public class Config implements Serializable { } return configHash; } + + /** Returns max depth for deserialization, when depth exceeds, an exception will be thrown. */ + public int maxDepth() { + return maxDepth; + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index cd3955a1c..69ee680b1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -38,6 +38,7 @@ import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.TimeSerializers; import org.apache.fory.serializer.collection.GuavaCollectionSerializers; import org.apache.fory.util.GraalvmSupport; +import org.apache.fory.util.Preconditions; /** Builder class to config and create {@link Fory}. */ // Method naming style for this builder: @@ -86,6 +87,7 @@ public final class ForyBuilder { boolean serializeEnumByName = false; int bufferSizeLimitBytes = 128 * 1024; MetaCompressor metaCompressor = new DeflaterMetaCompressor(); + int maxDepth = 50; public ForyBuilder() {} @@ -348,6 +350,16 @@ public final class ForyBuilder { return this; } + /** + * Set max depth for deserialization, when depth exceeds, an exception will be thrown. Default max + * depth is 50. + */ + public ForyBuilder withMaxDepth(int maxDepth) { + Preconditions.checkArgument(maxDepth >= 2, "maxDepth must >= 2 but got %s", maxDepth); + this.maxDepth = maxDepth; + return this; + } + /** Whether enable scala-specific serialization optimization. */ public ForyBuilder withScalaOptimizationEnabled(boolean enableScalaOptimization) { this.scalaOptimizationEnabled = enableScalaOptimization; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index dcdf71e5b..3f70c4a01 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -647,9 +647,9 @@ public class XtypeResolver implements TypeResolver { } private ClassInfo getListClassInfo() { - fory.incDepth(1); + fory.incReadDepth(); GenericType genericType = generics.nextGenericType(); - fory.incDepth(-1); + fory.decDepth(); if (genericType != null) { return getOrBuildClassInfo(genericType.getCls()); } @@ -657,9 +657,9 @@ public class XtypeResolver implements TypeResolver { } private ClassInfo getGenericClassInfo() { - fory.incDepth(1); + fory.incReadDepth(); GenericType genericType = generics.nextGenericType(); - fory.incDepth(-1); + fory.decDepth(); if (genericType != null) { return getOrBuildClassInfo(genericType.getCls()); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index bae348dc0..82b53321d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -89,15 +89,17 @@ public abstract class AbstractObjectSerializer<T> extends Serializer<T> { boolean isFinal, MemoryBuffer buffer) { Serializer<Object> serializer = fieldInfo.classInfo.getSerializer(); + binding.incReadDepth(); Object fieldValue; boolean nullable = fieldInfo.nullable; if (isFinal) { if (!fieldInfo.trackingRef) { - return binding.readNullable(buffer, serializer, nullable); + fieldValue = binding.readNullable(buffer, serializer, nullable); + } else { + // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still + // consistent with jit serializer. + fieldValue = binding.readRef(buffer, serializer); } - // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still - // consistent with jit serializer. - fieldValue = binding.readRef(buffer, serializer); } else { if (serializer.needToWriteRef()) { int nextReadRefId = refResolver.tryPreserveRefId(buffer); @@ -112,6 +114,7 @@ public abstract class AbstractObjectSerializer<T> extends Serializer<T> { if (nullable) { byte headFlag = buffer.readByte(); if (headFlag == Fory.NULL_FLAG) { + binding.decDepth(); return null; } } @@ -119,6 +122,7 @@ public abstract class AbstractObjectSerializer<T> extends Serializer<T> { fieldValue = serializer.read(buffer); } } + binding.decDepth(); return fieldValue; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 9c5ff111e..087189049 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -34,63 +34,88 @@ import org.apache.fory.resolver.XtypeResolver; // If it's used in other packages in fory, duplicate it in those packages. @SuppressWarnings({"rawtypes", "unchecked"}) // noinspection Duplicates -interface SerializationBinding { - <T> void writeRef(MemoryBuffer buffer, T obj); +abstract class SerializationBinding { + protected final Fory fory; + protected final RefResolver refResolver; - <T> void writeRef(MemoryBuffer buffer, T obj, Serializer<T> serializer); + SerializationBinding(Fory fory) { + this.fory = fory; + this.refResolver = fory.getRefResolver(); + } + + abstract <T> void writeRef(MemoryBuffer buffer, T obj); + + abstract <T> void writeRef(MemoryBuffer buffer, T obj, Serializer<T> serializer); - void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); + abstract void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); - void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo); + abstract void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo); - void writeNonRef(MemoryBuffer buffer, Object obj); + abstract void writeNonRef(MemoryBuffer buffer, Object obj); - void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo); + abstract void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo); - void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); + abstract void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); - void writeNullable(MemoryBuffer buffer, Object obj); + abstract void writeNullable(MemoryBuffer buffer, Object obj); - void writeNullable(MemoryBuffer buffer, Object obj, Serializer serializer); + abstract void writeNullable(MemoryBuffer buffer, Object obj, Serializer serializer); - void writeNullable(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); + abstract void writeNullable(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); - void writeNullable(MemoryBuffer buffer, Object obj, ClassInfo classInfo); + abstract void writeNullable(MemoryBuffer buffer, Object obj, ClassInfo classInfo); - void writeNullable( + abstract void writeNullable( MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder, boolean nullable); - void writeNullable(MemoryBuffer buffer, Object obj, Serializer serializer, boolean nullable); + abstract void writeNullable( + MemoryBuffer buffer, Object obj, Serializer serializer, boolean nullable); - void writeContainerFieldValue(MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo); + abstract void writeContainerFieldValue( + MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo); - void write(MemoryBuffer buffer, Serializer serializer, Object value); + abstract void write(MemoryBuffer buffer, Serializer serializer, Object value); - Object read(MemoryBuffer buffer, Serializer serializer); + abstract Object read(MemoryBuffer buffer, Serializer serializer); - <T> T readRef(MemoryBuffer buffer, Serializer<T> serializer); + abstract <T> T readRef(MemoryBuffer buffer, Serializer<T> serializer); - Object readRef(MemoryBuffer buffer, GenericTypeField field); + abstract Object readRef(MemoryBuffer buffer, GenericTypeField field); - Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); + abstract Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); - Object readRef(MemoryBuffer buffer); + abstract Object readRef(MemoryBuffer buffer); - Object readNonRef(MemoryBuffer buffer); + abstract Object readNonRef(MemoryBuffer buffer); - Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); + abstract Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); - Object readNonRef(MemoryBuffer buffer, GenericTypeField field); + abstract Object readNonRef(MemoryBuffer buffer, GenericTypeField field); - Object readNullable(MemoryBuffer buffer, Serializer<Object> serializer); + abstract Object readNullable(MemoryBuffer buffer, Serializer<Object> serializer); - Object readNullable(MemoryBuffer buffer, Serializer<Object> serializer, boolean nullable); + abstract Object readNullable( + MemoryBuffer buffer, Serializer<Object> serializer, boolean nullable); - Object readContainerFieldValue(MemoryBuffer buffer, GenericTypeField field); + abstract Object readContainerFieldValue(MemoryBuffer buffer, GenericTypeField field); - Object readContainerFieldValueRef(MemoryBuffer buffer, GenericTypeField fieldInfo); + abstract Object readContainerFieldValueRef(MemoryBuffer buffer, GenericTypeField fieldInfo); - int preserveRefId(int refId); + public int preserveRefId(int refId) { + return refResolver.preserveRefId(refId); + } + + void incReadDepth() { + fory.incReadDepth(); + } + + void incDepth() { + fory.incDepth(); + } + + void decDepth() { + fory.decDepth(); + } static SerializationBinding createBinding(Fory fory) { if (fory.isCrossLanguage()) { @@ -100,15 +125,12 @@ interface SerializationBinding { } } - final class JavaSerializationBinding implements SerializationBinding { - private final Fory fory; + static final class JavaSerializationBinding extends SerializationBinding { private final ClassResolver classResolver; - private final RefResolver refResolver; JavaSerializationBinding(Fory fory) { - this.fory = fory; + super(fory); classResolver = fory.getClassResolver(); - refResolver = fory.getRefResolver(); } @Override @@ -290,23 +312,14 @@ interface SerializationBinding { MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo) { fory.writeNonRef(buffer, fieldValue, classInfo); } - - @Override - public int preserveRefId(int refId) { - return refResolver.preserveRefId(refId); - } } - final class XlangSerializationBinding implements SerializationBinding { - - private final Fory fory; + static final class XlangSerializationBinding extends SerializationBinding { private final XtypeResolver xtypeResolver; - private final RefResolver refResolver; XlangSerializationBinding(Fory fory) { - this.fory = fory; + super(fory); xtypeResolver = fory.getXtypeResolver(); - refResolver = fory.getRefResolver(); } @Override @@ -500,10 +513,5 @@ interface SerializationBinding { MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo) { fory.xwriteData(buffer, classInfo, fieldValue); } - - @Override - public int preserveRefId(int refId) { - return refResolver.preserveRefId(refId); - } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java index 29568d4d1..43b38be79 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java @@ -662,7 +662,7 @@ public abstract class CollectionLikeSerializer<T> extends Serializer<T> { int flags, T collection, int numElements) { - fory.incDepth(1); + fory.incReadDepth(); if ((flags & CollectionFlags.TRACKING_REF) == CollectionFlags.TRACKING_REF) { for (int i = 0; i < numElements; i++) { collection.add(binding.readRef(buffer, serializer)); @@ -682,7 +682,7 @@ public abstract class CollectionLikeSerializer<T> extends Serializer<T> { } } } - fory.incDepth(-1); + fory.decDepth(); } /** Read elements whose type are different. */ diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java index c7aa2c59d..3dc89a074 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java @@ -689,11 +689,13 @@ public abstract class MapLikeSerializer<T> extends Serializer<T> { if (keySerializer == null) { key = readNonEmptyValueFromNullChunk(buffer, trackKeyRef, true); } else { + fory.incReadDepth(); if (trackKeyRef) { key = binding.readRef(buffer, keySerializer); } else { key = binding.read(buffer, keySerializer); } + fory.decDepth(); } } else { key = binding.readRef(buffer, keyClassInfoReadCache); @@ -729,11 +731,13 @@ public abstract class MapLikeSerializer<T> extends Serializer<T> { if (valueSerializer == null) { value = readNonEmptyValueFromNullChunk(buffer, trackValueRef, false); } else { + fory.incReadDepth(); if (trackValueRef) { value = binding.readRef(buffer, valueSerializer); } else { value = binding.read(buffer, valueSerializer); } + fory.decDepth(); } } else { value = binding.readRef(buffer, valueClassInfoReadCache); @@ -753,15 +757,15 @@ public abstract class MapLikeSerializer<T> extends Serializer<T> { } GenericType type = isKey ? genericType.getTypeParameter0() : genericType.getTypeParameter1(); generics.pushGenericType(type); - fory.incDepth(1); Serializer<?> serializer = type.getSerializer(typeResolver); Object v; + fory.incReadDepth(); if (trackRef) { v = binding.readRef(buffer, serializer); } else { v = binding.read(buffer, serializer); } - fory.incDepth(-1); + fory.decDepth(); generics.popGenericType(); return v; } @@ -781,8 +785,10 @@ public abstract class MapLikeSerializer<T> extends Serializer<T> { if (!valueHasNull) { return (size << 8) | chunkHeader; } else { + fory.incReadDepth(); Object key = binding.read(buffer, keySerializer); map.put(key, null); + fory.decDepth(); } } else { readNullKeyChunk(buffer, map, chunkHeader, valueSerializer, valueHasNull); @@ -815,6 +821,7 @@ public abstract class MapLikeSerializer<T> extends Serializer<T> { if (!valueIsDeclaredType) { valueSerializer = typeResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } + fory.incReadDepth(); for (int i = 0; i < chunkSize; i++) { Object key = trackKeyRef @@ -827,6 +834,7 @@ public abstract class MapLikeSerializer<T> extends Serializer<T> { map.put(key, value); size--; } + fory.decDepth(); return size > 0 ? (size << 8) | buffer.readUnsignedByte() : 0; } @@ -866,20 +874,20 @@ public abstract class MapLikeSerializer<T> extends Serializer<T> { if (keyGenericType.hasGenericParameters() || valueGenericType.hasGenericParameters()) { for (int i = 0; i < chunkSize; i++) { generics.pushGenericType(keyGenericType); - fory.incDepth(1); + fory.incReadDepth(); Object key = trackKeyRef ? binding.readRef(buffer, keySerializer) : binding.read(buffer, keySerializer); - fory.incDepth(-1); + fory.decDepth(); generics.popGenericType(); generics.pushGenericType(valueGenericType); - fory.incDepth(1); + fory.incReadDepth(); Object value = trackValueRef ? binding.readRef(buffer, valueSerializer) : binding.read(buffer, valueSerializer); - fory.incDepth(-1); + fory.decDepth(); generics.popGenericType(); map.put(key, value); size--; @@ -887,7 +895,7 @@ public abstract class MapLikeSerializer<T> extends Serializer<T> { } else { for (int i = 0; i < chunkSize; i++) { // increase depth to avoid read wrong outer generic type - fory.incDepth(1); + fory.incReadDepth(); Object key = trackKeyRef ? binding.readRef(buffer, keySerializer) @@ -896,7 +904,7 @@ public abstract class MapLikeSerializer<T> extends Serializer<T> { trackValueRef ? binding.readRef(buffer, valueSerializer) : binding.read(buffer, valueSerializer); - fory.incDepth(-1); + fory.decDepth(); map.put(key, value); size--; } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index dde3fca6a..2095d2000 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -20,6 +20,7 @@ package org.apache.fory.type; import java.lang.reflect.Array; +import java.lang.reflect.Field; import java.lang.reflect.GenericArrayType; import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; @@ -30,12 +31,27 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; +import java.time.Duration; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.MonthDay; +import java.time.OffsetDateTime; +import java.time.OffsetTime; +import java.time.Period; +import java.time.Year; +import java.time.YearMonth; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Collection; +import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -48,6 +64,8 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; +import java.util.TimeZone; +import java.util.WeakHashMap; import java.util.stream.Collectors; import org.apache.fory.collection.IdentityMap; import org.apache.fory.collection.Tuple2; @@ -834,4 +852,61 @@ public class TypeUtils { return pkg + "." + className; } } + + private static Set<Class<?>> leafTypes = new HashSet<>(); + + static { + leafTypes.addAll(sortedPrimitiveClasses); + leafTypes.addAll(sortedBoxedClasses); + leafTypes.add(String.class); + leafTypes.addAll( + Arrays.asList( + java.util.Date.class, + java.sql.Date.class, + Time.class, + Timestamp.class, + LocalDate.class, + LocalTime.class, + LocalDateTime.class, + Instant.class, + Duration.class, + ZoneId.class, + ZoneOffset.class, + ZonedDateTime.class, + Year.class, + YearMonth.class, + MonthDay.class, + Period.class, + OffsetTime.class, + OffsetDateTime.class, + Calendar.class, + GregorianCalendar.class, + TimeZone.class)); + } + + private static final WeakHashMap<Class<?>, Boolean> hasExpandableLeafsCache = new WeakHashMap<>(); + + public static synchronized boolean hasExpandableLeafs(Class<?> cls) { + return hasExpandableLeafsCache.computeIfAbsent( + cls, + k -> { + if (cls.isEnum()) { + return false; + } + if (leafTypes.contains(cls)) { + return false; + } + List<Field> fields = ReflectionUtils.getFields(cls, true); + if (fields.isEmpty()) { + return false; + } + for (Field field : fields) { + Class<?> fieldType = field.getType(); + if (!leafTypes.contains(fieldType) && !fieldType.isEnum()) { + return true; + } + } + return false; + }); + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index cbb6fb32c..01a2943cf 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -35,6 +35,7 @@ import java.nio.ByteBuffer; import java.sql.Timestamp; import java.time.Instant; import java.time.LocalDate; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -73,6 +74,7 @@ import org.apache.fory.serializer.Serializer; import org.apache.fory.test.bean.BeanA; import org.apache.fory.test.bean.Struct; import org.apache.fory.type.Descriptor; +import org.apache.fory.type.TypeUtils; import org.apache.fory.util.DateTimeUtils; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -690,4 +692,41 @@ public class ForyTest extends ForyTestBase { Assert.assertThrows( DeserializationException.class, () -> fory.deserializeJavaObject(bytes, Struct2.class)); } + + private Object maxDepthData() { + List<Object> list = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + List<Object> list1 = new ArrayList<>(); + list1.add("abc"); + list1.add(list); + list = list1; + } + return list; + } + + @Test + public void testMaxDepth() { + byte[] bytes = Fory.builder().requireClassRegistration(false).build().serialize(maxDepthData()); + Fory fory = + Fory.builder().requireClassRegistration(false).withName("fory1").withMaxDepth(3).build(); + assertThrows(InsecureException.class, () -> fory.deserialize(bytes)); + } + + @AllArgsConstructor + static class MaxDepth { + int f1; + Object f2; + } + + @Test + public void testMaxDepthCodegen() { + assertTrue(TypeUtils.hasExpandableLeafs(MaxDepth.class)); + MaxDepth maxDepth = + new MaxDepth( + 1, new MaxDepth(2, new MaxDepth(3, new MaxDepth(4, new MaxDepth(5, maxDepthData()))))); + byte[] bytes = Fory.builder().requireClassRegistration(false).build().serialize(maxDepth); + Fory fory = + Fory.builder().requireClassRegistration(false).withName("fory2").withMaxDepth(3).build(); + assertThrows(InsecureException.class, () -> fory.deserialize(bytes)); + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java b/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java index 6e156fce7..2bb24606e 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java @@ -187,34 +187,7 @@ public abstract class ForyTestBase { .requireClassRegistration(false) .suppressClassRegistrationWarnings(true) .build() - }, - { - Fory.builder() - .withLanguage(Language.JAVA) - .withRefTracking(false) - .withCodegen(false) - .requireClassRegistration(false) - .suppressClassRegistrationWarnings(true) - .build() - }, - { - Fory.builder() - .withLanguage(Language.JAVA) - .withRefTracking(true) - .withCodegen(true) - .requireClassRegistration(false) - .suppressClassRegistrationWarnings(true) - .build() - }, - { - Fory.builder() - .withLanguage(Language.JAVA) - .withRefTracking(false) - .withCodegen(true) - .requireClassRegistration(false) - .suppressClassRegistrationWarnings(true) - .build() - }, + } }; } diff --git a/java/fory-core/src/test/java/org/apache/fory/type/TypeUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/type/TypeUtilsTest.java index 1f22eec54..f084e396c 100644 --- a/java/fory-core/src/test/java/org/apache/fory/type/TypeUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/type/TypeUtilsTest.java @@ -20,9 +20,14 @@ package org.apache.fory.type; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Primitives; import java.lang.reflect.Type; +import java.time.Duration; +import java.time.Instant; import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; @@ -39,6 +44,7 @@ import org.apache.fory.collection.Tuple2; import org.apache.fory.reflect.TypeRef; import org.apache.fory.test.bean.BeanA; import org.apache.fory.test.bean.BeanB; +import org.apache.fory.test.bean.Foo; import org.testng.Assert; import org.testng.annotations.Test; @@ -289,4 +295,29 @@ public class TypeUtilsTest { assertEquals(allTypeArguments.size(), 3); assertEquals(allTypeArguments.get(2).getRawType(), BeanA.class); } + + static class SingleBasicFieldStruct { + int f1; + } + + static class SingleExpandableFieldStruct { + Object f1; + } + + @Test + public void testHasExpandableLeafs() { + for (Class<?> type : Primitives.allPrimitiveTypes()) { + assertFalse(TypeUtils.hasExpandableLeafs(type)); + } + for (Class<?> type : Primitives.allWrapperTypes()) { + assertFalse(TypeUtils.hasExpandableLeafs(type)); + } + assertFalse(TypeUtils.hasExpandableLeafs(Instant.class)); + assertFalse(TypeUtils.hasExpandableLeafs(Duration.class)); + assertFalse(TypeUtils.hasExpandableLeafs(Foo.class)); + assertTrue(TypeUtils.hasExpandableLeafs(BeanB.class)); + assertTrue(TypeUtils.hasExpandableLeafs(BeanA.class)); + assertFalse(TypeUtils.hasExpandableLeafs(SingleBasicFieldStruct.class)); + assertTrue(TypeUtils.hasExpandableLeafs(SingleExpandableFieldStruct.class)); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
