This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/main by this push:
new bcc01d7a fix(java): fix long type name meta string encoding (#1837)
bcc01d7a is described below
commit bcc01d7a0f2d8b14b5a6018491613369c18d0a3b
Author: Shawn Yang <[email protected]>
AuthorDate: Thu Sep 12 14:33:23 2024 +0800
fix(java): fix long type name meta string encoding (#1837)
## What does this PR do?
fix long type name meta string encoding
<!-- Describe the purpose of this PR. -->
## Related issues
Closes #1835
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fury/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
docs/specification/java_serialization_spec.md | 6 ++---
.../java/org/apache/fury/meta/ClassDefDecoder.java | 26 +++++++++---------
.../java/org/apache/fury/meta/ClassDefEncoder.java | 19 ++++++-------
.../org/apache/fury/meta/ClassDefEncoderTest.java | 31 +++++++++++++++++++++-
.../fury/serializer/MetaSharedCompatibleTest.java | 16 +++++++++++
5 files changed, 71 insertions(+), 27 deletions(-)
diff --git a/docs/specification/java_serialization_spec.md
b/docs/specification/java_serialization_spec.md
index a2357259..2d482565 100644
--- a/docs/specification/java_serialization_spec.md
+++ b/docs/specification/java_serialization_spec.md
@@ -155,11 +155,11 @@ Meta header is a 64 bits number value encoded in little
endian order.
fields info in meta for deserializing compatible fields.
- Package name encoding(omitted when class is registered):
- encoding algorithm: `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL`
- - Header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63`
will be used to indicate size `0~62`,
- the value `63` the size need more byte to read, the encoding will encode
`size - 62` as a varint next.
+ - Header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63`
will be used to indicate size `0~63`,
+ the value `63` the size need more byte to read, the encoding will encode
`size - 63` as a varint next.
- Class name encoding(omitted when class is registered):
- encoding algorithm:
`UTF8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL`
- - header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63`
will be used to indicate size `1~64`,
+ - header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63`
will be used to indicate size `0~63`,
the value `63` the size need more byte to read, the encoding will encode
`size - 63` as a varint next.
- Field info:
- header(8
diff --git
a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java
b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java
index cde7a50c..c5d90d94 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java
@@ -21,6 +21,7 @@ package org.apache.fury.meta;
import static org.apache.fury.meta.ClassDef.COMPRESSION_FLAG;
import static org.apache.fury.meta.ClassDef.SIZE_TWO_BYTES_FLAG;
+import static org.apache.fury.meta.ClassDefEncoder.BIG_NAME_THRESHOLD;
import static org.apache.fury.meta.Encoders.fieldNameEncodings;
import static org.apache.fury.meta.Encoders.pkgEncodings;
import static org.apache.fury.meta.Encoders.typeNameEncodings;
@@ -128,13 +129,10 @@ class ClassDefDecoder {
// - Package name encoding(omitted when class is registered):
// - encoding algorithm:
`UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL`
// - Header: `6 bits size | 2 bits encoding flags`.
- // The `6 bits size: 0~63` will be used to indicate size `0~62`,
- // the value `63` the size need more byte to read, the encoding will
encode `size - 62` as
+ // The `6 bits size: 0~63` will be used to indicate size `0~63`,
+ // the value `63` the size need more byte to read, the encoding will
encode `size - 63` as
// a varint next.
- int header = buffer.readByte() & 0xff;
- int encodingFlags = header & 0b11;
- Encoding encoding = pkgEncodings[encodingFlags];
- return readName(Encoders.PACKAGE_DECODER, buffer, header, encoding, 62);
+ return readName(Encoders.PACKAGE_DECODER, buffer, pkgEncodings);
}
private static String readTypeName(MemoryBuffer buffer) {
@@ -142,20 +140,20 @@ class ClassDefDecoder {
// - encoding algorithm:
//
`UTF8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL`
// - header: `6 bits size | 2 bits encoding flags`.
- // The `6 bits size: 0~63` will be used to indicate size `1~64`,
+ // The `6 bits size: 0~63` will be used to indicate size `0~63`,
// the value `63` the size need more byte to read, the encoding will
encode `size - 63` as
// a varint next.
- int header = buffer.readByte() & 0xff;
- int encodingFlags = header & 0b11;
- Encoding encoding = typeNameEncodings[encodingFlags];
- return readName(Encoders.TYPE_NAME_DECODER, buffer, header, encoding, 63);
+ return readName(Encoders.TYPE_NAME_DECODER, buffer, typeNameEncodings);
}
private static String readName(
- MetaStringDecoder decoder, MemoryBuffer buffer, int header, Encoding
encoding, int max) {
+ MetaStringDecoder decoder, MemoryBuffer buffer, Encoding[] encodings) {
+ int header = buffer.readByte() & 0xff;
+ int encodingFlags = header & 0b11;
+ Encoding encoding = encodings[encodingFlags];
int size = header >> 2;
- if (size == max) {
- size = buffer.readVarUint32Small7() + max;
+ if (size == BIG_NAME_THRESHOLD) {
+ size = buffer.readVarUint32Small7() + BIG_NAME_THRESHOLD;
}
return decoder.decode(buffer.readBytes(size), encoding);
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
index 46e2becf..c1ac5ebd 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefEncoder.java
@@ -277,8 +277,7 @@ class ClassDefEncoder {
// a varint next.
MetaString pkgMetaString = Encoders.encodePackage(pkg);
byte[] encoded = pkgMetaString.getBytes();
- int pkgHeader = (encoded.length << 2) |
pkgEncodingsList.indexOf(pkgMetaString.getEncoding());
- writeName(buffer, encoded, pkgHeader, 62);
+ writeName(buffer, encoded,
pkgEncodingsList.indexOf(pkgMetaString.getEncoding()));
}
private static void writeTypeName(MemoryBuffer buffer, String typeName) {
@@ -291,17 +290,19 @@ class ClassDefEncoder {
// a varint next.
MetaString metaString = Encoders.encodeTypeName(typeName);
byte[] encoded = metaString.getBytes();
- int header = (encoded.length << 2) |
typeNameEncodingsList.indexOf(metaString.getEncoding());
- writeName(buffer, encoded, header, 63);
+ writeName(buffer, encoded,
typeNameEncodingsList.indexOf(metaString.getEncoding()));
}
- private static void writeName(MemoryBuffer buffer, byte[] encoded, int
header, int max) {
- boolean bigSize = encoded.length > max;
+ static final int BIG_NAME_THRESHOLD = 0b111111;
+
+ private static void writeName(MemoryBuffer buffer, byte[] encoded, int
encoding) {
+ boolean bigSize = encoded.length >= BIG_NAME_THRESHOLD;
if (bigSize) {
- header |= 0b11111100;
- buffer.writeVarUint32Small7(header);
- buffer.writeVarUint32Small7(encoded.length - max);
+ int header = (BIG_NAME_THRESHOLD << 2) | encoding;
+ buffer.writeByte(header);
+ buffer.writeVarUint32Small7(encoded.length - BIG_NAME_THRESHOLD);
} else {
+ int header = (encoded.length << 2) | encoding;
buffer.writeByte(header);
}
buffer.writeBytes(encoded);
diff --git
a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java
b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java
index 45d3075f..6565afdf 100644
--- a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java
@@ -22,6 +22,7 @@ package org.apache.fury.meta;
import static org.apache.fury.meta.ClassDefEncoder.buildFieldsInfo;
import static org.apache.fury.meta.ClassDefEncoder.getClassFields;
+import java.io.Serializable;
import java.util.List;
import lombok.Data;
import org.apache.fury.Fury;
@@ -78,7 +79,35 @@ public class ClassDefEncoderTest {
ClassDef classDef1 =
ClassDef.readClassDef(
fury.getClassResolver(),
MemoryBuffer.fromByteArray(classDef.getEncoded()));
-
Assert.assertEquals(classDef, classDef1);
}
+
+ @Test
+ public void testBigClassNameObject() {
+ Fury fury = Fury.builder().withMetaShare(true).build();
+ ClassDef classDef =
+ ClassDef.buildClassDef(
+ fury,
+
TestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLength
+
.InnerClassTestLengthInnerClassTestLengthInnerClassTestLength.class);
+ ClassDef classDef1 =
+ ClassDef.readClassDef(
+ fury.getClassResolver(),
MemoryBuffer.fromByteArray(classDef.getEncoded()));
+ Assert.assertEquals(classDef1, classDef);
+ }
+
+ @Data
+ public static
+ class
TestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLength
+ implements Serializable {
+ private String name;
+ private InnerClassTestLengthInnerClassTestLengthInnerClassTestLength
innerClassTestLength;
+
+ @Data
+ public static class
InnerClassTestLengthInnerClassTestLengthInnerClassTestLength
+ implements Serializable {
+ private static final long serialVersionUID = -867612757789099089L;
+ private Long itemId;
+ }
+ }
}
diff --git
a/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java
b/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java
index e9b7d082..8a4eb683 100644
---
a/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java
+++
b/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java
@@ -34,6 +34,7 @@ import org.apache.fury.builder.MetaSharedCodecBuilder;
import org.apache.fury.config.CompatibleMode;
import org.apache.fury.config.FuryBuilder;
import org.apache.fury.config.Language;
+import org.apache.fury.meta.ClassDefEncoderTest;
import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.resolver.MetaContext;
import org.apache.fury.serializer.collection.UnmodifiableSerializersTest;
@@ -657,4 +658,19 @@ public class MetaSharedCompatibleTest extends FuryTestBase
{
Assert.assertEquals(o.getClass(), o1.getClass());
Assert.assertTrue(ReflectionUtils.objectFieldsEquals(o, o1));
}
+
+ @Test
+ public void testBigClassNameObject() {
+ Fury fury =
+ builder()
+ .withRefTracking(true)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .withScopedMetaShare(false)
+ .build();
+ Object o =
+ new ClassDefEncoderTest
+
.TestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLength
+ .InnerClassTestLengthInnerClassTestLengthInnerClassTestLength();
+ serDeCheck(fury, o);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
