This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/main by this push:
new 54b62fb6 feat(java): use varint for jdk compatible serializers (#1960)
54b62fb6 is described below
commit 54b62fb6ab5d7e557131efe07c7402c885f6e7c4
Author: Shawn Yang <[email protected]>
AuthorDate: Sun Dec 1 18:56:36 2024 +0800
feat(java): use varint for jdk compatible serializers (#1960)
## What does this PR do?
use varint for jdk compatible serializers to reduce serialized size
## Related issues
<!--
Is there any related issue? Please attach here.
- #xxxx0
- #xxxx1
- #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fury/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
.../org/apache/fury/io/MemoryBufferObjectInput.java | 10 ++++++++--
.../org/apache/fury/io/MemoryBufferObjectOutput.java | 16 ++++++++++++----
.../apache/fury/serializer/ObjectStreamSerializer.java | 16 ++++++++++++----
.../org/apache/fury/io/MemoryBufferObjectInputTest.java | 17 +++++++++++------
4 files changed, 43 insertions(+), 16 deletions(-)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/io/MemoryBufferObjectInput.java
b/java/fury-core/src/main/java/org/apache/fury/io/MemoryBufferObjectInput.java
index 026f18b2..c0885d7c 100644
---
a/java/fury-core/src/main/java/org/apache/fury/io/MemoryBufferObjectInput.java
+++
b/java/fury-core/src/main/java/org/apache/fury/io/MemoryBufferObjectInput.java
@@ -23,18 +23,24 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInput;
import org.apache.fury.Fury;
+import org.apache.fury.config.LongEncoding;
import org.apache.fury.memory.MemoryBuffer;
+import org.apache.fury.serializer.PrimitiveSerializers.LongSerializer;
import org.apache.fury.serializer.StringSerializer;
import org.apache.fury.util.Preconditions;
/** ObjectInput based on {@link Fury} and {@link MemoryBuffer}. */
public class MemoryBufferObjectInput extends InputStream implements
ObjectInput {
private final Fury fury;
+ private final boolean compressInt;
+ private final LongEncoding longEncoding;
private MemoryBuffer buffer;
private final StringSerializer stringSerializer;
public MemoryBufferObjectInput(Fury fury, MemoryBuffer buffer) {
this.fury = fury;
+ this.compressInt = fury.compressInt();
+ this.longEncoding = fury.longEncoding();
this.buffer = buffer;
this.stringSerializer = new StringSerializer(fury);
}
@@ -134,12 +140,12 @@ public class MemoryBufferObjectInput extends InputStream
implements ObjectInput
@Override
public int readInt() throws IOException {
- return buffer.readInt32();
+ return compressInt ? buffer.readVarInt32() : buffer.readInt32();
}
@Override
public long readLong() throws IOException {
- return buffer.readInt64();
+ return LongSerializer.readInt64(buffer, longEncoding);
}
@Override
diff --git
a/java/fury-core/src/main/java/org/apache/fury/io/MemoryBufferObjectOutput.java
b/java/fury-core/src/main/java/org/apache/fury/io/MemoryBufferObjectOutput.java
index 8f0dbf2f..8c74b17c 100644
---
a/java/fury-core/src/main/java/org/apache/fury/io/MemoryBufferObjectOutput.java
+++
b/java/fury-core/src/main/java/org/apache/fury/io/MemoryBufferObjectOutput.java
@@ -19,24 +19,28 @@
package org.apache.fury.io;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.io.ObjectOutput;
import java.io.OutputStream;
import org.apache.fury.Fury;
+import org.apache.fury.config.LongEncoding;
import org.apache.fury.memory.MemoryBuffer;
+import org.apache.fury.serializer.PrimitiveSerializers.LongSerializer;
import org.apache.fury.serializer.StringSerializer;
import org.apache.fury.util.Preconditions;
/** ObjectOutput based on {@link Fury} and {@link MemoryBuffer}. */
public class MemoryBufferObjectOutput extends OutputStream implements
ObjectOutput {
private final Fury fury;
- private final DataOutputStream utf8out = new DataOutputStream(this);
+ private final boolean compressInt;
+ private final LongEncoding longEncoding;
private final StringSerializer stringSerializer;
private MemoryBuffer buffer;
public MemoryBufferObjectOutput(Fury fury, MemoryBuffer buffer) {
this.fury = fury;
+ this.compressInt = fury.compressInt();
+ this.longEncoding = fury.longEncoding();
this.buffer = buffer;
this.stringSerializer = new StringSerializer(fury);
}
@@ -91,12 +95,16 @@ public class MemoryBufferObjectOutput extends OutputStream
implements ObjectOutp
@Override
public void writeInt(int v) throws IOException {
- buffer.writeInt32(v);
+ if (compressInt) {
+ buffer.writeVarInt32(v);
+ } else {
+ buffer.writeInt32(v);
+ }
}
@Override
public void writeLong(long v) throws IOException {
- buffer.writeInt64(v);
+ LongSerializer.writeInt64(buffer, v, longEncoding);
}
@Override
diff --git
a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectStreamSerializer.java
b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectStreamSerializer.java
index c8afee22..f3bf8f9f 100644
---
a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectStreamSerializer.java
+++
b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectStreamSerializer.java
@@ -413,6 +413,7 @@ public class ObjectStreamSerializer extends
AbstractObjectSerializer {
*/
private static class FuryObjectOutputStream extends ObjectOutputStream {
private final Fury fury;
+ private final boolean compressInt;
private final SlotsInfo slotsInfo;
private MemoryBuffer buffer;
private Object targetObject;
@@ -422,6 +423,7 @@ public class ObjectStreamSerializer extends
AbstractObjectSerializer {
super();
this.slotsInfo = slotsInfo;
this.fury = slotsInfo.slotsSerializer.fury;
+ this.compressInt = fury.compressInt();
}
@Override
@@ -628,12 +630,16 @@ public class ObjectStreamSerializer extends
AbstractObjectSerializer {
@Override
public void writeInt(int v) throws IOException {
- buffer.writeInt32(v);
+ if (compressInt) {
+ buffer.writeVarInt32(v);
+ } else {
+ buffer.writeInt32(v);
+ }
}
@Override
public void writeLong(long v) throws IOException {
- buffer.writeInt64(v);
+ fury.writeInt64(buffer, v);
}
@Override
@@ -692,6 +698,7 @@ public class ObjectStreamSerializer extends
AbstractObjectSerializer {
*/
private static class FuryObjectInputStream extends ObjectInputStream {
private final Fury fury;
+ private final boolean compressInt;
private final SlotsInfo slotsInfo;
private MemoryBuffer buffer;
private Object targetObject;
@@ -701,6 +708,7 @@ public class ObjectStreamSerializer extends
AbstractObjectSerializer {
protected FuryObjectInputStream(SlotsInfo slotsInfo) throws IOException {
this.fury = slotsInfo.slotsSerializer.fury;
+ this.compressInt = fury.compressInt();
this.slotsInfo = slotsInfo;
}
@@ -933,12 +941,12 @@ public class ObjectStreamSerializer extends
AbstractObjectSerializer {
@Override
public int readInt() throws IOException {
- return buffer.readInt32();
+ return compressInt ? buffer.readVarInt32() : buffer.readInt32();
}
@Override
public long readLong() throws IOException {
- return buffer.readInt64();
+ return fury.readInt64(buffer);
}
@Override
diff --git
a/java/fury-core/src/test/java/org/apache/fury/io/MemoryBufferObjectInputTest.java
b/java/fury-core/src/test/java/org/apache/fury/io/MemoryBufferObjectInputTest.java
index f5c965f0..361a1aa8 100644
---
a/java/fury-core/src/test/java/org/apache/fury/io/MemoryBufferObjectInputTest.java
+++
b/java/fury-core/src/test/java/org/apache/fury/io/MemoryBufferObjectInputTest.java
@@ -24,19 +24,24 @@ import static org.testng.Assert.assertTrue;
import java.io.IOException;
import org.apache.fury.Fury;
+import org.apache.fury.FuryTestBase;
import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.memory.MemoryUtils;
import org.testng.annotations.Test;
-public class MemoryBufferObjectInputTest {
+public class MemoryBufferObjectInputTest extends FuryTestBase {
- @Test
- public void testFuryObjectInput() throws IOException {
- Fury fury = Fury.builder().build();
+ @Test(dataProvider = "compressNumber")
+ public void testFuryObjectInput(boolean compressNumber) throws IOException {
+ Fury fury = Fury.builder().withNumberCompressed(compressNumber).build();
MemoryBuffer buffer = MemoryUtils.buffer(32);
buffer.writeByte(1);
- buffer.writeInt32(2);
- buffer.writeInt64(3);
+ if (compressNumber) {
+ buffer.writeVarInt32(2);
+ } else {
+ buffer.writeInt32(2);
+ }
+ fury.writeInt64(buffer, 3);
buffer.writeBoolean(true);
buffer.writeFloat32(4.1f);
buffer.writeFloat64(4.2);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
