chaokunyang commented on code in PR #1890:
URL: https://github.com/apache/fury/pull/1890#discussion_r1808666752
##########
java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java:
##########
@@ -235,75 +214,106 @@ public String readBytesString(MemoryBuffer buffer) {
}
@CodegenInvoke
- public String readCompressedCharsString(MemoryBuffer buffer) {
+ public String readCharsString(MemoryBuffer buffer) {
+ return newCharsStringZeroCopy(buffer.readCharsAndSize());
+ }
+
+ @CodegenInvoke
+ public String readCompressedBytesString(MemoryBuffer buffer) {
long header = buffer.readVarUint36Small();
byte coder = (byte) (header & 0b11);
int numBytes = (int) (header >>> 2);
- if (coder == LATIN1) {
- return newCharsStringZeroCopy(readLatinChars(buffer, numBytes));
- } else if (coder == UTF16) {
- return newCharsStringZeroCopy(readUTF16Chars(buffer, numBytes));
+ if (coder == UTF8) {
+ return newBytesStringZeroCopy(UTF16, readBytesUTF8(buffer, numBytes));
+ } else if (coder == LATIN1 || coder == UTF16) {
+ return newBytesStringZeroCopy(coder, readBytesUnCompressedUTF16(buffer,
numBytes));
} else {
- return readUtf8(buffer, coder, numBytes);
+ throw new RuntimeException("Unknown coder type " + coder);
}
}
- private String readUtf8(MemoryBuffer buffer, byte coder, int numBytes) {
- Preconditions.checkArgument(coder == UTF8, UTF8);
- byte[] bytes = buffer.readBytes(numBytes);
- return new String(bytes, 0, numBytes, StandardCharsets.UTF_8);
- }
-
- private byte[] getByteArray(int numElements) {
- byte[] byteArray = this.byteArray;
- if (byteArray.length < numElements) {
- byteArray = new byte[numElements];
- this.byteArray = byteArray;
- }
- if (byteArray.length > DEFAULT_BUFFER_SIZE) {
- smoothByteArrayLength =
- Math.max(((int) (smoothByteArrayLength * 0.9 + numElements * 0.1)),
DEFAULT_BUFFER_SIZE);
- if (smoothByteArrayLength <= DEFAULT_BUFFER_SIZE) {
- this.byteArray = new byte[DEFAULT_BUFFER_SIZE];
- }
+ @CodegenInvoke
+ public String readCompressedCharsString(MemoryBuffer buffer) {
+ long header = buffer.readVarUint36Small();
+ byte coder = (byte) (header & 0b11);
+ int numBytes = (int) (header >>> 2);
+ char[] chars;
+ if (coder == LATIN1) {
+ chars = readCharsLatin1(buffer, numBytes);
+ } else if (coder == UTF8) {
+ chars = readCharsUTF8(buffer, numBytes);
+ } else if (coder == UTF16) {
+ chars = readCharsUTF16(buffer, numBytes);
+ } else {
+ throw new RuntimeException("Unknown coder type " + coder);
}
- return byteArray;
+ return newCharsStringZeroCopy(chars);
}
// Invoked by fury JIT
public void writeJavaString(MemoryBuffer buffer, String value) {
if (STRING_VALUE_FIELD_IS_BYTES) {
- writeBytesString(buffer, value);
+ if (compressString) {
+ writeCompressedBytesString(buffer, value);
+ } else {
+ writeBytesString(buffer, value);
+ }
} else {
assert STRING_VALUE_FIELD_IS_CHARS;
- final char[] chars = (char[]) Platform.getObject(value,
STRING_VALUE_FIELD_OFFSET);
if (compressString) {
- if (StringUtils.isLatin(chars)) {
- writeCharsLatin(buffer, chars, chars.length);
- } else {
- writeCharsUTF16(buffer, chars, chars.length);
- }
+ writeCompressedCharsString(buffer, value);
} else {
- int numBytes = MathUtils.doubleExact(value.length());
- buffer.writePrimitiveArrayWithSize(chars, Platform.CHAR_ARRAY_OFFSET,
numBytes);
+ writeCharsString(buffer, value);
}
}
}
+ public void writeUTF8String(MemoryBuffer buffer, String value) {
+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+ buffer.writeVarUint32(bytes.length);
+ buffer.writeBytes(bytes);
+ }
+
// Invoked by fury JIT
public String readJavaString(MemoryBuffer buffer) {
if (STRING_VALUE_FIELD_IS_BYTES) {
- return readBytesString(buffer);
+ if (compressString) {
+ return readCompressedBytesString(buffer);
+ } else {
+ return readBytesString(buffer);
+ }
} else {
assert STRING_VALUE_FIELD_IS_CHARS;
if (compressString) {
return readCompressedCharsString(buffer);
} else {
- return newCharsStringZeroCopy(buffer.readCharsAndSize());
+ return readCharsString(buffer);
}
}
}
+ public void writeCompressedBytesString(MemoryBuffer buffer, String value) {
+ final byte[] bytes = (byte[]) Platform.getObject(value,
STRING_VALUE_FIELD_OFFSET);
+ final byte coder = Platform.getByte(value,
Offset.STRING_CODER_FIELD_OFFSET);
+ if (coder == LATIN1 || bestCoder(bytes) == UTF16) {
+ writeBytesString(buffer, value);
Review Comment:
Maybe we should make `writeBytesString` receive bytes and coder, and invoke
like `writeBytesString(buffer, coder, bytes)`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
