This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 5d777be7 feat(java): support meta string encoding for classname and
package name (#1527)
5d777be7 is described below
commit 5d777be7a675edaf01c19e2c7f88ac74ff4fdec6
Author: Shawn Yang <[email protected]>
AuthorDate: Wed Apr 17 02:03:57 2024 +0800
feat(java): support meta string encoding for classname and package name
(#1527)
## What does this PR do?
This PR supports meta string encoding for classname and package name in
fury java serialization.
Before this PR a unregistered class serialization:
```
Fury | MEDIA_CONTENT | false | array | 405 |
```
With this PR:
```
Fury | MEDIA_CONTENT | false | array | 390 |
```
This PR also fix buffer streaming readSliInt64 bugs introduced in #1451
## Related issues
<!--
Is there any related issue? Please attach here.
- #xxxx0
- #xxxx1
- #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/incubator-fury/issues/new/choose)
describing the need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
go/fury/type.go | 2 +-
.../java/org/apache/fury/memory/MemoryBuffer.java | 4 +-
.../main/java/org/apache/fury/meta/MetaString.java | 22 ++++++--
.../org/apache/fury/meta/MetaStringDecoder.java | 20 -------
.../org/apache/fury/meta/MetaStringEncoder.java | 20 +++++--
.../java/org/apache/fury/resolver/ClassInfo.java | 17 ++++--
.../org/apache/fury/resolver/ClassResolver.java | 7 +--
.../org/apache/fury/resolver/MetaStringBytes.java | 44 ++++++++++-----
.../apache/fury/resolver/MetaStringResolver.java | 31 +++++------
.../fury-core/native-image.properties | 6 +-
.../src/test/java/org/apache/fury/StreamTest.java | 65 ++++++++++++++++++++++
.../fury/resolver/MetaStringResolverTest.java | 6 +-
python/pyfury/_fury.py | 4 +-
python/pyfury/_serialization.pyx | 8 ++-
14 files changed, 179 insertions(+), 77 deletions(-)
diff --git a/go/fury/type.go b/go/fury/type.go
index b0239603..ac19d74f 100644
--- a/go/fury/type.go
+++ b/go/fury/type.go
@@ -557,7 +557,7 @@ func (r *typeResolver) writeMetaString(buffer *ByteBuffer,
str string) error {
if _, err := h.Write([]byte(str)); err != nil {
return err
}
- hash := int64(h.Sum64())
+ hash := int64(h.Sum64() & 0xffffffffffffff00)
buffer.WriteInt64(hash)
if len(str) > MaxInt16 {
return fmt.Errorf("too long string: %s", str)
diff --git
a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
index 293d1256..d2b860c7 100644
--- a/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
+++ b/java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
@@ -1471,6 +1471,7 @@ public final class MemoryBuffer {
readerIndex = readIdx + 4;
return i >> 1;
}
+ diff = size - readIdx;
if (diff < 9) {
streamReader.fillBuffer(9 - diff);
}
@@ -1493,6 +1494,7 @@ public final class MemoryBuffer {
readerIndex = readIdx + 4;
return i >> 1;
}
+ diff = size - readIdx;
if (diff < 9) {
streamReader.fillBuffer(9 - diff);
}
@@ -2511,8 +2513,6 @@ public final class MemoryBuffer {
+ writerIndex
+ ", heapMemory="
+ (heapMemory == null ? null : "len(" + heapMemory.length + ")")
- + ", heapData="
- + Arrays.toString(heapMemory)
+ ", heapOffset="
+ heapOffset
+ ", offHeapBuffer="
diff --git a/java/fury-core/src/main/java/org/apache/fury/meta/MetaString.java
b/java/fury-core/src/main/java/org/apache/fury/meta/MetaString.java
index 290c3788..57f86ef0 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/MetaString.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/MetaString.java
@@ -30,11 +30,11 @@ import java.util.Objects;
public class MetaString {
/** Defines the types of supported encodings for MetaStrings. */
public enum Encoding {
- LOWER_SPECIAL(0x00),
- LOWER_UPPER_DIGIT_SPECIAL(0x01),
- FIRST_TO_LOWER_SPECIAL(0x02),
- ALL_TO_LOWER_SPECIAL(0x03),
- UTF_8(0x04); // Using UTF-8 as the fallback
+ UTF_8(0x00), // Using UTF-8 as the fallback
+ LOWER_SPECIAL(0x01),
+ LOWER_UPPER_DIGIT_SPECIAL(0x02),
+ FIRST_TO_LOWER_SPECIAL(0x03),
+ ALL_TO_LOWER_SPECIAL(0x04);
private final int value;
@@ -61,6 +61,7 @@ public class MetaString {
private final char specialChar1;
private final char specialChar2;
private final byte[] bytes;
+ private final int numChars;
private final int numBits;
/**
@@ -76,12 +77,14 @@ public class MetaString {
char specialChar1,
char specialChar2,
byte[] bytes,
+ int numChars,
int numBits) {
this.string = string;
this.encoding = encoding;
this.specialChar1 = specialChar1;
this.specialChar2 = specialChar2;
this.bytes = bytes;
+ this.numChars = numChars;
this.numBits = numBits;
}
@@ -105,6 +108,10 @@ public class MetaString {
return bytes;
}
+ public int getNumChars() {
+ return numChars;
+ }
+
public int getNumBits() {
return numBits;
}
@@ -120,6 +127,7 @@ public class MetaString {
MetaString that = (MetaString) o;
return specialChar1 == that.specialChar1
&& specialChar2 == that.specialChar2
+ && numChars == that.numChars
&& numBits == that.numBits
&& encoding == that.encoding
&& Arrays.equals(bytes, that.bytes);
@@ -127,7 +135,7 @@ public class MetaString {
@Override
public int hashCode() {
- int result = Objects.hash(encoding, specialChar1, specialChar2, numBits);
+ int result = Objects.hash(encoding, specialChar1, specialChar2, numChars,
numBits);
result = 31 * result + Arrays.hashCode(bytes);
return result;
}
@@ -145,6 +153,8 @@ public class MetaString {
+ specialChar2
+ ", bytes="
+ Arrays.toString(bytes)
+ + ", numChars="
+ + numChars
+ ", numBits="
+ numBits
+ '}';
diff --git
a/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringDecoder.java
b/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringDecoder.java
index 35904d64..587cf758 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringDecoder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringDecoder.java
@@ -20,7 +20,6 @@
package org.apache.fury.meta;
import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
import org.apache.fury.meta.MetaString.Encoding;
import org.apache.fury.util.StringUtils;
@@ -40,25 +39,6 @@ public class MetaStringDecoder {
this.specialChar2 = specialChar2;
}
- /**
- * Decodes the encoded data back into a string based on the provided number
of bits and encoding
- * type.
- *
- * @param encodedData The encoded string data as a byte array.
- * @param numBits The number of bits used for encoding.
- * @return The decoded original string.
- */
- public String decode(byte[] encodedData, int numBits) {
- if (encodedData.length == 0) {
- return "";
- }
- // The very first byte signifies the encoding used
- Encoding chosenEncoding = Encoding.fromInt(encodedData[0] & 0xFF);
- // Extract actual data, skipping the first byte (encoding flag)
- encodedData = Arrays.copyOfRange(encodedData, 1, encodedData.length);
- return decode(encodedData, chosenEncoding, numBits);
- }
-
/**
* Decode data based on passed <code>encoding</code>. The data must be
encoded using passed
* encoding.
diff --git
a/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
b/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
index 60a6b393..5680a498 100644
--- a/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/meta/MetaStringEncoder.java
@@ -21,6 +21,7 @@ package org.apache.fury.meta;
import java.nio.charset.StandardCharsets;
import org.apache.fury.meta.MetaString.Encoding;
+import org.apache.fury.util.Preconditions;
/** Encodes plain text strings into MetaString objects with specified encoding
mechanisms. */
public class MetaStringEncoder {
@@ -48,7 +49,7 @@ public class MetaStringEncoder {
public MetaString encode(String input) {
if (input.isEmpty()) {
return new MetaString(
- input, Encoding.LOWER_SPECIAL, specialChar1, specialChar2, new
byte[0], 0);
+ input, Encoding.LOWER_SPECIAL, specialChar1, specialChar2, new
byte[0], 0, 0);
}
Encoding encoding = computeEncoding(input);
return encode(input, encoding);
@@ -62,15 +63,23 @@ public class MetaStringEncoder {
* @return A MetaString object representing the encoded string.
*/
public MetaString encode(String input, Encoding encoding) {
+ Preconditions.checkArgument(
+ input.length() < Short.MAX_VALUE, "Long meta string than 32767 is not
allowed");
if (input.isEmpty()) {
return new MetaString(
- input, Encoding.LOWER_SPECIAL, specialChar1, specialChar2, new
byte[0], 0);
+ input, Encoding.LOWER_SPECIAL, specialChar1, specialChar2, new
byte[0], 0, 0);
}
int length = input.length();
switch (encoding) {
case LOWER_SPECIAL:
return new MetaString(
- input, encoding, specialChar1, specialChar2,
encodeLowerSpecial(input), length * 5);
+ input,
+ encoding,
+ specialChar1,
+ specialChar2,
+ encodeLowerSpecial(input),
+ length,
+ length * 5);
case LOWER_UPPER_DIGIT_SPECIAL:
return new MetaString(
input,
@@ -78,6 +87,7 @@ public class MetaStringEncoder {
specialChar1,
specialChar2,
encodeLowerUpperDigitSpecial(input),
+ length,
length * 6);
case FIRST_TO_LOWER_SPECIAL:
return new MetaString(
@@ -86,6 +96,7 @@ public class MetaStringEncoder {
specialChar1,
specialChar2,
encodeFirstToLowerSpecial(input),
+ length,
length * 5);
case ALL_TO_LOWER_SPECIAL:
char[] chars = input.toCharArray();
@@ -96,11 +107,12 @@ public class MetaStringEncoder {
specialChar1,
specialChar2,
encodeAllToLowerSpecial(chars, upperCount),
+ length,
(upperCount + length) * 5);
default:
byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
return new MetaString(
- input, Encoding.UTF_8, specialChar1, specialChar2, bytes,
bytes.length * 8);
+ input, Encoding.UTF_8, specialChar1, specialChar2, bytes,
bytes.length * 8, 0);
}
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
index 07261396..25f67022 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
@@ -20,6 +20,8 @@
package org.apache.fury.resolver;
import org.apache.fury.config.Language;
+import org.apache.fury.meta.MetaString.Encoding;
+import org.apache.fury.meta.MetaStringEncoder;
import org.apache.fury.serializer.Serializer;
import org.apache.fury.util.Preconditions;
import org.apache.fury.util.ReflectionUtils;
@@ -73,7 +75,9 @@ public class ClassInfo {
this.serializer = serializer;
MetaStringResolver metaStringResolver =
classResolver.getMetaStringResolver();
if (cls != null && classResolver.getFury().getLanguage() != Language.JAVA)
{
- this.fullClassNameBytes =
metaStringResolver.getOrCreateMetaStringBytes(cls.getName());
+ this.fullClassNameBytes =
+ metaStringResolver.getOrCreateMetaStringBytes(
+ new MetaStringEncoder('.', '_').encode(cls.getName(),
Encoding.UTF_8));
} else {
this.fullClassNameBytes = null;
}
@@ -81,16 +85,21 @@ public class ClassInfo {
&& (classId == ClassResolver.NO_CLASS_ID || classId ==
ClassResolver.REPLACE_STUB_ID)) {
// REPLACE_STUB_ID for write replace class in `ClassSerializer`.
String packageName = ReflectionUtils.getPackage(cls);
- this.packageNameBytes =
metaStringResolver.getOrCreateMetaStringBytes(packageName);
+ this.packageNameBytes =
+ metaStringResolver.getOrCreateMetaStringBytes(
+ new MetaStringEncoder('.', '_').encode(packageName));
this.classNameBytes =
metaStringResolver.getOrCreateMetaStringBytes(
- ReflectionUtils.getClassNameWithoutPackage(cls));
+ new MetaStringEncoder('_', '$')
+ .encode(ReflectionUtils.getClassNameWithoutPackage(cls)));
} else {
this.packageNameBytes = null;
this.classNameBytes = null;
}
if (tag != null) {
- this.typeTagBytes = metaStringResolver.getOrCreateMetaStringBytes(tag);
+ this.typeTagBytes =
+ metaStringResolver.getOrCreateMetaStringBytes(
+ new MetaStringEncoder('.', '_').encode(tag, Encoding.UTF_8));
} else {
this.typeTagBytes = null;
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
index dc2087d5..de5ce261 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
@@ -36,7 +36,6 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.LocalDateTime;
@@ -1585,8 +1584,8 @@ public class ClassResolver {
new ClassNameBytes(packageBytes.hashCode,
simpleClassNameBytes.hashCode);
Class<?> cls = compositeClassNameBytes2Class.get(classNameBytes);
if (cls == null) {
- String packageName = new String(packageBytes.bytes,
StandardCharsets.UTF_8);
- String className = new String(simpleClassNameBytes.bytes,
StandardCharsets.UTF_8);
+ String packageName = packageBytes.decode('.', '_');
+ String className = simpleClassNameBytes.decode('.', '$');
String entireClassName;
if (StringUtils.isBlank(packageName)) {
entireClassName = className;
@@ -1612,7 +1611,7 @@ public class ClassResolver {
Class<?> cls = classNameBytes2Class.get(byteString);
if (cls == null) {
Preconditions.checkNotNull(byteString);
- String className = new String(byteString.bytes, StandardCharsets.UTF_8);
+ String className = byteString.decode('.', '_');
cls = loadClass(className);
classNameBytes2Class.put(byteString, cls);
}
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringBytes.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringBytes.java
index 167bb064..fec847d6 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringBytes.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringBytes.java
@@ -19,14 +19,15 @@
package org.apache.fury.resolver;
-import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.apache.fury.annotation.Internal;
+import org.apache.fury.meta.MetaString;
+import org.apache.fury.meta.MetaStringDecoder;
import org.apache.fury.util.MurmurHash3;
-import org.apache.fury.util.Preconditions;
@Internal
final class MetaStringBytes {
+ static final int STRIP_LAST_CHAR = 0b1000;
static final short DEFAULT_DYNAMIC_WRITE_STRING_ID = -1;
final byte[] bytes;
@@ -46,18 +47,35 @@ final class MetaStringBytes {
this.hashCode = hashCode;
}
- public MetaStringBytes(String string) {
- byte[] classNameBytes = string.getBytes(StandardCharsets.UTF_8);
- Preconditions.checkArgument(classNameBytes.length <= Short.MAX_VALUE);
+ public MetaStringBytes(MetaString metaString) {
+ this.bytes = metaString.getBytes();
// Set seed to ensure hash is deterministic.
- long hashCode =
- MurmurHash3.murmurhash3_x64_128(classNameBytes, 0,
classNameBytes.length, 47)[0];
+ long hashCode = MurmurHash3.murmurhash3_x64_128(bytes, 0, bytes.length,
47)[0];
if (hashCode == 0) {
// Ensure hashcode is not 0, so we can do some optimization to avoid
boxing.
- hashCode += 1;
+ hashCode += 256; // last byte is reserved for header.
}
- this.bytes = classNameBytes;
- this.hashCode = hashCode;
+ hashCode &= 0xffffffffffffff00L;
+ int header = metaString.getEncoding().getValue();
+ String decoded =
+ new MetaStringDecoder(metaString.getSpecialChar1(),
metaString.getSpecialChar2())
+ .decode(bytes, metaString.getEncoding(), bytes.length * 8);
+ if (decoded.length() > metaString.getString().length()) {
+ header |= STRIP_LAST_CHAR;
+ }
+ this.hashCode = hashCode | header;
+ }
+
+ public String decode(char specialChar1, char specialChar2) {
+ int header = (int) (hashCode & 0xff);
+ int encodingFlags = header & 0b111;
+ MetaString.Encoding encoding = MetaString.Encoding.values()[encodingFlags];
+ String str =
+ new MetaStringDecoder(specialChar1, specialChar2).decode(bytes,
encoding, bytes.length * 8);
+ if ((header & STRIP_LAST_CHAR) != 0) {
+ str = str.substring(0, str.length() - 1);
+ }
+ return str;
}
@Override
@@ -77,14 +95,10 @@ final class MetaStringBytes {
return hashCode == that.hashCode;
}
- public byte[] getBytes() {
- return bytes;
- }
-
@Override
public int hashCode() {
// equals will compare 8 byte hash code.
- return (int) hashCode;
+ return (int) (hashCode >> 1);
}
@Override
diff --git
a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java
b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java
index 53764086..658ddbe1 100644
---
a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java
+++
b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java
@@ -19,10 +19,10 @@
package org.apache.fury.resolver;
-import java.nio.charset.StandardCharsets;
import org.apache.fury.collection.LongMap;
import org.apache.fury.collection.ObjectMap;
import org.apache.fury.memory.MemoryBuffer;
+import org.apache.fury.meta.MetaString;
/**
* A resolver for limited string value writing. Currently, we only support
classname dynamic
@@ -43,7 +43,7 @@ public final class MetaStringResolver {
private final LongMap<MetaStringBytes> hash2MetaStringBytesMap =
new LongMap<>(initialCapacity, furyMapLoadFactor);
// Every enum bytes should be singleton at every fury, since we keep state
in it.
- private final ObjectMap<String, MetaStringBytes> metaString2BytesMap =
+ private final ObjectMap<MetaString, MetaStringBytes> metaString2BytesMap =
new ObjectMap<>(initialCapacity, furyMapLoadFactor);
private MetaStringBytes[] dynamicWrittenString = new MetaStringBytes[32];
private MetaStringBytes[] dynamicReadStringIds = new MetaStringBytes[32];
@@ -55,7 +55,7 @@ public final class MetaStringResolver {
dynamicReadStringId = 0;
}
- MetaStringBytes getOrCreateMetaStringBytes(String str) {
+ MetaStringBytes getOrCreateMetaStringBytes(MetaString str) {
MetaStringBytes metaStringBytes = metaString2BytesMap.get(str);
if (metaStringBytes == null) {
metaStringBytes = new MetaStringBytes(str);
@@ -64,20 +64,6 @@ public final class MetaStringResolver {
return metaStringBytes;
}
- public void writeMetaString(MemoryBuffer buffer, String str) {
- writeMetaStringBytes(buffer, getOrCreateMetaStringBytes(str));
- }
-
- public String readMetaString(MemoryBuffer buffer) {
- MetaStringBytes byteString = readMetaStringBytes(buffer);
- String str = metaStringBytes2StringMap.get(byteString);
- if (str == null) { // TODO use org.apache.fury.resolver.ObjectMap
- str = new String(byteString.bytes, StandardCharsets.UTF_8);
- metaStringBytes2StringMap.put(byteString, str);
- }
- return str;
- }
-
public void writeMetaStringBytesWithFlag(MemoryBuffer buffer,
MetaStringBytes byteString) {
short id = byteString.dynamicWriteStringId;
if (id == MetaStringBytes.DEFAULT_DYNAMIC_WRITE_STRING_ID) {
@@ -120,6 +106,17 @@ public final class MetaStringResolver {
return this.dynamicWrittenString = tmp;
}
+ public String readMetaString(MemoryBuffer buffer) {
+ MetaStringBytes byteString = readMetaStringBytes(buffer);
+ String str = metaStringBytes2StringMap.get(byteString);
+ if (str == null) {
+ // TODO support meta string in other languages.
+ str = byteString.decode('.', '_');
+ metaStringBytes2StringMap.put(byteString, str);
+ }
+ return str;
+ }
+
public MetaStringBytes readMetaStringBytesWithFlag(MemoryBuffer buffer, int
header) {
int len = header >>> 2;
if ((header & 0b10) == 0) {
diff --git
a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
index bcb3392d..4a696c71 100644
---
a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
+++
b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties
@@ -169,4 +169,8 @@
Args=--initialize-at-build-time=org.apache.fury.memory.MemoryBuffer,\
org.apache.fury.memory.MemoryUtils,\
org.apache.fury.type.DescriptorGrouper,\
sun.misc.Unsafe,\
- com.google.common.collect.Platform
+ com.google.common.collect.Platform,\
+ org.apache.fury.meta.MetaString,\
+ org.apache.fury.meta.MetaStringDecoder$1,\
+ org.apache.fury.meta.MetaStringEncoder$1,\
+ org.apache.fury.meta.MetaString$Encoding
diff --git a/java/fury-core/src/test/java/org/apache/fury/StreamTest.java
b/java/fury-core/src/test/java/org/apache/fury/StreamTest.java
index 400e83a1..95d783ac 100644
--- a/java/fury-core/src/test/java/org/apache/fury/StreamTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/StreamTest.java
@@ -32,12 +32,77 @@ import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.fury.io.FuryInputStream;
import org.apache.fury.io.FuryReadableChannel;
+import org.apache.fury.io.FuryStreamReader;
import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.test.bean.BeanA;
import org.apache.fury.util.ReflectionUtils;
import org.testng.annotations.Test;
public class StreamTest {
+ @Test
+ public void testBufferStream() {
+ MemoryBuffer buffer0 = MemoryBuffer.newHeapBuffer(10);
+ for (int i = 0; i < 10; i++) {
+ buffer0.writeByte(i);
+ buffer0.writeChar((char) i);
+ buffer0.writeInt16((short) i);
+ buffer0.writeInt32(i);
+ buffer0.writeInt64(i);
+ buffer0.writeFloat32(i);
+ buffer0.writeFloat64(i);
+ buffer0.writeVarInt32(i);
+ buffer0.writeVarInt32(Integer.MIN_VALUE);
+ buffer0.writeVarInt32(Integer.MAX_VALUE);
+ buffer0.writeVarUint32(i);
+ buffer0.writeVarUint32(Integer.MIN_VALUE);
+ buffer0.writeVarUint32(Integer.MAX_VALUE);
+ buffer0.writeVarInt64(i);
+ buffer0.writeVarInt64(Long.MIN_VALUE);
+ buffer0.writeVarInt64(Long.MAX_VALUE);
+ buffer0.writeVarUint64(i);
+ buffer0.writeVarUint64(Long.MIN_VALUE);
+ buffer0.writeVarUint64(Long.MAX_VALUE);
+ buffer0.writeSliInt64(i);
+ buffer0.writeSliInt64(Long.MIN_VALUE);
+ buffer0.writeSliInt64(Long.MAX_VALUE);
+ }
+ byte[] bytes = buffer0.getBytes(0, buffer0.writerIndex());
+ FuryInputStream stream =
+ FuryStreamReader.of(
+ new ByteArrayInputStream(bytes) {
+ @Override
+ public synchronized int read(byte[] b, int off, int len) {
+ buffer0.readBytes(b, off, 1);
+ return 1;
+ }
+ });
+ MemoryBuffer buffer = MemoryBuffer.fromByteArray(bytes, 0, 0, stream);
+ for (int i = 0; i < 10; i++) {
+ assertEquals(buffer.readByte(), i);
+ assertEquals(buffer.readChar(), i);
+ assertEquals(buffer.readInt16(), i);
+ assertEquals(buffer.readInt32(), i);
+ assertEquals(buffer.readInt64(), i);
+ assertEquals(buffer.readFloat32(), i);
+ assertEquals(buffer.readFloat64(), i);
+ assertEquals(buffer.readVarInt32(), i);
+ assertEquals(buffer.readVarInt32(), Integer.MIN_VALUE);
+ assertEquals(buffer.readVarInt32(), Integer.MAX_VALUE);
+ assertEquals(buffer.readVarUint32(), i);
+ assertEquals(buffer.readVarUint32(), Integer.MIN_VALUE);
+ assertEquals(buffer.readVarUint32(), Integer.MAX_VALUE);
+ assertEquals(buffer.readVarInt64(), i);
+ assertEquals(buffer.readVarInt64(), Long.MIN_VALUE);
+ assertEquals(buffer.readVarInt64(), Long.MAX_VALUE);
+ assertEquals(buffer.readVarUint64(), i);
+ assertEquals(buffer.readVarUint64(), Long.MIN_VALUE);
+ assertEquals(buffer.readVarUint64(), Long.MAX_VALUE);
+ assertEquals(buffer.readSliInt64(), i);
+ assertEquals(buffer.readSliInt64(), Long.MIN_VALUE);
+ assertEquals(buffer.readSliInt64(), Long.MAX_VALUE);
+ }
+ }
+
@Test
public void testBufferReset() {
Fury fury =
Fury.builder().withRefTracking(true).requireClassRegistration(false).build();
diff --git
a/java/fury-core/src/test/java/org/apache/fury/resolver/MetaStringResolverTest.java
b/java/fury-core/src/test/java/org/apache/fury/resolver/MetaStringResolverTest.java
index bc401526..9d8446d3 100644
---
a/java/fury-core/src/test/java/org/apache/fury/resolver/MetaStringResolverTest.java
+++
b/java/fury-core/src/test/java/org/apache/fury/resolver/MetaStringResolverTest.java
@@ -24,6 +24,8 @@ import static org.testng.Assert.assertTrue;
import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.memory.MemoryUtils;
+import org.apache.fury.meta.MetaString;
+import org.apache.fury.meta.MetaStringEncoder;
import org.apache.fury.util.StringUtils;
import org.testng.annotations.Test;
@@ -35,7 +37,9 @@ public class MetaStringResolverTest {
String str = StringUtils.random(128, 0);
MetaStringResolver stringResolver = new MetaStringResolver();
for (int i = 0; i < 128; i++) {
- stringResolver.writeMetaString(buffer, str);
+ MetaString metaString = new MetaStringEncoder('.', '_').encode(str);
+ stringResolver.writeMetaStringBytes(
+ buffer, stringResolver.getOrCreateMetaStringBytes(metaString));
}
for (int i = 0; i < 128; i++) {
String metaString = stringResolver.readMetaString(buffer);
diff --git a/python/pyfury/_fury.py b/python/pyfury/_fury.py
index 3d9b39ed..8f714cde 100644
--- a/python/pyfury/_fury.py
+++ b/python/pyfury/_fury.py
@@ -100,7 +100,9 @@ class MetaStringBytes:
def __init__(self, data, hashcode=None):
self.data = data
self.length = len(data)
- self.hashcode = hashcode or mmh3.hash_buffer(data, 47)[0]
+ if hashcode is None:
+ hashcode = mmh3.hash_buffer(data, 47)[0] & 0xFFFFFFFFFFFFFF00
+ self.hashcode = hashcode
self.dynamic_write_string_id = DEFAULT_DYNAMIC_WRITE_STRING_ID
def __eq__(self, other):
diff --git a/python/pyfury/_serialization.pyx b/python/pyfury/_serialization.pyx
index 108d0eaa..0da8beaa 100644
--- a/python/pyfury/_serialization.pyx
+++ b/python/pyfury/_serialization.pyx
@@ -759,7 +759,13 @@ cdef class MetaStringBytes:
def __init__(self, data, hashcode=None):
self.data = data
self.length = len(data)
- self.hashcode = hashcode or mmh3.hash_buffer(data, 47)[0]
+ if hashcode is None:
+ hashcodes = mmh3.hash_buffer(data, 47)
+ hashcode = hashcodes[0]
+ assert isinstance(hashcode, int)
+ # FIXME: why using & 0xffffffffffffff00 overflow in cython
+ hashcode = (hashcode >> 8) << 8
+ self.hashcode = hashcode
self.dynamic_write_string_id = DEFAULT_DYNAMIC_WRITE_STRING_ID
def __eq__(self, other):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
