This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8e814b0dbc6 Reuse UTF-8 scratch for DataTable dictionary decoding
(#19535)
8e814b0dbc6 is described below
commit 8e814b0dbc6e44ba37d84d2bcd8a5373bf923a63
Author: Xiang Fu <[email protected]>
AuthorDate: Mon Sep 21 17:56:59 2026 -0700
Reuse UTF-8 scratch for DataTable dictionary decoding (#19535)
* Reuse UTF-8 scratch for DataTable dictionary decoding
Decoding a string dictionary allocates temporary UTF-8 storage for every
entry. Reuse method-local scratch while preserving each returned String's
ownership and existing buffer and error behavior.
* Extract DataTableUtils.decodeStringArray and reuse it for DataBlock
dictionaries
* Remove unused BaseDataBlock dictionary ser/de; reuse scratch bytes in
ZeroCopyDataBlockSerde dictionary decoding
---
.../pinot/common/datablock/BaseDataBlock.java | 35 -----
.../common/datablock/ZeroCopyDataBlockSerde.java | 22 ++-
.../pinot/common/datatable/DataTableImplV4.java | 7 +-
.../pinot/common/datatable/DataTableUtils.java | 29 ++++
.../datablock/ZeroCopyDataBlockSerdeTest.java | 27 ++++
.../DataTableUtilsDecodeStringArrayTest.java | 174 +++++++++++++++++++++
.../datatable/DataTableDictionarySerDeTest.java | 95 +++++++++++
7 files changed, 347 insertions(+), 42 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/datablock/BaseDataBlock.java
b/pinot-common/src/main/java/org/apache/pinot/common/datablock/BaseDataBlock.java
index d3d88b3fd45..bdc3aefc015 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/datablock/BaseDataBlock.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/datablock/BaseDataBlock.java
@@ -19,7 +19,6 @@
package org.apache.pinot.common.datablock;
import com.google.common.base.Preconditions;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.math.BigDecimal;
@@ -29,9 +28,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.pinot.common.CustomObject;
-import org.apache.pinot.common.datatable.DataTableUtils;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.segment.spi.memory.DataBuffer;
import org.apache.pinot.segment.spi.memory.PinotByteBuffer;
@@ -41,8 +38,6 @@ import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.MapUtils;
import org.roaringbitmap.RoaringBitmap;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
/// Base data block mostly replicating implementation of
[org.apache.pinot.common.datatable.DataTableImplV4].
///
@@ -387,36 +382,6 @@ public abstract class BaseDataBlock implements DataBlock {
// Ser/De and exception handling
// --------------------------------------------------------------------------
- /// Helper method to serialize dictionary map.
- protected byte[] serializeStringDictionary()
- throws IOException {
- if (_stringDictionary.length == 0) {
- return new byte[4];
- }
- UnsynchronizedByteArrayOutputStream byteArrayOutputStream = new
UnsynchronizedByteArrayOutputStream(1024);
- DataOutputStream dataOutputStream = new
DataOutputStream(byteArrayOutputStream);
-
- dataOutputStream.writeInt(_stringDictionary.length);
- for (String entry : _stringDictionary) {
- byte[] valueBytes = entry.getBytes(UTF_8);
- dataOutputStream.writeInt(valueBytes.length);
- dataOutputStream.write(valueBytes);
- }
-
- return byteArrayOutputStream.toByteArray();
- }
-
- /// Helper method to deserialize dictionary map.
- protected String[] deserializeStringDictionary(ByteBuffer buffer)
- throws IOException {
- int dictionarySize = buffer.getInt();
- String[] stringDictionary = new String[dictionarySize];
- for (int i = 0; i < dictionarySize; i++) {
- stringDictionary[i] = DataTableUtils.decodeString(buffer);
- }
- return stringDictionary;
- }
-
@Override
public void addException(int errCode, String errMsg) {
_errCodeToExceptionMap.put(errCode, errMsg);
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/datablock/ZeroCopyDataBlockSerde.java
b/pinot-common/src/main/java/org/apache/pinot/common/datablock/ZeroCopyDataBlockSerde.java
index d83a537aafe..0b51eef6d04 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/datablock/ZeroCopyDataBlockSerde.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/datablock/ZeroCopyDataBlockSerde.java
@@ -30,6 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.function.LongConsumer;
import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.HashUtil;
import org.apache.pinot.segment.spi.memory.CompoundDataBuffer;
@@ -39,6 +40,8 @@ import org.apache.pinot.segment.spi.memory.PinotByteBuffer;
import org.apache.pinot.segment.spi.memory.PinotInputStream;
import org.apache.pinot.segment.spi.memory.PinotOutputStream;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
/// An efficient serde that implements [DataBlockSerde.Version#V1_V2] using
trying to make as fewer copies as
/// possible.
@@ -310,8 +313,25 @@ public class ZeroCopyDataBlockSerde implements
DataBlockSerde {
int dictionarySize = stream.readInt();
String[] stringDictionary = new String[dictionarySize];
+ // Reuse one scratch array across entries instead of allocating a
temporary byte array per entry as
+ // PinotInputStream.readInt4UTF() does.
+ byte[] bytes = null;
for (int i = 0; i < dictionarySize; i++) {
- stringDictionary[i] = stream.readInt4UTF();
+ int length = stream.readInt();
+ if (length == 0) {
+ stringDictionary[i] = StringUtils.EMPTY;
+ continue;
+ }
+ if (length < 0) {
+ // Preserve the exception raised by allocating the entry buffer in
readInt4UTF().
+ throw new NegativeArraySizeException(Integer.toString(length));
+ }
+ if (bytes == null || bytes.length < length) {
+ bytes = new byte[length];
+ }
+ stream.readFully(bytes, 0, length);
+ // String copies the decoded contents, so the next entry can reuse the
scratch bytes.
+ stringDictionary[i] = new String(bytes, 0, length, UTF_8);
}
return stringDictionary;
}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
index fc947e78121..d2ab8c43b23 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
@@ -415,12 +415,7 @@ public class DataTableImplV4 implements DataTable {
/// Helper method to deserialize dictionary map.
protected String[] deserializeStringDictionary(ByteBuffer buffer)
throws IOException {
- int dictionarySize = buffer.getInt();
- String[] stringDictionary = new String[dictionarySize];
- for (int i = 0; i < dictionarySize; i++) {
- stringDictionary[i] = DataTableUtils.decodeString(buffer);
- }
- return stringDictionary;
+ return DataTableUtils.decodeStringArray(buffer);
}
@Override
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableUtils.java
index 05b47f80e0a..929e52d2220 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableUtils.java
@@ -85,4 +85,33 @@ public class DataTableUtils {
return new String(bytes, UTF_8);
}
}
+
+ /// Decodes a string array serialized as an `int` array size followed by
entries in the [#decodeString] format.
+ ///
+ /// A single scratch byte array is reused across entries and grown only when
a longer entry is encountered, so
+ /// decoding does not allocate a temporary byte array per entry. Leaves the
buffer positioned after the last entry.
+ public static String[] decodeStringArray(ByteBuffer buffer)
+ throws IOException {
+ int size = buffer.getInt();
+ String[] strings = new String[size];
+ byte[] bytes = null;
+ for (int i = 0; i < size; i++) {
+ int length = buffer.getInt();
+ if (length == 0) {
+ strings[i] = StringUtils.EMPTY;
+ continue;
+ }
+ if (length < 0) {
+ // Preserve the exception raised by allocating the entry buffer in
decodeString().
+ throw new NegativeArraySizeException(Integer.toString(length));
+ }
+ if (bytes == null || bytes.length < length) {
+ bytes = new byte[length];
+ }
+ buffer.get(bytes, 0, length);
+ // String copies the decoded contents, so the next entry can reuse the
scratch bytes.
+ strings[i] = new String(bytes, 0, length, UTF_8);
+ }
+ return strings;
+ }
}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/datablock/ZeroCopyDataBlockSerdeTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/datablock/ZeroCopyDataBlockSerdeTest.java
index 604e3e9b246..7169e8865cc 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/datablock/ZeroCopyDataBlockSerdeTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/datablock/ZeroCopyDataBlockSerdeTest.java
@@ -19,9 +19,12 @@
package org.apache.pinot.common.datablock;
import com.google.common.collect.Lists;
+import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import org.apache.pinot.common.utils.DataSchema;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.segment.spi.memory.PinotByteBuffer;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.annotations.AfterSuite;
@@ -30,6 +33,7 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertSame;
import static org.testng.Assert.fail;
@@ -100,4 +104,27 @@ public class ZeroCopyDataBlockSerdeTest {
assertEquals(deserialized.getExceptions(), block.getExceptions(),
"Unexpected exceptions");
DataBlockEquals.checkSameContent(deserialized, block, "Unexpected data");
}
+
+ @Test
+ void testStringDictionary()
+ throws Exception {
+ // Longer, shorter and empty entries share the scratch bytes used while
decoding the dictionary.
+ String[] dictionary = {"", "long-" + "x".repeat(4097), "tiny", "ζ±δΊ¬", "ππ",
"", "\u0000tail", "final"};
+ ByteBuffer fixedSizeData = ByteBuffer.allocate(dictionary.length *
Integer.BYTES);
+ for (int dictId = dictionary.length - 1; dictId >= 0; dictId--) {
+ fixedSizeData.putInt(dictId);
+ }
+ DataSchema dataSchema = new DataSchema(new String[]{"value"}, new
ColumnDataType[]{ColumnDataType.STRING});
+ RowDataBlock block =
+ new RowDataBlock(dictionary.length, dataSchema, dictionary,
fixedSizeData.array(), new byte[0]);
+
+ DataBlock deserialized =
DataBlockUtils.deserialize(DataBlockUtils.serialize(block));
+ String[] actual = deserialized.getStringDictionary();
+ assertEquals(actual, dictionary);
+ assertSame(actual[0], "");
+ assertSame(actual[5], "");
+ for (int row = 0; row < dictionary.length; row++) {
+ assertEquals(deserialized.getString(row, 0),
dictionary[dictionary.length - 1 - row]);
+ }
+ }
}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/datatable/DataTableUtilsDecodeStringArrayTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/datatable/DataTableUtilsDecodeStringArrayTest.java
new file mode 100644
index 00000000000..6b1293d53a3
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/datatable/DataTableUtilsDecodeStringArrayTest.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.datatable;
+
+import java.io.IOException;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertSame;
+import static org.testng.Assert.expectThrows;
+
+
+/// Tests [DataTableUtils#decodeStringArray] decoding and buffer consumption.
Each invocation owns its buffers.
+public class DataTableUtilsDecodeStringArrayTest {
+ private static final int BUFFER_PREFIX_SIZE = 11;
+ private static final int TRAILING_SENTINEL = 0x12345678;
+
+ @DataProvider
+ public Object[][] bufferKinds() {
+ return new Object[][]{
+ {false, false, false}, {false, false, true}, {false, true, false},
{false, true, true},
+ {true, false, false}, {true, false, true}, {true, true, false}, {true,
true, true}
+ };
+ }
+
+ @Test(dataProvider = "bufferKinds")
+ public void testMixedStringsAndBufferPosition(boolean direct, boolean
readOnly, boolean sliced)
+ throws IOException {
+ String[] expected = {
+ "", "first", "Γ©", "ζ±δΊ¬", "ππ", "", "long-" + "x".repeat(4097), "tiny",
"\u0000tail", "final"
+ };
+ byte[][] entries = new byte[expected.length][];
+ for (int i = 0; i < expected.length; i++) {
+ entries[i] = expected[i].getBytes(UTF_8);
+ }
+ byte[] payload = dictionaryPayload(entries);
+ ByteBuffer buffer = inputBuffer(payload, direct, readOnly, sliced);
+ int initialLimit = buffer.limit();
+ buffer.mark();
+
+ String[] actual = DataTableUtils.decodeStringArray(buffer);
+
+ // Earlier entries must remain unchanged after longer, shorter and empty
entries reuse the scratch bytes.
+ assertEquals(actual, expected);
+ assertSame(actual[0], "");
+ assertSame(actual[5], "");
+ assertEquals(buffer.position(), BUFFER_PREFIX_SIZE + payload.length -
Integer.BYTES);
+ assertEquals(buffer.limit(), initialLimit);
+ assertEquals(buffer.getInt(), TRAILING_SENTINEL);
+ assertEquals(buffer.remaining(), 0);
+ buffer.reset();
+ assertEquals(buffer.position(), BUFFER_PREFIX_SIZE);
+ assertEquals(DataTableUtils.decodeStringArray(buffer), expected);
+ }
+
+ @Test(dataProvider = "bufferKinds")
+ public void testEmptyDictionary(boolean direct, boolean readOnly, boolean
sliced)
+ throws IOException {
+ ByteBuffer buffer = inputBuffer(dictionaryPayload(), direct, readOnly,
sliced);
+ assertEquals(DataTableUtils.decodeStringArray(buffer), new String[0]);
+ assertEquals(buffer.position(), BUFFER_PREFIX_SIZE + Integer.BYTES);
+ assertEquals(buffer.getInt(), TRAILING_SENTINEL);
+ assertEquals(buffer.remaining(), 0);
+ }
+
+ @Test(dataProvider = "bufferKinds")
+ public void testMalformedUtf8MatchesStandaloneDecoder(boolean direct,
boolean readOnly, boolean sliced)
+ throws IOException {
+ byte[][] entries = {
+ {(byte) 0xc3, 0x28}, {(byte) 0xf0, (byte) 0x9f}, {(byte) 0xed, (byte)
0xa0, (byte) 0x80},
+ {(byte) 0x80, 0x41}, {(byte) 0xef, (byte) 0xbf, (byte) 0xbd}, {},
"last".getBytes(UTF_8)
+ };
+ byte[] payload = dictionaryPayload(entries);
+ ByteBuffer buffer = inputBuffer(payload, direct, readOnly, sliced);
+ ByteBuffer reference = inputBuffer(payload, direct, readOnly, sliced);
+ String[] expected = new String[reference.getInt()];
+ for (int i = 0; i < expected.length; i++) {
+ expected[i] = DataTableUtils.decodeString(reference);
+ }
+
+ String[] actual = DataTableUtils.decodeStringArray(buffer);
+ assertEquals(actual, expected);
+ assertEquals(actual, new String[]{"\ufffd(", "\ufffd", "\ufffd",
"\ufffdA", "\ufffd", "", "last"});
+ assertEquals(buffer.position(), reference.position());
+ assertEquals(buffer.getInt(), TRAILING_SENTINEL);
+ }
+
+ @Test(dataProvider = "bufferKinds")
+ public void testNegativeDictionaryAndEntryLengths(boolean direct, boolean
readOnly, boolean sliced) {
+ assertDecodeFailure(ByteBuffer.allocate(4).putInt(-1).array(), direct,
readOnly, sliced,
+ NegativeArraySizeException.class, 4);
+ assertDecodeFailure(ByteBuffer.allocate(8).putInt(1).putInt(-1).array(),
direct, readOnly, sliced,
+ NegativeArraySizeException.class, 8);
+ // A negative length must retain its original failure even after a
previous entry allocated the scratch bytes.
+ assertDecodeFailure(ByteBuffer.allocate(13).putInt(2).putInt(1).put((byte)
'a').putInt(-1).array(), direct,
+ readOnly, sliced, NegativeArraySizeException.class, 13);
+ }
+
+ @Test(dataProvider = "bufferKinds")
+ public void testTruncatedDictionaryAndEntryLengths(boolean direct, boolean
readOnly, boolean sliced) {
+ for (int availableBytes = 0; availableBytes < Integer.BYTES;
availableBytes++) {
+ assertDecodeFailure(new byte[availableBytes], direct, readOnly, sliced,
BufferUnderflowException.class, 0);
+ assertDecodeFailure(ByteBuffer.allocate(4 +
availableBytes).putInt(1).array(), direct, readOnly, sliced,
+ BufferUnderflowException.class, 4);
+ }
+ }
+
+ @Test(dataProvider = "bufferKinds")
+ public void testTruncatedStringBytes(boolean direct, boolean readOnly,
boolean sliced) {
+ assertDecodeFailure(ByteBuffer.allocate(10).putInt(1).putInt(5).put((byte)
'a').put((byte) 'b').array(), direct,
+ readOnly, sliced, BufferUnderflowException.class, 8);
+ // The second entry fits the existing scratch array but exceeds the
remaining bytes in the source buffer.
+ assertDecodeFailure(ByteBuffer.allocate(17).putInt(2).putInt(3).put(new
byte[]{'a', 'b', 'c'}).putInt(3)
+ .put(new byte[]{'d', 'e'}).array(), direct, readOnly, sliced,
BufferUnderflowException.class, 15);
+ }
+
+ private static void assertDecodeFailure(byte[] payload, boolean direct,
boolean readOnly, boolean sliced,
+ Class<? extends Throwable> exceptionType, int consumedBytes) {
+ ByteBuffer buffer = inputBuffer(payload, direct, readOnly, sliced);
+ int initialLimit = buffer.limit();
+ Throwable exception = expectThrows(exceptionType,
+ () -> DataTableUtils.decodeStringArray(buffer));
+ assertSame(exception.getClass(), exceptionType);
+ assertEquals(buffer.position(), BUFFER_PREFIX_SIZE + consumedBytes);
+ assertEquals(buffer.limit(), initialLimit);
+ }
+
+ private static byte[] dictionaryPayload(byte[]... entries) {
+ int payloadSize = Integer.BYTES * 2;
+ for (byte[] entry : entries) {
+ payloadSize += Integer.BYTES + entry.length;
+ }
+ ByteBuffer payload =
ByteBuffer.allocate(payloadSize).putInt(entries.length);
+ for (byte[] entry : entries) {
+ payload.putInt(entry.length).put(entry);
+ }
+ return payload.putInt(TRAILING_SENTINEL).array();
+ }
+
+ private static ByteBuffer inputBuffer(byte[] payload, boolean direct,
boolean readOnly, boolean sliced) {
+ int sliceOffset = sliced ? 7 : 0;
+ int capacity = sliceOffset + BUFFER_PREFIX_SIZE + payload.length + 5;
+ ByteBuffer buffer = direct ? ByteBuffer.allocateDirect(capacity) :
ByteBuffer.allocate(capacity);
+ buffer.position(sliceOffset + BUFFER_PREFIX_SIZE);
+ buffer.put(payload);
+ buffer.limit(buffer.position());
+ buffer.position(sliceOffset);
+ if (sliced) {
+ buffer = buffer.slice();
+ }
+ buffer.position(BUFFER_PREFIX_SIZE);
+ return readOnly ? buffer.asReadOnlyBuffer() : buffer;
+ }
+}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableDictionarySerDeTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableDictionarySerDeTest.java
new file mode 100644
index 00000000000..84939773982
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableDictionarySerDeTest.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.core.common.datatable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.pinot.common.datatable.DataTable;
+import org.apache.pinot.common.datatable.DataTable.MetadataKey;
+import org.apache.pinot.common.datatable.DataTableFactory;
+import org.apache.pinot.common.utils.DataSchema;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+/// Exercises complete V4 messages with shared scalar/MV dictionaries across
transport buffer types.
+public class DataTableDictionarySerDeTest {
+ @DataProvider
+ public Object[][] bufferKinds() {
+ return new Object[][]{
+ {false, false, false}, {false, false, true}, {false, true, false},
{false, true, true},
+ {true, false, false}, {true, false, true}, {true, true, false}, {true,
true, true}
+ };
+ }
+
+ @Test(dataProvider = "bufferKinds")
+ public void testCompleteMessage(boolean direct, boolean readOnly, boolean
sliced)
+ throws IOException {
+ String[] values = {"", "first", "ζ±δΊ¬", "π", "long-" + "x".repeat(2049),
"tiny"};
+ DataSchema schema = new DataSchema(new String[]{"id", "value", "values"},
+ new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.STRING,
ColumnDataType.STRING_ARRAY});
+ DataTableBuilder builder = new DataTableBuilderV4(schema);
+ for (int row = 0; row < values.length; row++) {
+ builder.startRow();
+ builder.setColumn(0, row);
+ if (row == 0) {
+ builder.setNull(1);
+ } else {
+ builder.setColumn(1, values[row]);
+ }
+ builder.setColumn(2, new String[]{values[(row + 1) % values.length],
values[row]});
+ builder.finishRow();
+ }
+ DataTable original = builder.build();
+ original.getMetadata().put(MetadataKey.NUM_DOCS_SCANNED.getName(),
Integer.toString(values.length));
+ original.addException(QueryErrorCode.QUERY_EXECUTION, "test-ζΆζ―");
+ byte[] wire = original.toBytes();
+ int prefix = sliced ? 13 : 0;
+ ByteBuffer storage = direct
+ ? ByteBuffer.allocateDirect(prefix + wire.length)
+ : ByteBuffer.allocate(prefix + wire.length);
+ storage.position(prefix);
+ storage.put(wire).flip();
+ storage.position(prefix);
+ ByteBuffer input = sliced ? storage.slice() : storage;
+ if (readOnly) {
+ input = input.asReadOnlyBuffer();
+ }
+
+ DataTable decoded = DataTableFactory.getDataTable(input);
+ assertEquals(input.position(), wire.length);
+ assertEquals(decoded.getVersion(), DataTableFactory.VERSION_4);
+ assertEquals(decoded.getDataSchema(), schema);
+ assertEquals(decoded.getNumberOfRows(), values.length);
+
assertEquals(decoded.getMetadata().get(MetadataKey.NUM_DOCS_SCANNED.getName()),
Integer.toString(values.length));
+ assertEquals(decoded.getExceptions(), original.getExceptions());
+ assertTrue(decoded.getNullRowIds(1).contains(0));
+ for (int row = 0; row < values.length; row++) {
+ assertEquals(decoded.getInt(row, 0), row);
+ assertEquals(decoded.getString(row, 1), row == 0 ?
ColumnDataType.STRING.getNullPlaceholder() : values[row]);
+ assertEquals(decoded.getStringArray(row, 2), new String[]{values[(row +
1) % values.length], values[row]});
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
