This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new eb9bd11f1b Fix record extractor when ByteBuffer can be reused (#9549)
eb9bd11f1b is described below
commit eb9bd11f1bd04468b75a9f8bbfdfc38c5d4e73b2
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Thu Oct 6 22:14:05 2022 -0700
Fix record extractor when ByteBuffer can be reused (#9549)
---
.../plugin/inputformat/avro/AvroRecordExtractorTest.java | 13 +++++++++++++
.../apache/pinot/spi/data/readers/BaseRecordExtractor.java | 11 +++++------
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractorTest.java
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractorTest.java
index 7867aff1d1..e748bc68e4 100644
---
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractorTest.java
+++
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractorTest.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.MathContext;
+import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.sql.Timestamp;
import java.time.Instant;
@@ -34,6 +35,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.avro.Conversions;
import org.apache.avro.Schema;
import org.apache.avro.data.TimeConversions;
@@ -230,4 +232,15 @@ public class AvroRecordExtractorTest extends
AbstractRecordExtractorTest {
Assert.assertEquals(genericRow.getValue(timestampMicrosColName),
timestampMicrosColValue);
Assert.assertEquals(genericRow.getValue(timestampMicrosColName).getClass().getSimpleName(),
"Timestamp");
}
+
+ @Test
+ public void testReusedByteBuffer() {
+ byte[] content = new byte[100];
+ ThreadLocalRandom.current().nextBytes(content);
+ ByteBuffer byteBuffer = ByteBuffer.wrap(content);
+ AvroRecordExtractor avroRecordExtractor = new AvroRecordExtractor();
+ for (int i = 0; i < 10; i++) {
+ Assert.assertEquals(avroRecordExtractor.convertSingleValue(byteBuffer),
content);
+ }
+ }
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/BaseRecordExtractor.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/BaseRecordExtractor.java
index 1609dd89bd..90a29dbd5d 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/BaseRecordExtractor.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/BaseRecordExtractor.java
@@ -176,12 +176,11 @@ public abstract class BaseRecordExtractor<T> implements
RecordExtractor<T> {
*/
protected Object convertSingleValue(Object value) {
if (value instanceof ByteBuffer) {
- ByteBuffer byteBufferValue = (ByteBuffer) value;
-
- // Use byteBufferValue.remaining() instead of byteBufferValue.capacity()
so that it still works when buffer is
- // over-sized
- byte[] bytesValue = new byte[byteBufferValue.remaining()];
- byteBufferValue.get(bytesValue);
+ // NOTE: ByteBuffer might be reused in some record reader
implementation. Make a slice to ensure nothing is
+ // modified in the original buffer
+ ByteBuffer slice = ((ByteBuffer) value).slice();
+ byte[] bytesValue = new byte[slice.limit()];
+ slice.get(bytesValue);
return bytesValue;
}
if (value instanceof Number || value instanceof byte[]) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
