This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch hdfs_query_fix in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit ed6df0e8b9b517f5f3935da7635ad33f565d44ab Author: samperson1997 <[email protected]> AuthorDate: Mon May 4 10:58:07 2020 +0800 Try to fix benchmark query BufferUnderflowException in HDFS --- .../iotdb/tsfile/file/header/ChunkHeader.java | 20 ++++++++++++---- .../iotdb/tsfile/read/TsFileSequenceReader.java | 27 ++++++++++++++-------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java index af6cca4..00a74bc 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java @@ -19,6 +19,10 @@ package org.apache.iotdb.tsfile.file.header; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; @@ -26,13 +30,13 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.reader.TsFileInput; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ChunkHeader { + private static final Logger logger = LoggerFactory.getLogger(ChunkHeader.class); + private String measurementID; private int dataSize; private TSDataType dataType; @@ -111,11 +115,17 @@ public class ChunkHeader { // read chunk header from input to buffer ByteBuffer buffer = ByteBuffer.allocate(chunkHeaderSize); + int actualSize = input.read(buffer, offsetVar); input.read(buffer, offsetVar); buffer.flip(); // read measurementID - int size = buffer.getInt(); + int size = 0; + try { + size = buffer.getInt(); + } catch (Exception e) { + logger.error("Actual read size: {}", actualSize); + } String measurementID = ReadWriteIOUtils.readStringWithLength(buffer, size); int dataSize = ReadWriteIOUtils.readInt(buffer); TSDataType dataType = TSDataType.deserialize(ReadWriteIOUtils.readShort(buffer)); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 1e3c8a7..42f364a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -103,9 +103,9 @@ public class TsFileSequenceReader implements AutoCloseable { * @param loadMetadataSize -whether load meta data size */ public TsFileSequenceReader(String file, boolean loadMetadataSize) throws IOException { - if (resourceLogger.isDebugEnabled()) { - resourceLogger.debug("{} reader is opened. {}", file, getClass().getName()); - } +// if (resourceLogger.isDebugEnabled()) { + resourceLogger.debug("{} reader is opened. {}", file, getClass().getName()); +// } this.file = file; tsFileInput = FSFactoryProducer.getFileInputFactory().getTsFileInput(file); try { @@ -586,10 +586,17 @@ public class TsFileSequenceReader implements AutoCloseable { */ public Chunk readMemChunk(ChunkMetadata metaData) throws IOException { int chunkHeadSize = ChunkHeader.getSerializedSize(metaData.getMeasurementUid()); - ChunkHeader header = readChunkHeader(metaData.getOffsetOfChunkHeader(), chunkHeadSize, false); - ByteBuffer buffer = readChunk(metaData.getOffsetOfChunkHeader() + header.getSerializedSize(), - header.getDataSize()); - return new Chunk(header, buffer, metaData.getDeletedAt(), endianType); + try { + ChunkHeader header = readChunkHeader(metaData.getOffsetOfChunkHeader(), chunkHeadSize, false); + ByteBuffer buffer = readChunk(metaData.getOffsetOfChunkHeader() + header.getSerializedSize(), + header.getDataSize()); + return new Chunk(header, buffer, metaData.getDeletedAt(), endianType); + } catch (IOException e) { + logger.error("FileChannel size: " + tsFileInput.size()); + logger.error("Offset: " + metaData.getOffsetOfChunkHeader()); + logger.error("Size: " + chunkHeadSize); + throw e; + } } /** @@ -644,9 +651,9 @@ public class TsFileSequenceReader implements AutoCloseable { } public void close() throws IOException { - if (resourceLogger.isDebugEnabled()) { - resourceLogger.debug("{} reader is closed.", file); - } +// if (resourceLogger.isDebugEnabled()) { + resourceLogger.debug("{} reader is closed.", file); +// } this.tsFileInput.close(); }
