XNX02 commented on code in PR #16201:
URL: https://github.com/apache/iotdb/pull/16201#discussion_r2282729410
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java:
##########
@@ -359,25 +354,27 @@ private void initChunkReader(final
AbstractAlignedChunkMetadata alignedChunkMeta
measurementList.subList(deviceIdSize, measurementList.size()).clear();
dataTypeList.subList(deviceIdSize, dataTypeList.size()).clear();
- final int startOffset = offset;
for (; offset < alignedChunkMetadata.getValueChunkMetadataList().size();
++offset) {
final IChunkMetadata metadata =
alignedChunkMetadata.getValueChunkMetadataList().get(offset);
if (metadata != null) {
- // Record the column information corresponding to Meta to fill in
Tablet
- columnTypes.add(ColumnCategory.FIELD);
- measurementList.add(metadata.getMeasurementUid());
- dataTypeList.add(metadata.getDataType());
-
final Chunk chunk = reader.readMemChunk((ChunkMetadata) metadata);
size += PipeMemoryWeightUtil.calculateChunkRamBytesUsed(chunk);
- if (allocatedMemoryBlockForChunk.getMemoryUsageInBytes() < size) {
-
PipeDataNodeResourceManager.memory().forceResize(allocatedMemoryBlockForChunk,
size);
+ if (size > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) {
+ if (valueChunkList.size() == 1) {
+ // If the first chunk exceeds the memory limit, we need to
allocate more memory
+ PipeDataNodeResourceManager.memory()
+ .forceResize(
+ allocatedMemoryBlockForChunk,
+ size -
PipeMemoryWeightUtil.calculateChunkRamBytesUsed(chunk));
+ }
+ break;
+ } else {
+ // Record the column information corresponding to Meta to fill in
Tablet
+ columnTypes.add(ColumnCategory.FIELD);
+ measurementList.add(metadata.getMeasurementUid());
+ dataTypeList.add(metadata.getDataType());
+ valueChunkList.add(chunk);
}
-
- valueChunkList.add(chunk);
- }
- if (offset - startOffset >= pipeMaxAlignedSeriesNumInOneBatch) {
Review Comment:
Was it intentional that in the previous version, the last chunk would be
re-read in the next batch since the offset would not be updated if break ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
