This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 0bb345cd357 [To dev/1.3] Correct binary column memory calculation
0bb345cd357 is described below
commit 0bb345cd3575bb70212ace7c7df059a15ad59c00
Author: Jackie Tien <[email protected]>
AuthorDate: Mon Jun 16 21:56:37 2025 +0800
[To dev/1.3] Correct binary column memory calculation
---
.../db/queryengine/execution/exchange/SharedTsBlockQueue.java | 8 ++++----
.../db/queryengine/execution/exchange/sink/SinkChannel.java | 11 +++++------
.../execution/exchange/source/LocalSourceHandle.java | 4 +---
.../db/queryengine/execution/operator/AbstractOperator.java | 2 +-
.../queryengine/execution/operator/process/SortOperator.java | 4 ++--
.../apache/iotdb/db/queryengine/execution/exchange/Utils.java | 3 +++
.../db/queryengine/execution/operator/OperatorMemoryTest.java | 2 ++
pom.xml | 2 +-
8 files changed, 19 insertions(+), 17 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java
index 6c556c1ae55..154ffa9c714 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java
@@ -186,8 +186,8 @@ public class SharedTsBlockQueue {
localFragmentInstanceId.getQueryId(),
fullFragmentInstanceId,
localPlanNodeId,
- tsBlock.getRetainedSizeInBytes());
- bufferRetainedSizeInBytes -= tsBlock.getRetainedSizeInBytes();
+ tsBlock.getSizeInBytes());
+ bufferRetainedSizeInBytes -= tsBlock.getSizeInBytes();
// Every time LocalSourceHandle consumes a TsBlock, it needs to send the
event to
// corresponding LocalSinkChannel.
if (sinkChannel != null) {
@@ -226,10 +226,10 @@ public class SharedTsBlockQueue {
localFragmentInstanceId.getQueryId(),
fullFragmentInstanceId,
localPlanNodeId,
- tsBlock.getRetainedSizeInBytes(),
+ tsBlock.getSizeInBytes(),
maxBytesCanReserve);
blockedOnMemory = pair.left;
- bufferRetainedSizeInBytes += tsBlock.getRetainedSizeInBytes();
+ bufferRetainedSizeInBytes += tsBlock.getSizeInBytes();
// reserve memory failed, we should wait until there is enough memory
if (!Boolean.TRUE.equals(pair.right)) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java
index 9f29a8a753f..8915938eb03 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java
@@ -201,7 +201,7 @@ public class SinkChannel implements ISinkChannel {
if (noMoreTsBlocks) {
return;
}
- long retainedSizeInBytes = tsBlock.getRetainedSizeInBytes();
+ long sizeInBytes = tsBlock.getSizeInBytes();
int startSequenceId;
startSequenceId = nextSequenceId;
blocked =
@@ -211,17 +211,16 @@ public class SinkChannel implements ISinkChannel {
localFragmentInstanceId.getQueryId(),
fullFragmentInstanceId,
localPlanNodeId,
- retainedSizeInBytes,
+ sizeInBytes,
maxBytesCanReserve)
.left;
- bufferRetainedSizeInBytes += retainedSizeInBytes;
+ bufferRetainedSizeInBytes += sizeInBytes;
sequenceIdToTsBlock.put(nextSequenceId, new Pair<>(tsBlock,
currentTsBlockSize));
nextSequenceId += 1;
- currentTsBlockSize = retainedSizeInBytes;
+ currentTsBlockSize = sizeInBytes;
- // TODO: consider merge multiple NewDataBlockEvent for less network
traffic.
- submitSendNewDataBlockEventTask(startSequenceId,
ImmutableList.of(retainedSizeInBytes));
+ submitSendNewDataBlockEventTask(startSequenceId,
ImmutableList.of(sizeInBytes));
} finally {
DATA_EXCHANGE_COST_METRIC_SET.recordDataExchangeCost(
SINK_HANDLE_SEND_TSBLOCK_REMOTE, System.nanoTime() - startTime);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java
index 315e75cdfc5..c31f9d655de 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java
@@ -123,9 +123,7 @@ public class LocalSourceHandle implements ISourceHandle {
if (tsBlock != null) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
- "[GetTsBlockFromQueue] TsBlock:{} size:{}",
- currSequenceId,
- tsBlock.getRetainedSizeInBytes());
+ "[GetTsBlockFromQueue] TsBlock:{} size:{}", currSequenceId,
tsBlock.getSizeInBytes());
}
currSequenceId++;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java
index 0e2e739d5a0..e5d23d88d85 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java
@@ -44,7 +44,7 @@ public abstract class AbstractOperator implements Operator {
long oneTupleSize =
Math.max(
1,
- (tsBlock.getRetainedSizeInBytes() - tsBlock.getTotalInstanceSize())
+ (tsBlock.getSizeInBytes() - tsBlock.getTotalInstanceSize())
/ tsBlock.getPositionCount());
if (oneTupleSize > maxReturnSize) {
// make sure at least one-tuple-at-a-time
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
index 55453353f0d..834429cd1c3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
@@ -138,7 +138,7 @@ public class SortOperator implements ProcessOperator {
if (tsBlock == null) {
return null;
}
- dataSize += tsBlock.getRetainedSizeInBytes();
+ dataSize += tsBlock.getSizeInBytes();
cacheTsBlock(tsBlock);
} catch (IoTDBException e) {
clear();
@@ -184,7 +184,7 @@ public class SortOperator implements ProcessOperator {
}
private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
- long bytesSize = tsBlock.getRetainedSizeInBytes();
+ long bytesSize = tsBlock.getSizeInBytes();
if (bytesSize + cachedBytes < SORT_BUFFER_SIZE) {
cachedBytes += bytesSize;
for (int i = 0; i < tsBlock.getPositionCount(); i++) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java
index 0c6eac0dbed..327d4a34c39 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java
@@ -41,6 +41,7 @@ public class Utils {
for (int i = 0; i < numOfTsBlocks; i++) {
TsBlock mockTsBlock = Mockito.mock(TsBlock.class);
Mockito.when(mockTsBlock.getRetainedSizeInBytes()).thenReturn(mockTsBlockSize);
+ Mockito.when(mockTsBlock.getSizeInBytes()).thenReturn(mockTsBlockSize);
mockTsBlocks.add(mockTsBlock);
}
@@ -50,6 +51,7 @@ public class Utils {
public static TsBlock createMockTsBlock(long mockTsBlockSize) {
TsBlock mockTsBlock = Mockito.mock(TsBlock.class);
Mockito.when(mockTsBlock.getRetainedSizeInBytes()).thenReturn(mockTsBlockSize);
+ Mockito.when(mockTsBlock.getSizeInBytes()).thenReturn(mockTsBlockSize);
return mockTsBlock;
}
@@ -144,6 +146,7 @@ public class Utils {
TsBlockSerde mockTsBlockSerde = Mockito.mock(TsBlockSerde.class);
TsBlock mockTsBlock = Mockito.mock(TsBlock.class);
Mockito.when(mockTsBlock.getRetainedSizeInBytes()).thenReturn(mockTsBlockSize);
+ Mockito.when(mockTsBlock.getSizeInBytes()).thenReturn(mockTsBlockSize);
Mockito.when(mockTsBlockSerde.deserialize(Mockito.any(ByteBuffer.class)))
.thenReturn(mockTsBlock);
return mockTsBlockSerde;
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
index 39f79c9ca52..fdf9b459a11 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
@@ -249,6 +249,7 @@ public class OperatorMemoryTest {
public void lastCacheScanOperatorTest() {
TsBlock tsBlock = Mockito.mock(TsBlock.class);
Mockito.when(tsBlock.getRetainedSizeInBytes()).thenReturn(1024L);
+ Mockito.when(tsBlock.getSizeInBytes()).thenReturn(1024L);
LastCacheScanOperator lastCacheScanOperator = new
LastCacheScanOperator(null, null, tsBlock);
assertEquals(1024, lastCacheScanOperator.calculateMaxPeekMemory());
@@ -379,6 +380,7 @@ public class OperatorMemoryTest {
public void lastQuerySortOperatorTest() {
TsBlock tsBlock = Mockito.mock(TsBlock.class);
Mockito.when(tsBlock.getRetainedSizeInBytes()).thenReturn(16 * 1024L);
+ Mockito.when(tsBlock.getSizeInBytes()).thenReturn(16 * 1024L);
Mockito.when(tsBlock.getPositionCount()).thenReturn(16);
List<Operator> children = new ArrayList<>(4);
diff --git a/pom.xml b/pom.xml
index 9c67e1f1734..937e0afa44b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -175,7 +175,7 @@
<thrift.version>0.14.1</thrift.version>
<xz.version>1.9</xz.version>
<zstd-jni.version>1.5.6-3</zstd-jni.version>
- <tsfile.version>1.1.2-250603-SNAPSHOT</tsfile.version>
+ <tsfile.version>1.1.2-250616-SNAPSHOT</tsfile.version>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim
