This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/memoryEstimate in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f9f293309b485a30aa8572a757c8613f261f3190 Author: JackieTien97 <[email protected]> AuthorDate: Fri Jan 26 09:15:20 2024 +0800 Multiply 3 for ScanOperator to take unsequence data into account --- .../source/AbstractSeriesAggregationScanOperator.java | 2 +- .../operator/source/AlignedSeriesScanOperator.java | 4 +++- .../execution/operator/source/SeriesScanOperator.java | 3 ++- .../execution/operator/OperatorMemoryTest.java | 19 +++++++++++-------- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractSeriesAggregationScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractSeriesAggregationScanOperator.java index f6a92868d51..6ee4f0ee98a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractSeriesAggregationScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractSeriesAggregationScanOperator.java @@ -89,7 +89,7 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData this.timeRangeIterator = timeRangeIterator; this.cachedRawDataSize = - (1L + subSensorSize) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + (1L + subSensorSize) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3; this.maxReturnSize = maxReturnSize; this.outputEndTime = outputEndTime; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanOperator.java index 344b194a557..346c80139f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanOperator.java @@ -123,7 +123,9 @@ public class AlignedSeriesScanOperator extends AbstractDataSourceOperator { public long calculateMaxPeekMemory() { return Math.max( maxReturnSize, - (1L + valueColumnCount) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()); + (1L + valueColumnCount) + * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() + * 3L); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanOperator.java index acdbdf5ad6c..a5d66c1d2d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanOperator.java @@ -103,7 +103,8 @@ public class SeriesScanOperator extends AbstractDataSourceOperator { @Override public long calculateMaxPeekMemory() { - return Math.max(maxReturnSize, TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()); + return Math.max( + maxReturnSize, TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3L); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java index dbcf497d1fc..fd5eb9b8026 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java @@ -150,12 +150,14 @@ public class OperatorMemoryTest { scanOptionsBuilder.build()); assertEquals( - TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(), + TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3L, seriesScanOperator.calculateMaxPeekMemory()); assertEquals( TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(), seriesScanOperator.calculateMaxReturnSize()); - assertEquals(0, seriesScanOperator.calculateRetainedSizeAfterCallingNext()); + assertEquals( + TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 2L, + seriesScanOperator.calculateRetainedSizeAfterCallingNext()); } catch (IllegalPathException e) { e.printStackTrace(); @@ -199,7 +201,7 @@ public class OperatorMemoryTest { long maxPeekMemory = Math.max( TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(), - 4 * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()); + 4 * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3L); long maxReturnMemory = Math.min( TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(), @@ -1001,7 +1003,8 @@ public class OperatorMemoryTest { TimeColumn.SIZE_IN_BYTES_PER_POSITION + 512 * Byte.BYTES + LongColumn.SIZE_IN_BYTES_PER_POSITION; - long cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + long cachedRawDataSize = + 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3; assertEquals( expectedMaxReturnSize + cachedRawDataSize, @@ -1033,7 +1036,7 @@ public class OperatorMemoryTest { TimeColumn.SIZE_IN_BYTES_PER_POSITION + 512 * Byte.BYTES + 2 * LongColumn.SIZE_IN_BYTES_PER_POSITION; - cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3; assertEquals( expectedMaxReturnSize + cachedRawDataSize, @@ -1076,7 +1079,7 @@ public class OperatorMemoryTest { * (TimeColumn.SIZE_IN_BYTES_PER_POSITION + 512 * Byte.BYTES + LongColumn.SIZE_IN_BYTES_PER_POSITION); - cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3; assertEquals( expectedMaxReturnSize + cachedRawDataSize, @@ -1116,7 +1119,7 @@ public class OperatorMemoryTest { * (TimeColumn.SIZE_IN_BYTES_PER_POSITION + 512 * Byte.BYTES + LongColumn.SIZE_IN_BYTES_PER_POSITION)); - cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3; assertEquals( expectedMaxReturnSize + cachedRawDataSize, @@ -1154,7 +1157,7 @@ public class OperatorMemoryTest { typeProvider); expectedMaxReturnSize = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES; - cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3; assertEquals( expectedMaxReturnSize + cachedRawDataSize,
