This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3c01a93437 [IOTDB-5401] Reduce the estimated size of memory retained
after calling next in SeriesAggregationScanOperator
3c01a93437 is described below
commit 3c01a934376eacf655c23508ae85f7eee926772e
Author: liuminghui233 <[email protected]>
AuthorDate: Thu Jan 12 08:44:27 2023 +0800
[IOTDB-5401] Reduce the estimated size of memory retained after calling
next in SeriesAggregationScanOperator
---
.../AbstractSeriesAggregationScanOperator.java | 8 ++---
.../mpp/execution/operator/OperatorMemoryTest.java | 35 ++++++++++------------
2 files changed, 19 insertions(+), 24 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
index ee64166a4b..122247c85b 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
@@ -64,7 +64,7 @@ public abstract class AbstractSeriesAggregationScanOperator
extends AbstractData
protected boolean finished = false;
- private final long maxRetainedSize;
+ private final long cachedRawDataSize;
private final long maxReturnSize;
protected AbstractSeriesAggregationScanOperator(
@@ -92,14 +92,14 @@ public abstract class AbstractSeriesAggregationScanOperator
extends AbstractData
}
this.resultTsBlockBuilder = new TsBlockBuilder(dataTypes);
- this.maxRetainedSize =
+ this.cachedRawDataSize =
(1L + subSensorSize) *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
this.maxReturnSize = maxReturnSize;
}
@Override
public long calculateMaxPeekMemory() {
- return maxRetainedSize + maxReturnSize;
+ return cachedRawDataSize + maxReturnSize;
}
@Override
@@ -109,7 +109,7 @@ public abstract class AbstractSeriesAggregationScanOperator
extends AbstractData
@Override
public long calculateRetainedSizeAfterCallingNext() {
- return maxRetainedSize;
+ return isGroupByQuery ? cachedRawDataSize : 0;
}
@Override
diff --git
a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
index ce9df44997..f51e52b1ae 100644
---
a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
@@ -1014,16 +1014,13 @@ public class OperatorMemoryTest {
TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION;
- long expectedMaxRetainSize =
- 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ long cachedRawDataSize = 2L *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator1.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize,
seriesAggregationScanOperator1.calculateMaxReturnSize());
- assertEquals(
- expectedMaxRetainSize,
-
seriesAggregationScanOperator1.calculateRetainedSizeAfterCallingNext());
+ assertEquals(0,
seriesAggregationScanOperator1.calculateRetainedSizeAfterCallingNext());
// case2: without group by, step is PARTIAL
List<AggregationDescriptor> aggregationDescriptors2 =
@@ -1049,15 +1046,13 @@ public class OperatorMemoryTest {
TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ 2 * LongColumn.SIZE_IN_BYTES_PER_POSITION;
- expectedMaxRetainSize = 2L *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator2.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize,
seriesAggregationScanOperator2.calculateMaxReturnSize());
- assertEquals(
- expectedMaxRetainSize,
-
seriesAggregationScanOperator2.calculateRetainedSizeAfterCallingNext());
+ assertEquals(0,
seriesAggregationScanOperator2.calculateRetainedSizeAfterCallingNext());
long maxTsBlockLineNumber =
TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber();
@@ -1094,14 +1089,14 @@ public class OperatorMemoryTest {
* (TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION);
- expectedMaxRetainSize = 2L *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator3.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize,
seriesAggregationScanOperator3.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator3.calculateRetainedSizeAfterCallingNext());
// case4: with group by, total window num > 1000
@@ -1132,14 +1127,14 @@ public class OperatorMemoryTest {
* (TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION));
- expectedMaxRetainSize = 2L *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator4.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize,
seriesAggregationScanOperator4.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator4.calculateRetainedSizeAfterCallingNext());
// case5: over DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES
@@ -1168,14 +1163,14 @@ public class OperatorMemoryTest {
typeProvider);
expectedMaxReturnSize = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
- expectedMaxRetainSize = 2L *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L *
TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator5.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize,
seriesAggregationScanOperator5.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator5.calculateRetainedSizeAfterCallingNext());
} catch (IllegalPathException e) {
e.printStackTrace();
