This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch ycy/memoryIssueFix
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ycy/memoryIssueFix by this
push:
new f37d69d2603 feat: add memory control in analyze phrase for group by
time
f37d69d2603 is described below
commit f37d69d26035f2a11b3dee674680f48950347035
Author: ycycse <[email protected]>
AuthorDate: Wed Jun 12 17:16:28 2024 +0800
feat: add memory control in analyze phrase for group by time
---
.../queryengine/plan/analyze/AnalyzeVisitor.java | 8 ++++--
.../plan/analyze/TemplatedAggregationAnalyze.java | 2 +-
.../queryengine/plan/analyze/TemplatedAnalyze.java | 11 ++++----
.../plan/analyze/QueryTimePartitionTest.java | 33 ++++++++++++++--------
4 files changed, 33 insertions(+), 21 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 038514ee02f..914df45d4e6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -152,6 +152,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1993,7 +1994,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
long startTime = System.nanoTime();
try {
Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res =
- getTimePartitionSlotList(context.getGlobalTimeFilter());
+ getTimePartitionSlotList(context.getGlobalTimeFilter(), context);
// there is no satisfied time range
if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) {
return new DataPartition(
@@ -2030,7 +2031,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
public static Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>>
getTimePartitionSlotList(
- Filter timeFilter) {
+ Filter timeFilter, MPPQueryContext context) {
if (timeFilter == null) {
// (-oo, +oo)
return new Pair<>(Collections.emptyList(), new Pair<>(true, true));
@@ -2053,6 +2054,9 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
int index = 0;
int size = timeRangeList.size();
+ context.reserveMemoryForFrontEnd(
+ timeRangeList.size() *
RamUsageEstimator.shallowSizeOfInstance(TTimePartitionSlot.class));
+
if (timeRangeList.get(0).getMin() == Long.MIN_VALUE) {
needLeftAll = true;
endTime =
TimePartitionUtils.getTimePartitionUpperBound(timeRangeList.get(0).getMax());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java
index 3f617590812..6cdcefc77b9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java
@@ -120,7 +120,7 @@ public class TemplatedAggregationAnalyze {
context.generateGlobalTimeFilter(analysis);
// fetch partition information
- analyzeDataPartition(analysis, schemaTree, partitionFetcher,
context.getGlobalTimeFilter());
+ analyzeDataPartition(analysis, schemaTree, partitionFetcher, context);
return true;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
index a1fcb4c18ee..93defcafac7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
@@ -40,7 +40,6 @@ import
org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.schemaengine.template.Template;
import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.slf4j.Logger;
@@ -204,7 +203,7 @@ public class TemplatedAnalyze {
context.generateGlobalTimeFilter(analysis);
// fetch partition information
- analyzeDataPartition(analysis, schemaTree, partitionFetcher,
context.getGlobalTimeFilter());
+ analyzeDataPartition(analysis, schemaTree, partitionFetcher, context);
return true;
}
@@ -376,24 +375,24 @@ public class TemplatedAnalyze {
Analysis analysis,
ISchemaTree schemaTree,
IPartitionFetcher partitionFetcher,
- Filter globalTimeFilter) {
+ MPPQueryContext context) {
// TemplatedDevice has no views, so there is no need to use
outputDeviceToQueriedDevicesMap
Set<String> deviceSet =
analysis.getDeviceList().stream().map(PartialPath::getFullPath).collect(Collectors.toSet());
DataPartition dataPartition =
- fetchDataPartitionByDevices(deviceSet, schemaTree, globalTimeFilter,
partitionFetcher);
+ fetchDataPartitionByDevices(deviceSet, schemaTree, context,
partitionFetcher);
analysis.setDataPartitionInfo(dataPartition);
}
private static DataPartition fetchDataPartitionByDevices(
Set<String> deviceSet,
ISchemaTree schemaTree,
- Filter globalTimeFilter,
+ MPPQueryContext context,
IPartitionFetcher partitionFetcher) {
long startTime = System.nanoTime();
try {
Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res =
- getTimePartitionSlotList(globalTimeFilter);
+ getTimePartitionSlotList(context.getGlobalTimeFilter(), context);
// there is no satisfied time range
if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) {
return new DataPartition(
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java
index 30338d0d7d9..c49ad8cb4a4 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java
@@ -20,6 +20,8 @@ package org.apache.iotdb.db.queryengine.plan.analyze;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.filter.basic.Filter;
@@ -354,16 +356,17 @@ public class QueryTimePartitionTest {
@Test
public void testGetTimePartitionSlotList() {
-
+ MPPQueryContext context = new MPPQueryContext(new
QueryId("time-partition-test"));
// time >= 10 and time <= 9
Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res =
- getTimePartitionSlotList(FilterFactory.and(TimeFilterApi.gtEq(10),
TimeFilterApi.ltEq(9)));
+ getTimePartitionSlotList(
+ FilterFactory.and(TimeFilterApi.gtEq(10), TimeFilterApi.ltEq(9)),
context);
assertTrue(res.left.isEmpty());
assertFalse(res.right.left);
assertFalse(res.right.right);
// time >= 10
- res = getTimePartitionSlotList(TimeFilterApi.gtEq(10));
+ res = getTimePartitionSlotList(TimeFilterApi.gtEq(10), context);
assertEquals(1, res.left.size());
List<TTimePartitionSlot> expected = Collections.singletonList(new
TTimePartitionSlot(0));
assertEquals(expected.size(), res.left.size());
@@ -374,7 +377,7 @@ public class QueryTimePartitionTest {
assertTrue(res.right.right);
// time < 20
- res = getTimePartitionSlotList(TimeFilterApi.lt(20));
+ res = getTimePartitionSlotList(TimeFilterApi.lt(20), context);
assertEquals(1, res.left.size());
expected = Collections.singletonList(new TTimePartitionSlot(0));
assertEquals(expected.size(), res.left.size());
@@ -385,7 +388,9 @@ public class QueryTimePartitionTest {
assertFalse(res.right.right);
// time > 10 and time <= 20
- res = getTimePartitionSlotList(FilterFactory.and(TimeFilterApi.gt(10),
TimeFilterApi.ltEq(20)));
+ res =
+ getTimePartitionSlotList(
+ FilterFactory.and(TimeFilterApi.gt(10), TimeFilterApi.ltEq(20)),
context);
expected = Collections.singletonList(new TTimePartitionSlot(0));
assertEquals(expected.size(), res.left.size());
for (int i = 0; i < expected.size(); i++) {
@@ -401,8 +406,8 @@ public class QueryTimePartitionTest {
FilterFactory.and(
TimeFilterApi.gt(0),
TimeFilterApi.ltEq(
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() * 3
- + 1)));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() * 3 + 1)),
+ context);
expected =
Arrays.asList(
new TTimePartitionSlot(0),
@@ -427,7 +432,8 @@ public class QueryTimePartitionTest {
TimeFilterApi.gtEq(
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1),
TimeFilterApi.lt(
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)),
+ context);
expected =
Arrays.asList(
new TTimePartitionSlot(0),
@@ -446,7 +452,8 @@ public class QueryTimePartitionTest {
getTimePartitionSlotList(
TimeFilterApi.between(
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1,
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()),
+ context);
expected =
Arrays.asList(
new TTimePartitionSlot(0),
@@ -467,7 +474,8 @@ public class QueryTimePartitionTest {
TimeFilterApi.gtEq(
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()),
TimeFilterApi.ltEq(
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)),
+ context);
expected =
Collections.singletonList(
new TTimePartitionSlot(
@@ -485,7 +493,8 @@ public class QueryTimePartitionTest {
getTimePartitionSlotList(
TimeFilterApi.between(
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(),
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1),
+ context);
expected =
Collections.singletonList(
new TTimePartitionSlot(
@@ -547,7 +556,7 @@ public class QueryTimePartitionTest {
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() * 5
+ 10)));
- res = getTimePartitionSlotList(orFilter4);
+ res = getTimePartitionSlotList(orFilter4, context);
expected =
Arrays.asList(
new TTimePartitionSlot(0),
