This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new fb8f2fb09da [ISSUE 12499] Reject query with massive time span in
limited memroy
fb8f2fb09da is described below
commit fb8f2fb09dac42c8c81c1f05abebb956e2ecef22
Author: YangCaiyin <[email protected]>
AuthorDate: Thu Jun 13 14:42:42 2024 +0800
[ISSUE 12499] Reject query with massive time span in limited memroy
---
.../queryengine/plan/analyze/AnalyzeVisitor.java | 23 +++++++++++++--
.../plan/analyze/TemplatedAggregationAnalyze.java | 2 +-
.../queryengine/plan/analyze/TemplatedAnalyze.java | 11 ++++----
.../plan/analyze/QueryTimePartitionTest.java | 33 ++++++++++++++--------
.../iotdb/commons/utils/TimePartitionUtils.java | 4 +++
5 files changed, 52 insertions(+), 21 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 84708e52143..c6f7b240fd8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -152,6 +152,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1993,7 +1994,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
long startTime = System.nanoTime();
try {
Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res =
- getTimePartitionSlotList(context.getGlobalTimeFilter());
+ getTimePartitionSlotList(context.getGlobalTimeFilter(), context);
// there is no satisfied time range
if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) {
return new DataPartition(
@@ -2030,7 +2031,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
public static Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>>
getTimePartitionSlotList(
- Filter timeFilter) {
+ Filter timeFilter, MPPQueryContext context) {
if (timeFilter == null) {
// (-oo, +oo)
return new Pair<>(Collections.emptyList(), new Pair<>(true, true));
@@ -2071,6 +2072,9 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
}
List<TTimePartitionSlot> result = new ArrayList<>();
+ TimeRange currentTimeRange = timeRangeList.get(index);
+ reserveMemoryForTimePartitionSlot(
+ currentTimeRange.getMax(), currentTimeRange.getMin(), context);
while (index < size) {
long curLeft = timeRangeList.get(index).getMin();
long curRight = timeRangeList.get(index).getMax();
@@ -2086,6 +2090,11 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
endTime = endTime + TimePartitionUtils.getTimePartitionInterval();
} else {
index++;
+ if (index < size) {
+ currentTimeRange = timeRangeList.get(index);
+ reserveMemoryForTimePartitionSlot(
+ currentTimeRange.getMax(), currentTimeRange.getMin(), context);
+ }
}
}
result.add(timePartitionSlot);
@@ -2101,6 +2110,16 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
return new Pair<>(result, new Pair<>(needLeftAll, needRightAll));
}
+ private static void reserveMemoryForTimePartitionSlot(
+ long maxTime, long minTime, MPPQueryContext context) {
+ if (maxTime == Long.MAX_VALUE || minTime == Long.MIN_VALUE) {
+ return;
+ }
+ long size = TimePartitionUtils.getEstimateTimePartitionSize(minTime,
maxTime);
+ context.reserveMemoryForFrontEnd(
+ RamUsageEstimator.shallowSizeOfInstance(TTimePartitionSlot.class) *
size);
+ }
+
private void analyzeInto(
Analysis analysis,
QueryStatement queryStatement,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java
index 3f617590812..6cdcefc77b9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java
@@ -120,7 +120,7 @@ public class TemplatedAggregationAnalyze {
context.generateGlobalTimeFilter(analysis);
// fetch partition information
- analyzeDataPartition(analysis, schemaTree, partitionFetcher,
context.getGlobalTimeFilter());
+ analyzeDataPartition(analysis, schemaTree, partitionFetcher, context);
return true;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
index a1fcb4c18ee..93defcafac7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
@@ -40,7 +40,6 @@ import
org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.schemaengine.template.Template;
import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.slf4j.Logger;
@@ -204,7 +203,7 @@ public class TemplatedAnalyze {
context.generateGlobalTimeFilter(analysis);
// fetch partition information
- analyzeDataPartition(analysis, schemaTree, partitionFetcher,
context.getGlobalTimeFilter());
+ analyzeDataPartition(analysis, schemaTree, partitionFetcher, context);
return true;
}
@@ -376,24 +375,24 @@ public class TemplatedAnalyze {
Analysis analysis,
ISchemaTree schemaTree,
IPartitionFetcher partitionFetcher,
- Filter globalTimeFilter) {
+ MPPQueryContext context) {
// TemplatedDevice has no views, so there is no need to use
outputDeviceToQueriedDevicesMap
Set<String> deviceSet =
analysis.getDeviceList().stream().map(PartialPath::getFullPath).collect(Collectors.toSet());
DataPartition dataPartition =
- fetchDataPartitionByDevices(deviceSet, schemaTree, globalTimeFilter,
partitionFetcher);
+ fetchDataPartitionByDevices(deviceSet, schemaTree, context,
partitionFetcher);
analysis.setDataPartitionInfo(dataPartition);
}
private static DataPartition fetchDataPartitionByDevices(
Set<String> deviceSet,
ISchemaTree schemaTree,
- Filter globalTimeFilter,
+ MPPQueryContext context,
IPartitionFetcher partitionFetcher) {
long startTime = System.nanoTime();
try {
Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res =
- getTimePartitionSlotList(globalTimeFilter);
+ getTimePartitionSlotList(context.getGlobalTimeFilter(), context);
// there is no satisfied time range
if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) {
return new DataPartition(
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java
index 30338d0d7d9..1e85a1461ee 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java
@@ -20,6 +20,8 @@ package org.apache.iotdb.db.queryengine.plan.analyze;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.filter.basic.Filter;
@@ -354,16 +356,17 @@ public class QueryTimePartitionTest {
@Test
public void testGetTimePartitionSlotList() {
-
+ MPPQueryContext context = new MPPQueryContext(new
QueryId("time_partition_test"));
// time >= 10 and time <= 9
Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res =
- getTimePartitionSlotList(FilterFactory.and(TimeFilterApi.gtEq(10),
TimeFilterApi.ltEq(9)));
+ getTimePartitionSlotList(
+ FilterFactory.and(TimeFilterApi.gtEq(10), TimeFilterApi.ltEq(9)),
context);
assertTrue(res.left.isEmpty());
assertFalse(res.right.left);
assertFalse(res.right.right);
// time >= 10
- res = getTimePartitionSlotList(TimeFilterApi.gtEq(10));
+ res = getTimePartitionSlotList(TimeFilterApi.gtEq(10), context);
assertEquals(1, res.left.size());
List<TTimePartitionSlot> expected = Collections.singletonList(new
TTimePartitionSlot(0));
assertEquals(expected.size(), res.left.size());
@@ -374,7 +377,7 @@ public class QueryTimePartitionTest {
assertTrue(res.right.right);
// time < 20
- res = getTimePartitionSlotList(TimeFilterApi.lt(20));
+ res = getTimePartitionSlotList(TimeFilterApi.lt(20), context);
assertEquals(1, res.left.size());
expected = Collections.singletonList(new TTimePartitionSlot(0));
assertEquals(expected.size(), res.left.size());
@@ -385,7 +388,9 @@ public class QueryTimePartitionTest {
assertFalse(res.right.right);
// time > 10 and time <= 20
- res = getTimePartitionSlotList(FilterFactory.and(TimeFilterApi.gt(10),
TimeFilterApi.ltEq(20)));
+ res =
+ getTimePartitionSlotList(
+ FilterFactory.and(TimeFilterApi.gt(10), TimeFilterApi.ltEq(20)),
context);
expected = Collections.singletonList(new TTimePartitionSlot(0));
assertEquals(expected.size(), res.left.size());
for (int i = 0; i < expected.size(); i++) {
@@ -401,8 +406,8 @@ public class QueryTimePartitionTest {
FilterFactory.and(
TimeFilterApi.gt(0),
TimeFilterApi.ltEq(
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() * 3
- + 1)));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() * 3 + 1)),
+ context);
expected =
Arrays.asList(
new TTimePartitionSlot(0),
@@ -427,7 +432,8 @@ public class QueryTimePartitionTest {
TimeFilterApi.gtEq(
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1),
TimeFilterApi.lt(
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)),
+ context);
expected =
Arrays.asList(
new TTimePartitionSlot(0),
@@ -446,7 +452,8 @@ public class QueryTimePartitionTest {
getTimePartitionSlotList(
TimeFilterApi.between(
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1,
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()),
+ context);
expected =
Arrays.asList(
new TTimePartitionSlot(0),
@@ -467,7 +474,8 @@ public class QueryTimePartitionTest {
TimeFilterApi.gtEq(
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()),
TimeFilterApi.ltEq(
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)),
+ context);
expected =
Collections.singletonList(
new TTimePartitionSlot(
@@ -485,7 +493,8 @@ public class QueryTimePartitionTest {
getTimePartitionSlotList(
TimeFilterApi.between(
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(),
-
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1));
+
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1),
+ context);
expected =
Collections.singletonList(
new TTimePartitionSlot(
@@ -547,7 +556,7 @@ public class QueryTimePartitionTest {
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval() * 5
+ 10)));
- res = getTimePartitionSlotList(orFilter4);
+ res = getTimePartitionSlotList(orFilter4, context);
expected =
Arrays.asList(
new TTimePartitionSlot(0),
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
index fe20e6ba155..b0339cb6822 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
@@ -82,4 +82,8 @@ public class TimePartitionUtils {
public static void setTimePartitionInterval(long timePartitionInterval) {
TimePartitionUtils.timePartitionInterval = timePartitionInterval;
}
+
+ public static long getEstimateTimePartitionSize(long startTime, long
endTime) {
+ return (endTime - startTime) / timePartitionInterval + 1;
+ }
}
