This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.11 by this push:
new 8dee762 [To rel/0.11] [IOTDB-1500] Remove current dynamic query
memory control (#3631)
8dee762 is described below
commit 8dee762271dedb5703466bd945ae85b051596f61
Author: Xiangwei Wei <[email protected]>
AuthorDate: Tue Jul 27 19:59:31 2021 +0800
[To rel/0.11] [IOTDB-1500] Remove current dynamic query memory control
(#3631)
---
.../main/java/org/apache/iotdb/db/qp/Planner.java | 41 +++++-------------
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 36 ++++------------
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 20 +++++----
.../qp/strategy/optimizer/ILogicalOptimizer.java | 3 +-
.../db/query/control/QueryResourceManager.java | 36 +---------------
.../org/apache/iotdb/db/service/TSServiceImpl.java | 50 ++++------------------
.../db/integration/IoTDBSequenceDataQueryIT.java | 7 ++-
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 8 ++--
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 2 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 2 +-
.../apache/iotdb/spark/db/EnvironmentUtils.java | 5 +--
11 files changed, 50 insertions(+), 160 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
index cb2ab65..6460096 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
@@ -34,7 +34,6 @@ import
org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.DnfFilterOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.MergeSingleFilterOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.RemoveNotOptimizer;
-import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.service.rpc.thrift.TSRawDataQueryReq;
@@ -56,25 +55,15 @@ public class Planner {
@TestOnly
public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr) throws
QueryProcessException {
- return parseSQLToPhysicalPlan(sqlStr, ZoneId.systemDefault(), 1024);
+ return parseSQLToPhysicalPlan(sqlStr, ZoneId.systemDefault());
}
- /** @param fetchSize this parameter only take effect when it is a query plan
*/
- public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr, ZoneId zoneId, int
fetchSize)
+ public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr, ZoneId zoneId)
throws QueryProcessException {
Operator operator = logicalGenerator.generate(sqlStr, zoneId);
- int maxDeduplicatedPathNum =
-
QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(fetchSize);
- if (operator instanceof SFWOperator && ((SFWOperator)
operator).isLastQuery()) {
- // Dataset of last query actually has only three columns, so we
shouldn't limit the path num
- // while constructing logical plan
- // To avoid overflowing because logicalOptimize function may do
maxDeduplicatedPathNum + 1, we
- // set it to Integer.MAX_VALUE - 1
- maxDeduplicatedPathNum = Integer.MAX_VALUE - 1;
- }
- operator = logicalOptimize(operator, maxDeduplicatedPathNum);
+ operator = logicalOptimize(operator);
PhysicalGenerator physicalGenerator = new PhysicalGenerator();
- PhysicalPlan physicalPlan =
physicalGenerator.transformToPhysicalPlan(operator, fetchSize);
+ PhysicalPlan physicalPlan =
physicalGenerator.transformToPhysicalPlan(operator);
physicalPlan.setDebug(operator.isDebug());
return physicalPlan;
}
@@ -120,20 +109,10 @@ public class Planner {
queryOp.setFilterOperator(filterOp);
- int maxDeduplicatedPathNum =
-
QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(rawDataQueryReq.fetchSize);
- if (queryOp.isLastQuery()) {
- // Dataset of last query actually has only three columns, so we
shouldn't limit the path num
- // while constructing logical plan
- // To avoid overflowing because logicalOptimize function may do
maxDeduplicatedPathNum + 1, we
- // set it to Integer.MAX_VALUE - 1
- maxDeduplicatedPathNum = Integer.MAX_VALUE - 1;
- }
- SFWOperator op = (SFWOperator) logicalOptimize(queryOp,
maxDeduplicatedPathNum);
+ SFWOperator op = (SFWOperator) logicalOptimize(queryOp);
PhysicalGenerator physicalGenerator = new PhysicalGenerator();
- PhysicalPlan physicalPlan =
- physicalGenerator.transformToPhysicalPlan(op,
rawDataQueryReq.fetchSize);
+ PhysicalPlan physicalPlan = physicalGenerator.transformToPhysicalPlan(op);
physicalPlan.setDebug(op.isDebug());
return physicalPlan;
}
@@ -145,7 +124,7 @@ public class Planner {
* @return optimized logical operator
* @throws LogicalOptimizeException exception in logical optimizing
*/
- protected Operator logicalOptimize(Operator operator, int
maxDeduplicatedPathNum)
+ protected Operator logicalOptimize(Operator operator)
throws LogicalOperatorException, PathNumOverLimitException {
switch (operator.getType()) {
case AUTHOR:
@@ -178,7 +157,7 @@ public class Planner {
case UPDATE:
case DELETE:
SFWOperator root = (SFWOperator) operator;
- return optimizeSFWOperator(root, maxDeduplicatedPathNum);
+ return optimizeSFWOperator(root);
default:
throw new LogicalOperatorException(operator.getType().toString(), "");
}
@@ -191,10 +170,10 @@ public class Planner {
* @return optimized select-from-where operator
* @throws LogicalOptimizeException exception in SFW optimizing
*/
- private SFWOperator optimizeSFWOperator(SFWOperator root, int
maxDeduplicatedPathNum)
+ private SFWOperator optimizeSFWOperator(SFWOperator root)
throws LogicalOperatorException, PathNumOverLimitException {
ConcatPathOptimizer concatPathOptimizer = getConcatPathOptimizer();
- root = (SFWOperator) concatPathOptimizer.transform(root,
maxDeduplicatedPathNum);
+ root = (SFWOperator) concatPathOptimizer.transform(root);
FilterOperator filter = root.getFilterOperator();
if (filter == null) {
return root;
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index b88b11f..ad6f9d2 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.iotdb.db.auth.AuthException;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.PathNotExistException;
import org.apache.iotdb.db.exception.query.PathNumOverLimitException;
@@ -105,7 +106,6 @@ import
org.apache.iotdb.db.qp.physical.sys.ShowPlan.ShowContentType;
import org.apache.iotdb.db.qp.physical.sys.ShowStorageGroupPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowTTLPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
-import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.qp.physical.sys.TracingPlan;
import org.apache.iotdb.db.service.IoTDB;
import org.apache.iotdb.db.utils.SchemaUtils;
@@ -121,7 +121,7 @@ public class PhysicalGenerator {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
- public PhysicalPlan transformToPhysicalPlan(Operator operator, int fetchSize)
+ public PhysicalPlan transformToPhysicalPlan(Operator operator)
throws QueryProcessException {
List<PartialPath> paths;
switch (operator.getType()) {
@@ -203,7 +203,7 @@ public class PhysicalGenerator {
return new TracingPlan(tracingOperator.isTracingon());
case QUERY:
QueryOperator query = (QueryOperator) operator;
- return transformQuery(query, fetchSize);
+ return transformQuery(query);
case TTL:
switch (operator.getTokenIntType()) {
case SQLConstant.TOK_SET:
@@ -329,7 +329,7 @@ public class PhysicalGenerator {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
- private PhysicalPlan transformQuery(QueryOperator queryOperator, int
fetchSize)
+ private PhysicalPlan transformQuery(QueryOperator queryOperator)
throws QueryProcessException {
QueryPlan queryPlan;
@@ -541,8 +541,8 @@ public class PhysicalGenerator {
measurements = slimitTrimColumn(measurements, seriesSlimit,
seriesOffset);
}
- int maxDeduplicatedPathNum = QueryResourceManager.getInstance()
- .getMaxDeduplicatedPathNum(fetchSize);
+ int maxDeduplicatedPathNum = IoTDBDescriptor.getInstance()
+ .getConfig().getMaxQueryDeduplicatedPathNum();
if (measurements.size() > maxDeduplicatedPathNum) {
throw new PathNumOverLimitException(maxDeduplicatedPathNum,
measurements.size());
@@ -593,7 +593,7 @@ public class PhysicalGenerator {
}
}
try {
- deduplicate(queryPlan, fetchSize);
+ deduplicate(queryPlan);
} catch (MetadataException e) {
throw new QueryProcessException(e);
}
@@ -685,8 +685,7 @@ public class PhysicalGenerator {
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
- private void deduplicate(QueryPlan queryPlan, int fetchSize)
- throws MetadataException, PathNumOverLimitException {
+ private void deduplicate(QueryPlan queryPlan) throws MetadataException {
// generate dataType first
List<PartialPath> paths = queryPlan.getPaths();
List<TSDataType> dataTypes = getSeriesTypes(paths);
@@ -697,18 +696,6 @@ public class PhysicalGenerator {
return;
}
- if (queryPlan instanceof GroupByTimePlan) {
- GroupByTimePlan plan = (GroupByTimePlan) queryPlan;
- // the actual row number of group by query should be calculated from
startTime, endTime and interval.
- long interval = (plan.getEndTime() - plan.getStartTime()) /
plan.getInterval();
- if (interval > 0) {
- fetchSize = Math.min((int) (interval), fetchSize);
- }
- } else if (queryPlan instanceof AggregationPlan) {
- // the actual row number of aggregation query is 1
- fetchSize = 1;
- }
-
RawDataQueryPlan rawDataQueryPlan = (RawDataQueryPlan) queryPlan;
Set<String> columnSet = new HashSet<>();
// if it's a last query, no need to sort by device
@@ -737,9 +724,6 @@ public class PhysicalGenerator {
}
indexedPaths.sort(Comparator.comparing(pair -> pair.left));
- int maxDeduplicatedPathNum = QueryResourceManager.getInstance()
- .getMaxDeduplicatedPathNum(fetchSize);
- int deduplicatedPathNum = 0;
int index = 0;
for (Pair<PartialPath, Integer> indexedPath : indexedPaths) {
String column = indexedPath.left.getTsAlias();
@@ -755,10 +739,6 @@ public class PhysicalGenerator {
TSDataType seriesType = dataTypes.get(indexedPath.right);
rawDataQueryPlan.addDeduplicatedPaths(indexedPath.left);
rawDataQueryPlan.addDeduplicatedDataTypes(seriesType);
- deduplicatedPathNum++;
- if (deduplicatedPathNum > maxDeduplicatedPathNum) {
- throw new PathNumOverLimitException(maxDeduplicatedPathNum,
deduplicatedPathNum);
- }
columnSet.add(column);
rawDataQueryPlan.addPathToIndex(column, index++);
if (queryPlan instanceof AggregationPlan) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
index 3b84a5f..73b6a0b 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
@@ -22,6 +22,8 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.exception.query.PathNumOverLimitException;
@@ -52,7 +54,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer
{
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
@Override
- public Operator transform(Operator operator, int maxDeduplicatedPathNum)
+ public Operator transform(Operator operator)
throws LogicalOptimizeException, PathNumOverLimitException {
if (!(operator instanceof SFWOperator)) {
logger.warn("given operator isn't SFWOperator, cannot concat
seriesPath");
@@ -93,7 +95,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer
{
// concat paths and remove stars
int seriesLimit = ((QueryOperator) operator).getSeriesLimit();
int seriesOffset = ((QueryOperator) operator).getSeriesOffset();
- concatSelect(prefixPaths, select, seriesLimit, seriesOffset,
maxDeduplicatedPathNum);
+ concatSelect(prefixPaths, select, seriesLimit, seriesOffset);
} else {
isAlignByDevice = true;
for (PartialPath path : initialSuffixPaths) {
@@ -159,7 +161,7 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
* selectOperator's suffixPathList. Treat aggregations similarly.
*/
private void concatSelect(List<PartialPath> fromPaths, SelectOperator
selectOperator, int limit,
- int offset, int maxDeduplicatedPathNum)
+ int offset)
throws LogicalOptimizeException, PathNumOverLimitException {
List<PartialPath> suffixPaths = judgeSelectOperator(selectOperator);
@@ -180,8 +182,7 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
}
}
- removeStarsInPath(allPaths, afterConcatAggregations, selectOperator,
limit, offset,
- maxDeduplicatedPathNum);
+ removeStarsInPath(allPaths, afterConcatAggregations, selectOperator,
limit, offset);
}
private FilterOperator concatFilter(List<PartialPath> fromPaths,
FilterOperator operator,
@@ -274,11 +275,10 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
private void removeStarsInPath(List<PartialPath> paths, List<String>
afterConcatAggregations,
- SelectOperator selectOperator, int finalLimit, int finalOffset, int
maxDeduplicatedPathNum)
+ SelectOperator selectOperator, int finalLimit, int finalOffset)
throws LogicalOptimizeException, PathNumOverLimitException {
int offset = finalOffset;
- int limit = finalLimit == 0 || maxDeduplicatedPathNum < finalLimit
- ? maxDeduplicatedPathNum + 1 : finalLimit;
+ int limit = finalLimit == 0 ? Integer.MAX_VALUE : finalLimit;
int consumed = 0;
List<PartialPath> retPaths = new ArrayList<>();
List<String> newAggregations = new ArrayList<>();
@@ -313,7 +313,9 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
limit -= pair.right;
}
if (limit == 0) {
- if (retPaths.size() == maxDeduplicatedPathNum + 1) {
+ int maxDeduplicatedPathNum =
+
IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum();
+ if (maxDeduplicatedPathNum < retPaths.size()) {
throw new PathNumOverLimitException(maxDeduplicatedPathNum);
}
break;
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java
index 4467a8a..0a05517 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java
@@ -28,6 +28,5 @@ import org.apache.iotdb.db.qp.logical.Operator;
@FunctionalInterface
public interface ILogicalOptimizer {
- Operator transform(Operator operator, int maxDeduplicatedPathNum)
- throws LogicalOptimizeException, PathNumOverLimitException;
+ Operator transform(Operator operator) throws LogicalOptimizeException,
PathNumOverLimitException;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index e4491cd..1b65874 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -74,50 +74,22 @@ public class QueryResourceManager {
*/
private final Map<Long, List<IExternalSortFileDeserializer>>
externalSortFileMap;
- private final Map<Long, Long> queryIdEstimatedMemoryMap;
-
- // current total free memory for reading process(not including the cache
memory)
- private final AtomicLong totalFreeMemoryForRead;
-
- // estimated size for one point memory size, the unit is byte
- private static final long POINT_ESTIMATED_SIZE = 16L;
-
- private static final IoTDBConfig CONFIG =
IoTDBDescriptor.getInstance().getConfig();
-
private QueryResourceManager() {
filePathsManager = new QueryFileManager();
externalSortFileMap = new ConcurrentHashMap<>();
- queryIdEstimatedMemoryMap = new ConcurrentHashMap<>();
- totalFreeMemoryForRead = new AtomicLong(
-
IoTDBDescriptor.getInstance().getConfig().getAllocateMemoryForReadWithoutCache());
}
public static QueryResourceManager getInstance() {
return QueryTokenManagerHelper.INSTANCE;
}
- public int getMaxDeduplicatedPathNum(int fetchSize) {
- return Math.min((int) ((totalFreeMemoryForRead.get() / fetchSize) /
POINT_ESTIMATED_SIZE),
- CONFIG.getMaxQueryDeduplicatedPathNum());
- }
-
/**
* Register a new query. When a query request is created firstly, this
method must be invoked.
*/
- public long assignQueryId(boolean isDataQuery, int fetchSize, int
deduplicatedPathNum) {
+ public long assignQueryId(boolean isDataQuery) {
long queryId = queryIdAtom.incrementAndGet();
if (isDataQuery) {
filePathsManager.addQueryId(queryId);
- if (deduplicatedPathNum > 0) {
- long estimatedMemoryUsage =
- (long) deduplicatedPathNum * POINT_ESTIMATED_SIZE * (long)
fetchSize;
- // apply the memory successfully
- if (totalFreeMemoryForRead.addAndGet(-estimatedMemoryUsage) >= 0) {
- queryIdEstimatedMemoryMap.put(queryId, estimatedMemoryUsage);
- } else {
- totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage);
- }
- }
}
return queryId;
}
@@ -199,12 +171,6 @@ public class QueryResourceManager {
externalSortFileMap.remove(queryId);
}
- // put back the memory usage
- Long estimatedMemoryUsage = queryIdEstimatedMemoryMap.remove(queryId);
- if (estimatedMemoryUsage != null) {
- totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage);
- }
-
// remove usage of opened file paths of current thread
filePathsManager.removeUsedFilesForQuery(queryId);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index da62967..bba7a15 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -71,13 +71,11 @@ import org.apache.iotdb.db.qp.physical.crud.AggregationPlan;
import org.apache.iotdb.db.qp.physical.crud.AlignByDevicePlan;
import org.apache.iotdb.db.qp.physical.crud.AlignByDevicePlan.MeasurementType;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
-import org.apache.iotdb.db.qp.physical.crud.GroupByTimePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowsOfOneDevicePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
import org.apache.iotdb.db.qp.physical.crud.LastQueryPlan;
import org.apache.iotdb.db.qp.physical.crud.QueryPlan;
-import org.apache.iotdb.db.qp.physical.crud.RawDataQueryPlan;
import org.apache.iotdb.db.qp.physical.sys.AuthorPlan;
import org.apache.iotdb.db.qp.physical.sys.CreateMultiTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
@@ -464,7 +462,7 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
private boolean executeStatementInBatch(String statement, List<TSStatus>
result, long sessionId) {
try {
PhysicalPlan physicalPlan = processor
- .parseSQLToPhysicalPlan(statement,
sessionIdZoneIdMap.get(sessionId), DEFAULT_FETCH_SIZE);
+ .parseSQLToPhysicalPlan(statement,
sessionIdZoneIdMap.get(sessionId));
if (physicalPlan.isQuery()) {
throw new QueryInBatchStatementException(statement);
}
@@ -517,8 +515,7 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
String statement = req.getStatement();
PhysicalPlan physicalPlan = processor
- .parseSQLToPhysicalPlan(statement,
sessionIdZoneIdMap.get(req.getSessionId()),
- req.fetchSize);
+ .parseSQLToPhysicalPlan(statement,
sessionIdZoneIdMap.get(req.getSessionId()));
if (physicalPlan.isQuery()) {
return internalExecuteQueryStatement(statement, req.statementId,
physicalPlan,
req.fetchSize,
@@ -556,8 +553,7 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
PhysicalPlan physicalPlan;
try {
physicalPlan = processor
- .parseSQLToPhysicalPlan(statement,
sessionIdZoneIdMap.get(req.getSessionId()),
- req.fetchSize);
+ .parseSQLToPhysicalPlan(statement,
sessionIdZoneIdMap.get(req.getSessionId()));
} catch (QueryProcessException | SQLParserException e) {
logger.info(ERROR_PARSING_SQL, req.getStatement() + " " +
e.getMessage());
return
RpcUtils.getTSExecuteStatementResp(TSStatusCode.SQL_PARSE_ERROR,
e.getMessage());
@@ -640,11 +636,6 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
try {
TSExecuteStatementResp resp = getQueryResp(plan, username); // column
headers
- // In case users forget to set this field in query, use the default value
- if (fetchSize == 0) {
- fetchSize = DEFAULT_FETCH_SIZE;
- }
-
if (plan instanceof ShowTimeSeriesPlan) {
//If the user does not pass the limit, then set limit = fetchSize and
haslimit=false,else set haslimit = true
if (((ShowTimeSeriesPlan) plan).getLimit() == 0) {
@@ -667,36 +658,12 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
}
if (plan.getOperatorType() == OperatorType.AGGREGATION) {
resp.setIgnoreTimeStamp(true);
- // the actual row number of aggregation query is 1
- fetchSize = 1;
} // else default ignoreTimeStamp is false
- if (plan instanceof GroupByTimePlan) {
- GroupByTimePlan groupByTimePlan = (GroupByTimePlan) plan;
- // the actual row number of group by query should be calculated from
startTime, endTime and interval.
- fetchSize = Math.min(
- (int) ((groupByTimePlan.getEndTime() -
groupByTimePlan.getStartTime()) / groupByTimePlan
- .getInterval()), fetchSize);
- }
-
resp.setOperationType(plan.getOperatorType().toString());
- // get deduplicated path num
- int deduplicatedPathNum = -1;
- if (plan instanceof AlignByDevicePlan) {
- deduplicatedPathNum = ((AlignByDevicePlan)
plan).getMeasurements().size();
- } else if (plan instanceof LastQueryPlan) {
- // dataset of last query consists of three column: time column + value
column = 1 deduplicatedPathNum
- // and we assume that the memory which sensor name takes equals to 1
deduplicatedPathNum
- deduplicatedPathNum = 2;
- // last query's actual row number should be the minimum between the
number of series and fetchSize
- fetchSize = Math.min(((LastQueryPlan)
plan).getDeduplicatedPaths().size(), fetchSize);
- } else if (plan instanceof RawDataQueryPlan) {
- deduplicatedPathNum = ((RawDataQueryPlan)
plan).getDeduplicatedPaths().size();
- }
-
// generate the queryId for the operation
- queryId = generateQueryId(true, fetchSize, deduplicatedPathNum);
+ queryId = generateQueryId(true);
if (plan instanceof QueryPlan && config.isEnablePerformanceTracing()) {
if (!(plan instanceof AlignByDevicePlan)) {
TracingManager.getInstance()
@@ -1147,7 +1114,7 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
status = executeNonQueryPlan(plan);
TSExecuteStatementResp resp = RpcUtils.getTSExecuteStatementResp(status);
- long queryId = generateQueryId(false, DEFAULT_FETCH_SIZE, -1);
+ long queryId = generateQueryId(false);
resp.setQueryId(queryId);
return resp;
}
@@ -1166,7 +1133,7 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
PhysicalPlan physicalPlan;
try {
physicalPlan = processor
- .parseSQLToPhysicalPlan(statement,
sessionIdZoneIdMap.get(sessionId), DEFAULT_FETCH_SIZE);
+ .parseSQLToPhysicalPlan(statement,
sessionIdZoneIdMap.get(sessionId));
} catch (QueryProcessException | SQLParserException e) {
logger.warn(ERROR_PARSING_SQL, statement, e);
return RpcUtils.getTSExecuteStatementResp(TSStatusCode.SQL_PARSE_ERROR,
e.getMessage());
@@ -1809,9 +1776,8 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
}
- private long generateQueryId(boolean isDataQuery, int fetchSize, int
deduplicatedPathNum) {
- return QueryResourceManager.getInstance()
- .assignQueryId(isDataQuery, fetchSize, deduplicatedPathNum);
+ private long generateQueryId(boolean isDataQuery) {
+ return QueryResourceManager.getInstance().assignQueryId(isDataQuery);
}
protected List<TSDataType> getSeriesTypesByPaths(List<PartialPath> paths,
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
index 415fae6..29387c1 100644
---
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
@@ -185,8 +185,7 @@ public class IoTDBSequenceDataQueryIT {
pathList.add(new PartialPath(TestConstant.d1 +
TsFileConstant.PATH_SEPARATOR + TestConstant.s1));
dataTypes.add(TSDataType.INT64);
- TEST_QUERY_JOB_ID = QueryResourceManager.getInstance()
- .assignQueryId(true, 1024, pathList.size());
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
queryPlan.setDeduplicatedDataTypes(dataTypes);
@@ -218,7 +217,7 @@ public class IoTDBSequenceDataQueryIT {
GlobalTimeExpression globalTimeExpression = new
GlobalTimeExpression(TimeFilter.gtEq(800L));
TEST_QUERY_JOB_ID = QueryResourceManager.getInstance()
- .assignQueryId(true, 1024, pathList.size());
+ .assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
@@ -267,7 +266,7 @@ public class IoTDBSequenceDataQueryIT {
ValueFilter.gtEq(14));
TEST_QUERY_JOB_ID = QueryResourceManager.getInstance()
- .assignQueryId(true, 1024, pathList.size());
+ .assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
index ba2a46d..acceb8b 100644
---
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
@@ -280,7 +280,7 @@ public class IoTDBSeriesReaderIT {
dataTypes.add(TSDataType.INT64);
TEST_QUERY_JOB_ID = QueryResourceManager.getInstance()
- .assignQueryId(true, 1024, pathList.size());
+ .assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
@@ -311,7 +311,7 @@ public class IoTDBSeriesReaderIT {
ValueFilter.gtEq(20));
TEST_QUERY_JOB_ID = QueryResourceManager.getInstance()
- .assignQueryId(true, 1024, pathList.size());
+ .assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
@@ -338,7 +338,7 @@ public class IoTDBSeriesReaderIT {
List<TSDataType> dataTypes = Collections.singletonList(TSDataType.INT32);
SingleSeriesExpression expression = new SingleSeriesExpression(path,
TimeFilter.gt(22987L));
- TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true,
1024, 1);
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
RawDataQueryPlan queryPlan = new RawDataQueryPlan();
@@ -377,7 +377,7 @@ public class IoTDBSeriesReaderIT {
queryPlan.setDeduplicatedDataTypes(dataTypes);
TEST_QUERY_JOB_ID = QueryResourceManager.getInstance()
- .assignQueryId(true, 1024, pathList.size());
+ .assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
SingleSeriesExpression singleSeriesExpression = new
SingleSeriesExpression(path1,
diff --git
a/server/src/test/java/org/apache/iotdb/db/qp/plan/LogicalPlanSmallTest.java
b/server/src/test/java/org/apache/iotdb/db/qp/plan/LogicalPlanSmallTest.java
index f048a28..2b172e9 100644
--- a/server/src/test/java/org/apache/iotdb/db/qp/plan/LogicalPlanSmallTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/qp/plan/LogicalPlanSmallTest.java
@@ -182,7 +182,7 @@ public class LogicalPlanSmallTest {
.generate(sqlStr, ZoneId.systemDefault());
IoTDB.metaManager.init();
ConcatPathOptimizer concatPathOptimizer = new ConcatPathOptimizer();
- concatPathOptimizer.transform(operator, 1000);
+ concatPathOptimizer.transform(operator);
IoTDB.metaManager.clear();
// expected to throw LogicalOptimizeException: The value of SOFFSET (%d)
is equal to or exceeds the number of sequences (%d) that can actually be
returned.
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 634770b..da145c1 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -245,7 +245,7 @@ public class EnvironmentUtils {
createAllDir();
// disable the system monitor
config.setEnableStatMonitor(false);
- TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true,
1024, 0);
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
}
diff --git
a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
index afa66ba..39687a9 100644
---
a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
+++
b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
@@ -92,8 +92,7 @@ public class EnvironmentUtils {
private static IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
private static DirectoryManager directoryManager =
DirectoryManager.getInstance();
- public static long TEST_QUERY_JOB_ID = QueryResourceManager.getInstance()
- .assignQueryId(true, 1024, 0);
+ public static long TEST_QUERY_JOB_ID =
QueryResourceManager.getInstance().assignQueryId(true);
public static QueryContext TEST_QUERY_CONTEXT = new
QueryContext(TEST_QUERY_JOB_ID);
private static long oldTsFileThreshold = config.getTsFileSizeThreshold();
@@ -184,7 +183,7 @@ public class EnvironmentUtils {
StorageEngine.getInstance().reset();
MultiFileLogNodeManager.getInstance().start();
FlushManager.getInstance().start();
- TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true,
1024, 0);
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(true);
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
}
