This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 86be5ea16a [IOTDB-5803] Improve query performance by reducing cpu
consuming
86be5ea16a is described below
commit 86be5ea16a7a5f9331b3ef9424951fdc76355a1c
Author: Beyyes <[email protected]>
AuthorDate: Wed Apr 26 16:42:13 2023 +0800
[IOTDB-5803] Improve query performance by reducing cpu consuming
---
.../org/apache/iotdb/db/mpp/common/QueryId.java | 2 +-
.../mpp/execution/schedule/DriverTaskThread.java | 7 +-
.../iotdb/db/mpp/metric/QueryMetricsManager.java | 140 +++++++++++++--------
.../db/mpp/plan/analyze/cache/PartitionCache.java | 10 +-
.../db/mpp/plan/parser/StatementGenerator.java | 20 ++-
.../iotdb/db/query/context/QueryContext.java | 2 +
.../query/control/clientsession/ClientSession.java | 3 +-
thrift/src/main/thrift/client.thrift | 3 +
8 files changed, 120 insertions(+), 67 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/QueryId.java
b/server/src/main/java/org/apache/iotdb/db/mpp/common/QueryId.java
index be9462696f..2208e00672 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/QueryId.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/QueryId.java
@@ -52,7 +52,7 @@ public class QueryId {
}
public PlanNodeId genPlanNodeId() {
- return new PlanNodeId(String.format("%d", nextPlanNodeIndex++));
+ return new PlanNodeId(String.valueOf(nextPlanNodeIndex++));
}
public PlanFragmentId genPlanFragmentId() {
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
b/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
index 33028af21b..84e113fff4 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.mpp.execution.driver.IDriver;
import org.apache.iotdb.db.mpp.execution.schedule.queue.IndexedBlockingQueue;
import org.apache.iotdb.db.mpp.execution.schedule.task.DriverTask;
import org.apache.iotdb.db.utils.SetThreadName;
-import org.apache.iotdb.db.utils.stats.CpuTimer;
import com.google.common.base.Ticker;
import com.google.common.util.concurrent.ListenableFuture;
@@ -65,9 +64,9 @@ public class DriverTaskThread extends AbstractDriverThread {
return;
}
IDriver driver = task.getDriver();
- CpuTimer timer = new CpuTimer();
+ // CpuTimer timer = new CpuTimer();
ListenableFuture<?> future = driver.processFor(EXECUTION_TIME_SLICE);
- CpuTimer.CpuDuration duration = timer.elapsedTime();
+ // CpuTimer.CpuDuration duration = timer.elapsedTime();
// If the future is cancelled, the task is in an error and should be
thrown.
if (future.isCancelled()) {
task.setAbortCause(DriverTaskAbortedException.BY_ALREADY_BEING_CANCELLED);
@@ -76,7 +75,7 @@ public class DriverTaskThread extends AbstractDriverThread {
}
long quantaScheduledNanos = ticker.read() - startNanos;
ExecutionContext context = new ExecutionContext();
- context.setCpuDuration(duration);
+ // context.setCpuDuration(duration);
context.setScheduledTimeInNanos(quantaScheduledNanos);
context.setTimeSlice(EXECUTION_TIME_SLICE);
if (driver.isFinished()) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
index c6730d593c..adb6541e8c 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
@@ -22,93 +22,127 @@ package org.apache.iotdb.db.mpp.metric;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.utils.MetricInfo;
import org.apache.iotdb.metrics.utils.MetricLevel;
import java.util.concurrent.TimeUnit;
+import static org.apache.iotdb.metrics.utils.MetricLevel.DO_NOTHING;
+
public class QueryMetricsManager {
private final MetricService metricService = MetricService.getInstance();
public void recordPlanCost(String stage, long costTimeInNanos) {
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- Metric.QUERY_PLAN_COST.toString(),
- MetricLevel.IMPORTANT,
- Tag.STAGE.toString(),
- stage);
+ if (!DO_NOTHING.equals(
+
MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ Metric.QUERY_PLAN_COST.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.STAGE.toString(),
+ stage);
+ }
}
public void recordOperatorExecutionCost(String operatorType, long
costTimeInNanos) {
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- Metric.OPERATOR_EXECUTION_COST.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- operatorType);
+ if (!DO_NOTHING.equals(
+
MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ Metric.OPERATOR_EXECUTION_COST.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ operatorType);
+ }
}
public void recordOperatorExecutionCount(String operatorType, long count) {
- metricService.count(
- count,
- Metric.OPERATOR_EXECUTION_COUNT.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- operatorType);
+ if (!DO_NOTHING.equals(
+
MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.count(
+ count,
+ Metric.OPERATOR_EXECUTION_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ operatorType);
+ }
}
public void recordSeriesScanCost(String stage, long costTimeInNanos) {
- MetricInfo metricInfo = SeriesScanCostMetricSet.metricInfoMap.get(stage);
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- metricInfo.getName(),
- MetricLevel.IMPORTANT,
- metricInfo.getTagsInArray());
+ if (!DO_NOTHING.equals(
+
MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ MetricInfo metricInfo = SeriesScanCostMetricSet.metricInfoMap.get(stage);
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ metricInfo.getName(),
+ MetricLevel.IMPORTANT,
+ metricInfo.getTagsInArray());
+ }
}
public void recordExecutionCost(String stage, long costTimeInNanos) {
- MetricInfo metricInfo = QueryExecutionMetricSet.metricInfoMap.get(stage);
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- metricInfo.getName(),
- MetricLevel.IMPORTANT,
- metricInfo.getTagsInArray());
+ if (!DO_NOTHING.equals(
+
MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ MetricInfo metricInfo = QueryExecutionMetricSet.metricInfoMap.get(stage);
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ metricInfo.getName(),
+ MetricLevel.IMPORTANT,
+ metricInfo.getTagsInArray());
+ }
}
public void recordQueryResourceNum(String type, int count) {
- metricService.histogram(
- count, Metric.QUERY_RESOURCE.toString(), MetricLevel.IMPORTANT,
Tag.TYPE.toString(), type);
+ if (!DO_NOTHING.equals(
+
MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.histogram(
+ count,
+ Metric.QUERY_RESOURCE.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.TYPE.toString(),
+ type);
+ }
}
public void recordDataExchangeCost(String stage, long costTimeInNanos) {
- MetricInfo metricInfo = DataExchangeCostMetricSet.metricInfoMap.get(stage);
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- metricInfo.getName(),
- MetricLevel.IMPORTANT,
- metricInfo.getTagsInArray());
+ if (!DO_NOTHING.equals(
+
MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ MetricInfo metricInfo =
DataExchangeCostMetricSet.metricInfoMap.get(stage);
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ metricInfo.getName(),
+ MetricLevel.IMPORTANT,
+ metricInfo.getTagsInArray());
+ }
}
public void recordDataBlockNum(String type, int num) {
- MetricInfo metricInfo = DataExchangeCountMetricSet.metricInfoMap.get(type);
- metricService.histogram(
- num, metricInfo.getName(), MetricLevel.IMPORTANT,
metricInfo.getTagsInArray());
+ if (!DO_NOTHING.equals(
+
MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ MetricInfo metricInfo =
DataExchangeCountMetricSet.metricInfoMap.get(type);
+ metricService.histogram(
+ num, metricInfo.getName(), MetricLevel.IMPORTANT,
metricInfo.getTagsInArray());
+ }
}
public void recordTaskQueueTime(String name, long queueTimeInNanos) {
- metricService.timer(
- queueTimeInNanos,
- TimeUnit.NANOSECONDS,
- Metric.DRIVER_SCHEDULER.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- name);
+ if (!DO_NOTHING.equals(
+
MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.timer(
+ queueTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ Metric.DRIVER_SCHEDULER.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ name);
+ }
}
public static QueryMetricsManager getInstance() {
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
index 8449e4fc05..ed787adb33 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
@@ -673,10 +673,12 @@ public class PartitionCache {
SeriesPartitionTable cachedSeriesPartitionTable =
cachedStorageGroupPartitionMap.get(seriesPartitionSlot);
if (null == cachedSeriesPartitionTable) {
- logger.debug(
- "[{} Cache] miss when search device {}",
- DATA_PARTITION_CACHE_NAME,
- dataPartitionQueryParam.getDevicePath());
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] miss when search device {}",
+ DATA_PARTITION_CACHE_NAME,
+ dataPartitionQueryParam.getDevicePath());
+ }
return false;
}
Map<TTimePartitionSlot, List<TConsensusGroupId>> cachedTimePartitionSlot =
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
index f29b8016e5..a450673702 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
@@ -138,7 +138,12 @@ public class StatementGenerator {
// iterate the path list and add it to from operator
for (String pathStr : rawDataQueryReq.getPaths()) {
- PartialPath path = new PartialPath(pathStr);
+ PartialPath path;
+ if (rawDataQueryReq.isLegalPathNodes()) {
+ path = new PartialPath(pathStr.split("\\."));
+ } else {
+ path = new PartialPath(pathStr);
+ }
fromComponent.addPrefixPath(path);
}
selectComponent.addResultColumn(
@@ -176,7 +181,12 @@ public class StatementGenerator {
// iterate the path list and add it to from operator
for (String pathStr : lastDataQueryReq.getPaths()) {
- PartialPath path = new PartialPath(pathStr);
+ PartialPath path;
+ if (lastDataQueryReq.isLegalPathNodes()) {
+ path = new PartialPath(pathStr.split("\\."));
+ } else {
+ path = new PartialPath(pathStr);
+ }
fromComponent.addPrefixPath(path);
}
selectComponent.addResultColumn(
@@ -212,7 +222,11 @@ public class StatementGenerator {
SelectComponent selectComponent = new SelectComponent(zoneId);
List<PartialPath> selectPaths = new ArrayList<>();
for (String pathStr : req.getPaths()) {
- selectPaths.add(new PartialPath(pathStr));
+ if (req.isLegalPathNodes()) {
+ selectPaths.add(new PartialPath(pathStr.split("\\.")));
+ } else {
+ selectPaths.add(new PartialPath(pathStr));
+ }
}
List<TAggregationType> aggregations = req.getAggregations();
for (int i = 0; i < aggregations.size(); i++) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index c3583f4070..2a44762fdd 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -93,6 +93,8 @@ public class QueryContext {
* them from 'modFile' and put then into the cache.
*/
public List<Modification> getPathModifications(ModificationFile modFile,
PartialPath path) {
+ // TODO change a way to do the existing check to avoid this IO call each
time.
+
// if the mods file does not exist, do not add it to the cache
if (!modFile.exists()) {
return Collections.emptyList();
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
index c52a27b535..49bb8e790f 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
@@ -25,7 +25,6 @@ import java.net.Socket;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArraySet;
/** Client Session is the only identity for a connection. */
public class ClientSession extends IClientSession {
@@ -65,7 +64,7 @@ public class ClientSession extends IClientSession {
@Override
public void addStatementId(long statementId) {
- statementIdToQueryId.computeIfAbsent(statementId, sid -> new
CopyOnWriteArraySet<>());
+ statementIdToQueryId.computeIfAbsent(statementId, sid ->
ConcurrentHashMap.newKeySet());
}
@Override
diff --git a/thrift/src/main/thrift/client.thrift
b/thrift/src/main/thrift/client.thrift
index ebb1ef1880..6779bbf89c 100644
--- a/thrift/src/main/thrift/client.thrift
+++ b/thrift/src/main/thrift/client.thrift
@@ -323,6 +323,7 @@ struct TSRawDataQueryReq {
7: optional bool enableRedirectQuery
8: optional bool jdbcQuery
9: optional i64 timeout
+ 10: optional bool legalPathNodes
}
struct TSLastDataQueryReq {
@@ -334,6 +335,7 @@ struct TSLastDataQueryReq {
6: optional bool enableRedirectQuery
7: optional bool jdbcQuery
8: optional i64 timeout
+ 9: optional bool legalPathNodes
}
struct TSAggregationQueryReq {
@@ -347,6 +349,7 @@ struct TSAggregationQueryReq {
8: optional i64 slidingStep
9: optional i32 fetchSize
10: optional i64 timeout
+ 11: optional bool legalPathNodes
}
struct TSCreateMultiTimeseriesReq {
