This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/imporve_query in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5cdc4a4a96b0c13aea7d8f719dd0c186dd40d87b Author: Beyyes <[email protected]> AuthorDate: Sun Apr 23 14:55:57 2023 +0800 fix some query code impl to improve query performance --- .../src/assembly/resources/conf/iotdb-common.properties | 4 ++++ .../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +++++++++++ .../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++++ .../iotdb/db/mpp/execution/schedule/DriverTaskThread.java | 7 +++---- .../iotdb/db/mpp/plan/analyze/cache/PartitionCache.java | 10 ++++++---- .../apache/iotdb/db/mpp/plan/parser/StatementGenerator.java | 2 +- .../java/org/apache/iotdb/db/query/context/QueryContext.java | 2 ++ .../iotdb/db/query/control/clientsession/ClientSession.java | 3 +-- 8 files changed, 34 insertions(+), 11 deletions(-) diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties index 038b1f894d..0b31292a7f 100644 --- a/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -366,6 +366,10 @@ cluster_name=defaultCluster # Datatype: boolean # meta_data_cache_enable=true +# Whether to enable query metrics monitoring or not. Enable this may cause a little query performance degradation. +# Datatype: boolean +# query_metric_enable = true + # Read memory Allocation Ratio: BloomFilterCache : ChunkCache : TimeSeriesMetadataCache : Coordinator : Operators : DataExchange : timeIndex in TsFileResourceList : others. # The parameter form is a:b:c:d:e:f:g:h, where a, b, c, d, e, f, g and h are integers. for example: 1:1:1:1:1:1:1:1 , 1:100:200:50:200:200:200:50 # chunk_timeseriesmeta_free_memory_proportion=1:100:200:50:200:200:200:50 diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 4e39691ed0..8237040620 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -511,6 +511,9 @@ public class IoTDBConfig { /** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */ private boolean metaDataCacheEnable = true; + /** whether to enable query metrics monitoring or not. */ + private boolean queryMetricsEnable = true; + /** Memory allocated for bloomFilter cache in read process */ private long allocateMemoryForBloomFilterCache = allocateMemoryForRead / 1001; @@ -2065,6 +2068,14 @@ public class IoTDBConfig { this.metaDataCacheEnable = metaDataCacheEnable; } + public boolean isQueryMetricsEnable() { + return queryMetricsEnable; + } + + public void setQueryMetricsEnable(boolean queryMetricsEnable) { + this.queryMetricsEnable = queryMetricsEnable; + } + public long getAllocateMemoryForBloomFilterCache() { return allocateMemoryForBloomFilterCache; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 5581bacde9..e54db728d0 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -338,6 +338,12 @@ public class IoTDBDescriptor { "meta_data_cache_enable", Boolean.toString(conf.isMetaDataCacheEnable())) .trim())); + conf.setQueryMetricsEnable( + Boolean.parseBoolean( + properties + .getProperty("query_metric_enable", Boolean.toString(conf.isQueryMetricsEnable())) + .trim())); + initMemoryAllocate(properties); loadWALProps(properties); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java index 33028af21b..84e113fff4 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java @@ -24,7 +24,6 @@ import org.apache.iotdb.db.mpp.execution.driver.IDriver; import org.apache.iotdb.db.mpp.execution.schedule.queue.IndexedBlockingQueue; import org.apache.iotdb.db.mpp.execution.schedule.task.DriverTask; import org.apache.iotdb.db.utils.SetThreadName; -import org.apache.iotdb.db.utils.stats.CpuTimer; import com.google.common.base.Ticker; import com.google.common.util.concurrent.ListenableFuture; @@ -65,9 +64,9 @@ public class DriverTaskThread extends AbstractDriverThread { return; } IDriver driver = task.getDriver(); - CpuTimer timer = new CpuTimer(); + // CpuTimer timer = new CpuTimer(); ListenableFuture<?> future = driver.processFor(EXECUTION_TIME_SLICE); - CpuTimer.CpuDuration duration = timer.elapsedTime(); + // CpuTimer.CpuDuration duration = timer.elapsedTime(); // If the future is cancelled, the task is in an error and should be thrown. if (future.isCancelled()) { task.setAbortCause(DriverTaskAbortedException.BY_ALREADY_BEING_CANCELLED); @@ -76,7 +75,7 @@ public class DriverTaskThread extends AbstractDriverThread { } long quantaScheduledNanos = ticker.read() - startNanos; ExecutionContext context = new ExecutionContext(); - context.setCpuDuration(duration); + // context.setCpuDuration(duration); context.setScheduledTimeInNanos(quantaScheduledNanos); context.setTimeSlice(EXECUTION_TIME_SLICE); if (driver.isFinished()) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java index 8449e4fc05..ed787adb33 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java @@ -673,10 +673,12 @@ public class PartitionCache { SeriesPartitionTable cachedSeriesPartitionTable = cachedStorageGroupPartitionMap.get(seriesPartitionSlot); if (null == cachedSeriesPartitionTable) { - logger.debug( - "[{} Cache] miss when search device {}", - DATA_PARTITION_CACHE_NAME, - dataPartitionQueryParam.getDevicePath()); + if (logger.isDebugEnabled()) { + logger.debug( + "[{} Cache] miss when search device {}", + DATA_PARTITION_CACHE_NAME, + dataPartitionQueryParam.getDevicePath()); + } return false; } Map<TTimePartitionSlot, List<TConsensusGroupId>> cachedTimePartitionSlot = diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java index 432b76a718..12c330a300 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java @@ -208,7 +208,7 @@ public class StatementGenerator { SelectComponent selectComponent = new SelectComponent(zoneId); List<PartialPath> selectPaths = new ArrayList<>(); for (String pathStr : req.getPaths()) { - selectPaths.add(new PartialPath(pathStr)); + selectPaths.add(new PartialPath(pathStr.split("\\."))); } List<TAggregationType> aggregations = req.getAggregations(); for (int i = 0; i < aggregations.size(); i++) { diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java index c3583f4070..33dc7af63a 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java +++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java @@ -93,6 +93,8 @@ public class QueryContext { * them from 'modFile' and put then into the cache. */ public List<Modification> getPathModifications(ModificationFile modFile, PartialPath path) { + // TODO if no delete operation exists globally, return false + // if the mods file does not exist, do not add it to the cache if (!modFile.exists()) { return Collections.emptyList(); diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java index c52a27b535..49bb8e790f 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java +++ b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java @@ -25,7 +25,6 @@ import java.net.Socket; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArraySet; /** Client Session is the only identity for a connection. */ public class ClientSession extends IClientSession { @@ -65,7 +64,7 @@ public class ClientSession extends IClientSession { @Override public void addStatementId(long statementId) { - statementIdToQueryId.computeIfAbsent(statementId, sid -> new CopyOnWriteArraySet<>()); + statementIdToQueryId.computeIfAbsent(statementId, sid -> ConcurrentHashMap.newKeySet()); } @Override
