This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch LogQueryMemoryControl in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 55dc66c99437d909a336d5b4bd7d41fbe9d87084 Author: JackieTien97 <[email protected]> AuthorDate: Mon Jun 7 12:45:08 2021 +0800 add logs for memory control of query --- .../db/query/control/QueryResourceManager.java | 35 ++++++++++++++++++++++ .../org/apache/iotdb/db/service/TSServiceImpl.java | 7 ++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java index e4491cd..5c96197 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java +++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java @@ -122,6 +122,34 @@ public class QueryResourceManager { return queryId; } + /** + * Register a new query. When a query request is created firstly, this method must be invoked. + */ + public long assignQueryId(boolean isDataQuery, int fetchSize, int deduplicatedPathNum, + String sql) { + long queryId = queryIdAtom.incrementAndGet(); + if (isDataQuery) { + filePathsManager.addQueryId(queryId); + if (deduplicatedPathNum > 0) { + long estimatedMemoryUsage = + (long) deduplicatedPathNum * POINT_ESTIMATED_SIZE * (long) fetchSize; + // apply the memory successfully + if (totalFreeMemoryForRead.addAndGet(-estimatedMemoryUsage) >= 0) { + queryIdEstimatedMemoryMap.put(queryId, estimatedMemoryUsage); + logger.info( + "query id {}, sql is {}, apply the memory successfully, used {} memory, left {} memory", + queryId, sql, estimatedMemoryUsage, totalFreeMemoryForRead.get()); + } else { + totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage); + logger.info( + "query id {}, sql is {}, apply the memory failed, used {} memory, left {} memory", + queryId, sql, estimatedMemoryUsage, totalFreeMemoryForRead.get()); + } + } + } + return queryId; + } + public Map<Long, Integer> getChunkNumMap() { return chunkNumMap; } @@ -203,6 +231,13 @@ public class QueryResourceManager { Long estimatedMemoryUsage = queryIdEstimatedMemoryMap.remove(queryId); if (estimatedMemoryUsage != null) { totalFreeMemoryForRead.addAndGet(estimatedMemoryUsage); + logger.info( + "query id {}, put back the memory successfully, memory size is {}, current memory is {}", + queryId, estimatedMemoryUsage, totalFreeMemoryForRead.get()); + } else { + logger.info( + "query id {}, put back the memory failed, current memory is {}", + queryId, totalFreeMemoryForRead.get()); } // remove usage of opened file paths of current thread diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index da62967..71677d5 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@ -696,7 +696,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } // generate the queryId for the operation - queryId = generateQueryId(true, fetchSize, deduplicatedPathNum); + queryId = generateQueryId(true, fetchSize, deduplicatedPathNum, statement); if (plan instanceof QueryPlan && config.isEnablePerformanceTracing()) { if (!(plan instanceof AlignByDevicePlan)) { TracingManager.getInstance() @@ -1814,6 +1814,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { .assignQueryId(isDataQuery, fetchSize, deduplicatedPathNum); } + private long generateQueryId(boolean isDataQuery, int fetchSize, int deduplicatedPathNum, String sql) { + return QueryResourceManager.getInstance() + .assignQueryId(isDataQuery, fetchSize, deduplicatedPathNum, sql); + } + protected List<TSDataType> getSeriesTypesByPaths(List<PartialPath> paths, List<String> aggregations) throws MetadataException {
