This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch QueryMetrics in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7ef20b3d1e5a499df27d7fb41a2c9a5f6b3e6c08 Author: JackieTien97 <[email protected]> AuthorDate: Sat Nov 5 18:32:35 2022 +0800 Add query metrics --- .../apache/iotdb/commons/conf/IoTDBConstant.java | 3 + .../assembly/resources/conf/logback-datanode.xml | 20 ++++ .../iotdb/db/mpp/statistics/QueryStatistics.java | 118 +++++++++++++++++++++ 3 files changed, 141 insertions(+) diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index d5a26f4982..da571d8c68 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -77,6 +77,9 @@ public class IoTDBConstant { public static final String AUDIT_LOGGER_NAME = "IoTDB_AUDIT_LOGGER"; public static final String SLOW_SQL_LOGGER_NAME = "SLOW_SQL"; + + public static final String QUERY_STATISTICS_LOGGER_NAME = "QUERY_STATISTICS"; + public static final String COMPACTION_LOGGER_NAME = "COMPACTION"; public static final String IOTDB_JMX_PORT = "iotdb.jmx.port"; diff --git a/server/src/assembly/resources/conf/logback-datanode.xml b/server/src/assembly/resources/conf/logback-datanode.xml index 4f7373e1c2..aeae186f94 100644 --- a/server/src/assembly/resources/conf/logback-datanode.xml +++ b/server/src/assembly/resources/conf/logback-datanode.xml @@ -265,6 +265,23 @@ <level>INFO</level> </filter> </appender> + <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="QUERY_STATISTICS"> + <file>${IOTDB_HOME}/logs/log_datanode_query_statistics.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${IOTDB_HOME}/logs/log-datanode-slow-sql-%d{yyyyMMdd}.%i.log.gz</fileNamePattern> + <maxFileSize>10MB</maxFileSize> + <maxHistory>168</maxHistory> + <totalSizeCap>512MB</totalSizeCap> + </rollingPolicy> + <append>true</append> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern> + <charset>utf-8</charset> + </encoder> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>INFO</level> + </filter> + </appender> <root level="info"> <appender-ref ref="FILETRACE"/> <appender-ref ref="FILEDEBUG"/> @@ -299,4 +316,7 @@ <logger level="info" name="COMPACTION"> <appender-ref ref="COMPACTION"/> </logger> + <logger level="info" name="QUERY_STATISTICS"> + <appender-ref ref="QUERY_STATISTICS"/> + </logger> </configuration> diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java new file mode 100644 index 0000000000..f6ba8b3050 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.mpp.statistics; + +import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; +import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; +import org.apache.iotdb.commons.conf.IoTDBConstant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +@ThreadSafe +public class QueryStatistics { + + private static final long QUERY_STATISTICS_PRINT_INTERVAL_IN_MS = 10_000; + + private static final Logger QUERY_STATISTICS_LOGGER = + LoggerFactory.getLogger(IoTDBConstant.QUERY_STATISTICS_LOGGER_NAME); + + private final AtomicBoolean tracing = new AtomicBoolean(false); + + private final Map<String, OperationStatistic> operationStatistics = new ConcurrentHashMap<>(); + + private QueryStatistics() { + ScheduledExecutorService scheduledExecutor = + IoTDBThreadPoolFactory.newScheduledThreadPool(1, "Query-Statistics-Print"); + ScheduledExecutorUtil.safelyScheduleAtFixedRate( + scheduledExecutor, + this::printQueryStatistics, + 0, + QUERY_STATISTICS_PRINT_INTERVAL_IN_MS, + TimeUnit.MILLISECONDS); + } + + private void printQueryStatistics() { + if (tracing.get()) { + operationStatistics.forEach( + (k, v) -> { + QUERY_STATISTICS_LOGGER.info("Operation: {}, Statistics: {}", k, v); + }); + } + } + + public void addCost(String key, long costTimeInNanos) { + if (tracing.get()) { + operationStatistics + .computeIfAbsent(key, k -> new OperationStatistic()) + .addTimeCost(costTimeInNanos); + } + } + + public void traceOff() { + tracing.set(false); + operationStatistics.clear(); + } + + public void tradeOn() { + tracing.set(true); + operationStatistics.clear(); + } + + private static class OperationStatistic { + // accumulated operation time in ns + private final AtomicLong totalTime; + private final AtomicLong totalCount; + + public OperationStatistic() { + this.totalTime = new AtomicLong(0); + this.totalCount = new AtomicLong(0); + } + + public void addTimeCost(long costTimeInNanos) { + totalTime.addAndGet(costTimeInNanos); + totalCount.incrementAndGet(); + } + + @Override + public String toString() { + long time = totalTime.get() / 1_000; + long count = totalCount.get(); + return "{" + + "totalTime=" + + time + + "us" + + ", totalCount=" + + count + + ", avgOperationTime=" + + (time / count) + + "us" + + '}'; + } + } +}
