This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch QueryMetrics in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 72020ef96b74c12de0d9eebd03b216e734872d4f Author: Minghui Liu <[email protected]> AuthorDate: Tue Dec 6 23:13:16 2022 +0800 add io metrics --- .../apache/iotdb/db/engine/cache/ChunkCache.java | 5 + .../db/engine/cache/TimeSeriesMetadataCache.java | 6 + .../iotdb/db/mpp/statistics/QueryStatistics.java | 7 + .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 259 +++++++++++---------- 4 files changed, 159 insertions(+), 118 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java index 0fa63a5fdf..a47f9ecffe 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.mpp.statistics.QueryStatistics; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; @@ -71,6 +72,7 @@ public class ChunkCache { .recordStats() .build( chunkMetadata -> { + long startTime = System.nanoTime(); try { TsFileSequenceReader reader = FileReaderManager.getInstance() @@ -79,6 +81,9 @@ public class ChunkCache { } catch (IOException e) { logger.error("Something wrong happened in reading {}", chunkMetadata, e); throw e; + } finally { + QueryStatistics.getInstance() + .addCost(QueryStatistics.CHUNK_CACHE_MISS, System.nanoTime() - startTime); } }); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java index d70b2faf41..c9610d3354 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.mpp.statistics.QueryStatistics; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; @@ -50,6 +51,8 @@ import java.util.Set; import java.util.WeakHashMap; import java.util.concurrent.atomic.AtomicLong; +import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.TIME_SERIES_METADATA_CACHE_MISS; + /** * This class is used to cache <code>TimeSeriesMetadata</code> in IoTDB. The caching strategy is * LRU. @@ -137,6 +140,7 @@ public class TimeSeriesMetadataCache { TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key); if (timeseriesMetadata == null) { + long startTime = System.nanoTime(); if (debug) { DEBUG_LOGGER.info( "Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath); @@ -180,6 +184,8 @@ public class TimeSeriesMetadataCache { } } } + QueryStatistics.getInstance() + .addCost(TIME_SERIES_METADATA_CACHE_MISS, System.nanoTime() - startTime); } if (timeseriesMetadata == null) { if (debug) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java index 9b0dd7e985..7f6074d709 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java @@ -84,6 +84,13 @@ public class QueryStatistics { public static final String SERVER_RPC_RT = "ServerRpcRT"; + public static final String LOAD_TIME_SERIES_METADATA_ALIGNED = "loadTimeSeriesMetadata-aligned"; + public static final String LOAD_TIME_SERIES_METADATA = "loadTimeSeriesMetadata"; + public static final String LOAD_CHUNK_METADATA_LIST = "loadChunkMetadataList"; + public static final String LOAD_PAGE_READER_LIST = "loadPageReaderList"; + public static final String TIME_SERIES_METADATA_CACHE_MISS = "TimeSeriesMetadataCacheMiss"; + public static final String CHUNK_CACHE_MISS = "ChunkCacheMiss"; + private QueryStatistics() { ScheduledExecutorService scheduledExecutor = IoTDBThreadPoolFactory.newScheduledThreadPool(1, "Query-Statistics-Print"); diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index bfd6a0bf80..bdaffb8360 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache.TimeSeriesMetada import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; +import org.apache.iotdb.db.mpp.statistics.QueryStatistics; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.reader.chunk.metadata.DiskAlignedChunkMetadataLoader; import org.apache.iotdb.db.query.reader.chunk.metadata.DiskChunkMetadataLoader; @@ -124,51 +125,56 @@ public class FileLoaderUtils { Filter filter, Set<String> allSensors) throws IOException { - - // common path - TimeseriesMetadata timeSeriesMetadata; - // If the tsfile is closed, we need to load from tsfile - if (resource.isClosed()) { - // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex - // we should not ignore the non-exist of device in TsFileMetadata - timeSeriesMetadata = - TimeSeriesMetadataCache.getInstance() - .get( - new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey( - resource.getTsFilePath(), - seriesPath.getDevice(), - seriesPath.getMeasurement()), - allSensors, - resource.getTimeIndexType() != 1, - context.isDebug()); - if (timeSeriesMetadata != null) { - timeSeriesMetadata.setChunkMetadataLoader( - new DiskChunkMetadataLoader(resource, seriesPath, context, filter)); - } - } else { // if the tsfile is unclosed, we just get it directly from TsFileResource - timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath); - if (timeSeriesMetadata != null) { - timeSeriesMetadata.setChunkMetadataLoader( - new MemChunkMetadataLoader(resource, seriesPath, context, filter)); + long startTime = System.nanoTime(); + try { + // common path + TimeseriesMetadata timeSeriesMetadata; + // If the tsfile is closed, we need to load from tsfile + if (resource.isClosed()) { + // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex + // we should not ignore the non-exist of device in TsFileMetadata + timeSeriesMetadata = + TimeSeriesMetadataCache.getInstance() + .get( + new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey( + resource.getTsFilePath(), + seriesPath.getDevice(), + seriesPath.getMeasurement()), + allSensors, + resource.getTimeIndexType() != 1, + context.isDebug()); + if (timeSeriesMetadata != null) { + timeSeriesMetadata.setChunkMetadataLoader( + new DiskChunkMetadataLoader(resource, seriesPath, context, filter)); + } + } else { // if the tsfile is unclosed, we just get it directly from TsFileResource + timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath); + if (timeSeriesMetadata != null) { + timeSeriesMetadata.setChunkMetadataLoader( + new MemChunkMetadataLoader(resource, seriesPath, context, filter)); + } } - } - if (timeSeriesMetadata != null) { - List<Modification> pathModifications = - context.getPathModifications(resource.getModFile(), seriesPath); - timeSeriesMetadata.setModified(!pathModifications.isEmpty()); - if (timeSeriesMetadata.getStatistics().getStartTime() - > timeSeriesMetadata.getStatistics().getEndTime()) { - return null; - } - if (filter != null - && !filter.satisfyStartEndTime( - timeSeriesMetadata.getStatistics().getStartTime(), - timeSeriesMetadata.getStatistics().getEndTime())) { - return null; + if (timeSeriesMetadata != null) { + List<Modification> pathModifications = + context.getPathModifications(resource.getModFile(), seriesPath); + timeSeriesMetadata.setModified(!pathModifications.isEmpty()); + if (timeSeriesMetadata.getStatistics().getStartTime() + > timeSeriesMetadata.getStatistics().getEndTime()) { + return null; + } + if (filter != null + && !filter.satisfyStartEndTime( + timeSeriesMetadata.getStatistics().getStartTime(), + timeSeriesMetadata.getStatistics().getEndTime())) { + return null; + } } + return timeSeriesMetadata; + } finally { + QueryStatistics.getInstance() + .addCost(QueryStatistics.LOAD_TIME_SERIES_METADATA, System.nanoTime() - startTime); } - return timeSeriesMetadata; } /** @@ -181,87 +187,94 @@ public class FileLoaderUtils { public static AlignedTimeSeriesMetadata loadTimeSeriesMetadata( TsFileResource resource, AlignedPath vectorPath, QueryContext context, Filter filter) throws IOException { - AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null; - // If the tsfile is closed, we need to load from tsfile - if (resource.isClosed()) { - // load all the TimeseriesMetadata of vector, the first one is for time column and the - // remaining is for sub sensors - // the order of timeSeriesMetadata list is same as subSensorList's order - TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); - List<String> valueMeasurementList = vectorPath.getMeasurementList(); - Set<String> allSensors = new HashSet<>(valueMeasurementList); - allSensors.add(""); - boolean isDebug = context.isDebug(); - String filePath = resource.getTsFilePath(); - String deviceId = vectorPath.getDevice(); + long startTime = System.nanoTime(); + try { + AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null; + // If the tsfile is closed, we need to load from tsfile + if (resource.isClosed()) { + // load all the TimeseriesMetadata of vector, the first one is for time column and the + // remaining is for sub sensors + // the order of timeSeriesMetadata list is same as subSensorList's order + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + List<String> valueMeasurementList = vectorPath.getMeasurementList(); + Set<String> allSensors = new HashSet<>(valueMeasurementList); + allSensors.add(""); + boolean isDebug = context.isDebug(); + String filePath = resource.getTsFilePath(); + String deviceId = vectorPath.getDevice(); - // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex - // we should not ignore the non-exist of device in TsFileMetadata - TimeseriesMetadata timeColumn = - cache.get( - new TimeSeriesMetadataCacheKey(filePath, deviceId, ""), - allSensors, - resource.getTimeIndexType() != 1, - isDebug); - if (timeColumn != null) { - List<TimeseriesMetadata> valueTimeSeriesMetadataList = - new ArrayList<>(valueMeasurementList.size()); - // if all the queried aligned sensors does not exist, we will return null - boolean exist = false; - for (String valueMeasurement : valueMeasurementList) { - TimeseriesMetadata valueColumn = - cache.get( - new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement), - allSensors, - resource.getTimeIndexType() != 1, - isDebug); - exist = (exist || (valueColumn != null)); - valueTimeSeriesMetadataList.add(valueColumn); + // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex + // we should not ignore the non-exist of device in TsFileMetadata + TimeseriesMetadata timeColumn = + cache.get( + new TimeSeriesMetadataCacheKey(filePath, deviceId, ""), + allSensors, + resource.getTimeIndexType() != 1, + isDebug); + if (timeColumn != null) { + List<TimeseriesMetadata> valueTimeSeriesMetadataList = + new ArrayList<>(valueMeasurementList.size()); + // if all the queried aligned sensors does not exist, we will return null + boolean exist = false; + for (String valueMeasurement : valueMeasurementList) { + TimeseriesMetadata valueColumn = + cache.get( + new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement), + allSensors, + resource.getTimeIndexType() != 1, + isDebug); + exist = (exist || (valueColumn != null)); + valueTimeSeriesMetadataList.add(valueColumn); + } + if (exist) { + alignedTimeSeriesMetadata = + new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList); + alignedTimeSeriesMetadata.setChunkMetadataLoader( + new DiskAlignedChunkMetadataLoader(resource, vectorPath, context, filter)); + } } - if (exist) { - alignedTimeSeriesMetadata = - new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList); + } else { // if the tsfile is unclosed, we just get it directly from TsFileResource + alignedTimeSeriesMetadata = + (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(vectorPath); + if (alignedTimeSeriesMetadata != null) { alignedTimeSeriesMetadata.setChunkMetadataLoader( - new DiskAlignedChunkMetadataLoader(resource, vectorPath, context, filter)); + new MemAlignedChunkMetadataLoader(resource, vectorPath, context, filter)); } } - } else { // if the tsfile is unclosed, we just get it directly from TsFileResource - alignedTimeSeriesMetadata = - (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(vectorPath); - if (alignedTimeSeriesMetadata != null) { - alignedTimeSeriesMetadata.setChunkMetadataLoader( - new MemAlignedChunkMetadataLoader(resource, vectorPath, context, filter)); - } - } - if (alignedTimeSeriesMetadata != null) { - if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime() - > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) { - return null; - } - if (filter != null - && !filter.satisfyStartEndTime( - alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(), - alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) { - return null; - } + if (alignedTimeSeriesMetadata != null) { + if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime() + > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) { + return null; + } + if (filter != null + && !filter.satisfyStartEndTime( + alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(), + alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) { + return null; + } - // set modifications to each aligned path - List<TimeseriesMetadata> valueTimeSeriesMetadataList = - alignedTimeSeriesMetadata.getValueTimeseriesMetadataList(); - boolean modified = false; - for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { - if (valueTimeSeriesMetadataList.get(i) != null) { - List<Modification> pathModifications = - context.getPathModifications( - resource.getModFile(), vectorPath.getPathWithMeasurement(i)); - valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty()); - modified = (modified || !pathModifications.isEmpty()); + // set modifications to each aligned path + List<TimeseriesMetadata> valueTimeSeriesMetadataList = + alignedTimeSeriesMetadata.getValueTimeseriesMetadataList(); + boolean modified = false; + for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { + if (valueTimeSeriesMetadataList.get(i) != null) { + List<Modification> pathModifications = + context.getPathModifications( + resource.getModFile(), vectorPath.getPathWithMeasurement(i)); + valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty()); + modified = (modified || !pathModifications.isEmpty()); + } } + alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified); } - alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified); + return alignedTimeSeriesMetadata; + } finally { + QueryStatistics.getInstance() + .addCost( + QueryStatistics.LOAD_TIME_SERIES_METADATA_ALIGNED, System.nanoTime() - startTime); } - return alignedTimeSeriesMetadata; } /** @@ -271,7 +284,11 @@ public class FileLoaderUtils { */ public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) throws IOException { - return timeSeriesMetadata.loadChunkMetadataList(); + long startTime = System.nanoTime(); + List<IChunkMetadata> chunkMetadataList = timeSeriesMetadata.loadChunkMetadataList(); + QueryStatistics.getInstance() + .addCost(QueryStatistics.LOAD_CHUNK_METADATA_LIST, System.nanoTime() - startTime); + return chunkMetadataList; } /** @@ -282,11 +299,17 @@ public class FileLoaderUtils { */ public static List<IPageReader> loadPageReaderList( IChunkMetadata chunkMetaData, Filter timeFilter) throws IOException { - if (chunkMetaData == null) { - throw new IOException("Can't init null chunkMeta"); + long startTime = System.nanoTime(); + try { + if (chunkMetaData == null) { + throw new IOException("Can't init null chunkMeta"); + } + IChunkLoader chunkLoader = chunkMetaData.getChunkLoader(); + IChunkReader chunkReader = chunkLoader.getChunkReader(chunkMetaData, timeFilter); + return chunkReader.loadPageReaderList(); + } finally { + QueryStatistics.getInstance() + .addCost(QueryStatistics.LOAD_PAGE_READER_LIST, System.nanoTime() - startTime); } - IChunkLoader chunkLoader = chunkMetaData.getChunkLoader(); - IChunkReader chunkReader = chunkLoader.getChunkReader(chunkMetaData, timeFilter); - return chunkReader.loadPageReaderList(); } }
