This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/fix_mods_modification in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fc5cbf6b336e92a70840a7bd27848f2bc06067ef Author: Beyyes <[email protected]> AuthorDate: Wed Jul 10 22:52:11 2024 +0800 fix modification mods in loadTimeseriesMetadata --- .../fragment/FragmentInstanceContext.java | 66 +++--- .../execution/fragment/QueryStatistics.java | 224 +++++++++++++++++---- .../execution/operator/source/FileLoaderUtils.java | 96 +++++---- .../execution/operator/source/SeriesScanUtil.java | 22 +- .../read/reader/chunk/DiskAlignedChunkLoader.java | 4 +- .../read/reader/chunk/DiskChunkLoader.java | 4 +- .../read/reader/chunk/MemAlignedChunkLoader.java | 4 +- .../read/reader/chunk/MemChunkLoader.java | 4 +- 8 files changed, 296 insertions(+), 128 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 7c83518fcf0..2cd67f1321b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -631,57 +631,57 @@ public class FragmentInstanceContext extends QueryContext { SeriesScanCostMetricSet.getInstance() .recordNonAlignedTimeSeriesMetadataCount( - getQueryStatistics().loadTimeSeriesMetadataDiskSeqCount.get(), - getQueryStatistics().loadTimeSeriesMetadataDiskUnSeqCount.get(), - getQueryStatistics().loadTimeSeriesMetadataMemSeqCount.get(), - getQueryStatistics().loadTimeSeriesMetadataMemUnSeqCount.get()); + getQueryStatistics().getLoadTimeSeriesMetadataDiskSeqCount().get(), + getQueryStatistics().getLoadTimeSeriesMetadataDiskUnSeqCount().get(), + getQueryStatistics().getLoadTimeSeriesMetadataMemSeqCount().get(), + getQueryStatistics().getLoadTimeSeriesMetadataMemUnSeqCount().get()); SeriesScanCostMetricSet.getInstance() .recordNonAlignedTimeSeriesMetadataTime( - getQueryStatistics().loadTimeSeriesMetadataDiskSeqTime.get(), - getQueryStatistics().loadTimeSeriesMetadataDiskUnSeqTime.get(), - getQueryStatistics().loadTimeSeriesMetadataMemSeqTime.get(), - getQueryStatistics().loadTimeSeriesMetadataMemUnSeqTime.get()); + getQueryStatistics().getLoadTimeSeriesMetadataDiskSeqTime().get(), + getQueryStatistics().getLoadTimeSeriesMetadataDiskUnSeqTime().get(), + getQueryStatistics().getLoadTimeSeriesMetadataMemSeqTime().get(), + getQueryStatistics().getLoadTimeSeriesMetadataMemUnSeqTime().get()); SeriesScanCostMetricSet.getInstance() .recordAlignedTimeSeriesMetadataCount( - getQueryStatistics().loadTimeSeriesMetadataAlignedDiskSeqCount.get(), - getQueryStatistics().loadTimeSeriesMetadataAlignedDiskUnSeqCount.get(), - getQueryStatistics().loadTimeSeriesMetadataAlignedMemSeqCount.get(), - getQueryStatistics().loadTimeSeriesMetadataAlignedMemUnSeqCount.get()); + getQueryStatistics().getLoadTimeSeriesMetadataAlignedDiskSeqCount().get(), + getQueryStatistics().getLoadTimeSeriesMetadataAlignedDiskUnSeqCount().get(), + getQueryStatistics().getLoadTimeSeriesMetadataAlignedMemSeqCount().get(), + getQueryStatistics().getLoadTimeSeriesMetadataAlignedMemUnSeqCount().get()); SeriesScanCostMetricSet.getInstance() .recordAlignedTimeSeriesMetadataTime( - getQueryStatistics().loadTimeSeriesMetadataAlignedDiskSeqTime.get(), - getQueryStatistics().loadTimeSeriesMetadataAlignedDiskUnSeqTime.get(), - getQueryStatistics().loadTimeSeriesMetadataAlignedMemSeqTime.get(), - getQueryStatistics().loadTimeSeriesMetadataAlignedMemUnSeqTime.get()); + getQueryStatistics().getLoadTimeSeriesMetadataAlignedDiskSeqTime().get(), + getQueryStatistics().getLoadTimeSeriesMetadataAlignedDiskUnSeqTime().get(), + getQueryStatistics().getLoadTimeSeriesMetadataAlignedMemSeqTime().get(), + getQueryStatistics().getLoadTimeSeriesMetadataAlignedMemUnSeqTime().get()); SeriesScanCostMetricSet.getInstance() .recordConstructChunkReadersCount( - getQueryStatistics().constructAlignedChunkReadersMemCount.get(), - getQueryStatistics().constructAlignedChunkReadersDiskCount.get(), - getQueryStatistics().constructNonAlignedChunkReadersMemCount.get(), - getQueryStatistics().constructNonAlignedChunkReadersDiskCount.get()); + getQueryStatistics().getConstructAlignedChunkReadersMemCount().get(), + getQueryStatistics().getConstructAlignedChunkReadersDiskCount().get(), + getQueryStatistics().getConstructNonAlignedChunkReadersMemCount().get(), + getQueryStatistics().getConstructNonAlignedChunkReadersDiskCount().get()); SeriesScanCostMetricSet.getInstance() .recordConstructChunkReadersTime( - getQueryStatistics().constructAlignedChunkReadersMemTime.get(), - getQueryStatistics().constructAlignedChunkReadersDiskTime.get(), - getQueryStatistics().constructNonAlignedChunkReadersMemTime.get(), - getQueryStatistics().constructNonAlignedChunkReadersDiskTime.get()); + getQueryStatistics().getConstructAlignedChunkReadersMemTime().get(), + getQueryStatistics().getConstructAlignedChunkReadersDiskTime().get(), + getQueryStatistics().getConstructNonAlignedChunkReadersMemTime().get(), + getQueryStatistics().getConstructNonAlignedChunkReadersDiskTime().get()); SeriesScanCostMetricSet.getInstance() .recordPageReadersDecompressCount( - getQueryStatistics().pageReadersDecodeAlignedMemCount.get(), - getQueryStatistics().pageReadersDecodeAlignedDiskCount.get(), - getQueryStatistics().pageReadersDecodeNonAlignedMemCount.get(), - getQueryStatistics().pageReadersDecodeNonAlignedDiskCount.get()); + getQueryStatistics().getPageReadersDecodeAlignedMemCount().get(), + getQueryStatistics().getPageReadersDecodeAlignedDiskCount().get(), + getQueryStatistics().getPageReadersDecodeNonAlignedMemCount().get(), + getQueryStatistics().getPageReadersDecodeNonAlignedDiskCount().get()); SeriesScanCostMetricSet.getInstance() .recordPageReadersDecompressTime( - getQueryStatistics().pageReadersDecodeAlignedMemTime.get(), - getQueryStatistics().pageReadersDecodeAlignedDiskTime.get(), - getQueryStatistics().pageReadersDecodeNonAlignedMemTime.get(), - getQueryStatistics().pageReadersDecodeNonAlignedDiskTime.get()); + getQueryStatistics().getPageReadersDecodeAlignedMemTime().get(), + getQueryStatistics().getPageReadersDecodeAlignedDiskTime().get(), + getQueryStatistics().getPageReadersDecodeNonAlignedMemTime().get(), + getQueryStatistics().getPageReadersDecodeNonAlignedDiskTime().get()); SeriesScanCostMetricSet.getInstance() - .updatePageReaderMemoryUsage(getQueryStatistics().pageReaderMaxUsedMemorySize.get()); + .updatePageReaderMemoryUsage(getQueryStatistics().getPageReaderMaxUsedMemorySize().get()); } private synchronized void releaseDataNodeQueryContext() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java index 82094467034..acba03740ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java @@ -30,46 +30,200 @@ import java.util.concurrent.atomic.AtomicLong; public class QueryStatistics { // statistics for count and time of load timeseriesmetadata - public AtomicLong loadTimeSeriesMetadataDiskSeqCount = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataDiskUnSeqCount = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataMemSeqCount = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataMemUnSeqCount = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataAlignedDiskSeqCount = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataAlignedDiskUnSeqCount = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataAlignedMemSeqCount = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataAlignedMemUnSeqCount = new AtomicLong(0); - - public AtomicLong loadTimeSeriesMetadataDiskSeqTime = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataDiskUnSeqTime = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataMemSeqTime = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataMemUnSeqTime = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataAlignedDiskSeqTime = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataAlignedDiskUnSeqTime = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataAlignedMemSeqTime = new AtomicLong(0); - public AtomicLong loadTimeSeriesMetadataAlignedMemUnSeqTime = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataDiskSeqCount = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataDiskUnSeqCount = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataMemSeqCount = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataMemUnSeqCount = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataAlignedDiskSeqCount = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataAlignedDiskUnSeqCount = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataAlignedMemSeqCount = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataAlignedMemUnSeqCount = new AtomicLong(0); + + private final AtomicLong loadTimeSeriesMetadataDiskSeqTime = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataDiskUnSeqTime = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataMemSeqTime = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataMemUnSeqTime = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataAlignedDiskSeqTime = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataAlignedDiskUnSeqTime = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataAlignedMemSeqTime = new AtomicLong(0); + private final AtomicLong loadTimeSeriesMetadataAlignedMemUnSeqTime = new AtomicLong(0); // statistics for count and time of construct chunk readers(disk io and decompress) - public AtomicLong constructNonAlignedChunkReadersDiskCount = new AtomicLong(0); - public AtomicLong constructNonAlignedChunkReadersMemCount = new AtomicLong(0); - public AtomicLong constructAlignedChunkReadersDiskCount = new AtomicLong(0); - public AtomicLong constructAlignedChunkReadersMemCount = new AtomicLong(0); + private final AtomicLong constructNonAlignedChunkReadersDiskCount = new AtomicLong(0); + private final AtomicLong constructNonAlignedChunkReadersMemCount = new AtomicLong(0); + private final AtomicLong constructAlignedChunkReadersDiskCount = new AtomicLong(0); + private final AtomicLong constructAlignedChunkReadersMemCount = new AtomicLong(0); - public AtomicLong constructNonAlignedChunkReadersDiskTime = new AtomicLong(0); - public AtomicLong constructNonAlignedChunkReadersMemTime = new AtomicLong(0); - public AtomicLong constructAlignedChunkReadersDiskTime = new AtomicLong(0); - public AtomicLong constructAlignedChunkReadersMemTime = new AtomicLong(0); + private final AtomicLong constructNonAlignedChunkReadersDiskTime = new AtomicLong(0); + private final AtomicLong constructNonAlignedChunkReadersMemTime = new AtomicLong(0); + private final AtomicLong constructAlignedChunkReadersDiskTime = new AtomicLong(0); + private final AtomicLong constructAlignedChunkReadersMemTime = new AtomicLong(0); // statistics for count and time of page decode - public AtomicLong pageReadersDecodeAlignedDiskCount = new AtomicLong(0); - public AtomicLong pageReadersDecodeAlignedDiskTime = new AtomicLong(0); - public AtomicLong pageReadersDecodeAlignedMemCount = new AtomicLong(0); - public AtomicLong pageReadersDecodeAlignedMemTime = new AtomicLong(0); - public AtomicLong pageReadersDecodeNonAlignedDiskCount = new AtomicLong(0); - public AtomicLong pageReadersDecodeNonAlignedDiskTime = new AtomicLong(0); - public AtomicLong pageReadersDecodeNonAlignedMemCount = new AtomicLong(0); - public AtomicLong pageReadersDecodeNonAlignedMemTime = new AtomicLong(0); - - public AtomicLong pageReaderMaxUsedMemorySize = new AtomicLong(0); + private final AtomicLong pageReadersDecodeAlignedDiskCount = new AtomicLong(0); + private final AtomicLong pageReadersDecodeAlignedDiskTime = new AtomicLong(0); + private final AtomicLong pageReadersDecodeAlignedMemCount = new AtomicLong(0); + private final AtomicLong pageReadersDecodeAlignedMemTime = new AtomicLong(0); + private final AtomicLong pageReadersDecodeNonAlignedDiskCount = new AtomicLong(0); + private final AtomicLong pageReadersDecodeNonAlignedDiskTime = new AtomicLong(0); + private final AtomicLong pageReadersDecodeNonAlignedMemCount = new AtomicLong(0); + private final AtomicLong pageReadersDecodeNonAlignedMemTime = new AtomicLong(0); + + private final AtomicLong timeSeriesMetadataModificationCount = new AtomicLong(0); + private final AtomicLong timeSeriesMetadataModificationTime = new AtomicLong(0); + private final AtomicLong alignedTimeSeriesMetadataModificationCount = new AtomicLong(0); + private final AtomicLong alignedTimeSeriesMetadataModificationTime = new AtomicLong(0); + + // statistics for count and time of page decode + private final AtomicLong pageReaderMaxUsedMemorySize = new AtomicLong(0); + + public AtomicLong getLoadTimeSeriesMetadataDiskSeqCount() { + return loadTimeSeriesMetadataDiskSeqCount; + } + + public AtomicLong getLoadTimeSeriesMetadataDiskUnSeqCount() { + return loadTimeSeriesMetadataDiskUnSeqCount; + } + + public AtomicLong getLoadTimeSeriesMetadataMemSeqCount() { + return loadTimeSeriesMetadataMemSeqCount; + } + + public AtomicLong getLoadTimeSeriesMetadataMemUnSeqCount() { + return loadTimeSeriesMetadataMemUnSeqCount; + } + + public AtomicLong getLoadTimeSeriesMetadataAlignedDiskSeqCount() { + return loadTimeSeriesMetadataAlignedDiskSeqCount; + } + + public AtomicLong getLoadTimeSeriesMetadataAlignedDiskUnSeqCount() { + return loadTimeSeriesMetadataAlignedDiskUnSeqCount; + } + + public AtomicLong getLoadTimeSeriesMetadataAlignedMemSeqCount() { + return loadTimeSeriesMetadataAlignedMemSeqCount; + } + + public AtomicLong getLoadTimeSeriesMetadataAlignedMemUnSeqCount() { + return loadTimeSeriesMetadataAlignedMemUnSeqCount; + } + + public AtomicLong getLoadTimeSeriesMetadataDiskSeqTime() { + return loadTimeSeriesMetadataDiskSeqTime; + } + + public AtomicLong getLoadTimeSeriesMetadataDiskUnSeqTime() { + return loadTimeSeriesMetadataDiskUnSeqTime; + } + + public AtomicLong getLoadTimeSeriesMetadataMemSeqTime() { + return loadTimeSeriesMetadataMemSeqTime; + } + + public AtomicLong getLoadTimeSeriesMetadataMemUnSeqTime() { + return loadTimeSeriesMetadataMemUnSeqTime; + } + + public AtomicLong getLoadTimeSeriesMetadataAlignedDiskSeqTime() { + return loadTimeSeriesMetadataAlignedDiskSeqTime; + } + + public AtomicLong getLoadTimeSeriesMetadataAlignedDiskUnSeqTime() { + return loadTimeSeriesMetadataAlignedDiskUnSeqTime; + } + + public AtomicLong getLoadTimeSeriesMetadataAlignedMemSeqTime() { + return loadTimeSeriesMetadataAlignedMemSeqTime; + } + + public AtomicLong getLoadTimeSeriesMetadataAlignedMemUnSeqTime() { + return loadTimeSeriesMetadataAlignedMemUnSeqTime; + } + + public AtomicLong getConstructNonAlignedChunkReadersDiskCount() { + return constructNonAlignedChunkReadersDiskCount; + } + + public AtomicLong getConstructNonAlignedChunkReadersMemCount() { + return constructNonAlignedChunkReadersMemCount; + } + + public AtomicLong getConstructAlignedChunkReadersDiskCount() { + return constructAlignedChunkReadersDiskCount; + } + + public AtomicLong getConstructAlignedChunkReadersMemCount() { + return constructAlignedChunkReadersMemCount; + } + + public AtomicLong getConstructNonAlignedChunkReadersDiskTime() { + return constructNonAlignedChunkReadersDiskTime; + } + + public AtomicLong getConstructNonAlignedChunkReadersMemTime() { + return constructNonAlignedChunkReadersMemTime; + } + + public AtomicLong getConstructAlignedChunkReadersDiskTime() { + return constructAlignedChunkReadersDiskTime; + } + + public AtomicLong getConstructAlignedChunkReadersMemTime() { + return constructAlignedChunkReadersMemTime; + } + + public AtomicLong getPageReadersDecodeAlignedDiskCount() { + return pageReadersDecodeAlignedDiskCount; + } + + public AtomicLong getPageReadersDecodeAlignedDiskTime() { + return pageReadersDecodeAlignedDiskTime; + } + + public AtomicLong getPageReadersDecodeAlignedMemCount() { + return pageReadersDecodeAlignedMemCount; + } + + public AtomicLong getPageReadersDecodeAlignedMemTime() { + return pageReadersDecodeAlignedMemTime; + } + + public AtomicLong getPageReadersDecodeNonAlignedDiskCount() { + return pageReadersDecodeNonAlignedDiskCount; + } + + public AtomicLong getPageReadersDecodeNonAlignedDiskTime() { + return pageReadersDecodeNonAlignedDiskTime; + } + + public AtomicLong getPageReadersDecodeNonAlignedMemCount() { + return pageReadersDecodeNonAlignedMemCount; + } + + public AtomicLong getPageReadersDecodeNonAlignedMemTime() { + return pageReadersDecodeNonAlignedMemTime; + } + + public AtomicLong getTimeSeriesMetadataModificationCount() { + return timeSeriesMetadataModificationCount; + } + + public AtomicLong getTimeSeriesMetadataModificationTime() { + return timeSeriesMetadataModificationTime; + } + + public AtomicLong getAlignedTimeSeriesMetadataModificationCount() { + return alignedTimeSeriesMetadataModificationCount; + } + + public AtomicLong getAlignedTimeSeriesMetadataModificationTime() { + return alignedTimeSeriesMetadataModificationTime; + } + + public AtomicLong getPageReaderMaxUsedMemorySize() { + return pageReaderMaxUsedMemorySize; + } public TQueryStatistics toThrift() { return new TQueryStatistics( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index b45990a7f21..4d782f5c900 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -51,7 +51,6 @@ import java.util.List; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_ALIGNED; import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_NONALIGNED; public class FileLoaderUtils { @@ -102,10 +101,19 @@ public class FileLoaderUtils { resource.getTimeIndexType() != 1, context.isDebug()); if (timeSeriesMetadata != null) { + long t2 = System.nanoTime(); List<Modification> pathModifications = context.getPathModifications(resource, seriesPath); timeSeriesMetadata.setModified(!pathModifications.isEmpty()); timeSeriesMetadata.setChunkMetadataLoader( new DiskChunkMetadataLoader(resource, context, globalTimeFilter, pathModifications)); + long costTime = System.nanoTime() - t2; + SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( + TIMESERIES_METADATA_MODIFICATION_NONALIGNED, costTime); + context + .getQueryStatistics() + .getTimeSeriesMetadataModificationCount() + .getAndAdd(pathModifications.size()); + context.getQueryStatistics().getTimeSeriesMetadataModificationTime().getAndAdd(costTime); } } else { // if the tsfile is unclosed, we just get it directly from TsFileResource loadFromMem = true; @@ -118,18 +126,12 @@ public class FileLoaderUtils { } if (timeSeriesMetadata != null) { - long t2 = System.nanoTime(); - try { - if (timeSeriesMetadata.getStatistics().getStartTime() - > timeSeriesMetadata.getStatistics().getEndTime()) { - return null; - } - if (globalTimeFilter != null && globalTimeFilter.canSkip(timeSeriesMetadata)) { - return null; - } - } finally { - SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( - TIMESERIES_METADATA_MODIFICATION_NONALIGNED, System.nanoTime() - t2); + if (timeSeriesMetadata.getStatistics().getStartTime() + > timeSeriesMetadata.getStatistics().getEndTime()) { + return null; + } + if (globalTimeFilter != null && globalTimeFilter.canSkip(timeSeriesMetadata)) { + return null; } } return timeSeriesMetadata; @@ -137,19 +139,19 @@ public class FileLoaderUtils { long costTime = System.nanoTime() - t1; if (loadFromMem) { if (isSeq) { - context.getQueryStatistics().loadTimeSeriesMetadataMemSeqCount.getAndAdd(1); - context.getQueryStatistics().loadTimeSeriesMetadataMemSeqTime.getAndAdd(costTime); + context.getQueryStatistics().getLoadTimeSeriesMetadataMemSeqCount().getAndAdd(1); + context.getQueryStatistics().getLoadTimeSeriesMetadataMemSeqTime().getAndAdd(costTime); } else { - context.getQueryStatistics().loadTimeSeriesMetadataMemUnSeqCount.getAndAdd(1); - context.getQueryStatistics().loadTimeSeriesMetadataMemUnSeqTime.getAndAdd(costTime); + context.getQueryStatistics().getLoadTimeSeriesMetadataMemUnSeqCount().getAndAdd(1); + context.getQueryStatistics().getLoadTimeSeriesMetadataMemUnSeqTime().getAndAdd(costTime); } } else { if (isSeq) { - context.getQueryStatistics().loadTimeSeriesMetadataDiskSeqCount.getAndAdd(1); - context.getQueryStatistics().loadTimeSeriesMetadataDiskSeqTime.getAndAdd(costTime); + context.getQueryStatistics().getLoadTimeSeriesMetadataDiskSeqCount().getAndAdd(1); + context.getQueryStatistics().getLoadTimeSeriesMetadataDiskSeqTime().getAndAdd(costTime); } else { - context.getQueryStatistics().loadTimeSeriesMetadataDiskUnSeqCount.getAndAdd(1); - context.getQueryStatistics().loadTimeSeriesMetadataDiskUnSeqTime.getAndAdd(costTime); + context.getQueryStatistics().getLoadTimeSeriesMetadataDiskUnSeqCount().getAndAdd(1); + context.getQueryStatistics().getLoadTimeSeriesMetadataDiskUnSeqTime().getAndAdd(costTime); } } } @@ -190,18 +192,12 @@ public class FileLoaderUtils { } if (alignedTimeSeriesMetadata != null) { - final long t2 = System.nanoTime(); - try { - if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime() - > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) { - return null; - } - if (globalTimeFilter != null && globalTimeFilter.canSkip(alignedTimeSeriesMetadata)) { - return null; - } - } finally { - SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( - TIMESERIES_METADATA_MODIFICATION_ALIGNED, System.nanoTime() - t2); + if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime() + > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) { + return null; + } + if (globalTimeFilter != null && globalTimeFilter.canSkip(alignedTimeSeriesMetadata)) { + return null; } } return alignedTimeSeriesMetadata; @@ -209,24 +205,33 @@ public class FileLoaderUtils { long costTime = System.nanoTime() - t1; if (loadFromMem) { if (isSeq) { - context.getQueryStatistics().loadTimeSeriesMetadataAlignedMemSeqCount.getAndAdd(1); - context.getQueryStatistics().loadTimeSeriesMetadataAlignedMemSeqTime.getAndAdd(costTime); + context.getQueryStatistics().getLoadTimeSeriesMetadataAlignedMemSeqCount().getAndAdd(1); + context + .getQueryStatistics() + .getLoadTimeSeriesMetadataAlignedMemSeqTime() + .getAndAdd(costTime); } else { - context.getQueryStatistics().loadTimeSeriesMetadataAlignedMemUnSeqCount.getAndAdd(1); + context.getQueryStatistics().getLoadTimeSeriesMetadataAlignedMemUnSeqCount().getAndAdd(1); context .getQueryStatistics() - .loadTimeSeriesMetadataAlignedMemUnSeqTime + .getLoadTimeSeriesMetadataAlignedMemUnSeqTime() .getAndAdd(costTime); } } else { if (isSeq) { - context.getQueryStatistics().loadTimeSeriesMetadataAlignedDiskSeqCount.getAndAdd(1); - context.getQueryStatistics().loadTimeSeriesMetadataAlignedDiskSeqTime.getAndAdd(costTime); + context.getQueryStatistics().getLoadTimeSeriesMetadataAlignedDiskSeqCount().getAndAdd(1); + context + .getQueryStatistics() + .getLoadTimeSeriesMetadataAlignedDiskSeqTime() + .getAndAdd(costTime); } else { - context.getQueryStatistics().loadTimeSeriesMetadataAlignedDiskUnSeqCount.getAndAdd(1); context .getQueryStatistics() - .loadTimeSeriesMetadataAlignedDiskUnSeqTime + .getLoadTimeSeriesMetadataAlignedDiskUnSeqCount() + .getAndAdd(1); + context + .getQueryStatistics() + .getLoadTimeSeriesMetadataAlignedDiskUnSeqTime() .getAndAdd(costTime); } } @@ -306,6 +311,7 @@ public class FileLoaderUtils { AlignedTimeSeriesMetadata alignedTimeSeriesMetadata, AlignedPath alignedPath, QueryContext context) { + long startTime = System.nanoTime(); List<TimeseriesMetadata> valueTimeSeriesMetadataList = alignedTimeSeriesMetadata.getValueTimeseriesMetadataList(); List<List<Modification>> res = new ArrayList<>(); @@ -317,11 +323,19 @@ public class FileLoaderUtils { valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty()); res.add(pathModifications); modified = (modified || !pathModifications.isEmpty()); + context + .getQueryStatistics() + .getAlignedTimeSeriesMetadataModificationCount() + .getAndAdd(pathModifications.size()); } else { res.add(Collections.emptyList()); } } alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified); + long costTime = System.nanoTime() - startTime; + SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( + TIMESERIES_METADATA_MODIFICATION_NONALIGNED, costTime); + context.getQueryStatistics().getAlignedTimeSeriesMetadataModificationTime().getAndAdd(costTime); return res; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 66c833401a8..2c687dcc1a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -811,7 +811,7 @@ public class SeriesScanUtil implements Accountable { orderUtils.getOverlapCheckTime(firstPageReader.getStatistics())); context .getQueryStatistics() - .pageReaderMaxUsedMemorySize + .getPageReaderMaxUsedMemorySize() .updateAndGet(v -> Math.max(v, mergeReader.getUsedMemorySize())); currentPageEndPointTime = updateEndPointTime(currentPageEndPointTime, firstPageReader); @@ -839,7 +839,7 @@ public class SeriesScanUtil implements Accountable { orderUtils.getOverlapCheckTime(pageReader.getStatistics())); context .getQueryStatistics() - .pageReaderMaxUsedMemorySize + .getPageReaderMaxUsedMemorySize() .updateAndGet(v -> Math.max(v, mergeReader.getUsedMemorySize())); currentPageEndPointTime = updateEndPointTime(currentPageEndPointTime, pageReader); } @@ -1019,7 +1019,7 @@ public class SeriesScanUtil implements Accountable { orderUtils.getOverlapCheckTime(pageReader.getStatistics())); context .getQueryStatistics() - .pageReaderMaxUsedMemorySize + .getPageReaderMaxUsedMemorySize() .updateAndGet(v -> Math.max(v, mergeReader.getUsedMemorySize())); } @@ -1233,19 +1233,19 @@ public class SeriesScanUtil implements Accountable { long time = System.nanoTime() - startTime; if (isAligned) { if (isMem) { - context.getQueryStatistics().pageReadersDecodeAlignedMemCount.getAndAdd(1); - context.getQueryStatistics().pageReadersDecodeAlignedMemTime.getAndAdd(time); + context.getQueryStatistics().getPageReadersDecodeAlignedMemCount().getAndAdd(1); + context.getQueryStatistics().getPageReadersDecodeAlignedMemTime().getAndAdd(time); } else { - context.getQueryStatistics().pageReadersDecodeAlignedDiskCount.getAndAdd(1); - context.getQueryStatistics().pageReadersDecodeAlignedDiskTime.getAndAdd(time); + context.getQueryStatistics().getPageReadersDecodeAlignedDiskCount().getAndAdd(1); + context.getQueryStatistics().getPageReadersDecodeAlignedDiskTime().getAndAdd(time); } } else { if (isMem) { - context.getQueryStatistics().pageReadersDecodeNonAlignedMemCount.getAndAdd(1); - context.getQueryStatistics().pageReadersDecodeNonAlignedMemTime.getAndAdd(time); + context.getQueryStatistics().getPageReadersDecodeNonAlignedMemCount().getAndAdd(1); + context.getQueryStatistics().getPageReadersDecodeNonAlignedMemTime().getAndAdd(time); } else { - context.getQueryStatistics().pageReadersDecodeNonAlignedDiskCount.getAndAdd(1); - context.getQueryStatistics().pageReadersDecodeNonAlignedDiskTime.getAndAdd(time); + context.getQueryStatistics().getPageReadersDecodeNonAlignedDiskCount().getAndAdd(1); + context.getQueryStatistics().getPageReadersDecodeNonAlignedDiskTime().getAndAdd(time); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java index 7340204bea6..c69dc59607d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java @@ -109,8 +109,8 @@ public class DiskAlignedChunkLoader implements IChunkLoader { return chunkReader; } finally { long time = System.nanoTime() - t1; - context.getQueryStatistics().constructAlignedChunkReadersDiskCount.getAndAdd(1); - context.getQueryStatistics().constructAlignedChunkReadersDiskTime.getAndAdd(time); + context.getQueryStatistics().getConstructAlignedChunkReadersDiskCount().getAndAdd(1); + context.getQueryStatistics().getConstructAlignedChunkReadersDiskTime().getAndAdd(time); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java index cc702414bf4..38cc7215d21 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java @@ -95,8 +95,8 @@ public class DiskChunkLoader implements IChunkLoader { return chunkReader; } finally { long time = System.nanoTime() - t1; - context.getQueryStatistics().constructNonAlignedChunkReadersDiskCount.getAndAdd(1); - context.getQueryStatistics().constructNonAlignedChunkReadersDiskTime.getAndAdd(time); + context.getQueryStatistics().getConstructNonAlignedChunkReadersDiskCount().getAndAdd(1); + context.getQueryStatistics().getConstructNonAlignedChunkReadersDiskTime().getAndAdd(time); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoader.java index 6b25deaeb31..c9c92bc7398 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoader.java @@ -62,8 +62,8 @@ public class MemAlignedChunkLoader implements IChunkLoader { return new MemAlignedChunkReader(chunk, globalTimeFilter); } finally { long duration = System.nanoTime() - startTime; - context.getQueryStatistics().constructAlignedChunkReadersMemCount.getAndAdd(1); - context.getQueryStatistics().constructAlignedChunkReadersMemTime.getAndAdd(duration); + context.getQueryStatistics().getConstructAlignedChunkReadersMemCount().getAndAdd(1); + context.getQueryStatistics().getConstructAlignedChunkReadersMemTime().getAndAdd(duration); SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(INIT_CHUNK_READER_ALIGNED_MEM, duration); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkLoader.java index 56a4ec4e81e..7683b38a23f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkLoader.java @@ -62,8 +62,8 @@ public class MemChunkLoader implements IChunkLoader { return new MemChunkReader(chunk, globalTimeFilter); } finally { long duration = System.nanoTime() - startTime; - context.getQueryStatistics().constructNonAlignedChunkReadersMemCount.getAndAdd(1); - context.getQueryStatistics().constructNonAlignedChunkReadersMemTime.getAndAdd(duration); + context.getQueryStatistics().getConstructNonAlignedChunkReadersMemCount().getAndAdd(1); + context.getQueryStatistics().getConstructNonAlignedChunkReadersMemTime().getAndAdd(duration); SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(INIT_CHUNK_READER_NONALIGNED_MEM, duration); } }
