This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch QueryDebug in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f836c4bb7c60adacad79b22a82cea191e6e22698 Author: JackieTien97 <[email protected]> AuthorDate: Mon Mar 15 11:10:39 2021 +0800 add explain grammar --- grafana/pom.xml | 2 +- jdbc/pom.xml | 2 +- pom.xml | 6 +++--- .../org/apache/iotdb/db/engine/cache/ChunkCache.java | 7 ++++++- .../iotdb/db/engine/cache/ChunkMetadataCache.java | 9 +++++++-- .../db/engine/cache/TimeSeriesMetadataCache.java | 15 ++++++++++----- .../iotdb/db/engine/modification/Modification.java | 9 +++++++++ .../engine/storagegroup/StorageGroupProcessor.java | 14 +++++++------- .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 10 ++++++++++ .../apache/iotdb/db/query/context/QueryContext.java | 11 +++++++++++ .../groupby/GroupByWithValueFilterDataSet.java | 2 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 20 ++++++++++++++++++-- .../org/apache/iotdb/db/service/TSServiceImpl.java | 8 ++++---- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +- 14 files changed, 89 insertions(+), 28 deletions(-) diff --git a/grafana/pom.xml b/grafana/pom.xml index 357350a..b605e65 100644 --- a/grafana/pom.xml +++ b/grafana/pom.xml @@ -165,7 +165,7 @@ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> <resource>META-INF/spring.schemas</resource> </transformer> - <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> + <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass>${start-class}</mainClass> </transformer> diff --git a/jdbc/pom.xml b/jdbc/pom.xml index b44bb46..090fdb0 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -223,7 +223,7 @@ </goals> </pluginExecutionFilter> <action> - <ignore /> + <ignore/> </action> </pluginExecution> </pluginExecutions> diff --git a/pom.xml b/pom.xml index 3a15f55..ec39eb2 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ <sonar.exclusions>**/generated-sources</sonar.exclusions> <!-- By default, the argLine is empty--> <gson.version>2.8.6</gson.version> - <argLine /> + <argLine/> </properties> <!-- if we claim dependencies in dependencyManagement, then we do not claim @@ -599,7 +599,7 @@ <id>enforce-version-convergence</id> <configuration> <rules> - <dependencyConvergence /> + <dependencyConvergence/> </rules> </configuration> <goals> @@ -645,7 +645,7 @@ </requireJavaVersion> <!-- Disabled for now as it breaks the ability to build single modules --> <!--reactorModuleConvergence/--> - <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies" /> + <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies"/> </rules> </configuration> </execution> diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java index 6fccb8b..545b666 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java @@ -84,6 +84,11 @@ public class ChunkCache { } public Chunk get(ChunkMetadata chunkMetaData, TsFileSequenceReader reader) throws IOException { + return get(chunkMetaData, reader, false); + } + + public Chunk get(ChunkMetadata chunkMetaData, TsFileSequenceReader reader, boolean debug) + throws IOException { if (!CACHE_ENABLE) { Chunk chunk = reader.readMemChunk(chunkMetaData); return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), @@ -118,7 +123,7 @@ public class ChunkCache { } } - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info("get chunk from cache whose meta data is: " + chunkMetaData); } return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeleteIntervalList()); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java index f8fd5f3..356da90 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java @@ -102,11 +102,16 @@ public class ChunkMetadataCache { return ChunkMetadataCacheSingleton.INSTANCE; } + public List<ChunkMetadata> get(String filePath, Path seriesPath, + TimeseriesMetadata timeseriesMetadata) throws IOException { + return get(filePath, seriesPath, timeseriesMetadata, false); + } + /** * get {@link ChunkMetadata}. THREAD SAFE. */ public List<ChunkMetadata> get(String filePath, Path seriesPath, - TimeseriesMetadata timeseriesMetadata) throws IOException { + TimeseriesMetadata timeseriesMetadata, boolean debug) throws IOException { if (timeseriesMetadata == null) { return Collections.emptyList(); } @@ -144,7 +149,7 @@ public class ChunkMetadataCache { lock.writeLock().unlock(); } } - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info( "Chunk meta data list size: " + chunkMetadataList.size() + " key is: " + key.getString()); chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString())); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java index d24160e..9aef0db 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java @@ -106,9 +106,14 @@ public class TimeSeriesMetadataCache { return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE; } - @SuppressWarnings("squid:S1860") // Suppress synchronize warning public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String> allSensors) throws IOException { + return get(key, allSensors, false); + } + + @SuppressWarnings("squid:S1860") // Suppress synchronize warning + public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String> allSensors, boolean debug) + throws IOException { if (!CACHE_ENABLE) { // bloom filter part TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); @@ -134,7 +139,7 @@ public class TimeSeriesMetadataCache { cacheHitNum.incrementAndGet(); printCacheLog(true); } else { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info( "Cache miss: " + key.device + "." + key.measurement + " metadata in file: " + key.filePath); @@ -159,7 +164,7 @@ public class TimeSeriesMetadataCache { TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); BloomFilter bloomFilter = reader.readBloomFilter(); if (bloomFilter != null && !bloomFilter.contains(path.getFullPath())) { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key); } return null; @@ -185,12 +190,12 @@ public class TimeSeriesMetadataCache { } } if (timeseriesMetadata == null) { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info("The file doesn't have this time series {}", key); } return null; } else { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info( "Get timeseries: {}. {} metadata in file: {} from cache: {}", key.device, key.measurement, key.filePath, timeseriesMetadata); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java b/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java index 3e3f46e..2c7a757 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java @@ -94,4 +94,13 @@ public abstract class Modification { public int hashCode() { return Objects.hash(type, path, versionNum); } + + @Override + public String toString() { + return "Modification{" + + "type=" + type + + ", path=" + path + + ", versionNum=" + versionNum + + '}'; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java index 751789f..4161201 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java @@ -1415,7 +1415,7 @@ public class StorageGroupProcessor { boolean isSeq) throws MetadataException { - if (config.isDebugOn()) { + if (context.isDebug()) { DEBUG_LOGGER .info("Path: {}.{}, get tsfile list: {} isSeq: {} timefilter: {}", deviceId.getFullPath(), measurementId, tsFileResources, isSeq, (timeFilter == null ? "null" : timeFilter)); @@ -1429,7 +1429,7 @@ public class StorageGroupProcessor { context.setQueryTimeLowerBound(timeLowerBound); for (TsFileResource tsFileResource : tsFileResources) { - if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq)) { + if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq, context.isDebug())) { continue; } closeQueryLock.readLock().lock(); @@ -1450,7 +1450,7 @@ public class StorageGroupProcessor { } // for upgrade files and old files must be closed for (TsFileResource tsFileResource : upgradeTsFileResources) { - if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq)) { + if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq, context.isDebug())) { continue; } closeQueryLock.readLock().lock(); @@ -1467,9 +1467,9 @@ public class StorageGroupProcessor { * @return true if the device is contained in the TsFile and it lives beyond TTL */ private boolean isTsFileResourceSatisfied(TsFileResource tsFileResource, String deviceId, - Filter timeFilter, boolean isSeq) { + Filter timeFilter, boolean isSeq, boolean debug) { if (!tsFileResource.containsDevice(deviceId)) { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info("Path: {} file {} is not satisfied because of no device!", deviceId, tsFileResource); } @@ -1482,7 +1482,7 @@ public class StorageGroupProcessor { : Long.MAX_VALUE; if (!isAlive(endTime)) { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER .info("Path: {} file {} is not satisfied because of ttl!", deviceId, tsFileResource); } @@ -1491,7 +1491,7 @@ public class StorageGroupProcessor { if (timeFilter != null) { boolean res = timeFilter.satisfyStartEndTime(startTime, endTime); - if (config.isDebugOn() && !res) { + if (debug && !res) { DEBUG_LOGGER.info("Path: {} file {} is not satisfied because of time filter!", deviceId, tsFileResource); } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java index c2842f1..1beb2c9 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java @@ -63,6 +63,8 @@ public abstract class PhysicalPlan { // a bridge from a cluster raft log to a physical plan protected long index; + private boolean debug; + /** * whether the plan can be split into more than one Plans. Only used in the cluster mode. */ @@ -171,6 +173,14 @@ public abstract class PhysicalPlan { this.loginUserName = loginUserName; } + public boolean isDebug() { + return debug; + } + + public void setDebug(boolean debug) { + this.debug = debug; + } + public static class Factory { private Factory() { diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java index 863d4c3..a8c11fd 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java +++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java @@ -50,6 +50,8 @@ public class QueryContext { private long queryTimeLowerBound = Long.MIN_VALUE; + private boolean debug; + public QueryContext() { } @@ -57,6 +59,15 @@ public class QueryContext { this.queryId = queryId; } + public QueryContext(long queryId, boolean debug) { + this.queryId = queryId; + this.debug = debug; + } + + public boolean isDebug() { + return debug; + } + /** * Find the modifications of timeseries 'path' in 'modFile'. If they are not in the cache, read * them from 'modFile' and put then into the cache. diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java index bd37495..64bad5f 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java @@ -74,7 +74,7 @@ public class GroupByWithValueFilterDataSet extends GroupByEngineDataSet { } public GroupByWithValueFilterDataSet(long queryId, GroupByTimePlan groupByTimePlan) { - super(new QueryContext(queryId), groupByTimePlan); + super(new QueryContext(queryId, groupByTimePlan.isDebug()), groupByTimePlan); this.allDataReaderList = new ArrayList<>(); this.timeStampFetchSize = IoTDBDescriptor.getInstance().getConfig().getBatchSize(); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java index 95466e9..8318365 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java @@ -31,9 +31,13 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DiskChunkMetadataLoader implements IChunkMetadataLoader { + private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG"); + private TsFileResource resource; private PartialPath seriesPath; private QueryContext context; @@ -51,8 +55,8 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { @Override public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) throws IOException { - List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache - .getInstance().get(resource.getTsFilePath(), seriesPath, timeseriesMetadata); + List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache.getInstance() + .get(resource.getTsFilePath(), seriesPath, timeseriesMetadata, context.isDebug()); setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context); @@ -82,10 +86,22 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { List<Modification> pathModifications = context.getPathModifications(resource.getModFile(), seriesPath); + if (context.isDebug()) { + DEBUG_LOGGER.info("Modifications file Path: {} ", resource.getTsFilePath()); + pathModifications.forEach(c -> DEBUG_LOGGER.info(c.toString())); + } + if (!pathModifications.isEmpty()) { QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications); } + if (context.isDebug()) { + if (context.isDebug()) { + DEBUG_LOGGER.info("After modification Chunk meta data list is: "); + chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString())); + } + } + for (ChunkMetadata data : chunkMetadataList) { data.setChunkLoader(new DiskChunkLoader(resource)); } diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index 500ae7a..da62967 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@ -767,7 +767,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { if (costTime >= config.getSlowQueryThreshold()) { SLOW_SQL_LOGGER.info("Cost: " + costTime + " ms, sql is " + statement); } - if (config.isDebugOn()) { + if (plan.isDebug()) { SLOW_SQL_LOGGER.info("ChunkCache used memory proportion: " + ChunkCache.getInstance() .getUsedMemoryProportion() + "\nChunkMetadataCache used memory proportion: " + ChunkMetadataCache.getInstance().getUsedMemoryProportion() @@ -1113,14 +1113,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { throws QueryProcessException, QueryFilterOptimizationException, StorageEngineException, IOException, MetadataException, SQLException, TException, InterruptedException { - QueryContext context = genQueryContext(queryId); + QueryContext context = genQueryContext(queryId, physicalPlan.isDebug()); QueryDataSet queryDataSet = executor.processQuery(physicalPlan, context); queryId2DataSet.put(queryId, queryDataSet); return queryDataSet; } - protected QueryContext genQueryContext(long queryId) { - return new QueryContext(queryId); + protected QueryContext genQueryContext(long queryId, boolean debug) { + return new QueryContext(queryId, debug); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index b3af734..dea1b60 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -94,7 +94,7 @@ public class FileLoaderUtils { } timeSeriesMetadata = TimeSeriesMetadataCache.getInstance() .get(new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(resource.getTsFilePath(), - seriesPath.getDevice(), seriesPath.getMeasurement()), allSensors); + seriesPath.getDevice(), seriesPath.getMeasurement()), allSensors, context.isDebug()); if (timeSeriesMetadata != null) { timeSeriesMetadata.setChunkMetadataLoader( new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
