This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/time_recording in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7348120296795b43585721e70fc0f93ec6fb5ff1 Merge: 4bf62c08acf 0cc841b7639 Author: Beyyes <[email protected]> AuthorDate: Fri Dec 1 09:39:40 2023 +0800 merge master .github/workflows/grafana-plugin.yml | 2 +- example/schema/pom.xml | 2 +- .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 25 ++- .../db/it/auth/IoTDBTemplatePermissionIT.java | 5 + .../org/apache/iotdb/db/it/utils/TestUtils.java | 12 +- .../iotdb/pipe/PipeEnvironmentException.java | 12 +- .../apache/iotdb/pipe/it/AbstractPipeDualIT.java | 60 ++++++ .../apache/iotdb/pipe/it/IoTDBPipeClusterIT.java | 65 +++---- .../pipe/it/IoTDBPipeConnectorParallelIT.java | 28 +-- .../apache/iotdb/pipe/it/IoTDBPipeDataSinkIT.java | 29 +-- .../apache/iotdb/pipe/it/IoTDBPipeExtractorIT.java | 71 +++---- .../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 29 +-- ...orParallelIT.java => IoTDBPipeProcessorIT.java} | 68 +++---- .../apache/iotdb/pipe/it/IoTDBPipeProtocolIT.java | 27 ++- .../iotdb/pipe/it/IoTDBPipeSingleEnvDemoIT.java | 16 +- .../iotdb/pipe/it/IoTDBPipeSwitchStatusIT.java | 29 +-- .../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 29 +-- .../java/org/apache/iotdb/pipe/api/access/Row.java | 10 +- iotdb-connector/grafana-plugin/backend-compile.bat | 2 +- iotdb-connector/grafana-plugin/backend-compile.sh | 31 ++- iotdb-connector/grafana-plugin/go.mod | 35 ++-- .../iotdb/confignode/manager/ConfigManager.java | 2 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 19 +- .../impl/pipe/task/DropPipeProcedureV2.java | 8 +- .../impl/pipe/task/StartPipeProcedureV2.java | 20 +- .../impl/pipe/task/StopPipeProcedureV2.java | 19 +- .../consensus/iot/IoTConsensusServerImpl.java | 23 +-- .../consensus/iot/IoTConsensusServerMetrics.java | 2 +- .../consensus/iot/client/DispatchLogHandler.java | 15 +- .../iot/logdispatcher/IndexController.java | 2 - .../consensus/iot/logdispatcher/LogDispatcher.java | 14 +- .../service/IoTConsensusRPCServiceProcessor.java | 2 +- .../apache/iotdb/consensus/ratis/utils/Utils.java | 19 +- .../apache/iotdb/consensus/iot/ReplicateTest.java | 30 ++- .../iotdb/consensus/ratis/RecoverReadTest.java | 8 +- .../agent/plugin/PipeProcessorConstructor.java | 3 + .../config/constant/PipeProcessorConstant.java | 12 ++ .../env/PipeTaskConnectorRuntimeEnvironment.java | 9 +- .../env/PipeTaskExtractorRuntimeEnvironment.java | 9 +- ...va => PipeTaskProcessorRuntimeEnvironment.java} | 13 +- .../plugin/env/PipeTaskRuntimeEnvironment.java | 8 +- .../iotdb/db/pipe/event/common/row/PipeRow.java | 5 + .../PipeHistoricalDataRegionTsFileExtractor.java | 59 +++--- .../downsampling/DownSamplingProcessor.java | 213 +++++++++++++++++++++ .../downsampling/PartialPathLastTimeCache.java | 111 +++++++++++ .../db/pipe/resource/memory/PipeMemoryBlock.java | 14 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 6 +- .../common/schematree/ClusterSchemaTree.java | 8 +- .../queryengine/common/schematree/ISchemaTree.java | 11 +- .../db/queryengine/execution/driver/Driver.java | 4 - .../execution/exchange/sink/LocalSinkChannel.java | 39 ++-- .../execution/exchange/sink/ShuffleSinkHandle.java | 27 ++- .../execution/exchange/sink/SinkChannel.java | 73 ++++--- .../exchange/source/LocalSourceHandle.java | 15 +- .../execution/exchange/source/SourceHandle.java | 20 +- .../fragment/FragmentInstanceContext.java | 12 +- .../fragment/FragmentInstanceExecution.java | 3 +- .../execution/fragment/QueryContext.java | 4 + .../queryengine/execution/memory/MemoryPool.java | 6 +- .../queryengine/execution/operator/Operator.java | 12 +- .../execution/operator/process/TopKOperator.java | 10 + .../operator/schema/SchemaFetchScanOperator.java | 13 +- .../source/AbstractDataSourceOperator.java | 5 + .../operator/source/LastCacheScanOperator.java | 5 + .../operator/source/ShowQueriesOperator.java | 5 + .../schedule/DriverTaskTimeoutSentinelThread.java | 2 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 26 ++- .../analyze/schema/AutoCreateSchemaExecutor.java | 61 ++++-- .../analyze/schema/ClusterSchemaFetchExecutor.java | 17 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 13 +- .../plan/analyze/schema/ISchemaFetcher.java | 6 +- .../config/executor/ClusterConfigTaskExecutor.java | 3 +- .../plan/planner/LogicalPlanBuilder.java | 6 +- .../plan/planner/LogicalPlanVisitor.java | 3 +- .../plan/planner/OperatorTreeGenerator.java | 3 +- .../planner/distribution/ExchangeNodeAdder.java | 41 ++-- .../node/metedata/read/SchemaFetchScanNode.java | 17 +- .../statement/internal/SchemaFetchStatement.java | 11 +- .../template/AlterSchemaTemplateStatement.java | 19 -- .../binary/CompareBinaryColumnTransformer.java | 33 ++-- .../column/ternary/BetweenColumnTransformer.java | 21 +- .../binary/CompareEqualToTransformer.java | 5 +- .../binary/CompareGreaterEqualTransformer.java | 5 +- .../binary/CompareGreaterThanTransformer.java | 5 +- .../binary/CompareLessEqualTransformer.java | 5 +- .../binary/CompareLessThanTransformer.java | 5 +- .../binary/CompareNonEqualTransformer.java | 5 +- .../transformer/ternary/BetweenTransformer.java | 15 +- .../transformation/dag/util/TransformUtils.java | 41 ++-- .../schemaengine/schemaregion/ISchemaRegion.java | 5 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 9 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 9 +- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 14 +- .../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 14 +- .../reader/impl/TimeseriesReaderWithViewFetch.java | 3 +- .../iotdb/db/storageengine/buffer/ChunkCache.java | 10 +- .../db/storageengine/dataregion/DataRegion.java | 2 - .../io/LocalTextModificationAccessor.java | 3 - .../read/reader/chunk/DiskAlignedChunkLoader.java | 6 +- .../read/reader/chunk/DiskChunkLoader.java | 6 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 8 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 10 +- .../schemaRegion/SchemaRegionBasicTest.java | 11 +- .../schemaRegion/SchemaRegionTemplateTest.java | 9 +- .../execution/operator/FillOperatorTest.java | 15 ++ .../operator/LastQueryMergeOperatorTest.java | 20 ++ .../execution/operator/LinearFillOperatorTest.java | 35 ++++ .../execution/operator/OperatorMemoryTest.java | 8 +- .../schema/SchemaFetchScanOperatorTest.java | 4 +- .../plan/analyze/ExpressionAnalyzerTest.java | 2 +- .../plan/analyze/FakeSchemaFetcherImpl.java | 8 +- .../queryengine/plan/plan/distribution/Util.java | 6 +- .../queryengine/plan/plan/distribution/Util2.java | 6 +- .../metadata/read/SchemaFetchMergeNodeTest.java | 1 + .../metadata/read/SchemaFetchScanNodeTest.java | 2 + .../db/storageengine/buffer/ChunkCacheTest.java | 6 +- .../inner/InnerSpaceCompactionSelectorTest.java | 41 ++-- .../apache/iotdb/commons/path/PathPatternTree.java | 8 + .../pipe/plugin/builtin/BuiltinPipePlugin.java | 47 +++-- .../builtin/processor/DownSamplingProcessor.java} | 22 +-- .../builtin/processor/PlaceHolderProcessor.java | 68 +++++++ .../org/apache/iotdb/tsfile/utils/BytesUtils.java | 10 + pom.xml | 4 +- 123 files changed, 1480 insertions(+), 824 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 9af34817427,195986aae3f..0dc79dd5c48 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@@ -447,16 -445,6 +448,23 @@@ public class FragmentInstanceContext ex unClosedFilePaths = null; } + long endTime = System.currentTimeMillis(); + releaseTime.set(endTime); + durationTime.set(endTime - executionStartTime.get()); + LOGGER.warn( - " ===== FragmentInstanceContext released, id: {}, releaseTime: {}, startTime: {}, durationTime:{}ms", ++ " ===== FragmentInstanceContext released, id: {}, " ++ + "releaseTime: {}, " ++ + "startTime: {}, " ++ + "durationTime: {}ms, " ++ + "loadTimeseriesMetadataDiskCount: {}, " ++ + "loadTimeseriesMetadataDiskTime: {}", + id, + releaseTime.get(), + executionStartTime.get(), - durationTime.get()); ++ durationTime.get(), ++ loadTimeSeriesMetadataDiskCount, ++ loadTimeSeriesMetadataDiskCount); + dataRegion = null; globalTimeFilter = null; sourcePaths = null; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java index b22f2b5f3cc,b22f2b5f3cc..2e2babe80ea --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java @@@ -40,6 -40,6 +40,10 @@@ import java.util.Set /** QueryContext contains the shared information with in a query. */ public class QueryContext { ++ public int loadTimeSeriesMetadataDiskCount = 0; ++ public long loadTimeSeriesMetadataDiskTime = 0; ++ public int loadTimeSeriesMetadataMemCount = 0; ++ public long loadTimeSeriesMetadataMemTime = 0; /** * The key is the path of a ModificationFile and the value is all Modifications in this file. We diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index 0af2cd7a8c7,7a08d5b3e39..05fa39658a5 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@@ -190,11 -185,11 +190,19 @@@ public class FileLoaderUtils } return timeSeriesMetadata; } finally { ++ long time = System.nanoTime() - t1; ++ if (loadFromMem) { ++ context.loadTimeSeriesMetadataMemCount += 1; ++ context.loadTimeSeriesMetadataMemTime += time; ++ } else { ++ context.loadTimeSeriesMetadataDiskCount += 1; ++ context.loadTimeSeriesMetadataDiskTime += time; ++ } SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( loadFromMem ? LOAD_TIMESERIES_METADATA_NONALIGNED_MEM : LOAD_TIMESERIES_METADATA_NONALIGNED_DISK, -- System.nanoTime() - t1); ++ time); } }
