This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch lmh/PredicatePushDown in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dd58965f4a96a85699c58935c6fa93b679c3096f Merge: ad1e882fb8b 50789004d35 Author: Minghui Liu <[email protected]> AuthorDate: Sun Jan 7 23:09:17 2024 +0800 Merge remote-tracking branch 'origin/master' into lmh/PredicatePushDown .github/workflows/cluster-it-1c1d.yml | 2 + .github/workflows/cluster-it-1c3d.yml | 2 + .github/workflows/pipe-it-2cluster.yml | 2 + .github/workflows/unit-test.yml | 2 + .gitignore | 2 + .../it/env/cluster/config/MppCommonConfig.java | 5 +- .../it/env/cluster/config/MppDataNodeConfig.java | 9 + .../it/env/remote/config/RemoteDataNodeConfig.java | 6 + .../apache/iotdb/itbase/env/DataNodeConfig.java | 3 + .../it/cluster/IoTDBClusterNodeGetterIT.java | 2 +- .../confignode/it/utils/ConfigNodeTestUtils.java | 2 +- .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 35 ++- .../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 4 +- .../IoTDBAlignByDeviceWithTemplateIT.java | 308 ++++++++++++++++++- .../IoTDBAlignedSeriesQueryWithDeletionIT.java | 28 ++ .../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 54 +++- .../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 4 +- .../api/customizer/parameter/PipeParameters.java | 33 ++- .../api/exception/PipeConnectionException.java | 3 + .../assembly/resources/tools/collection-info.bat | 275 +++++++++++++++++ .../assembly/resources/tools/collection-info.sh | 268 +++++++++++++++++ .../java/org/apache/iotdb/session/Session.java | 2 +- .../iotdb/session/SessionCacheLeaderTest.java | 1 + .../iotdb/session/SessionConnectionTest.java | 26 +- .../java/org/apache/iotdb/session/SessionTest.java | 9 +- .../session/pool/SessionPoolExceptionTest.java | 9 +- .../apache/iotdb/session/pool/SessionPoolTest.java | 13 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 3 +- .../src/assembly/resources/conf/confignode-env.sh | 2 +- .../assembly/resources/conf/logback-confignode.xml | 52 +--- .../assembly/resources/sbin/register-confignode.sh | 80 +++++ .../exception/NotEnoughDataNodeException.java | 13 +- .../iotdb/confignode/manager/ConfigManager.java | 2 + .../iotdb/confignode/manager/ProcedureManager.java | 26 +- .../manager/consensus/ConsensusManager.java | 37 ++- .../manager/load/balancer/RegionBalancer.java | 2 +- .../region/GreedyCopySetRegionGroupAllocator.java | 181 ++++++++---- .../confignode/manager/load/cache/LoadCache.java | 10 +- .../manager/node/ClusterNodeStartUtils.java | 5 - .../iotdb/confignode/manager/node/NodeManager.java | 2 +- .../manager/partition/PartitionManager.java | 10 +- .../manager/pipe/agent/PipeConfigNodeAgent.java | 2 +- .../agent/plugin/PipePluginConfigNodeAgent.java | 14 +- .../persistence/schema/ConfigMTreeStore.java | 5 + .../iotdb/confignode/procedure/Procedure.java | 46 ++- .../confignode/procedure/ProcedureExecutor.java | 25 +- .../confignode/procedure/ProcedureMetrics.java | 184 ++++++++++++ .../statemachine/CreateRegionGroupsProcedure.java | 62 ++-- .../impl/statemachine/StateMachineProcedure.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 25 +- .../region/AllocatorScatterWidthManualTest.java | 136 +++++++++ .../GreedyCopySetRegionGroupAllocatorTest.java | 15 +- .../procedure/TestProcedureExecutor.java | 2 +- .../procedure/entity/StuckProcedure.java | 4 +- .../src/test/resources/confignode1conf/logback.xml | 52 +--- .../src/test/resources/confignode2conf/logback.xml | 52 +--- .../src/test/resources/confignode3conf/logback.xml | 52 +--- .../consensus/iot/IoTConsensusServerImpl.java | 2 +- .../src/assembly/resources/conf/datanode-env.sh | 2 +- .../assembly/resources/conf/logback-datanode.xml | 122 +++----- .../assembly/resources/sbin/register-datanode.sh | 80 +++++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 74 +++-- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 16 +- .../dataregion/DataExecutionVisitor.java | 6 + .../LoadRuntimeOutOfMemoryException.java} | 9 +- .../pipe/agent/plugin/PipePluginDataNodeAgent.java | 8 +- .../dataregion/PipeDataRegionPluginAgent.java | 23 +- .../schemaregion/PipeSchemaRegionPluginAgent.java | 14 +- .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 16 +- .../db/pipe/agent/task/PipeTaskDataNodeAgent.java | 4 +- .../protocol/legacy/IoTDBLegacyPipeConnector.java | 27 +- ...eManager.java => IoTDBThriftClientManager.java} | 19 +- .../protocol/thrift/LeaderCacheManager.java | 80 ++++- .../async/IoTDBThriftAsyncClientManager.java | 200 +++++++++++++ .../thrift/async/IoTDBThriftAsyncConnector.java | 220 ++++---------- .../PipeTransferTabletInsertNodeEventHandler.java | 12 + .../PipeTransferTabletInsertionEventHandler.java | 17 +- .../handler/PipeTransferTabletRawEventHandler.java | 7 + .../thrift/sync/IoTDBThriftSyncClientManager.java | 68 +++-- .../thrift/sync/IoTDBThriftSyncConnector.java | 2 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 44 ++- .../pipe/resource/wal/PipeWALResourceManager.java | 4 +- .../impl/DataNodeInternalRPCServiceImpl.java | 25 +- .../db/queryengine/common/MPPQueryContext.java | 10 + .../iotdb/db/queryengine/common/NodeRef.java | 4 + .../execution/exchange/source/SourceHandle.java | 2 +- .../execution/executor/RegionWriteExecutor.java | 9 +- .../db/queryengine/execution/load/ChunkData.java | 3 + .../queryengine/execution/load/DeletionData.java | 17 +- .../execution/load/LoadTsFileManager.java | 65 +++- .../db/queryengine/execution/load/TsFileData.java | 3 - .../operator/schema/source/DeviceSchemaSource.java | 35 ++- .../load/LoadTsFileAbstractMemoryBlock.java | 66 +++++ .../load/LoadTsFileAnalyzeSchemaMemoryBlock.java | 98 ++++++ .../load/LoadTsFileDataCacheMemoryBlock.java | 147 +++++++++ .../queryengine/load/LoadTsFileMemoryManager.java | 149 ++++++++++ .../queryengine/metric/LoadTsFileMemMetricSet.java | 101 +++++++ .../iotdb/db/queryengine/plan/Coordinator.java | 8 +- .../db/queryengine/plan/analyze/Analysis.java | 2 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 29 +- .../plan/analyze/ExpressionTypeAnalyzer.java | 118 +++++--- .../plan/analyze/LoadTsfileAnalyzer.java | 284 +++++++++++++++--- .../queryengine/plan/analyze/TemplatedAnalyze.java | 61 +--- .../db/queryengine/plan/analyze/TemplatedInfo.java | 104 ++++++- .../analyze/cache/schema/DataNodeSchemaCache.java | 44 +++ .../plan/analyze/schema/ClusterSchemaFetcher.java | 9 + .../visitor/ColumnTransformerVisitor.java | 14 +- ...edConcatRemoveUnExistentMeasurementVisitor.java | 48 +++ ...ncatDeviceAndBindSchemaForPredicateVisitor.java | 1 + .../plan/optimization/LimitOffsetPushDown.java | 8 + .../db/queryengine/plan/parser/ASTVisitor.java | 23 +- .../plan/planner/LocalExecutionPlanner.java | 43 ++- .../plan/planner/OperatorTreeGenerator.java | 169 ++++++++++- .../plan/planner/SubPlanTypeExtractor.java | 13 + .../plan/planner/TemplatedLogicalPlan.java | 181 ++++++------ .../plan/planner/TemplatedLogicalPlanBuilder.java | 11 +- .../plan/planner/plan/node/PlanNodeType.java | 2 + .../plan/node/load/LoadSingleTsFileNode.java | 8 - .../plan/planner/plan/node/process/FilterNode.java | 30 ++ .../planner/plan/node/process/TransformNode.java | 2 +- .../plan/node/write/InsertMultiTabletsNode.java | 4 + .../planner/plan/node/write/InsertRowsNode.java | 4 + .../plan/node/write/InsertRowsOfOneDeviceNode.java | 4 + .../scheduler/load/LoadTsFileDispatcherImpl.java | 37 ++- .../plan/scheduler/load/LoadTsFileScheduler.java | 117 +++++--- .../apache/iotdb/db/schemaengine/SchemaEngine.java | 10 +- .../metric/SchemaEngineCachedMetric.java | 23 +- .../schemaengine/metric/SchemaMetricManager.java | 13 +- .../metric/SchemaRegionCachedMetric.java | 328 ++++++++++++++++----- .../schemaengine/metric/SchemaRegionMemMetric.java | 31 ++ .../rescon/CachedSchemaRegionStatistics.java | 9 +- .../schemaengine/schemaregion/ISchemaRegion.java | 2 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 12 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 12 +- .../schemaregion/mtree/IMTreeStore.java | 2 + .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 9 +- .../schemaregion/mtree/impl/mem/MemMTreeStore.java | 24 +- .../mtree/impl/pbtree/CachedMTreeStore.java | 51 +++- .../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 14 +- .../mtree/impl/pbtree/PBTreeFactory.java | 19 +- .../pbtree/ReentrantReadOnlyCachedMTreeStore.java | 5 + .../impl/pbtree/flush/PBTreeFlushExecutor.java | 19 +- .../mtree/impl/pbtree/flush/Scheduler.java | 79 +++-- .../mtree/impl/pbtree/lock/LockManager.java | 4 + .../impl/pbtree/memcontrol/MemoryStatistics.java | 11 +- .../mtree/impl/pbtree/memory/IMemoryManager.java | 3 +- .../mtree/impl/pbtree/memory/MemoryManager.java | 11 +- .../impl/pbtree/memory/ReleaseFlushMonitor.java | 82 ++---- .../mtree/impl/pbtree/schemafile/SchemaFile.java | 36 ++- .../schemafile/pagemgr/BTreePageManager.java | 79 ++++- .../pbtree/schemafile/pagemgr/PageManager.java | 36 ++- .../schemaregion/mtree/traverser/Traverser.java | 6 +- .../read/resp/info/impl/ShowDevicesResult.java | 13 +- .../utils/filter/DeviceFilterVisitor.java | 29 +- .../db/service/metrics/DataNodeMetricsHelper.java | 4 + .../iotdb/db/service/metrics/WritingMetrics.java | 169 +++++++++++ .../iotdb/db/storageengine/StorageEngine.java | 14 +- .../db/storageengine/dataregion/DataRegion.java | 227 +++++++++++--- .../CompactionFileCountExceededException.java | 5 + .../CompactionLastTimeCheckFailedException.java | 5 + .../CompactionMemoryNotEnoughException.java | 5 + .../CompactionValidationFailedException.java | 5 + .../performer/impl/FastCompactionPerformer.java | 4 + .../execute/task/AbstractCompactionTask.java | 11 +- .../execute/task/CrossSpaceCompactionTask.java | 3 +- .../execute/task/InnerSpaceCompactionTask.java | 3 +- .../compaction/schedule/CompactionWorker.java | 2 +- .../dataregion/flush/CompressionRatio.java | 3 - .../dataregion/flush/MemTableFlushTask.java | 18 -- .../memtable/AlignedWritableMemChunk.java | 2 +- .../dataregion/memtable/TsFileProcessor.java | 65 ++-- .../dataregion/modification/Deletion.java | 4 + .../dataregion/modification/ModificationFile.java | 19 ++ .../io/LocalTextModificationAccessor.java | 9 +- .../modification/io/ModificationWriter.java | 9 + .../dataregion/utils/TsFileResourceUtils.java | 2 - .../storageengine/dataregion/wal/node/WALNode.java | 5 +- .../dataregion/wal/recover/WALRecoverManager.java | 11 +- .../dataregion/wal/utils/WALFileUtils.java | 14 + .../db/storageengine/rescon/memory/SystemInfo.java | 43 +-- .../db/utils/datastructure/AlignedTVList.java | 6 +- .../db/metadata/mtree/schemafile/MonitorTest.java | 13 +- .../schema/SchemaQueryScanOperatorTest.java | 3 +- .../compaction/ReadChunkInnerCompactionTest.java | 49 +++ .../cross/CrossSpaceCompactionExceptionTest.java | 79 +++++ .../dataregion/memtable/TsFileProcessorTest.java | 16 +- .../src/test/resources/datanode1conf/logback.xml | 122 +++----- .../src/test/resources/datanode2conf/logback.xml | 122 +++----- .../src/test/resources/datanode3conf/logback.xml | 122 +++----- .../resources/conf/iotdb-cluster.properties | 30 ++ .../src/assembly/resources/sbin/clean-all.bat | 38 +++ .../src/assembly/resources/sbin/clean-all.sh | 95 ++++++ .../assembly/resources/sbin/clean-confignode.bat | 82 ++++++ .../assembly/resources/sbin/clean-confignode.sh | 55 ++++ .../src/assembly/resources/sbin/clean-datanode.bat | 203 +++++++++++++ .../src/assembly/resources/sbin/clean-datanode.sh | 70 +++++ .../src/assembly/resources/sbin/start-all.sh | 81 +++++ .../src/assembly/resources/sbin/stop-all.sh | 79 +++++ .../async/AsyncPipeDataTransferServiceClient.java | 8 + .../apache/iotdb/commons/conf/CommonConfig.java | 9 + .../iotdb/commons/conf/CommonDescriptor.java | 5 + .../apache/iotdb/commons/conf/IoTDBConstant.java | 1 - .../commons/pipe/agent/plugin/PipePluginAgent.java | 18 +- .../iotdb/commons/pipe/config/PipeConfig.java | 7 + .../builtin/connector/iotdb/IoTDBConnector.java | 81 +++-- .../connector/iotdb/thrift/IoTDBMetaConnector.java | 32 +- .../iotdb/commons/pipe/task/meta/PipeMeta.java | 29 ++ .../commons/pipe/task/meta/PipeStaticMeta.java | 6 +- .../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 8 + .../iotdb/commons/service/metric/enums/Metric.java | 18 +- iotdb-core/tsfile/pom.xml | 56 +++- .../antlr4/org/apache/tsfile/parser/PathLexer.g4 | 212 +++++++++++++ .../antlr4/org/apache/tsfile/parser}/PathParser.g4 | 9 +- .../tsfile/common/constant/TsFileConstant.java | 4 - .../apache/iotdb/tsfile/compress/ICompressor.java | 14 +- .../iotdb/tsfile/compress/IUnCompressor.java | 9 +- ...leSequenceReaderTimeseriesMetadataIterator.java | 42 ++- .../read/common/parser/PathNodesGenerator.java | 8 +- .../tsfile/read/common/parser/PathVisitor.java | 6 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 3 - ...quenceReaderTimeseriesMetadataIteratorTest.java | 8 +- .../src/main/thrift/datanode.thrift | 1 + pom.xml | 4 +- 223 files changed, 7018 insertions(+), 1878 deletions(-)
