This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/optimize_for_multiply_device_limit_query in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 15d6e9c393ae88ea33a496e81623809f6190037c Merge: ee050df9428 9bcef1c22f8 Author: Beyyes <[email protected]> AuthorDate: Tue Jan 9 12:19:04 2024 +0800 merge .github/workflows/cluster-it-1c1d.yml | 2 + .github/workflows/cluster-it-1c3d.yml | 2 + .github/workflows/grafana-plugin.yml | 2 +- .github/workflows/pipe-it-2cluster.yml | 2 + .github/workflows/unit-test.yml | 2 + .gitignore | 2 + Jenkinsfile | 4 + example/client-cpp-example/README.md | 4 +- example/schema/pom.xml | 2 +- .../java/org/apache/iotdb/tsfile/TsFileRead.java | 16 +- .../apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +- .../it/env/cluster/config/MppCommonConfig.java | 5 +- .../it/env/cluster/config/MppDataNodeConfig.java | 21 + .../it/env/cluster/node/AbstractNodeWrapper.java | 26 +- .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 11 + .../it/env/remote/config/RemoteDataNodeConfig.java | 16 + .../apache/iotdb/itbase/constant/TestConstant.java | 1 + .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 1 + .../apache/iotdb/itbase/env/DataNodeConfig.java | 7 + .../it/cluster/IoTDBClusterNodeGetterIT.java | 2 +- .../it/cluster/IoTDBClusterRestartIT.java | 26 +- .../confignode/it/cluster/IoTDBClusterStartIT.java | 93 + .../it/partition/IoTDBPartitionCreationIT.java | 4 +- .../it/partition/IoTDBPartitionGetterIT.java | 59 +- .../confignode/it/utils/ConfigNodeTestUtils.java | 2 +- .../apache/iotdb/db/it/GrafanaApiServiceIT.java | 451 +++++ .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 41 +- .../org/apache/iotdb/db/it/IoTDBMultiDeviceIT.java | 34 + .../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 2119 ++++++++++++++++++++ .../IoTDBAlignByDeviceWithTemplateIT.java | 406 +++- .../IoTDBAlignedSeriesQueryWithDeletionIT.java | 28 + .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 25 +- .../iotdb/db/it/auth/IoTDBClusterAuthorityIT.java | 5 +- .../iotdb/db/it/auth/IoTDBSeriesPermissionIT.java | 4 +- .../iotdb/db/it/auth/IoTDBSystemPermissionIT.java | 75 +- .../db/it/auth/IoTDBTemplatePermissionIT.java | 5 + .../org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java | 60 +- .../apache/iotdb/db/it/cq/IoTDBCQExecInNsIT.java | 60 +- .../apache/iotdb/db/it/cq/IoTDBCQExecInUsIT.java | 60 +- .../apache/iotdb/db/it/groupby/IOTDBGroupByIT.java | 253 +++ .../db/it/groupby/IoTDBGroupByNaturalMonthIT.java | 44 +- .../IoTDBGroupByNaturalMonthNsPrecisionIT.java | 12 +- .../IoTDBGroupByNaturalMonthUsPrecisionIT.java | 12 +- .../apache/iotdb/db/it/metric/IoTDBMetricIT.java | 71 + .../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 54 +- .../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 7 +- .../db/it/schema/IoTDBCreateTimeseriesIT.java | 4 +- .../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 85 +- .../org/apache/iotdb/db/it/utils/TestUtils.java | 89 +- .../iotdb/pipe/PipeEnvironmentException.java | 31 + .../apache/iotdb/pipe/it/AbstractPipeDualIT.java | 60 + .../apache/iotdb/pipe/it/IoTDBPipeClusterIT.java | 254 ++- .../pipe/it/IoTDBPipeConnectorParallelIT.java | 77 +- .../apache/iotdb/pipe/it/IoTDBPipeDataSinkIT.java | 183 ++ .../apache/iotdb/pipe/it/IoTDBPipeDataSyncIT.java | 202 -- .../apache/iotdb/pipe/it/IoTDBPipeExtractorIT.java | 842 ++++++++ .../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 163 +- .../apache/iotdb/pipe/it/IoTDBPipeProcessorIT.java | 104 + .../apache/iotdb/pipe/it/IoTDBPipeProtocolIT.java | 61 +- .../iotdb/pipe/it/IoTDBPipeSingleEnvDemoIT.java | 16 +- .../iotdb/pipe/it/IoTDBPipeSwitchStatusIT.java | 51 +- .../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 31 +- .../pipe/it/extractor/IoTDBPipeExtractorIT.java | 700 ------- .../session/it/IoTDBSessionSyntaxConventionIT.java | 34 + .../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 10 +- .../java/org/apache/iotdb/pipe/api/access/Row.java | 10 +- .../parameter/PipeParameterValidator.java | 6 +- .../api/customizer/parameter/PipeParameters.java | 137 +- .../api/exception/PipeConnectionException.java | 3 + iotdb-client/cli/pom.xml | 17 + .../assembly/resources/tools/collection-info.bat | 275 +++ .../assembly/resources/tools/collection-info.sh | 268 +++ .../java/org/apache/iotdb/cli/AbstractCli.java | 2 +- .../org/apache/iotdb/isession/INodeSupplier.java | 30 + .../java/org/apache/iotdb/isession/ISession.java | 3 +- .../org/apache/iotdb/isession/SessionConfig.java | 2 + .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 15 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 + .../apache/iotdb/session/DummyNodesSupplier.java | 45 + .../org/apache/iotdb/session/NodesSupplier.java | 239 +++ .../java/org/apache/iotdb/session/Session.java | 115 +- .../apache/iotdb/session/SessionConnection.java | 23 +- .../org/apache/iotdb/session/ThriftConnection.java | 192 ++ .../org/apache/iotdb/session/pool/SessionPool.java | 104 +- .../iotdb/session/SessionCacheLeaderTest.java | 1 + .../iotdb/session/SessionConnectionTest.java | 39 +- .../java/org/apache/iotdb/session/SessionTest.java | 21 +- .../session/pool/SessionPoolExceptionTest.java | 21 +- .../apache/iotdb/session/pool/SessionPoolTest.java | 13 +- .../flink/sql/client/IoTDBWebSocketClient.java | 20 +- .../org/apache/iotdb/flink/sql/common/Options.java | 7 + .../sql/factory/IoTDBDynamicTableFactory.java | 1 + .../flink/sql/function/IoTDBCDCSourceFunction.java | 103 +- iotdb-connector/grafana-plugin/backend-compile.bat | 2 +- iotdb-connector/grafana-plugin/backend-compile.sh | 31 +- iotdb-connector/grafana-plugin/go.mod | 35 +- .../iotdb/spark/tsfile/NarrowConverter.scala | 56 +- .../apache/iotdb/spark/tsfile/WideConverter.scala | 56 +- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 16 +- .../org/apache/iotdb/db/qp/sql/PathParser.g4 | 62 - .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + .../src/assembly/resources/conf/confignode-env.sh | 2 +- .../resources/conf/iotdb-confignode.properties | 2 + .../assembly/resources/conf/logback-confignode.xml | 52 +- .../assembly/resources/sbin/register-confignode.sh | 80 + .../async/AsyncDataNodeHeartbeatClientPool.java | 4 +- .../heartbeat/DataNodeHeartbeatHandler.java | 8 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 2 +- .../consensus/request/ConfigPhysicalPlan.java | 8 + .../consensus/request/ConfigPhysicalPlanType.java | 5 +- .../request/read/region/GetRegionIdPlan.java | 35 +- .../write/confignode/UpdateClusterIdPlan.java | 76 + .../request/write/pipe/PipeEnrichedPlan.java | 79 + .../statemachine/ConfigRegionStateMachine.java | 6 +- .../exception/NotEnoughDataNodeException.java | 13 +- .../iotdb/confignode/manager/ClusterManager.java | 67 + .../iotdb/confignode/manager/ConfigManager.java | 27 +- .../apache/iotdb/confignode/manager/IManager.java | 12 +- .../iotdb/confignode/manager/ProcedureManager.java | 43 +- .../manager/consensus/ConsensusManager.java | 37 +- .../manager/load/balancer/RegionBalancer.java | 23 +- .../region/CopySetRegionGroupAllocator.java | 160 -- .../region/GreedyCopySetRegionGroupAllocator.java | 246 +++ .../region/GreedyRegionGroupAllocator.java | 31 +- .../confignode/manager/load/cache/LoadCache.java | 10 +- .../load/cache/node/NodeHeartbeatSample.java | 7 +- .../manager/load/cache/node/NodeStatistics.java | 4 +- .../manager/load/service/HeartbeatService.java | 8 +- .../manager/node/ClusterNodeStartUtils.java | 5 - .../iotdb/confignode/manager/node/NodeManager.java | 13 +- .../manager/partition/PartitionManager.java | 60 +- .../manager/partition/PartitionMetrics.java | 36 + .../iotdb/confignode/manager/pipe/PipeManager.java | 54 - .../manager/pipe/agent/PipeConfigNodeAgent.java | 54 + .../PipeConfigRegionConnectorConstructor.java | 68 + .../PipeConfigRegionExtractorConstructor.java | 49 + .../PipeConfigRegionProcessorConstructor.java | 41 + .../agent/plugin/PipePluginConfigNodeAgent.java | 49 + .../pipe/agent/task/PipeTaskConfigNodeAgent.java | 84 + .../pipe/connector/IoTDBConfigRegionConnector.java | 30 + .../payload/request/PipeTransferConfigPlanReq.java | 78 + .../manager/pipe/coordinator/PipeManager.java | 54 + .../coordinator/plugin/PipePluginCoordinator.java | 110 + .../coordinator/runtime/PipeHeartbeatParser.java | 270 +++ .../runtime/PipeHeartbeatScheduler.java | 123 ++ .../runtime/PipeLeaderChangeHandler.java | 94 + .../pipe/coordinator/runtime/PipeMetaSyncer.java | 182 ++ .../runtime/PipeRuntimeCoordinator.java | 98 + .../pipe/coordinator/task/PipeTaskCoordinator.java | 209 ++ .../coordinator/task/PipeTaskCoordinatorLock.java | 107 + .../pipe/execution/PipeConfigNodeSubtask.java | 314 +++ .../execution/PipeConfigNodeSubtaskExecutor.java | 50 + .../pipe/extractor/IoTDBConfigRegionExtractor.java | 30 + .../manager/pipe/metric/PipeConfigNodeMetrics.java | 47 + .../manager/pipe/metric/PipeProcedureMetrics.java | 97 + .../manager/pipe/metric/PipeTaskInfoMetrics.java | 99 + .../manager/pipe/plugin/PipePluginCoordinator.java | 110 - .../manager/pipe/runtime/PipeHeartbeatParser.java | 270 --- .../pipe/runtime/PipeHeartbeatScheduler.java | 123 -- .../pipe/runtime/PipeLeaderChangeHandler.java | 94 - .../manager/pipe/runtime/PipeMetaSyncer.java | 187 -- .../pipe/runtime/PipeRuntimeCoordinator.java | 98 - .../manager/pipe/task/PipeConfigNodeTask.java | 51 + .../pipe/task/PipeConfigNodeTaskBuilder.java | 72 + .../manager/pipe/task/PipeConfigNodeTaskStage.java | 75 + .../manager/pipe/task/PipeTaskCoordinator.java | 182 -- .../manager/pipe/task/PipeTaskCoordinatorLock.java | 107 - .../iotdb/confignode/persistence/ClusterInfo.java | 109 + .../persistence/executor/ConfigPlanExecutor.java | 13 + .../confignode/persistence/node/NodeInfo.java | 29 +- .../partition/DatabasePartitionTable.java | 44 +- .../persistence/partition/PartitionInfo.java | 39 +- .../persistence/pipe/PipePluginInfo.java | 23 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 50 +- .../confignode/persistence/schema/ConfigMTree.java | 2 +- .../persistence/schema/ConfigMTreeStore.java | 12 + .../persistence/schema/mnode/IConfigMNode.java | 28 + .../schema/mnode/basic/ConfigBasicMNode.java | 50 +- .../mnode/impl/ConfigBasicInternalMNode.java | 37 +- .../iotdb/confignode/procedure/Procedure.java | 46 +- .../confignode/procedure/ProcedureExecutor.java | 25 +- .../confignode/procedure/ProcedureMetrics.java | 184 ++ .../impl/pipe/AbstractOperatePipeProcedureV2.java | 21 +- .../procedure/impl/pipe/PipeTaskOperation.java | 24 +- .../pipe/plugin/CreatePipePluginProcedure.java | 2 +- .../impl/pipe/plugin/DropPipePluginProcedure.java | 2 +- .../runtime/PipeHandleLeaderChangeProcedure.java | 3 +- .../runtime/PipeHandleMetaChangeProcedure.java | 3 +- .../impl/pipe/runtime/PipeMetaSyncProcedure.java | 3 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 28 +- .../impl/pipe/task/DropPipeProcedureV2.java | 12 +- .../impl/pipe/task/StartPipeProcedureV2.java | 25 +- .../impl/pipe/task/StopPipeProcedureV2.java | 25 +- .../statemachine/CreateRegionGroupsProcedure.java | 62 +- .../impl/statemachine/StateMachineProcedure.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 28 +- .../confignode/service/ConfigNodeShutdownHook.java | 2 + .../thrift/ConfigNodeRPCServiceProcessor.java | 51 +- .../request/ConfigPhysicalPlanSerDeTest.java | 12 + .../response/pipe/PipePluginTableRespTest.java | 4 +- .../region/AllocatorScatterWidthManualTest.java | 136 ++ .../GreedyCopySetRegionGroupAllocatorTest.java | 196 ++ .../router/priority/GreedyPriorityTest.java | 4 +- .../priority/LeaderPriorityBalancerTest.java | 4 +- .../manager/load/cache/NodeCacheTest.java | 8 +- .../connector/IoTDBConfigRegionConnectorTest.java | 53 + .../PipeConfigNodeSubtaskExecutorTest.java | 181 ++ .../extractor/IoTDBConfigRegionExtractorTest.java | 59 + .../confignode/persistence/ClusterInfoTest.java | 67 + .../confignode/persistence/PartitionInfoTest.java | 70 +- .../procedure/TestProcedureExecutor.java | 2 +- .../procedure/entity/StuckProcedure.java | 4 +- .../src/test/resources/confignode1conf/logback.xml | 52 +- .../src/test/resources/confignode2conf/logback.xml | 52 +- .../src/test/resources/confignode3conf/logback.xml | 52 +- iotdb-core/consensus/pom.xml | 10 +- .../apache/iotdb/consensus/config/RatisConfig.java | 2 +- .../exception/RatisReadUnavailableException.java | 31 + .../exception/RatisUnderRecoveryException.java | 32 - .../apache/iotdb/consensus/iot/IoTConsensus.java | 31 +- .../consensus/iot/IoTConsensusServerImpl.java | 39 +- .../consensus/iot/IoTConsensusServerMetrics.java | 2 +- .../consensus/iot/client/DispatchLogHandler.java | 17 +- .../iot/logdispatcher/IndexController.java | 5 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 28 +- .../consensus/iot/logdispatcher/SyncStatus.java | 4 +- .../service/IoTConsensusRPCServiceProcessor.java | 2 +- .../ratis/ApplicationStateMachineProxy.java | 17 +- .../iotdb/consensus/ratis/RatisConsensus.java | 103 +- .../consensus/ratis/metrics/CounterProxy.java | 6 +- .../iotdb/consensus/ratis/metrics/GaugeProxy.java | 24 +- .../ratis/metrics/IoTDBMetricRegistry.java | 121 +- .../ratis/metrics/MetricRegistryManager.java | 2 +- .../ratis/metrics/RatisMetricsManager.java | 20 +- .../iotdb/consensus/ratis/metrics/TimerProxy.java | 32 +- .../apache/iotdb/consensus/ratis/utils/Utils.java | 27 +- .../apache/iotdb/consensus/iot/ReplicateTest.java | 30 +- .../iot/logdispatcher/IndexControllerTest.java | 14 +- .../iotdb/consensus/ratis/DiskGuardianTest.java | 3 +- .../iotdb/consensus/ratis/RatisConsensusTest.java | 25 +- .../iotdb/consensus/ratis/RecoverReadTest.java | 16 +- .../apache/iotdb/consensus/ratis/TestUtils.java | 29 +- .../src/assembly/resources/conf/datanode-env.sh | 2 +- .../resources/conf/iotdb-datanode.properties | 10 +- .../assembly/resources/conf/logback-datanode.xml | 122 +- .../assembly/resources/sbin/register-datanode.sh | 80 + .../org/apache/iotdb/db/audit/AuditLogger.java | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 130 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 60 +- .../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 1 + .../db/conf/rest/IoTDBRestServiceDescriptor.java | 119 +- .../db/consensus/SchemaRegionConsensusImpl.java | 1 + .../dataregion/DataExecutionVisitor.java | 91 +- .../dataregion/DataRegionStateMachine.java | 49 +- .../schemaregion/SchemaExecutionVisitor.java | 14 + .../exception/LoadRuntimeOutOfMemoryException.java | 26 + .../org/apache/iotdb/db/pipe/agent/PipeAgent.java | 20 +- .../agent/plugin/PipeConnectorConstructor.java | 112 -- .../agent/plugin/PipeExtractorConstructor.java | 58 - .../db/pipe/agent/plugin/PipePluginAgent.java | 263 --- .../pipe/agent/plugin/PipePluginConstructor.java | 85 - .../pipe/agent/plugin/PipePluginDataNodeAgent.java | 217 ++ .../agent/plugin/PipeProcessorConstructor.java | 52 - .../PipeDataRegionConnectorConstructor.java | 91 + .../PipeDataRegionExtractorConstructor.java | 46 + .../dataregion/PipeDataRegionPluginAgent.java | 53 + .../PipeDataRegionProcessorConstructor.java | 41 + .../PipeSchemaRegionConnectorConstructor.java | 68 + .../PipeSchemaRegionExtractorConstructor.java | 49 + .../schemaregion/PipeSchemaRegionPluginAgent.java | 49 + .../PipeSchemaRegionProcessorConstructor.java | 41 + .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 17 +- .../SimpleConsensusProgressIndexAssigner.java | 113 -- .../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 921 --------- .../db/pipe/agent/task/PipeTaskDataNodeAgent.java | 285 +++ .../db/pipe/commit/PipeEventCommitManager.java | 106 - .../iotdb/db/pipe/commit/PipeEventCommitter.java | 79 - .../config/constant/PipeConnectorConstant.java | 107 - .../config/constant/PipeExtractorConstant.java | 62 - .../config/constant/PipeProcessorConstant.java | 29 - .../configuraion/PipeTaskRuntimeConfiguration.java | 42 - .../env/PipeTaskConnectorRuntimeEnvironment.java | 34 - .../env/PipeTaskExtractorRuntimeEnvironment.java | 44 - .../plugin/env/PipeTaskRuntimeEnvironment.java | 43 - .../payload/evolvable/PipeRequestType.java | 63 - .../builder/PipeTransferBatchReqBuilder.java | 17 +- .../request/PipeTransferFilePieceReq.java | 4 +- .../evolvable/request/PipeTransferFileSealReq.java | 4 +- .../request/PipeTransferHandshakeReq.java | 4 +- .../request/PipeTransferSchemaPlanReq.java | 105 + .../request/PipeTransferTabletBatchReq.java | 30 +- .../request/PipeTransferTabletBinaryReq.java | 8 +- .../request/PipeTransferTabletInsertNodeReq.java | 4 +- .../request/PipeTransferTabletRawReq.java | 4 +- .../db/pipe/connector/protocol/IoTDBConnector.java | 128 -- .../protocol/IoTDBConnectorRequestVersion.java | 35 - .../protocol/airgap/IoTDBAirGapConnector.java | 22 +- .../protocol/legacy/IoTDBLegacyPipeConnector.java | 61 +- .../connector/protocol/opcua/OpcUaConnector.java | 30 +- .../protocol/opcua/OpcUaServerBuilder.java | 2 +- .../protocol/thrift/IoTDBThriftClientManager.java | 39 + .../protocol/thrift/LeaderCacheManager.java | 110 + .../async/IoTDBThriftAsyncClientManager.java | 200 ++ .../thrift/async/IoTDBThriftAsyncConnector.java | 240 +-- .../PipeTransferTabletInsertNodeEventHandler.java | 12 + .../PipeTransferTabletInsertionEventHandler.java | 17 +- .../handler/PipeTransferTabletRawEventHandler.java | 7 + .../thrift/sync/IoTDBThriftSyncClientManager.java | 246 +++ .../thrift/sync/IoTDBThriftSyncConnector.java | 391 ++-- .../sync/IoTDBThriftSyncConnectorClient.java | 84 - .../protocol/websocket/WebSocketConnector.java | 97 +- .../websocket/WebSocketConnectorServer.java | 467 +++-- .../protocol/writeback/WriteBackConnector.java | 56 +- .../apache/iotdb/db/pipe/event/EnrichedEvent.java | 35 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 59 +- .../iotdb/db/pipe/event/common/row/PipeRow.java | 5 + .../tablet/PipeInsertNodeTabletInsertionEvent.java | 58 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 54 +- .../tablet/TabletInsertionDataContainer.java | 229 ++- .../common/tsfile/PipeTsFileInsertionEvent.java | 40 +- .../tsfile/TsFileInsertionDataContainer.java | 6 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 34 +- .../executor/PipeAssignerSubtaskExecutor.java | 10 +- .../executor/PipeConnectorSubtaskExecutor.java | 10 +- .../executor/PipeProcessorSubtaskExecutor.java | 10 +- .../execution/executor/PipeSubtaskExecutor.java | 158 -- .../executor/PipeSubtaskExecutorManager.java | 47 +- .../PipeDataRegionAssignerSubtaskExecutor.java | 33 + .../PipeDataRegionConnectorSubtaskExecutor.java | 33 + .../PipeDataRegionProcessorSubtaskExecutor.java | 33 + .../PipeSchemaRegionAssignerSubtaskExecutor.java | 31 + .../PipeSchemaRegionConnectorSubtaskExecutor.java | 31 + .../PipeSchemaRegionProcessorSubtaskExecutor.java | 31 + .../execution/scheduler/PipeSubtaskScheduler.java | 90 - .../pipe/extractor/IoTDBDataRegionExtractor.java | 112 +- .../pipe/extractor/IoTDBSchemaRegionExtractor.java | 30 + .../PipeHistoricalDataRegionTsFileExtractor.java | 136 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 141 +- .../realtime/assigner/DisruptorQueue.java | 5 +- .../realtime/assigner/PipeDataRegionAssigner.java | 6 +- .../listener/PipeTimePartitionListener.java | 112 ++ .../iotdb/db/pipe/metric/PipeConnectorMetrics.java | 22 + .../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 71 + .../db/pipe/metric/PipeDataRegionEventCounter.java | 86 + .../db/pipe/metric/PipeEventCommitMetrics.java | 141 ++ .../iotdb/db/pipe/metric/PipeEventCounter.java | 79 - .../apache/iotdb/db/pipe/metric/PipeMetrics.java | 69 - .../pipe/metric/PipeWALInsertNodeCacheMetrics.java | 24 + .../downsampling/DownSamplingProcessor.java | 213 ++ .../downsampling/PartialPathLastTimeCache.java | 111 + .../SimpleConsensusProgressIndexAssigner.java | 113 ++ .../progress/committer/PipeEventCommitManager.java | 110 + .../progress/committer/PipeEventCommitter.java | 98 + .../pipe/receiver/PipePlanToStatementVisitor.java | 186 ++ .../pipe/receiver/airgap/IoTDBAirGapReceiver.java | 2 +- .../pipe/receiver/thrift/IoTDBThriftReceiver.java | 2 +- .../receiver/thrift/IoTDBThriftReceiverAgent.java | 2 +- .../receiver/thrift/IoTDBThriftReceiverV1.java | 55 +- .../db/pipe/resource/memory/PipeMemoryBlock.java | 14 +- .../resource/tsfile/PipeTsFileResourceManager.java | 11 +- .../pipe/resource/wal/PipeWALResourceManager.java | 8 +- .../org/apache/iotdb/db/pipe/task/PipeBuilder.java | 64 - .../iotdb/db/pipe/task/PipeDataNodeTask.java | 84 + .../org/apache/iotdb/db/pipe/task/PipeTask.java | 79 - .../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 74 - .../apache/iotdb/db/pipe/task/PipeTaskManager.java | 122 -- .../db/pipe/task/builder/PipeDataNodeBuilder.java | 79 + .../pipe/task/builder/PipeDataNodeTaskBuilder.java | 88 + .../builder/PipeDataNodeTaskDataRegionBuilder.java | 38 + .../PipeDataNodeTaskSchemaRegionBuilder.java | 38 + .../pipe/task/connection/BlockingPendingQueue.java | 129 -- .../connection/BoundedBlockingPendingQueue.java | 31 - .../db/pipe/task/connection/EnrichedDeque.java | 4 +- .../db/pipe/task/connection/EventSupplier.java | 34 - .../pipe/task/connection/PipeEventCollector.java | 3 +- .../connection/UnboundedBlockingPendingQueue.java | 39 - .../db/pipe/task/stage/PipeTaskConnectorStage.java | 12 +- .../db/pipe/task/stage/PipeTaskExtractorStage.java | 9 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 24 +- .../iotdb/db/pipe/task/stage/PipeTaskStage.java | 144 -- .../iotdb/db/pipe/task/subtask/DecoratingLock.java | 44 - .../db/pipe/task/subtask/PipeDataNodeSubtask.java | 133 ++ .../iotdb/db/pipe/task/subtask/PipeSubtask.java | 255 --- .../subtask/connector/PipeConnectorSubtask.java | 16 +- .../connector/PipeConnectorSubtaskLifeCycle.java | 2 +- .../connector/PipeConnectorSubtaskManager.java | 16 +- .../subtask/processor/PipeProcessorSubtask.java | 8 +- .../processor/PipeProcessorSubtaskWorker.java | 2 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 37 +- .../rest/v1/handler/RequestValidationHandler.java | 12 - .../v1/handler/StatementConstructionHandler.java | 59 - .../protocol/rest/v1/impl/RestApiServiceImpl.java | 47 - .../rest/v2/handler/QueryDataSetHandler.java | 8 +- .../rest/v2/handler/RequestValidationHandler.java | 43 + .../protocol/rest/v2/impl/RestApiServiceImpl.java | 36 +- .../iotdb/db/protocol/session/SessionManager.java | 2 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 9 +- .../impl/DataNodeInternalRPCServiceImpl.java | 33 +- .../db/queryengine/common/MPPQueryContext.java | 10 + .../iotdb/db/queryengine/common/NodeRef.java | 4 + .../common/header/ColumnHeaderConstant.java | 16 +- .../common/header/DatasetHeaderFactory.java | 4 + .../common/schematree/ClusterSchemaTree.java | 8 +- .../queryengine/common/schematree/ISchemaTree.java | 11 +- .../timerangeiterator/AggrWindowIterator.java | 39 +- .../db/queryengine/execution/driver/Driver.java | 4 - .../execution/exchange/sink/LocalSinkChannel.java | 39 +- .../execution/exchange/sink/ShuffleSinkHandle.java | 27 +- .../execution/exchange/sink/SinkChannel.java | 73 +- .../exchange/source/LocalSourceHandle.java | 15 +- .../execution/exchange/source/SourceHandle.java | 22 +- .../execution/executor/RegionWriteExecutor.java | 285 ++- .../fragment/FragmentInstanceContext.java | 73 +- .../fragment/FragmentInstanceExecution.java | 3 +- .../fragment/FragmentInstanceManager.java | 15 +- .../execution/fragment/QueryContext.java | 14 +- .../execution/fragment/QueryStatistics.java | 69 + .../db/queryengine/execution/load/ChunkData.java | 3 + .../queryengine/execution/load/DeletionData.java | 17 +- .../execution/load/LoadTsFileManager.java | 69 +- .../db/queryengine/execution/load/TsFileData.java | 3 - .../queryengine/execution/load/TsFileSplitter.java | 2 +- .../queryengine/execution/memory/MemoryPool.java | 6 +- .../execution/operator/AggregationUtil.java | 16 +- .../queryengine/execution/operator/Operator.java | 12 +- .../operator/process/AggregationOperator.java | 19 +- .../operator/process/ColumnInjectOperator.java | 115 ++ .../process/SlidingWindowAggregationOperator.java | 18 +- .../execution/operator/process/TopKOperator.java | 10 + .../process/join/FullOuterTimeJoinOperator.java | 332 +++ .../process/join/InnerTimeJoinOperator.java | 407 ++++ .../process/join/LeftOuterTimeJoinOperator.java | 338 ++++ .../process/join/RowBasedTimeJoinOperator.java | 332 --- .../process/join/merge/AscTimeComparator.java | 12 +- .../process/join/merge/DescTimeComparator.java | 12 +- .../process/join/merge/TimeComparator.java | 10 +- .../operator/schema/SchemaFetchScanOperator.java | 13 +- .../operator/schema/source/DeviceSchemaSource.java | 24 + .../source/AbstractDataSourceOperator.java | 5 + .../AbstractSeriesAggregationScanOperator.java | 52 +- .../AlignedSeriesAggregationScanOperator.java | 25 + .../operator/source/AlignedSeriesScanUtil.java | 163 +- .../execution/operator/source/FileLoaderUtils.java | 350 ++++ .../operator/source/LastCacheScanOperator.java | 5 + .../source/SeriesAggregationScanOperator.java | 25 + .../execution/operator/source/SeriesScanUtil.java | 605 +++--- .../operator/source/ShowQueriesOperator.java | 5 + .../schedule/DriverTaskTimeoutSentinelThread.java | 2 +- .../load/LoadTsFileAbstractMemoryBlock.java | 66 + .../load/LoadTsFileAnalyzeSchemaMemoryBlock.java | 98 + .../load/LoadTsFileDataCacheMemoryBlock.java | 147 ++ .../queryengine/load/LoadTsFileMemoryManager.java | 149 ++ .../queryengine/metric/LoadTsFileMemMetricSet.java | 101 + .../metric/QueryExecutionMetricSet.java | 24 +- .../queryengine/metric/QueryPlanCostMetricSet.java | 10 +- .../metric/QueryRelatedResourceMetricSet.java | 144 +- .../metric/SeriesScanCostMetricSet.java | 825 +++++--- .../iotdb/db/queryengine/plan/Coordinator.java | 8 +- .../db/queryengine/plan/analyze/Analysis.java | 28 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 458 +++-- .../plan/analyze/ColumnPaginationController.java | 19 +- .../plan/analyze/ExpressionAnalyzer.java | 15 +- .../plan/analyze/ExpressionTypeAnalyzer.java | 118 +- .../plan/analyze/GroupByLevelController.java | 208 -- .../plan/analyze/GroupByLevelHelper.java | 223 ++ .../plan/analyze/LoadTsfileAnalyzer.java | 323 ++- .../queryengine/plan/analyze/PredicateUtils.java | 21 + .../queryengine/plan/analyze/TemplatedAnalyze.java | 99 +- .../db/queryengine/plan/analyze/TemplatedInfo.java | 104 +- .../analyze/cache/partition/PartitionCache.java | 12 +- .../analyze/cache/schema/DataNodeSchemaCache.java | 44 + .../analyze/schema/AutoCreateSchemaExecutor.java | 61 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 17 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 22 +- .../plan/analyze/schema/ISchemaFetcher.java | 6 +- .../queryengine/plan/execution/QueryExecution.java | 11 +- .../plan/execution/config/ConfigTaskVisitor.java | 8 + .../config/executor/ClusterConfigTaskExecutor.java | 120 +- .../config/executor/IConfigTaskExecutor.java | 10 + .../config/metadata/ShowClusterIdTask.java | 65 + .../visitor/ColumnTransformerVisitor.java | 14 +- .../ReplaceRawPathWithGroupedPathVisitor.java | 44 +- .../visitor/ReplaceSubTreeWithViewVisitor.java | 59 + ...edConcatRemoveUnExistentMeasurementVisitor.java | 48 + ...ncatDeviceAndBindSchemaForPredicateVisitor.java | 1 + .../ConvertPredicateToTimeFilterVisitor.java | 30 +- .../plan/optimization/ColumnInjectionPushDown.java | 127 ++ .../plan/optimization/LimitOffsetPushDown.java | 95 +- .../db/queryengine/plan/parser/ASTVisitor.java | 86 +- .../plan/parser/StatementGenerator.java | 5 + .../plan/planner/LocalExecutionPlanContext.java | 16 + .../plan/planner/LocalExecutionPlanner.java | 45 +- .../plan/planner/LogicalPlanBuilder.java | 44 +- .../plan/planner/LogicalPlanVisitor.java | 78 +- .../plan/planner/OperatorTreeGenerator.java | 348 +++- .../plan/planner/SubPlanTypeExtractor.java | 13 + .../plan/planner/TemplatedLogicalPlan.java | 181 +- .../plan/planner/TemplatedLogicalPlanBuilder.java | 11 +- .../planner/distribution/DistributionPlanner.java | 9 +- .../planner/distribution/ExchangeNodeAdder.java | 82 +- .../plan/planner/distribution/SourceRewriter.java | 205 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 64 +- .../plan/planner/plan/node/PlanNode.java | 11 + .../plan/planner/plan/node/PlanNodeType.java | 37 +- .../plan/planner/plan/node/PlanVisitor.java | 61 +- .../plan/node/load/LoadSingleTsFileNode.java | 8 - .../planner/plan/node/load/LoadTsFileNode.java | 7 +- .../node/metedata/read/SchemaFetchScanNode.java | 17 +- .../node/metedata/write/CreateTimeSeriesNode.java | 2 +- .../metedata/write/view/AlterLogicalViewNode.java | 43 +- .../node/pipe/PipeEnrichedConfigSchemaNode.java | 161 ++ .../plan/node/pipe/PipeEnrichedDeleteDataNode.java | 169 ++ .../plan/node/pipe/PipeEnrichedInsertNode.java | 292 +++ .../node/pipe/PipeEnrichedWriteSchemaNode.java | 192 ++ .../planner/plan/node/process/AggregationNode.java | 4 + .../plan/node/process/ColumnInjectNode.java | 145 ++ .../plan/planner/plan/node/process/FilterNode.java | 30 + .../node/process/SlidingWindowAggregationNode.java | 51 +- .../planner/plan/node/process/TimeJoinNode.java | 129 -- .../planner/plan/node/process/TransformNode.java | 2 +- .../plan/node/process/TwoChildProcessNode.java | 101 + .../node/process/join/FullOuterTimeJoinNode.java | 143 ++ .../plan/node/process/join/InnerTimeJoinNode.java | 239 +++ .../node/process/join/LeftOuterTimeJoinNode.java | 133 ++ .../source/AlignedSeriesAggregationScanNode.java | 34 +- .../node/source/SeriesAggregationScanNode.java | 34 +- .../node/source/SeriesAggregationSourceNode.java | 38 +- .../planner/plan/node/write/DeleteDataNode.java | 6 +- .../plan/node/write/InsertMultiTabletsNode.java | 4 + .../plan/planner/plan/node/write/InsertNode.java | 10 - .../planner/plan/node/write/InsertRowsNode.java | 4 + .../plan/node/write/InsertRowsOfOneDeviceNode.java | 4 + .../plan/node/write/PipeEnrichedInsertNode.java | 277 --- .../planner/plan/parameter/SeriesScanOptions.java | 10 +- .../scheduler/load/LoadTsFileDispatcherImpl.java | 37 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 117 +- .../queryengine/plan/statement/StatementType.java | 3 +- .../plan/statement/StatementVisitor.java | 18 +- .../statement/component/GroupByLevelComponent.java | 18 - .../crud/PipeEnrichedInsertBaseStatement.java | 220 -- .../crud/PipeEnrichedLoadTsFileStatement.java | 137 -- .../statement/internal/SchemaFetchStatement.java | 11 +- .../statement/metadata/GetRegionIdStatement.java | 28 +- .../statement/metadata/ShowClusterIdStatement.java | 36 + .../template/AlterSchemaTemplateStatement.java | 19 - .../metadata/view/CreateLogicalViewStatement.java | 10 +- .../plan/statement/pipe/PipeEnrichedStatement.java | 72 + .../binary/CompareBinaryColumnTransformer.java | 33 +- .../column/ternary/BetweenColumnTransformer.java | 21 +- .../binary/CompareEqualToTransformer.java | 5 +- .../binary/CompareGreaterEqualTransformer.java | 5 +- .../binary/CompareGreaterThanTransformer.java | 5 +- .../binary/CompareLessEqualTransformer.java | 5 +- .../binary/CompareLessThanTransformer.java | 5 +- .../binary/CompareNonEqualTransformer.java | 5 +- .../transformer/ternary/BetweenTransformer.java | 15 +- .../transformation/dag/util/TransformUtils.java | 41 +- .../apache/iotdb/db/schemaengine/SchemaEngine.java | 49 +- .../metric/SchemaEngineCachedMetric.java | 82 +- .../schemaengine/metric/SchemaMetricManager.java | 17 +- .../metric/SchemaRegionCachedMetric.java | 328 ++- .../schemaengine/metric/SchemaRegionMemMetric.java | 31 + .../rescon/CachedSchemaRegionStatistics.java | 19 +- .../schemaengine/rescon/SchemaResourceManager.java | 6 +- .../schemaengine/schemaregion/ISchemaRegion.java | 7 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 21 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 31 +- .../schemaregion/mtree/IMTreeStore.java | 5 + .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 74 +- .../schemaregion/mtree/impl/mem/MemMTreeStore.java | 56 +- .../mtree/impl/mem/mnode/basic/BasicMNode.java | 26 +- .../impl/mem/mnode/factory/LogicalViewMNode.java | 36 + .../impl/mem/mnode/factory/MemMNodeFactory.java | 11 +- .../impl/mem/mnode/impl/BasicInternalMNode.java | 83 +- .../impl/mem/mnode/impl/DatabaseDeviceMNode.java | 37 - .../mtree/impl/mem/mnode/impl/DatabaseMNode.java | 17 +- .../mtree/impl/mem/mnode/impl/DeviceMNode.java | 36 - .../mtree/impl/mem/mnode/info/LogicalViewInfo.java | 2 +- .../impl/mem/snapshot/MemMTreeSnapshotUtil.java | 75 +- .../mtree/impl/pbtree/CachedMTreeStore.java | 381 ++-- .../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 127 +- .../mtree/impl/pbtree/PBTreeFactory.java | 107 + .../pbtree/ReentrantReadOnlyCachedMTreeStore.java | 15 +- .../impl/pbtree/StampedWriterPreferredLock.java | 220 -- .../mtree/impl/pbtree/cache/CacheEntry.java | 48 - .../mtree/impl/pbtree/cache/CacheManager.java | 741 ------- .../impl/pbtree/cache/CacheMemoryManager.java | 363 ---- .../mtree/impl/pbtree/cache/ICacheManager.java | 61 - .../mtree/impl/pbtree/cache/LRUCacheManager.java | 287 --- .../mtree/impl/pbtree/cache/PlainCacheManager.java | 83 - .../impl/pbtree/flush/PBTreeFlushExecutor.java | 135 +- .../mtree/impl/pbtree/flush/Scheduler.java | 270 +++ .../mtree/impl/pbtree/lock/LockEntry.java | 45 + .../mtree/impl/pbtree/lock/LockManager.java | 151 ++ .../pbtree/lock/StampedWriterPreferredLock.java | 242 +++ .../pbtree/memcontrol/IReleaseFlushStrategy.java | 3 - .../mtree/impl/pbtree/memcontrol/MemManager.java | 84 - .../impl/pbtree/memcontrol/MemoryStatistics.java | 93 + .../ReleaseFlushStrategyNumBasedImpl.java | 5 - .../ReleaseFlushStrategySizeBasedImpl.java | 11 +- .../mtree/impl/pbtree/memory/IMemoryManager.java | 93 + .../mtree/impl/pbtree/memory/MemoryManager.java | 603 ++++++ .../impl/pbtree/memory/ReleaseFlushMonitor.java | 378 ++++ .../impl/pbtree/memory/buffer/INodeBuffer.java | 53 + .../impl/pbtree/memory/buffer/NodeBuffer.java | 188 ++ .../mtree/impl/pbtree/memory/cache/CacheEntry.java | 62 + .../mtree/impl/pbtree/memory/cache/INodeCache.java | 43 + .../impl/pbtree/memory/cache/LRUNodeCache.java | 271 +++ .../impl/pbtree/memory/cache/PlainNodeCache.java | 73 + .../mtree/impl/pbtree/mnode/ICachedMNode.java | 7 +- .../impl/pbtree/mnode/basic/CachedBasicMNode.java | 38 +- .../mnode/container/CachedMNodeContainer.java | 8 +- .../pbtree/mnode/factory/CacheMNodeFactory.java | 11 +- .../mnode/impl/CachedAboveDatabaseMNode.java | 13 +- .../mnode/impl/CachedBasicInternalMNode.java | 91 +- .../mnode/impl/CachedDatabaseDeviceMNode.java | 49 - .../pbtree/mnode/impl/CachedDatabaseMNode.java | 31 +- .../impl/pbtree/mnode/impl/CachedDeviceMNode.java | 48 - .../pbtree/mnode/impl/CachedMeasurementMNode.java | 13 +- .../impl/pbtree/mnode/info/CacheMNodeInfo.java | 15 +- .../mtree/impl/pbtree/schemafile/ISchemaPage.java | 39 +- .../impl/pbtree/schemafile/ISegmentedPage.java | 2 +- .../mtree/impl/pbtree/schemafile/InternalPage.java | 27 +- .../impl/pbtree/schemafile/MockSchemaFile.java | 20 +- .../mtree/impl/pbtree/schemafile/SchemaFile.java | 45 +- .../mtree/impl/pbtree/schemafile/SchemaPage.java | 59 +- .../mtree/impl/pbtree/schemafile/Segment.java | 478 ----- .../impl/pbtree/schemafile/SegmentedPage.java | 16 +- .../impl/pbtree/schemafile/WrappedSegment.java | 708 +++++-- .../schemafile/pagemgr/BTreePageManager.java | 452 +++-- .../pbtree/schemafile/pagemgr/IPageManager.java | 9 +- .../pbtree/schemafile/pagemgr/PageManager.java | 886 +++++--- .../schemaregion/mtree/traverser/Traverser.java | 11 +- .../read/resp/info/IDeviceSchemaInfo.java | 2 + .../read/resp/info/impl/ShowDevicesResult.java | 16 +- .../reader/impl/TimeseriesReaderWithViewFetch.java | 3 +- .../schemaregion/utils/MNodeUtils.java | 54 +- .../schemaregion/utils/ResourceByPathUtils.java | 21 +- .../utils/filter/DeviceFilterVisitor.java | 21 + .../apache/iotdb/db/service/IoTDBShutdownHook.java | 3 +- .../db/service/metrics/CompactionMetrics.java | 2 +- .../db/service/metrics/DataNodeMetricsHelper.java | 10 +- .../iotdb/db/service/metrics/WritingMetrics.java | 169 ++ .../db/service/metrics/file/TsFileMetrics.java | 8 +- .../iotdb/db/storageengine/StorageEngine.java | 14 +- .../db/storageengine/dataregion/DataRegion.java | 596 ++++-- .../dataregion/HashLastFlushTimeMap.java | 115 +- .../dataregion/IDataRegionForQuery.java | 6 +- .../dataregion/ILastFlushTimeMap.java | 31 +- .../dataregion/VirtualDataRegion.java | 6 +- .../CompactionFileCountExceededException.java | 5 + .../CompactionLastTimeCheckFailedException.java | 39 + .../CompactionMemoryNotEnoughException.java | 5 + .../CompactionValidationFailedException.java | 5 + .../performer/impl/FastCompactionPerformer.java | 4 + .../execute/task/AbstractCompactionTask.java | 13 +- .../execute/task/CrossSpaceCompactionTask.java | 3 +- .../execute/task/InnerSpaceCompactionTask.java | 11 +- .../fast/AlignedSeriesCompactionExecutor.java | 19 +- .../fast/NonAlignedSeriesCompactionExecutor.java | 15 +- .../executor/fast/SeriesCompactionExecutor.java | 46 +- .../executor/fast/element/AlignedPageElement.java | 94 + .../fast/element/NonAlignedPageElement.java | 74 + .../utils/executor/fast/element/PageElement.java | 96 +- .../utils/executor/fast/element/PointElement.java | 11 +- .../fast/reader/CompactionAlignedChunkReader.java | 122 ++ .../fast/reader/CompactionChunkReader.java | 94 + .../readchunk/AlignedSeriesCompactionExecutor.java | 12 + .../readchunk/SingleSeriesCompactionExecutor.java | 15 +- .../execute/utils/reader/PointPriorityReader.java | 2 +- .../utils/writer/AbstractCompactionWriter.java | 11 + .../writer/AbstractCrossCompactionWriter.java | 11 +- .../writer/AbstractInnerCompactionWriter.java | 8 +- .../utils/writer/FastInnerCompactionWriter.java | 12 +- .../compaction/schedule/CompactionScheduler.java | 13 + .../compaction/schedule/CompactionTaskManager.java | 2 +- .../compaction/schedule/CompactionWorker.java | 11 +- .../estimator/AbstractCompactionEstimator.java | 27 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 6 +- .../impl/SizeTieredCompactionSelector.java | 11 +- .../compaction/settle/SettleRequestHandler.java | 24 +- .../dataregion/flush/CompressionRatio.java | 3 - .../dataregion/flush/MemTableFlushTask.java | 18 - .../dataregion/flush/NotifyFlushMemTable.java | 7 + .../dataregion/memtable/AbstractMemTable.java | 21 +- .../memtable/AlignedReadOnlyMemChunk.java | 21 +- .../memtable/AlignedWritableMemChunk.java | 2 +- .../dataregion/memtable/IMemTable.java | 10 +- .../dataregion/memtable/ReadOnlyMemChunk.java | 11 +- .../dataregion/memtable/TsFileProcessor.java | 82 +- .../dataregion/modification/Deletion.java | 4 + .../dataregion/modification/ModificationFile.java | 19 + .../io/LocalTextModificationAccessor.java | 12 +- .../modification/io/ModificationWriter.java | 9 + .../read/reader/chunk/DiskAlignedChunkLoader.java | 24 +- .../read/reader/chunk/DiskChunkLoader.java | 25 +- .../read/reader/chunk/MemAlignedChunkLoader.java | 22 +- .../read/reader/chunk/MemAlignedChunkReader.java | 6 +- .../read/reader/chunk/MemAlignedPageReader.java | 127 +- .../read/reader/chunk/MemChunkLoader.java | 15 +- .../read/reader/chunk/MemChunkReader.java | 13 +- .../read/reader/chunk/MemPageReader.java | 281 +-- .../metadata/DiskAlignedChunkMetadataLoader.java | 57 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 50 +- .../metadata/MemAlignedChunkMetadataLoader.java | 40 +- .../chunk/metadata/MemChunkMetadataLoader.java | 33 +- .../dataregion/snapshot/SnapshotTaker.java | 14 +- .../dataregion/tsfile/TsFileManager.java | 34 +- .../dataregion/tsfile/TsFileResource.java | 8 +- .../dataregion/utils/TsFileResourceUtils.java | 50 +- .../dataregion/wal/buffer/AbstractWALBuffer.java | 14 +- .../storageengine/dataregion/wal/node/WALNode.java | 7 +- .../dataregion/wal/recover/WALRecoverManager.java | 11 +- .../file/AbstractTsFileRecoverPerformer.java | 4 +- .../dataregion/wal/utils/WALEntryHandler.java | 14 +- .../dataregion/wal/utils/WALFileUtils.java | 22 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 18 +- .../db/storageengine/rescon/memory/SystemInfo.java | 43 +- .../rescon/memory/TimePartitionManager.java | 29 + .../quotas/DataNodeThrottleQuotaManager.java | 2 +- .../rescon/quotas/DefaultOperationQuota.java | 16 +- .../apache/iotdb/db/tools/TsFileSketchTool.java | 80 +- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 2 +- .../org/apache/iotdb/db/tools/TsFileSplitTool.java | 2 +- .../db/tools/validate/TsFileValidationTool.java | 2 +- .../db/trigger/executor/TriggerFireVisitor.java | 17 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 12 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 378 ---- .../db/utils/columngenerator/ColumnGenerator.java | 28 + .../utils/columngenerator/ColumnGeneratorType.java | 34 + .../SlidingTimeColumnGenerator.java | 52 + .../parameter/ColumnGeneratorParameter.java | 82 + .../SlidingTimeColumnGeneratorParameter.java | 80 + .../db/utils/datastructure/AlignedTVList.java | 6 +- .../dataregion/DataRegionStateMachineTest.java | 78 + .../apache/iotdb/db/metadata/mnode/MNodeTest.java | 31 +- .../mtree/lock/StampedWriterPreferredLockTest.java | 16 +- .../db/metadata/mtree/schemafile/MonitorTest.java | 105 + .../mtree/schemafile/SchemaFileLogTest.java | 4 +- .../metadata/mtree/schemafile/SchemaFileTest.java | 3 +- .../schemaRegion/SchemaRegionBasicTest.java | 53 +- .../schemaRegion/SchemaRegionTemplateTest.java | 9 +- .../schemaRegion/SchemaStatisticsTest.java | 14 +- .../db/pipe/agent/plugin/PipePluginAgentTest.java | 120 -- .../agent/plugin/PipePluginDataNodeAgentTest.java | 123 ++ .../iotdb/db/pipe/connector/PipeConnectorTest.java | 2 +- .../connector/PipeDataNodeThriftRequestTest.java | 248 +++ .../db/pipe/connector/PipeThriftRequestTest.java | 170 -- .../pipe/event/PipeTabletInsertionEventTest.java | 46 + .../PipeConnectorSubtaskExecutorTest.java | 6 +- .../PipeProcessorSubtaskExecutorTest.java | 6 +- .../db/pipe/execution/PipeSubtaskExecutorTest.java | 4 +- .../extractor/CachedSchemaPatternMatcherTest.java | 6 +- .../extractor/IoTDBDataRegionExtractorTest.java | 50 +- .../db/pipe/extractor/PipeRealtimeExtractTest.java | 6 +- .../db/protocol/rest/IoTDBRestServiceTest.java | 15 + .../db/queryengine/execution/DataDriverTest.java | 11 +- .../aggregation/TimeRangeIteratorTest.java | 66 +- .../operator/AggregationOperatorTest.java | 1 + .../AlignedSeriesAggregationScanOperatorTest.java | 12 +- .../operator/AlignedSeriesScanOperatorTest.java | 14 +- .../execution/operator/FillOperatorTest.java | 15 + .../operator/LastQueryMergeOperatorTest.java | 20 + .../execution/operator/LimitOperatorTest.java | 8 +- .../execution/operator/LinearFillOperatorTest.java | 35 + .../execution/operator/MergeSortOperatorTest.java | 50 +- .../execution/operator/OffsetOperatorTest.java | 26 +- .../execution/operator/OperatorMemoryTest.java | 22 +- .../operator/RawDataAggregationOperatorTest.java | 12 +- .../SeriesAggregationScanOperatorTest.java | 12 +- .../operator/SingleDeviceViewOperatorTest.java | 8 +- .../SlidingWindowAggregationOperatorTest.java | 1 + .../execution/operator/SortOperatorTest.java | 8 +- .../execution/operator/TopKOperatorTest.java | 62 +- .../operator/UpdateLastCacheOperatorTest.java | 6 +- .../process/join/InnerTimeJoinOperatorTest.java | 1652 +++++++++++++++ .../join/LeftOuterTimeJoinOperatorTest.java | 631 ++++++ .../schema/SchemaFetchScanOperatorTest.java | 4 +- .../schema/SchemaQueryScanOperatorTest.java | 7 +- .../ConvertPredicateToTimeFilterTest.java | 46 +- .../plan/analyze/ExpressionAnalyzerTest.java | 2 +- .../plan/analyze/FakeSchemaFetcherImpl.java | 8 +- .../plan/analyze/QueryTimePartitionTest.java | 108 +- .../optimization/ColumnInjectionPushDownTest.java | 657 ++++++ .../plan/optimization/LimitOffsetPushDownTest.java | 45 +- .../plan/optimization/OptimizationTestUtil.java | 75 + .../plan/optimization/TestPlanBuilder.java | 155 +- .../db/queryengine/plan/plan/ExpressionTest.java | 68 - .../plan/plan/FragmentInstanceIdTest.java | 38 - .../plan/plan/FragmentInstanceSerdeTest.java | 159 -- .../queryengine/plan/plan/LogicalPlannerTest.java | 843 -------- .../queryengine/plan/plan/PipelineBuilderTest.java | 949 --------- .../plan/plan/QueryLogicalPlanUtil.java | 1016 ---------- .../db/queryengine/plan/plan/QueryPlannerTest.java | 106 - .../distribution/AggregationDistributionTest.java | 910 --------- .../AlignByDeviceOrderByLimitOffsetTest.java | 278 --- .../plan/distribution/AlignedByDeviceTest.java | 1491 -------------- .../distribution/DistributionPlannerBasicTest.java | 306 --- .../distribution/DistributionPlannerCycleTest.java | 82 - .../plan/plan/distribution/LastQueryTest.java | 215 -- .../distribution/NoDataRegionPlanningTest.java | 51 - .../queryengine/plan/plan/distribution/Util.java | 417 ---- .../queryengine/plan/plan/distribution/Util2.java | 311 --- .../plan/plan/node/PlanGraphPrinterTest.java | 70 - .../plan/plan/node/PlanNodeDeserializeHelper.java | 37 - .../plan/plan/node/load/LoadTsFileNodeTest.java | 89 - .../read/DeviceSchemaScanNodeSerdeTest.java | 85 - .../NodeManagementMemoryMergeNodeSerdeTest.java | 122 -- .../read/PathsUsingTemplateScanNodeTest.java | 54 - .../metadata/read/SchemaCountNodeSerdeTest.java | 148 -- .../metadata/read/SchemaFetchMergeNodeTest.java | 60 - .../metadata/read/SchemaFetchScanNodeTest.java | 57 - .../read/TimeSeriesSchemaScanNodeSerdeTest.java | 86 - .../metadata/write/MetadataWriteNodeSerDeTest.java | 385 ---- .../node/process/AggregationNodeSerdeTest.java | 315 --- .../plan/node/process/DeviceViewNodeSerdeTest.java | 61 - .../plan/node/process/ExchangeNodeSerdeTest.java | 69 - .../plan/plan/node/process/FillNodeSerdeTest.java | 54 - .../plan/node/process/FilterNodeSerdeTest.java | 63 - .../node/process/GroupByLevelNodeSerdeTest.java | 116 -- .../plan/node/process/GroupByTagNodeSerdeTest.java | 131 -- .../plan/plan/node/process/IntoNodeSerdeTest.java | 83 - .../plan/plan/node/process/LimitNodeSerdeTest.java | 58 - .../plan/node/process/OffsetNodeSerdeTest.java | 56 - .../process/SingleDeviceViewNodeSerdeTest.java | 61 - .../plan/plan/node/process/SortNodeSerdeTest.java | 64 - .../plan/node/process/TimeJoinNodeSerdeTest.java | 67 - .../plan/node/sink/IdentitySinkNodeSerdeTest.java | 60 - .../node/sink/ShuffleSinkHandleNodeSerdeTest.java | 59 - .../source/SeriesAggregationScanNodeSerdeTest.java | 75 - .../plan/node/source/SeriesScanNodeSerdeTest.java | 55 - .../plan/node/write/DeleteDataNodeSerdeTest.java | 67 - .../write/InsertMultiTabletsNodeSerdeTest.java | 96 - .../plan/node/write/InsertRowNodeSerdeTest.java | 202 -- .../plan/node/write/InsertRowsNodeSerdeTest.java | 72 - .../write/InsertRowsOfOneDeviceNodeSerdeTest.java | 82 - .../plan/node/write/InsertTabletNodeSerdeTest.java | 178 -- .../plan/node/write/WritePlanNodeSplitTest.java | 404 ---- .../plan/statement/sys/pipe/PipeStatementTest.java | 90 - .../queryengine/plan/planner/ExpressionTest.java | 68 + .../plan/planner/FragmentInstanceIdTest.java | 38 + .../plan/planner/FragmentInstanceSerdeTest.java | 160 ++ .../plan/planner/LogicalPlannerTest.java | 842 ++++++++ .../plan/planner/PipelineBuilderTest.java | 1116 +++++++++++ .../plan/planner/QueryLogicalPlanUtil.java | 1016 ++++++++++ .../queryengine/plan/planner/QueryPlannerTest.java | 106 + .../distribution/AggregationDistributionTest.java | 917 +++++++++ .../AlignByDeviceOrderByLimitOffsetTest.java | 277 +++ .../planner/distribution/AlignedByDeviceTest.java | 1490 ++++++++++++++ .../distribution/DistributionPlannerBasicTest.java | 305 +++ .../distribution/DistributionPlannerCycleTest.java | 81 + .../plan/planner/distribution/LastQueryTest.java | 214 ++ .../distribution/NoDataRegionPlanningTest.java | 50 + .../distribution/SplitTimePartitionTest.java | 273 +++ .../plan/planner/distribution/Util.java | 419 ++++ .../plan/planner/distribution/Util2.java | 313 +++ .../plan/planner/node/PlanGraphPrinterTest.java | 70 + .../planner/node/PlanNodeDeserializeHelper.java | 37 + .../plan/planner/node/load/LoadTsFileNodeTest.java | 89 + .../read/DeviceSchemaScanNodeSerdeTest.java | 85 + .../NodeManagementMemoryMergeNodeSerdeTest.java | 122 ++ .../read/PathsUsingTemplateScanNodeTest.java | 54 + .../metadata/read/SchemaCountNodeSerdeTest.java | 148 ++ .../metadata/read/SchemaFetchMergeNodeTest.java | 61 + .../metadata/read/SchemaFetchScanNodeTest.java | 59 + .../read/TimeSeriesSchemaScanNodeSerdeTest.java | 86 + .../metadata/write/MetadataWriteNodeSerDeTest.java | 385 ++++ .../node/process/AggregationNodeSerdeTest.java | 315 +++ .../node/process/DeviceViewNodeSerdeTest.java | 63 + .../node/process/ExchangeNodeSerdeTest.java | 70 + .../planner/node/process/FillNodeSerdeTest.java | 55 + .../planner/node/process/FilterNodeSerdeTest.java | 64 + .../node/process/GroupByLevelNodeSerdeTest.java | 116 ++ .../node/process/GroupByTagNodeSerdeTest.java | 131 ++ .../planner/node/process/IntoNodeSerdeTest.java | 83 + .../planner/node/process/LimitNodeSerdeTest.java | 58 + .../planner/node/process/OffsetNodeSerdeTest.java | 56 + .../process/SingleDeviceViewNodeSerdeTest.java | 61 + .../planner/node/process/SortNodeSerdeTest.java | 64 + .../node/process/TimeJoinNodeSerdeTest.java | 106 + .../node/sink/IdentitySinkNodeSerdeTest.java | 60 + .../node/sink/ShuffleSinkHandleNodeSerdeTest.java | 59 + .../source/SeriesAggregationScanNodeSerdeTest.java | 75 + .../node/source/SeriesScanNodeSerdeTest.java | 55 + .../node/write/DeleteDataNodeSerdeTest.java | 67 + .../write/InsertMultiTabletsNodeSerdeTest.java | 96 + .../planner/node/write/InsertRowNodeSerdeTest.java | 202 ++ .../node/write/InsertRowsNodeSerdeTest.java | 72 + .../write/InsertRowsOfOneDeviceNodeSerdeTest.java | 82 + .../node/write/InsertTabletNodeSerdeTest.java | 178 ++ .../planner/node/write/WritePlanNodeSplitTest.java | 404 ++++ .../statement/sys/pipe/PipeStatementTest.java | 90 + .../storageengine/dataregion/DataRegionTest.java | 163 +- .../dataregion/LastFlushTimeMapTest.java | 2 + .../iotdb/db/storageengine/dataregion/TTLTest.java | 3 + .../compaction/CompactionOverlapCheckTest.java | 269 +++ .../compaction/CompactionSchedulerTest.java | 24 +- .../CompactionSchedulerWithFastPerformerTest.java | 24 +- ...InconsistentCompressionTypeAndEncodingTest.java | 4 +- .../compaction/ReadChunkInnerCompactionTest.java | 49 + .../ReadPointCompactionPerformerTest.java | 2 +- .../cross/CrossSpaceCompactionExceptionTest.java | 79 + .../inner/InnerSpaceCompactionSelectorTest.java | 63 +- .../compaction/utils/CompactionCheckerUtils.java | 7 +- .../utils/CompactionUpdateFileCountTest.java | 2 +- .../dataregion/memtable/PrimitiveMemTableTest.java | 17 +- .../dataregion/memtable/TsFileProcessorTest.java | 16 +- .../reader/chunk/AlignedMemPageReaderTest.java | 250 +++ .../reader/chunk/MemAlignedChunkLoaderTest.java | 15 +- .../read/reader/chunk/MemChunkLoaderTest.java | 25 +- .../read/reader/chunk/MemPageReaderTest.java | 134 ++ .../series/AbstractAlignedSeriesScanTest.java | 435 ++++ .../read/reader/series/AbstractSeriesScanTest.java | 236 +++ .../AlignedSeriesScanLimitOffsetPushDownTest.java | 404 +--- .../AlignedSeriesScanPredicatePushDownTest.java | 225 +++ ...gleColumnSeriesScanLimitOffsetPushDownTest.java | 201 ++ .../series/SeriesScanLimitOffsetPushDownTest.java | 210 +- .../series/SeriesScanPredicatePushDownTest.java | 318 +++ .../dataregion/snapshot/IoTDBSnapshotTest.java | 4 + .../wal/recover/file/TsFilePlanRedoerTest.java | 25 +- .../datanode1conf/iotdb-datanode.properties | 2 +- .../src/test/resources/datanode1conf/logback.xml | 122 +- .../datanode2conf/iotdb-datanode.properties | 2 +- .../src/test/resources/datanode2conf/logback.xml | 122 +- .../datanode3conf/iotdb-datanode.properties | 2 +- .../src/test/resources/datanode3conf/logback.xml | 122 +- .../datanode/src/test/resources/logback-test.xml | 2 + iotdb-core/node-commons/pom.xml | 5 + .../resources/conf/iotdb-cluster.properties | 30 + .../resources/conf/iotdb-common.properties | 30 +- .../src/assembly/resources/sbin/clean-all.bat | 38 + .../src/assembly/resources/sbin/clean-all.sh | 95 + .../assembly/resources/sbin/clean-confignode.bat | 82 + .../assembly/resources/sbin/clean-confignode.sh | 55 + .../src/assembly/resources/sbin/clean-datanode.bat | 203 ++ .../src/assembly/resources/sbin/clean-datanode.sh | 70 + .../src/assembly/resources/sbin/start-all.sh | 81 + .../src/assembly/resources/sbin/stop-all.sh | 79 + .../iotdb/commons/client/ClientPoolFactory.java | 2 +- .../async/AsyncPipeDataTransferServiceClient.java | 8 + .../commons/concurrent/IoTDBThreadPoolFactory.java | 22 + .../iotdb/commons/concurrent/ThreadName.java | 40 +- .../threadpool/WrappedThreadPoolExecutor.java | 18 + .../apache/iotdb/commons/conf/CommonConfig.java | 30 +- .../iotdb/commons/conf/CommonDescriptor.java | 125 +- .../iotdb/commons/partition/DataPartition.java | 77 +- .../commons/partition/DataPartitionTable.java | 16 +- .../commons/partition/SeriesPartitionTable.java | 26 +- .../apache/iotdb/commons/path/PathPatternTree.java | 8 + .../agent/plugin/PipeConnectorConstructor.java | 53 + .../agent/plugin/PipeExtractorConstructor.java | 53 + .../commons/pipe/agent/plugin/PipePluginAgent.java | 117 ++ .../pipe/agent/plugin/PipePluginConstructor.java | 99 + .../agent/plugin/PipeProcessorConstructor.java | 50 + .../commons/pipe/agent/task/PipeTaskAgent.java | 798 ++++++++ .../iotdb/commons/pipe/config/PipeConfig.java | 18 +- .../config/constant/PipeConnectorConstant.java | 111 + .../config/constant/PipeExtractorConstant.java | 76 + .../config/constant/PipeProcessorConstant.java | 41 + .../configuraion/PipeTaskRuntimeConfiguration.java | 42 + .../env/PipeTaskConnectorRuntimeEnvironment.java | 27 + .../env/PipeTaskExtractorRuntimeEnvironment.java | 37 + .../env/PipeTaskProcessorRuntimeEnvironment.java | 27 + .../plugin/env/PipeTaskRuntimeEnvironment.java | 49 + .../env/PipeTaskTemporaryRuntimeEnvironment.java | 28 + .../client/IoTDBThriftSyncConnectorClient.java | 86 + .../request/IoTDBConnectorRequestVersion.java | 35 + .../connector/payload/request/PipeRequestType.java | 69 + .../request/PipeTransferSnapshotPieceReq.java | 132 ++ .../request/PipeTransferSnapshotSealReq.java | 119 ++ .../payload/request/TransferConfigPlanReq.java | 24 + .../response/PipeTransferSnapshotPieceResp.java | 81 + .../ConcurrentIterableLinkedQueue.java | 345 ++++ .../execution/executor/PipeSubtaskExecutor.java | 158 ++ .../execution/scheduler/PipeSubtaskScheduler.java | 90 + .../commons/pipe/metric/PipeEventCounter.java | 43 + .../commons/pipe/metric/PipeFakeEventCounter.java | 34 + .../pipe/plugin/builtin/BuiltinPipePlugin.java | 83 +- .../builtin/connector/DoNothingConnector.java | 72 - .../builtin/connector/IoTDBAirGapConnector.java | 28 - .../connector/IoTDBLegacyPipeConnector.java | 28 - .../connector/IoTDBThriftAsyncConnector.java | 28 - .../builtin/connector/IoTDBThriftConnector.java | 28 - .../connector/IoTDBThriftSyncConnector.java | 28 - .../plugin/builtin/connector/OpcUaConnector.java | 28 - .../builtin/connector/WebSocketConnector.java | 28 - .../builtin/connector/WriteBackConnector.java | 28 - .../connector/donothing/DoNothingConnector.java | 72 + .../builtin/connector/iotdb/IoTDBConnector.java | 157 ++ .../iotdb/airgap/IoTDBAirGapConnector.java | 30 + .../iotdb/thrift/IoTDBLegacyPipeConnector.java | 30 + .../connector/iotdb/thrift/IoTDBMetaConnector.java | 263 +++ .../iotdb/thrift/IoTDBSchemaRegionConnector.java | 22 + .../iotdb/thrift/IoTDBThriftAsyncConnector.java | 28 + .../iotdb/thrift/IoTDBThriftConnector.java | 30 + .../iotdb/thrift/IoTDBThriftSslConnector.java | 30 + .../iotdb/thrift/IoTDBThriftSyncConnector.java | 28 + .../builtin/connector/opcua/OpcUaConnector.java | 30 + .../connector/websocket/WebSocketConnector.java | 30 + .../connector/writeback/WriteBackConnector.java | 30 + .../plugin/builtin/extractor/IoTDBExtractor.java | 63 - .../extractor/donothing/DoNothingExtractor.java | 55 + .../builtin/extractor/iotdb/IoTDBExtractor.java | 64 + .../extractor/iotdb/IoTDBMetaExtractor.java | 150 ++ .../builtin/processor/DoNothingProcessor.java | 67 - .../builtin/processor/PlaceHolderProcessor.java | 68 + .../processor/donothing/DoNothingProcessor.java | 67 + .../downsampling/DownSamplingProcessor.java | 30 + .../meta/ConfigNodePipePluginMetaKeeper.java | 2 +- .../plugin/meta/DataNodePipePluginMetaKeeper.java | 45 +- .../pipe/plugin/meta/PipePluginMetaKeeper.java | 28 + .../iotdb/commons/pipe/task/DecoratingLock.java | 44 + .../iotdb/commons/pipe/task/EventSupplier.java | 34 + .../apache/iotdb/commons/pipe/task/PipeTask.java | 31 + .../iotdb/commons/pipe/task/PipeTaskManager.java | 121 ++ .../pipe/task/connection/BlockingPendingQueue.java | 130 ++ .../connection/BoundedBlockingPendingQueue.java | 32 + .../connection/UnboundedBlockingPendingQueue.java | 40 + .../iotdb/commons/pipe/task/meta/PipeMeta.java | 29 + .../commons/pipe/task/meta/PipeMetaKeeper.java | 32 + .../commons/pipe/task/meta/PipeStaticMeta.java | 6 +- .../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 8 + .../commons/pipe/task/stage/PipeTaskStage.java | 144 ++ .../commons/pipe/task/subtask/PipeSubtask.java | 155 ++ .../iotdb/commons/schema/filter/SchemaFilter.java | 3 + .../commons/schema/filter/SchemaFilterFactory.java | 5 + .../commons/schema/filter/SchemaFilterType.java | 3 + .../commons/schema/filter/SchemaFilterVisitor.java | 5 + .../commons/schema/filter/impl/TemplateFilter.java | 73 + .../apache/iotdb/commons/schema/node/IMNode.java | 12 +- .../node/common/AbstractAboveDatabaseMNode.java | 18 +- .../node/common/AbstractDatabaseDeviceMNode.java | 299 --- .../schema/node/common/AbstractDatabaseMNode.java | 26 +- .../schema/node/common/AbstractDeviceMNode.java | 289 --- .../node/common/AbstractMeasurementMNode.java | 19 +- .../schema/node/common/DeviceMNodeWrapper.java | 263 +++ .../commons/schema/node/role/IDeviceMNode.java | 2 +- .../commons/schema/node/role/IInternalMNode.java | 55 + .../commons/schema/node/visitor/MNodeVisitor.java | 7 - .../apache/iotdb/commons/service/JMXService.java | 8 +- .../iotdb/commons/service/metric/enums/Metric.java | 27 +- .../apache/iotdb/commons/utils/NodeUrlUtils.java | 7 +- .../commons/utils/ThriftConfigNodeSerDeUtils.java | 3 + .../iotdb/commons/pipe/PipeMetaDeSerTest.java | 133 -- .../ConcurrentIterableLinkedQueueTest.java | 439 ++++ .../pipe/plugin/builtin/BuiltinPipePluginTest.java | 10 +- .../commons/pipe/task/meta/PipeMetaDeSerTest.java | 128 ++ iotdb-core/tsfile/pom.xml | 56 +- .../antlr4/org/apache/tsfile/parser/PathLexer.g4 | 212 ++ .../antlr4/org/apache/tsfile/parser/PathParser.g4 | 67 + .../tsfile/common/constant/TsFileConstant.java | 4 - .../apache/iotdb/tsfile/compress/ICompressor.java | 14 +- .../iotdb/tsfile/compress/IUnCompressor.java | 9 +- .../iotdb/tsfile/file/header/ChunkGroupHeader.java | 2 +- .../iotdb/tsfile/file/header/PageHeader.java | 23 +- .../tsfile/file/metadata/AlignedChunkMetadata.java | 44 +- .../file/metadata/AlignedTimeSeriesMetadata.java | 59 +- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 42 + .../iotdb/tsfile/file/metadata/IChunkMetadata.java | 6 +- .../iotdb/tsfile/file/metadata/IMetadata.java | 44 + .../tsfile/file/metadata/ITimeSeriesMetadata.java | 9 +- .../tsfile/file/metadata/MetadataIndexEntry.java | 7 + .../tsfile/file/metadata/MetadataIndexNode.java | 13 + .../tsfile/file/metadata/TimeseriesMetadata.java | 59 + .../apache/iotdb/tsfile/read/TimeValuePair.java | 20 + .../read/TsFileAlignedSeriesReaderIterator.java | 3 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 177 +- ...leSequenceReaderTimeseriesMetadataIterator.java | 69 +- .../apache/iotdb/tsfile/read/common/TimeRange.java | 10 +- .../iotdb/tsfile/read/common/block/TsBlock.java | 32 +- .../tsfile/read/common/block/TsBlockBuilder.java | 7 +- .../read/common/parser/PathNodesGenerator.java | 8 +- .../tsfile/read/common/parser/PathVisitor.java | 16 +- .../read/controller/CachedChunkLoaderImpl.java | 4 +- .../iotdb/tsfile/read/controller/IChunkLoader.java | 3 +- .../read/filter/basic/BinaryLogicalFilter.java | 68 + .../filter/basic/DisableStatisticsTimeFilter.java | 33 + .../filter/basic/DisableStatisticsValueFilter.java | 46 + .../iotdb/tsfile/read/filter/basic/Filter.java | 80 +- .../filter/basic/IDisableStatisticsTimeFilter.java | 31 - .../basic/IDisableStatisticsValueFilter.java | 33 - .../tsfile/read/filter/basic/IStatefulFilter.java | 32 - .../tsfile/read/filter/basic/ITimeFilter.java | 33 - .../tsfile/read/filter/basic/IValueFilter.java | 40 - .../tsfile/read/filter/basic/OperatorType.java | 68 +- .../iotdb/tsfile/read/filter/basic/TimeFilter.java | 68 + .../tsfile/read/filter/basic/ValueFilter.java | 137 ++ .../tsfile/read/filter/factory/FilterFactory.java | 21 +- .../tsfile/read/filter/factory/TimeFilter.java | 101 - .../tsfile/read/filter/factory/TimeFilterApi.java | 101 + .../tsfile/read/filter/factory/ValueFilter.java | 168 -- .../tsfile/read/filter/factory/ValueFilterApi.java | 186 ++ .../iotdb/tsfile/read/filter/operator/And.java | 30 +- .../tsfile/read/filter/operator/GroupByFilter.java | 8 +- .../read/filter/operator/GroupByMonthFilter.java | 18 +- .../iotdb/tsfile/read/filter/operator/Not.java | 40 +- .../iotdb/tsfile/read/filter/operator/Or.java | 30 +- .../read/filter/operator/TimeFilterOperators.java | 196 +- .../read/filter/operator/ValueFilterOperators.java | 649 +++--- .../filter/operator/base/BinaryLogicalFilter.java | 70 - .../filter/operator/base/ColumnCompareFilter.java | 51 - .../operator/base/ColumnPatternMatchFilter.java | 50 - .../filter/operator/base/ColumnRangeFilter.java | 51 - .../read/filter/operator/base/ColumnSetFilter.java | 50 - .../tsfile/read/reader/IAlignedPageReader.java | 29 - .../iotdb/tsfile/read/reader/IPageReader.java | 9 +- .../read/reader/chunk/AbstractChunkReader.java | 83 + .../read/reader/chunk/AlignedChunkReader.java | 474 ++--- .../tsfile/read/reader/chunk/ChunkReader.java | 239 +-- .../tsfile/read/reader/page/AlignedPageReader.java | 517 ++--- .../iotdb/tsfile/read/reader/page/PageReader.java | 402 ++-- .../tsfile/read/reader/page/ValuePageReader.java | 3 +- .../reader/series/AbstractFileSeriesReader.java | 16 +- .../read/reader/series/EmptyFileSeriesReader.java | 4 +- .../read/reader/series/FileSeriesReader.java | 6 +- .../reader/series/FileSeriesReaderByTimestamp.java | 4 +- .../read/reader/series/PaginationController.java | 1 - .../org/apache/iotdb/tsfile/utils/BytesUtils.java | 10 + .../apache/iotdb/tsfile/utils/Preconditions.java | 39 + .../apache/iotdb/tsfile/utils/TimeDuration.java | 93 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 3 - .../iotdb/tsfile/common/block/TsBlockTest.java | 77 + .../file/metadata/TimeseriesMetadataTest.java | 34 +- .../tsfile/file/metadata/utils/TestHelper.java | 4 +- .../apache/iotdb/tsfile/read/ExpressionTest.java | 36 +- .../iotdb/tsfile/read/ReadInPartitionTest.java | 8 +- .../org/apache/iotdb/tsfile/read/ReadTest.java | 60 +- .../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 64 +- .../apache/iotdb/tsfile/read/TsFileReaderTest.java | 66 +- ...quenceReaderTimeseriesMetadataIteratorTest.java | 8 +- .../apache/iotdb/tsfile/read/common/PathTest.java | 32 + .../tsfile/read/filter/FilterSerializeTest.java | 70 +- .../iotdb/tsfile/read/filter/FilterTestUtil.java | 105 + .../tsfile/read/filter/GroupByFilterTest.java | 33 +- .../tsfile/read/filter/GroupByMonthFilterTest.java | 35 +- .../read/filter/IExpressionOptimizerTest.java | 83 +- .../read/filter/MinTimeMaxTimeFilterTest.java | 70 +- .../iotdb/tsfile/read/filter/OperatorTest.java | 95 +- .../filter/PredicateRemoveNotRewriterTest.java | 62 +- .../tsfile/read/filter/StatisticsFilterTest.java | 300 +-- .../read/query/executor/QueryExecutorTest.java | 11 +- .../read/query/timegenerator/ReadWriteTest.java | 10 +- .../timegenerator/TimeGeneratorReadEmptyTest.java | 10 +- .../timegenerator/TimeGeneratorReadWriteTest.java | 10 +- .../query/timegenerator/TimeGeneratorTest.java | 10 +- .../read/reader/AlignedPageReaderPushDownTest.java | 322 +++ .../iotdb/tsfile/read/reader/ChunkReaderTest.java | 4 +- .../tsfile/read/reader/PageReaderPushDownTest.java | 145 ++ .../iotdb/tsfile/read/reader/PageReaderTest.java | 6 +- .../iotdb/tsfile/read/reader/ReaderTest.java | 8 +- .../iotdb/tsfile/utils/TimeDurationTest.java | 40 +- .../tsfile/write/TsFileIntegrityCheckingTool.java | 7 +- iotdb-doap.rdf | 33 + .../openapi/src/main/openapi3/iotdb_rest_v1.yaml | 52 - .../openapi/src/main/openapi3/iotdb_rest_v2.yaml | 4 + .../src/main/thrift/confignode.thrift | 52 +- .../src/main/thrift/datanode.thrift | 9 +- pom.xml | 27 +- 1159 files changed, 65411 insertions(+), 34782 deletions(-) diff --cc integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java index 78cda6b40e6,b87f5df60f3..ed451eb920a --- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java @@@ -575,24 -842,121 +842,137 @@@ public class IoTDBAlignByDeviceWithTemp }; resultSetEqualTest( "select count(s1+1) from root.sg1.** align by device;", expectedHeader, retArray); + expectedHeader = new String[] {"Device,count(s1 + 1)"}; + retArray = + new String[] { + "root.sg2.d1,2,", "root.sg2.d2,2,", "root.sg2.d3,2,", "root.sg2.d4,2,", + }; + resultSetEqualTest( + "select count(s1+1) from root.sg2.** align by device;", expectedHeader, retArray); + + assertTestFail( + "select s1 from root.sg1.** where s1 align by device;", + "The output type of the expression in WHERE clause should be BOOLEAN, actual data type: FLOAT."); + + assertTestFail( + "select s1 from root.sg2.** where s1 align by device;", + "The output type of the expression in WHERE clause should be BOOLEAN, actual data type: FLOAT."); + } + + @Test + public void sLimitOffsetTest() { + // 1. original + String[] expectedHeader = new String[] {"Time,Device,s1"}; + String[] retArray = + new String[] { + "1,root.sg1.d1,1.1,", + "2,root.sg1.d1,2.2,", + "1,root.sg1.d2,11.1,", + "2,root.sg1.d2,22.2,", + "1,root.sg1.d3,111.1,", + "4,root.sg1.d3,444.4,", + "1,root.sg1.d4,1111.1,", + "5,root.sg1.d4,5555.5,", + }; + resultSetEqualTest( + "SELECT * FROM root.sg1.** slimit 1 soffset 1 ALIGN BY DEVICE;", expectedHeader, retArray); + retArray = + new String[] { + "1,root.sg2.d1,1.1,", + "2,root.sg2.d1,2.2,", + "1,root.sg2.d2,11.1,", + "2,root.sg2.d2,22.2,", + "1,root.sg2.d3,111.1,", + "4,root.sg2.d3,444.4,", + "1,root.sg2.d4,1111.1,", + "5,root.sg2.d4,5555.5,", + }; + resultSetEqualTest( + "SELECT * FROM root.sg2.** slimit 1 soffset 1 ALIGN BY DEVICE;", expectedHeader, retArray); + + expectedHeader = new String[] {"Time,Device,s1,s2"}; + retArray = + new String[] { + "1,root.sg1.d1,1.1,false,", + "2,root.sg1.d1,2.2,false,", + "1,root.sg1.d2,11.1,false,", + "2,root.sg1.d2,22.2,false,", + "1,root.sg1.d3,111.1,true,", + "4,root.sg1.d3,444.4,true,", + "1,root.sg1.d4,1111.1,true,", + "5,root.sg1.d4,5555.5,false,", + }; + resultSetEqualTest( + "SELECT *, s1 FROM root.sg1.** slimit 2 soffset 1 ALIGN BY DEVICE;", + expectedHeader, + retArray); + retArray = + new String[] { + "1,root.sg2.d1,1.1,false,", + "2,root.sg2.d1,2.2,false,", + "1,root.sg2.d2,11.1,false,", + "2,root.sg2.d2,22.2,false,", + "1,root.sg2.d3,111.1,true,", + "4,root.sg2.d3,444.4,true,", + "1,root.sg2.d4,1111.1,true,", + "5,root.sg2.d4,5555.5,false,", + }; + resultSetEqualTest( + "SELECT *, s1 FROM root.sg2.** slimit 2 soffset 1 ALIGN BY DEVICE;", + expectedHeader, + retArray); + + expectedHeader = new String[] {"Time,Device,s1"}; + retArray = + new String[] { + "1,root.sg1.d1,1.1,", "2,root.sg1.d1,2.2,", "1,root.sg1.d2,11.1,", "2,root.sg1.d2,22.2,", + }; + resultSetEqualTest( + "SELECT * FROM root.sg1.d1,root.sg1.d2,root.sg1.d6 soffset 1 slimit 1 ALIGN BY DEVICE;", + expectedHeader, + retArray); + retArray = + new String[] { + "1,root.sg2.d1,1.1,", "2,root.sg2.d1,2.2,", "1,root.sg2.d2,11.1,", "2,root.sg2.d2,22.2,", + }; + resultSetEqualTest( + "SELECT * FROM root.sg2.d1,root.sg2.d2,root.sg2.d6 soffset 1 slimit 1 ALIGN BY DEVICE;", + expectedHeader, + retArray); + } + + @Test + public void emptyResultTest() { + String[] expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + String[] retArray = new String[] {}; + resultSetEqualTest( + "SELECT * FROM root.sg1.** where time>=now()-1d and time<=now() " + + "ORDER BY TIME DESC ALIGN BY DEVICE;", + expectedHeader, + retArray); + resultSetEqualTest( + "SELECT * FROM root.sg2.** where time>=now()-1d and time<=now() " + + "ORDER BY TIME DESC ALIGN BY DEVICE;", + expectedHeader, + retArray); } + @Test + public void emptyResultTest() { + String[] expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + String[] retArray = new String[] {}; + resultSetEqualTest( + "SELECT * FROM root.sg1.** where time>=now()-1d and time<=now() " + + "ORDER BY TIME DESC ALIGN BY DEVICE;", + expectedHeader, + retArray); + resultSetEqualTest( + "SELECT * FROM root.sg2.** where time>=now()-1d and time<=now() " + + "ORDER BY TIME DESC ALIGN BY DEVICE;", + expectedHeader, + retArray); + } + private static void insertData() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/factory/LogicalViewMNode.java index 00000000000,00000000000..2aeb63c3bb7 new file mode 100644 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/factory/LogicalViewMNode.java @@@ -1,0 -1,0 +1,36 @@@ ++package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.factory; ++ ++import org.apache.iotdb.commons.schema.node.common.AbstractMeasurementMNode; ++import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; ++import org.apache.iotdb.commons.schema.node.utils.IMNodeContainer; ++import org.apache.iotdb.commons.schema.view.LogicalViewSchema; ++import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; ++import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.basic.BasicMNode; ++import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.container.MemMNodeContainer; ++import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.LogicalViewInfo; ++ ++public class LogicalViewMNode extends AbstractMeasurementMNode<IMemMNode, BasicMNode> ++ implements IMemMNode { ++ ++ public LogicalViewMNode( ++ IDeviceMNode<IMemMNode> parent, String name, LogicalViewSchema measurementSchema) { ++ super( ++ new BasicMNode(parent == null ? null : parent.getAsMNode(), name), ++ new LogicalViewInfo(measurementSchema)); ++ } ++ ++ @Override ++ public IMNodeContainer<IMemMNode> getChildren() { ++ return MemMNodeContainer.emptyMNodeContainer(); ++ } ++ ++ @Override ++ public IMemMNode getAsMNode() { ++ return this; ++ } ++ ++ @Override ++ public final boolean isLogicalView() { ++ return true; ++ } ++}
