This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/TableModelGrammar in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 39912f3ce8044f5813f13c793512c74716055ed6 Merge: e400a45fd90 347e42d59c7 Author: JackieTien97 <[email protected]> AuthorDate: Sun Apr 7 11:51:13 2024 +0800 Merge remote-tracking branch 'origin/master' into ty/TableModelGrammar .github/workflows/pipe-it-2cluster.yml | 84 +- .../apache/iotdb/opcua/ClientExampleRunner.java | 13 +- .../java/org/apache/iotdb/opcua/ClientTest.java | 19 +- .../iotdb/opcua/IoTDBKeyStoreLoaderClient.java | 14 +- .../apache/iotdb/SubscriptionSessionExample.java | 145 +++ .../apache/iotdb/tsfile/TsFileSequenceRead.java | 3 +- .../java/org/apache/iotdb/udf/UDAFExample.java | 16 +- integration-test/pom.xml | 34 +- .../apache/iotdb/db/query/udf/example/UDAFAvg.java | 16 +- .../iotdb/db/query/udf/example/UDAFCount.java | 2 +- .../apache/iotdb/db/query/udf/example/UDAFSum.java | 16 +- .../iotdb/it/env/cluster/config/MppBaseConfig.java | 4 +- .../iotdb/it/env/cluster/env/AbstractEnv.java | 44 +- .../it/env/cluster/node/AbstractNodeWrapper.java | 15 +- .../iotdb/it/env/remote/env/RemoteServerEnv.java | 16 + ...2.java => MultiClusterIT2AutoCreateSchema.java} | 2 +- ...java => MultiClusterIT2ManualCreateSchema.java} | 2 +- ...erIT2.java => MultiClusterIT2Subscription.java} | 2 +- .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 7 + .../apache/iotdb/cli/it/StartClientScriptIT.java | 11 +- ...oTDBMultiDBRegionGroupLeaderDistributionIT.java | 154 +++ ...a => IoTDBRegionGroupLeaderDistributionIT.java} | 123 +- .../partition/IoTDBAutoRegionGroupExtensionIT.java | 119 +- .../confignode/it/procedure/IoTDBProcedureIT.java | 6 +- .../IoTDBRegionMigrateReliabilityITFramework.java | 511 ++++++++ .../CoordinatorRemoveRemotePeerCrashIT.java | 51 + .../IoTDBRegionMigrateDataNodeCrashIT.java | 72 ++ .../pass/IoTDBRegionMigrateConfigNodeCrashIT.java | 92 ++ .../pass/IoTDBRegionMigrateNormalIT.java | 25 +- .../pass/IoTDBRegionMigrateOtherIT.java | 45 + .../db/it/IOTDBInsertWithTimeAtAnyIndexIT.java | 6 +- .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 67 + .../java/org/apache/iotdb/db/it/IoTDBFilterIT.java | 4 +- .../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 6 +- .../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 2 +- .../db/it/aligned/IoTDBInsertAlignedValuesIT.java | 30 +- .../scalar/IoTDBSubStringFunctionIT.java | 36 +- .../it/last/IoTDBLastQueryWithLimitOffsetIT.java | 6 +- .../iotdb/db/it/last/IoTDBLastWithTTLIT.java | 8 +- .../iotdb/db/it/query/IoTDBCaseWhenThenIT.java | 4 +- .../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 16 +- .../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 4 +- .../iotdb/db/it/udf/IoTDBUDFManagementIT.java | 2 +- .../org/apache/iotdb/db/it/utils/TestUtils.java | 28 + .../pipe/it/autocreate/AbstractPipeDualAutoIT.java | 16 +- .../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java | 171 ++- .../it/autocreate/IoTDBPipeAutoConflictIT.java | 76 +- .../pipe/it/autocreate/IoTDBPipeClusterIT.java | 222 ++-- .../autocreate/IoTDBPipeConnectorParallelIT.java | 22 +- .../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 36 +- .../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 204 +-- .../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 51 +- .../pipe/it/autocreate/IoTDBPipeLifeCycleIT.java | 184 +-- .../pipe/it/autocreate/IoTDBPipeNullValueIT.java | 33 +- .../it/autocreate/IoTDBPipePatternFormatIT.java | 61 +- .../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 44 +- .../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 106 +- .../it/autocreate/IoTDBPipeSwitchStatusIT.java | 62 +- .../pipe/it/autocreate/IoTDBPipeSyntaxIT.java | 222 ++-- .../pipe/it/manual/AbstractPipeDualManualIT.java | 16 +- .../iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java | 54 +- .../pipe/it/manual/IoTDBPipeManualConflictIT.java | 58 +- .../pipe/it/manual/IoTDBPipeMetaHistoricalIT.java | 239 ++++ .../it/manual/IoTDBPipeMetaLeaderChangeIT.java | 40 +- .../pipe/it/manual/IoTDBPipeMetaRestartIT.java | 46 +- .../it/manual/IoTDBPipeMultiSchemaRegionIT.java | 35 +- .../AbstractPipeSingleIT.java} | 21 +- .../iotdb/pipe/it/single/IoTDBPipeAggregateIT.java | 121 ++ .../it/{ => single}/IoTDBPipeSingleEnvDemoIT.java | 30 +- .../session/it/IoTDBSessionAlignedInsertIT.java | 4 +- .../iotdb/session/it/IoTDBSessionSimpleIT.java | 51 +- .../it/dual/AbstractSubscriptionDualIT.java} | 5 +- .../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 826 ++++++++++++ .../it/dual/IoTDBSubscriptionTopicIT.java | 376 ++++++ .../it/local/IoTDBSubscriptionBasicIT.java | 157 +++ .../it/local/IoTDBSubscriptionIdempotentIT.java | 162 +++ .../it/local/IoTDBSubscriptionRestartIT.java | 200 +++ ...{ExportCsvTestIT.java => ExportDataTestIT.java} | 59 +- ...{ImportCsvTestIT.java => ImportDataTestIT.java} | 6 +- .../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 8 +- .../org/apache/iotdb/tsfile/utils/Accountable.java | 13 +- .../iotdb/tsfile/utils/RamUsageEstimator.java | 4 +- .../org/apache/iotdb/pipe/api/PipeExtractor.java | 2 +- .../parameter/PipeParameterValidator.java | 22 +- .../api/customizer/parameter/PipeParameters.java | 21 +- .../event/dml/insertion/TabletInsertionEvent.java | 12 +- .../main/java/org/apache/iotdb/udf/api/UDAF.java | 2 +- iotdb-client/cli/pom.xml | 4 + .../cli/src/assembly/resources/sbin/start-cli.sh | 4 +- .../tools/{export-csv.bat => export-data.bat} | 2 +- .../tools/{export-csv.sh => export-data.sh} | 2 +- .../tools/{import-csv.bat => import-data.bat} | 2 +- .../tools/{import-csv.sh => import-data.sh} | 2 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 8 +- ...{AbstractCsvTool.java => AbstractDataTool.java} | 7 +- .../iotdb/tool/{ExportCsv.java => ExportData.java} | 178 ++- .../iotdb/tool/{ImportCsv.java => ImportData.java} | 64 +- ...riteCsvFileTest.java => WriteDataFileTest.java} | 6 +- iotdb-client/client-py/tests/test_dataframe.py | 2 +- .../apache/iotdb/isession/ISessionDataSet.java} | 15 +- .../org/apache/iotdb/isession/SessionConfig.java | 5 +- .../org/apache/iotdb/isession/SessionDataSet.java | 6 +- iotdb-client/service-rpc/pom.xml | 5 + .../java/org/apache/iotdb/rpc/TSStatusCode.java | 4 +- .../rpc/subscription}/SubscriptionException.java | 15 +- .../rpc}/subscription/config/ConsumerConfig.java | 32 +- .../rpc/subscription/config/ConsumerConstant.java | 50 + .../iotdb/rpc/subscription/config/TopicConfig.java | 94 ++ .../rpc}/subscription/config/TopicConstant.java | 6 +- .../payload/{response => }/EnrichedTablets.java | 53 +- .../payload/request/ConsumerConfig.java | 85 -- .../payload/request/PipeSubscribeCommitReq.java | 35 +- .../payload/request/PipeSubscribeHandshakeReq.java | 3 +- .../payload/request/PipeSubscribeHeartbeatReq.java | 3 +- .../payload/request/PipeSubscribePollReq.java | 30 +- .../payload/request/PipeSubscribeSubscribeReq.java | 16 +- .../request/PipeSubscribeUnsubscribeReq.java | 16 +- .../response/PipeSubscribeHandshakeResp.java | 42 +- .../payload/response/PipeSubscribePollResp.java | 78 +- .../java/org/apache/iotdb/session/Session.java | 250 +++- .../apache/iotdb/session/SessionConnection.java | 4 +- .../org/apache/iotdb/session/pool/SessionPool.java | 11 + .../subscription/ConsumerHeartbeatWorker.java | 53 + .../subscription/PullConsumerAutoCommitWorker.java | 64 + .../session/subscription/SubscriptionConsumer.java | 279 ++++ .../session/subscription/SubscriptionMessage.java | 85 ++ .../subscription/SubscriptionMessagePayload.java | 4 +- .../session/subscription/SubscriptionProvider.java | 91 ++ .../subscription/SubscriptionPullConsumer.java | 310 +++++ .../subscription/SubscriptionPushConsumer.java | 39 +- .../session/subscription/SubscriptionSession.java | 185 +++ .../SubscriptionSessionConnection.java | 167 +++ .../subscription/SubscriptionSessionDataSet.java | 175 +++ .../subscription/SubscriptionSessionDataSets.java | 53 + .../session/subscription/model/Subscription.java | 41 +- .../iotdb/session/subscription/model/Topic.java} | 28 +- .../iotdb/session/SessionCacheLeaderTest.java | 6 +- .../java/org/apache/iotdb/session/SessionTest.java | 5 +- .../apache/iotdb/session/pool/SessionPoolTest.java | 2 + .../flink/sql/client/IoTDBWebSocketClient.java | 9 +- .../flink/sql/function/IoTDBCDCSourceFunction.java | 1 + .../tsfile/qp/optimizer/PhysicalOptimizer.java | 14 +- .../apache/iotdb/spark/tsfile/DefaultSource.scala | 3 +- .../confignode/client/DataNodeRequestType.java | 6 +- .../client/async/AsyncDataNodeClientPool.java | 35 +- .../client/async/handlers/AsyncClientHandler.java | 5 +- .../heartbeat/DataNodeHeartbeatHandler.java | 4 + .../ConsumerGroupPushMetaRPCHandler.java | 4 +- .../client/sync/SyncDataNodeClientPool.java | 5 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 4 +- .../confignode/conf/ConfigNodeDescriptor.java | 11 +- .../confignode/conf/ConfigNodeRemoveCheck.java | 4 +- .../confignode/conf/ConfigNodeStartupCheck.java | 2 +- .../confignode/conf/SystemPropertiesUtils.java | 7 +- .../consensus/request/ConfigPhysicalPlan.java | 30 +- .../consensus/request/ConfigPhysicalPlanType.java | 11 +- .../consensus/request/auth/AuthorPlan.java | 22 +- ...ocationPlan.java => AddRegionLocationPlan.java} | 40 +- ...tionPlan.java => RemoveRegionLocationPlan.java} | 46 +- .../write/partition/UpdateRegionLocationPlan.java | 1 + .../pipe/runtime/PipeHandleMetaChangePlan.java | 18 + .../pipe/task/OperateMultiplePipesPlanV2.java | 125 ++ .../write/procedure/UpdateProcedurePlan.java | 9 +- .../runtime/ConsumerGroupHandleMetaChangePlan.java | 85 ++ .../topic/AlterMultipleTopicsPlan.java | 101 ++ .../topic/runtime/TopicHandleMetaChangePlan.java} | 53 +- .../response/pipe/task/PipeTableResp.java | 3 + .../subscription/SubscriptionTableResp.java | 20 +- .../statemachine/ConfigRegionStateMachine.java | 68 +- .../iotdb/confignode/manager/ConfigManager.java | 20 +- .../apache/iotdb/confignode/manager/IManager.java | 9 - .../confignode/manager/PermissionManager.java | 1 + .../iotdb/confignode/manager/ProcedureManager.java | 424 +++--- .../manager/consensus/ConsensusManager.java | 5 + .../manager/load/balancer/RegionBalancer.java | 9 +- .../manager/load/balancer/RouteBalancer.java | 28 +- .../region/GreedyCopySetRegionGroupAllocator.java | 100 +- .../region/GreedyRegionGroupAllocator.java | 40 +- .../balancer/region/IRegionGroupAllocator.java | 3 + .../router/leader/GreedyLeaderBalancer.java | 129 +- .../balancer/router/leader/ILeaderBalancer.java | 6 +- .../router/leader/MinCostFlowLeaderBalancer.java | 211 +-- .../confignode/manager/load/cache/LoadCache.java | 14 + .../manager/load/service/HeartbeatService.java | 25 + .../manager/load/service/StatisticsService.java | 26 +- .../iotdb/confignode/manager/node/NodeManager.java | 15 +- .../manager/partition/PartitionManager.java | 70 +- .../{transfer => }/agent/PipeConfigNodeAgent.java | 10 +- .../agent/plugin/PipeConfigNodePluginAgent.java | 12 +- .../PipeConfigRegionConnectorConstructor.java | 8 +- .../PipeConfigRegionExtractorConstructor.java | 6 +- .../PipeConfigRegionProcessorConstructor.java | 4 +- .../receiver/IoTDBConfigNodeReceiverAgent.java | 26 +- .../agent/runtime/PipeConfigNodeRuntimeAgent.java | 12 +- .../agent/runtime/PipeConfigRegionListener.java | 6 +- .../agent/task/PipeConfigNodeTaskAgent.java | 15 +- .../client/IoTDBConfigNodeSyncClientManager.java | 14 +- .../PipeTransferConfigNodeHandshakeV1Req.java | 2 +- .../PipeTransferConfigNodeHandshakeV2Req.java | 2 +- .../payload}/PipeTransferConfigPlanReq.java | 2 +- .../PipeTransferConfigSnapshotPieceReq.java | 2 +- .../payload/PipeTransferConfigSnapshotSealReq.java | 113 ++ .../protocol/IoTDBConfigRegionAirGapConnector.java | 186 +++ .../protocol}/IoTDBConfigRegionConnector.java | 149 +-- .../runtime/PipeHeartbeatScheduler.java | 2 +- .../runtime/PipeLeaderChangeHandler.java | 23 +- .../pipe/coordinator/runtime/PipeMetaSyncer.java | 2 +- .../runtime/PipeRuntimeCoordinator.java | 6 +- .../pipe/coordinator/task/PipeTaskCoordinator.java | 4 - .../pipe/event/PipeConfigRegionSnapshotEvent.java | 189 ++- .../execution/PipeConfigNodeSubtask.java | 35 +- .../execution/PipeConfigNodeSubtaskExecutor.java | 2 +- .../extractor/ConfigRegionListeningFilter.java | 15 +- .../extractor/ConfigRegionListeningQueue.java | 42 +- .../extractor/IoTDBConfigRegionExtractor.java | 51 +- .../protocol}/IoTDBConfigNodeReceiver.java | 126 +- .../PipeConfigPhysicalPlanExceptionVisitor.java | 2 +- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 14 +- .../PipeConfigNodeCopiedFileDirStartupCleaner.java | 52 + .../{transfer => }/task/PipeConfigNodeTask.java | 2 +- .../task/PipeConfigNodeTaskBuilder.java | 11 +- .../task/PipeConfigNodeTaskStage.java | 10 +- .../config/IoTDBConfigRegionAirGapConnector.java | 160 --- .../request/PipeTransferConfigSnapshotSealReq.java | 70 - .../manager/schema/ClusterSchemaManager.java | 2 +- .../subscription/SubscriptionCoordinator.java | 33 +- .../subscription/SubscriptionMetaSyncer.java | 117 ++ .../iotdb/confignode/persistence/AuthorInfo.java | 4 +- .../confignode/persistence/ProcedureInfo.java | 237 +++- .../persistence/executor/ConfigPlanExecutor.java | 31 +- .../partition/DatabasePartitionTable.java | 16 +- .../persistence/partition/PartitionInfo.java | 88 +- .../confignode/persistence/pipe/PipeInfo.java | 13 +- .../persistence/pipe/PipePluginInfo.java | 4 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 55 +- .../schema/CNPhysicalPlanGenerator.java | 423 ++++++ .../persistence/schema/CNSnapshotFileType.java} | 35 +- .../schema/ConfignodeSnapshotParser.java | 167 +++ .../persistence/schema/TemplateTable.java | 2 +- .../persistence/subscription/SubscriptionInfo.java | 200 ++- .../procedure/CompletedProcedureRecycler.java | 6 +- .../iotdb/confignode/procedure/Procedure.java | 14 +- .../confignode/procedure/ProcedureExecutor.java | 117 +- .../procedure/TimeoutExecutorThread.java | 2 +- .../procedure/env/ConfigNodeProcedureEnv.java | 185 ++- ...moveHandler.java => RegionMaintainHandler.java} | 181 ++- .../procedure/impl/StateMachineProcedure.java | 14 +- .../impl/node/AddConfigNodeProcedure.java | 1 - .../impl/node/RemoveConfigNodeProcedure.java | 1 - .../impl/node/RemoveDataNodeProcedure.java | 22 +- .../impl/pipe/AbstractOperatePipeProcedureV2.java | 20 +- .../impl/pipe/runtime/PipeMetaSyncProcedure.java | 26 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 31 +- .../impl/pipe/task/DropPipeProcedureV2.java | 16 +- .../impl/pipe/task/StartPipeProcedureV2.java | 4 +- .../impl/pipe/task/StopPipeProcedureV2.java | 4 +- .../impl/region/AddRegionPeerProcedure.java | 207 +++ .../impl/region/RegionMigrateProcedure.java | 184 +-- .../impl/region/RemoveRegionPeerProcedure.java | 192 +++ .../AbstractOperateSubscriptionProcedure.java | 80 +- .../impl/subscription/SubscriptionOperation.java | 2 + .../consumer/AlterConsumerGroupProcedure.java | 13 +- .../consumer/CreateConsumerProcedure.java | 6 +- .../runtime/ConsumerGroupMetaSyncProcedure.java | 158 +++ ...bstractOperateSubscriptionAndPipeProcedure.java | 159 +++ .../subscription/CreateSubscriptionProcedure.java | 226 +++- .../subscription/DropSubscriptionProcedure.java | 244 +++- .../subscription/topic/AlterTopicProcedure.java | 22 +- .../subscription/topic/CreateTopicProcedure.java | 2 +- .../subscription/topic/DropTopicProcedure.java | 2 +- .../topic/runtime/TopicMetaSyncProcedure.java | 157 +++ .../impl/sync/AuthOperationProcedure.java | 21 +- .../AddNeverFinishSubProcedureProcedure.java | 6 + .../impl/testonly/NeverFinishProcedure.java | 17 + .../procedure/state/AddRegionPeerState.java | 19 +- .../procedure/state/RegionTransitionState.java | 5 +- ...sitionState.java => RemoveRegionPeerState.java} | 8 +- .../procedure/store/ConfigProcedureStore.java | 44 +- .../procedure/store/IProcedureStore.java | 17 +- .../procedure/store/ProcedureFactory.java | 24 + .../confignode/procedure/store/ProcedureType.java | 8 +- .../confignode/procedure/store/ProcedureWAL.java | 46 +- .../iotdb/confignode/service/ConfigNode.java | 11 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 6 - .../request/ConfigPhysicalPlanSerDeTest.java | 140 +- .../confignode/consensus/request/TestOnlyPlan.java | 19 +- .../region/AllocatorScatterWidthManualTest.java | 5 + .../GreedyCopySetRegionGroupAllocatorTest.java | 115 +- .../region/GreedyRegionGroupAllocatorTest.java | 4 + ...alancerTest.java => CFDLeaderBalancerTest.java} | 66 +- .../router/leader/GreedyLeaderBalancerTest.java | 6 +- .../leader/LeaderBalancerComparisonTest.java | 4 +- .../connector/IoTDBConfigRegionConnectorTest.java | 2 +- .../PipeConfigNodeThriftRequestTest.java | 25 +- .../PipeConfigNodeSubtaskExecutorTest.java | 2 - .../extractor/IoTDBConfigRegionExtractorTest.java | 1 - .../persistence/CNPhysicalPlanGeneratorTest.java | 472 +++++++ .../confignode/persistence/ClusterInfoTest.java | 2 +- .../ConfigRegionListeningQueueTest.java | 2 +- .../confignode/persistence/ProcedureInfoTest.java | 75 ++ .../confignode/procedure/NoopProcedureStore.java | 31 +- .../UpgradeFromWALToConsensusLayerTest.java | 137 ++ ...reTest.java => AddRegionPeerProcedureTest.java} | 16 +- .../impl/region/RegionMigrateProcedureTest.java | 16 +- ...est.java => RemoveRegionPeerProcedureTest.java} | 16 +- .../ConsumerGroupMetaSyncProcedureTest.java | 55 + .../CreateSubscriptionProcedureTest.java | 10 +- .../DropSubscriptionProcedureTest.java | 3 +- .../topic/runtime/TopicMetaSyncProcedureTest.java | 55 + .../procedure/util/ProcedureTestUtil.java | 11 - .../org/apache/iotdb/consensus/IConsensus.java | 19 +- .../org/apache/iotdb/consensus/common/Peer.java | 21 + .../apache/iotdb/consensus/iot/IoTConsensus.java | 66 +- .../consensus/iot/IoTConsensusServerImpl.java | 131 +- .../consensus/iot/client/DispatchLogHandler.java | 23 +- .../service/IoTConsensusRPCServiceProcessor.java | 5 + .../apache/iotdb/consensus/ratis/DiskGuardian.java | 19 +- .../iotdb/consensus/ratis/RatisConsensus.java | 39 +- .../iotdb/consensus/ratis/SnapshotStorage.java | 8 +- .../iotdb/consensus/ratis/utils/Retriable.java | 11 +- .../apache/iotdb/consensus/ratis/utils/Utils.java | 1 + .../iotdb/consensus/simple/SimpleConsensus.java | 9 +- .../apache/iotdb/consensus/iot/StabilityTest.java | 10 +- .../iotdb/consensus/ratis/DiskGuardianTest.java | 4 +- .../iotdb/consensus/ratis/RatisConsensusTest.java | 2 +- .../apache/iotdb/consensus/ratis/TestUtils.java | 6 +- .../consensus/simple/SimpleConsensusTest.java | 2 +- .../src/assembly/resources/conf/datanode-env.bat | 36 +- .../src/assembly/resources/conf/datanode-env.sh | 38 +- .../org/apache/iotdb/db/audit/AuditLogger.java | 12 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 100 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 42 +- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 34 +- .../db/conf/rest/IoTDBRestServiceDescriptor.java | 4 +- .../dataregion/DataExecutionVisitor.java | 11 +- .../schemaregion/SchemaRegionStateMachine.java | 68 +- .../PipeDataRegionConnectorConstructor.java | 2 + .../dataregion/PipeDataRegionPluginAgent.java | 10 +- .../PipeDataRegionProcessorConstructor.java | 11 + .../PipeSchemaRegionConnectorConstructor.java | 2 +- .../PipeSchemaRegionExtractorConstructor.java | 2 +- .../schemaregion/PipeSchemaRegionPluginAgent.java | 10 +- .../PipeSchemaRegionProcessorConstructor.java | 2 +- .../agent/receiver/PipeDataNodeReceiverAgent.java | 27 +- .../agent/runtime/PipeDataNodeRuntimeAgent.java | 16 +- .../agent/runtime/PipePeriodicalJobExecutor.java | 9 +- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 3 +- .../client/IoTDBDataNodeAsyncClientManager.java | 87 +- .../client/IoTDBDataNodeSyncClientManager.java | 5 +- .../request/PipeTransferSchemaSnapshotSealReq.java | 52 +- .../PipeTransferTsFilePieceWithModReq.java} | 24 +- .../request/PipeTransferTsFileSealReq.java | 4 +- ....java => PipeTransferTsFileSealWithModReq.java} | 40 +- .../connector/payload/legacy/DeletionPipeData.java | 4 +- .../db/pipe/connector/payload/legacy/PipeData.java | 2 +- .../connector/payload/legacy/TsFilePipeData.java | 4 +- .../airgap/IoTDBDataNodeAirGapConnector.java | 16 +- .../airgap/IoTDBDataRegionAirGapConnector.java | 122 +- .../airgap/IoTDBSchemaRegionAirGapConnector.java | 111 +- .../protocol/legacy/IoTDBLegacyPipeConnector.java | 6 +- .../connector/protocol/opcua/OpcUaConnector.java | 24 +- .../protocol/opcua/OpcUaKeyStoreLoader.java | 20 +- .../protocol/opcua/OpcUaServerBuilder.java | 44 +- .../async/IoTDBDataRegionAsyncConnector.java | 11 +- .../PipeTransferTabletBatchEventHandler.java | 13 +- .../PipeTransferTabletInsertionEventHandler.java | 11 +- .../PipeTransferTsFileInsertionEventHandler.java | 82 +- .../thrift/sync/IoTDBDataNodeSyncConnector.java | 22 +- .../thrift/sync/IoTDBDataRegionSyncConnector.java | 178 +-- .../thrift/sync/IoTDBSchemaRegionConnector.java | 113 +- .../protocol/websocket/WebSocketConnector.java | 6 +- .../protocol/writeback/WriteBackConnector.java | 2 +- .../pipe/event/common/row/PipeResetTabletRow.java | 54 + .../db/pipe/event/common/row/PipeRowCollector.java | 9 +- .../schema/PipeSchemaRegionSnapshotEvent.java | 157 ++- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 14 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 23 +- .../tablet/TabletInsertionDataContainer.java | 2 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 134 +- .../tsfile/TsFileInsertionDataContainer.java | 40 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 2 +- .../PipeConnectorSubtaskExecutor.java | 6 +- .../PipeProcessorSubtaskExecutor.java | 2 +- .../{executor => }/PipeSubtaskExecutorManager.java | 10 +- .../dataregion/IoTDBDataRegionExtractor.java | 32 +- .../PipeHistoricalDataRegionTsFileExtractor.java | 80 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 39 +- .../PipeRealtimeDataRegionLogExtractor.java | 2 +- .../realtime/assigner/PipeDataRegionAssigner.java | 11 +- .../realtime/epoch/TsFileEpochManager.java | 8 +- .../listener/PipeInsertionDataNodeListener.java | 4 +- .../schemaregion/IoTDBSchemaRegionExtractor.java | 56 +- .../schemaregion/SchemaRegionListeningFilter.java | 6 +- .../schemaregion/SchemaRegionListeningQueue.java | 18 +- .../{matcher => }/CachedSchemaPatternMatcher.java | 2 +- .../{matcher => }/PipeDataRegionMatcher.java | 2 +- .../aggregate/AbstractFormalProcessor.java | 54 + .../processor/aggregate/AggregateProcessor.java | 725 +++++++++++ .../aggregate/TimeSeriesRuntimeState.java | 396 ++++++ .../aggregatedresult/AggregatedResultOperator.java | 62 + .../standardstatistics/AverageOperator.java | 56 + .../ClearanceFactorOperator.java | 60 + .../standardstatistics/CrestFactorOperator.java | 59 + .../standardstatistics/FormFactorOperator.java | 57 + .../standardstatistics/KurtosisOperator.java | 67 + .../standardstatistics/PeakOperator.java | 52 + .../standardstatistics/PulseFactorOperator.java | 57 + .../standardstatistics/RootMeanSquareOperator.java | 57 + .../standardstatistics/SkewnessOperator.java | 63 + .../standardstatistics/VarianceOperator.java | 59 + .../CustomizedReadableIntermediateResults.java | 158 +++ .../IntermediateResultOperator.java | 113 ++ .../sametype/numeric/AbsoluteMaxOperator.java | 71 + .../numeric/AbstractSameTypeNumericOperator.java | 152 +++ .../numeric/IntegralPoweredSumOperator.java | 77 ++ .../doubletype/FractionPoweredSumOperator.java | 137 ++ .../specifictype/integertype/CountOperator.java | 125 ++ .../processor/AbstractOperatorProcessor.java | 73 ++ .../StandardStatisticsOperatorProcessor.java | 76 ++ .../window/datastructure/TimeSeriesWindow.java | 492 +++++++ .../window/datastructure/WindowOutput.java | 71 + .../window/datastructure/WindowState.java | 75 ++ .../AbstractSimpleTimeWindowingProcessor.java | 102 ++ .../processor/AbstractWindowingProcessor.java | 119 ++ .../processor/TumblingWindowingProcessor.java | 110 ++ .../sdt/SwingingDoorTrendingFilter.java | 8 +- .../sdt/SwingingDoorTrendingSamplingProcessor.java | 6 +- .../SimpleConsensusProgressIndexAssigner.java | 8 +- .../{ => protocol}/airgap/IoTDBAirGapReceiver.java | 63 +- .../airgap/IoTDBAirGapReceiverAgent.java | 2 +- .../legacy/IoTDBLegacyPipeReceiverAgent.java | 2 +- .../legacy/loader/DeletionLoader.java | 2 +- .../{ => protocol}/legacy/loader/ILoader.java | 2 +- .../{ => protocol}/legacy/loader/TsFileLoader.java | 2 +- .../thrift/IoTDBDataNodeReceiver.java | 129 +- .../thrift/IoTDBDataNodeReceiverAgent.java | 2 +- .../{ => visitor}/PipePlanToStatementVisitor.java | 7 +- .../PipeStatementExceptionVisitor.java | 56 +- .../PipeStatementTSStatusVisitor.java | 31 +- ...NodeHardlinkOrCopiedFileDirStartupCleaner.java} | 26 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 2 +- .../pipe/resource/memory/PipeMemoryWeighUtil.java | 17 +- .../pipe/resource/tsfile/PipeTsFileResource.java | 16 +- .../resource/tsfile/PipeTsFileResourceManager.java | 75 +- .../db/pipe/task/builder/PipeDataNodeBuilder.java | 28 +- .../pipe/task/builder/PipeDataNodeTaskBuilder.java | 52 +- .../pipe/task/connection/PipeEventCollector.java | 2 +- .../db/pipe/task/stage/PipeTaskConnectorStage.java | 21 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 49 +- .../subtask/connector/PipeConnectorSubtask.java | 4 +- .../connector/PipeConnectorSubtaskLifeCycle.java | 6 +- .../connector/PipeConnectorSubtaskManager.java | 4 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 13 +- .../iotdb/db/protocol/client/ConfigNodeInfo.java | 43 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 36 +- .../impl/DataNodeInternalRPCServiceImpl.java | 215 +++- .../db/queryengine/common/DataNodeEndPoints.java | 26 + .../execution/aggregation/UDAFAccumulator.java | 8 + .../queryengine/execution/driver/DataDriver.java | 2 + .../db/queryengine/execution/driver/Driver.java | 2 +- .../fragment/FragmentInstanceContext.java | 1 + .../fragment/FragmentInstanceExecution.java | 2 +- .../fragment/FragmentInstanceManager.java | 12 +- .../execution/load/LoadTsFileManager.java | 5 +- .../queryengine/execution/load/TsFileSplitter.java | 24 +- .../schema/source/TimeSeriesSchemaSource.java | 2 +- .../execution/operator/source/FileLoaderUtils.java | 6 +- .../execution/operator/source/SeriesScanUtil.java | 100 +- .../iotdb/db/queryengine/plan/Coordinator.java | 3 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 11 +- .../plan/analyze/LoadTsfileAnalyzer.java | 51 +- .../analyze/cache/partition/PartitionCache.java | 12 +- .../analyze/schema/AutoCreateSchemaExecutor.java | 6 +- .../config/executor/ClusterConfigTaskExecutor.java | 191 ++- .../config/metadata/ShowFunctionsTask.java | 3 +- .../plan/optimization/LimitOffsetPushDown.java | 48 +- .../OrderByExpressionWithLimitChangeToTopK.java | 68 +- .../db/queryengine/plan/parser/ASTVisitor.java | 81 +- .../SimpleFragmentParallelPlanner.java | 5 +- .../plan/planner/plan/node/PlanNodeType.java | 4 +- .../plan/node/load/LoadSingleTsFileNode.java | 5 +- .../node/metedata/write/DeleteTimeSeriesNode.java | 6 +- .../write/InternalCreateTimeSeriesNode.java | 6 +- .../plan/node/pipe/PipeEnrichedInsertNode.java | 2 +- .../plan/planner/plan/node/write/InsertNode.java | 38 +- .../planner/plan/node/write/InsertTabletNode.java | 3 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 23 +- .../queryengine/plan/statement/StatementType.java | 12 +- .../metadata/AlterTimeSeriesStatement.java | 4 +- .../metadata/CreateAlignedTimeSeriesStatement.java | 2 +- .../metadata/CreateMultiTimeSeriesStatement.java | 2 +- .../metadata/CreateTimeSeriesStatement.java | 2 +- .../metadata/DeleteTimeSeriesStatement.java | 2 +- .../apache/iotdb/db/schemaengine/SchemaEngine.java | 13 +- .../schemaengine/metric/SchemaRegionMemMetric.java | 2 +- .../rescon/DataNodeSchemaQuotaManager.java | 30 +- .../rescon/ISchemaRegionStatistics.java | 2 +- .../rescon/MemSchemaEngineStatistics.java | 29 +- .../rescon/MemSchemaRegionStatistics.java | 40 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 37 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 49 +- .../schemaregion/mtree/traverser/Traverser.java | 13 +- .../schemaregion/utils/ResourceByPathUtils.java | 8 +- .../java/org/apache/iotdb/db/service/DataNode.java | 76 +- .../apache/iotdb/db/service/IoTDBShutdownHook.java | 25 +- .../iotdb/db/service/RegionMigrateService.java | 169 ++- .../iotdb/db/storageengine/StorageEngine.java | 8 +- .../buffer/TimeSeriesMetadataCache.java | 29 +- .../db/storageengine/dataregion/DataRegion.java | 141 +- .../dataregion/DeviceLastFlushTime.java | 8 +- .../dataregion/HashLastFlushTimeMap.java | 45 +- .../storageengine/dataregion/ILastFlushTime.java | 6 +- .../dataregion/ILastFlushTimeMap.java | 16 +- .../dataregion/PartitionLastFlushTime.java | 6 +- .../performer/impl/FastCompactionPerformer.java | 9 +- .../impl/ReadChunkCompactionPerformer.java | 56 +- .../impl/ReadPointCompactionPerformer.java | 20 +- .../execute/task/AbstractCompactionTask.java | 5 +- .../execute/task/InnerSpaceCompactionTask.java | 4 + .../subtask/FastCompactionPerformerSubTask.java | 7 +- .../task/subtask/ReadPointPerformerSubTask.java | 5 +- .../execute/utils/CompactionPathUtils.java | 12 +- .../compaction/execute/utils/CompactionUtils.java | 3 +- .../execute/utils/MultiTsFileDeviceIterator.java | 29 +- .../fast/AlignedSeriesCompactionExecutor.java | 25 +- .../fast/NonAlignedSeriesCompactionExecutor.java | 3 +- .../executor/fast/SeriesCompactionExecutor.java | 5 +- .../readchunk/AlignedSeriesCompactionExecutor.java | 8 +- .../ReadChunkAlignedSeriesCompactionExecutor.java | 21 +- .../readchunk/SingleSeriesCompactionExecutor.java | 8 +- .../utils/writer/AbstractCompactionWriter.java | 10 +- .../writer/AbstractCrossCompactionWriter.java | 3 +- .../writer/AbstractInnerCompactionWriter.java | 3 +- .../compaction/io/CompactionTsFileReader.java | 59 +- .../compaction/repair/RepairDataFileScanUtil.java | 30 +- .../repair/RepairTimePartitionScanTask.java | 12 + .../schedule/CompactionScheduleTaskManager.java | 4 +- .../DefaultCompactionTaskComparatorImpl.java | 21 +- .../estimator/AbstractCompactionEstimator.java | 5 +- .../estimator/CompactionEstimateUtils.java | 5 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 7 +- .../utils/CrossSpaceCompactionCandidate.java | 3 +- .../compaction/selector/utils/DeviceInfo.java | 6 +- .../selector/utils/TsFileResourceCandidate.java | 11 +- .../compaction/tool/SingleSequenceFileTask.java | 3 +- .../compaction/tool/TsFileStatisticReader.java | 11 +- .../compaction/tool/UnseqSpaceStatistics.java | 18 +- .../dataregion/flush/MemTableFlushTask.java | 14 +- .../dataregion/memtable/AbstractMemTable.java | 41 +- .../dataregion/memtable/DeviceIDFactory.java | 7 +- .../dataregion/memtable/IMemTable.java | 3 +- .../dataregion/memtable/PrimitiveMemTable.java | 2 + .../dataregion/memtable/TsFileProcessor.java | 219 ++-- .../dataregion/read/QueryDataSource.java | 115 +- .../dataregion/read/control/FileReaderManager.java | 4 +- .../dataregion/snapshot/SnapshotTaker.java | 2 +- .../dataregion/tsfile/TsFileManager.java | 8 +- .../dataregion/tsfile/TsFileResource.java | 33 +- .../tsfile/timeindex/DeviceTimeIndex.java | 56 +- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 23 +- .../dataregion/tsfile/timeindex/ITimeIndex.java | 29 +- .../dataregion/utils/TsFileResourceUtils.java | 25 +- .../storageengine/dataregion/wal/WALManager.java | 5 +- .../wal/allocation/FirstCreateStrategy.java | 2 +- .../dataregion/wal/recover/WALNodeRecoverTask.java | 2 +- .../wal/recover/file/TsFilePlanRedoer.java | 14 +- .../file/UnsealedTsFileRecoverPerformer.java | 30 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 2 +- .../dataregion/wal/utils/WALWriteUtils.java | 18 + .../quotas/DataNodeThrottleQuotaManager.java | 14 +- .../db/subscription/agent/SubscriptionAgent.java | 32 +- .../agent/SubscriptionAgentLauncher.java | 170 +++ .../agent/SubscriptionBrokerAgent.java | 133 ++ .../agent/SubscriptionConsumerAgent.java | 203 +++ .../agent/SubscriptionReceiverAgent.java | 108 ++ .../agent/SubscriptionRuntimeAgent.java | 91 ++ ...TopicAgent.java => SubscriptionTopicAgent.java} | 72 +- .../broker/EnrichedTabletsBinaryCache.java | 62 + .../broker/SerializedEnrichedEvent.java | 124 ++ .../db/subscription/broker/SubscriptionBroker.java | 124 ++ .../broker/SubscriptionPrefetchingQueue.java | 250 ++++ .../executor/SubscriptionSubtaskExecutor.java} | 14 +- .../receiver/SubscriptionReceiver.java} | 14 +- .../receiver/SubscriptionReceiverV1.java | 555 ++++++++ .../stage/SubscriptionTaskConnectorStage.java} | 42 +- .../task/subtask/SubscriptionConnectorSubtask.java | 75 ++ .../SubscriptionConnectorSubtaskLifeCycle.java} | 58 +- .../SubscriptionConnectorSubtaskManager.java} | 167 ++- .../subscription/timer/SubscriptionPollTimer.java | 79 ++ .../apache/iotdb/db/tools/IoTDBDataDirViewer.java | 5 +- .../iotdb/db/tools/TsFileResourcePrinter.java | 6 +- .../apache/iotdb/db/tools/TsFileSelfCheckTool.java | 24 +- .../apache/iotdb/db/tools/TsFileSketchTool.java | 52 +- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 27 +- .../org/apache/iotdb/db/tools/TsFileSplitTool.java | 11 +- .../db/tools/schema/SRStatementGenerator.java | 60 +- .../tools/schema/SchemaRegionSnapshotParser.java | 76 +- .../TsFileOverlapValidationAndRepairTool.java | 14 +- .../db/tools/validate/TsFileValidationTool.java | 15 +- .../java/org/apache/iotdb/db/utils/MemUtils.java | 9 +- .../apache/iotdb/db/utils/TypeInferenceUtils.java | 40 +- .../db/metadata/idtable/entry/DeviceIDTest.java | 2 +- .../schemaRegion/AbstractSchemaRegionTest.java | 22 +- .../schemaRegion/SchemaStatisticsTest.java | 12 +- .../connector/PipeDataNodeThriftRequestTest.java | 36 +- .../iotdb/db/pipe/connector/PipeReceiverTest.java | 2 +- .../event/TsFileInsertionDataContainerTest.java | 7 +- .../PipeConnectorSubtaskExecutorTest.java | 1 - .../PipeProcessorSubtaskExecutorTest.java | 1 - .../db/pipe/extractor/PipeRealtimeExtractTest.java | 6 +- .../pattern/CachedSchemaPatternMatcherTest.java | 1 - .../PipeDataNodeSnapshotResourceManagerTest.java | 25 +- .../resource/PipeTsFileResourceManagerTest.java | 2 +- .../PipeWALHardlinkResourceManagerTest.java | 2 +- .../execution/operator/AlignedSeriesTestUtil.java | 13 +- .../plan/planner/FragmentInstanceSerdeTest.java | 1 + .../db/storageengine/buffer/ChunkCacheTest.java | 5 +- .../storageengine/dataregion/DataRegionTest.java | 4 +- .../dataregion/LastFlushTimeMapTest.java | 31 +- .../TsFileResourceProgressIndexTest.java | 11 +- .../compaction/AbstractCompactionTest.java | 26 +- .../compaction/CompactionDataTypeNotMatchTest.java | 212 +++ .../compaction/CompactionValidationTest.java | 33 +- .../CrossSpaceCompactionWithUnusualCasesTest.java | 15 +- .../compaction/FastAlignedCrossCompactionTest.java | 1267 ++++++++++++------ .../FastCompactionPerformerWithEmptyPageTest.java | 4 +- ...InconsistentCompressionTypeAndEncodingTest.java | 5 +- .../FastCrossCompactionPerformerTest.java | 1152 +++++++++++------ .../FastInnerCompactionPerformerTest.java | 63 +- .../FastNonAlignedCrossCompactionTest.java | 1267 ++++++++++++------ .../compaction/ReadChunkInnerCompactionTest.java | 55 +- .../ReadPointAlignedCrossCompactionTest.java | 1267 ++++++++++++------ .../ReadPointCompactionPerformerTest.java | 1357 +++++++++++++------- .../ReadPointNonAlignedCrossCompactionTest.java | 1267 ++++++++++++------ .../compaction/TestUtilsForAlignedSeries.java | 9 +- .../dataregion/compaction/TsFileIdentifierUT.java | 2 +- .../TsFileValidationCorrectnessTests.java | 25 +- ...eCompactionWithFastPerformerValidationTest.java | 16 +- ...actionWithReadPointPerformerValidationTest.java | 11 +- .../InsertionCrossSpaceCompactionRecoverTest.java | 22 +- .../InsertionCrossSpaceCompactionSelectorTest.java | 156 +-- .../dataregion/compaction/cross/MergeTest.java | 12 +- .../compaction/cross/MergeUpgradeTest.java | 5 +- .../cross/RewriteCompactionFileSelectorTest.java | 17 +- .../inner/AbstractInnerSpaceCompactionTest.java | 5 +- .../inner/InnerCompactionMoreDataTest.java | 5 +- .../compaction/inner/InnerCompactionTest.java | 5 +- ...nkCompactionPerformerWithAlignedSeriesTest.java | 2 +- .../ReadChunkCompactionPerformerAlignedTest.java | 3 +- .../SizeTieredCompactionSelectorTest.java | 5 +- .../inner/sizetiered/SizeTieredCompactionTest.java | 5 +- .../repair/RepairDataFileScanUtilTest.java | 29 + .../repair/RepairUnsortedFileCompactionTest.java | 3 +- .../compaction/tools/UnseqSpaceStatisticsTest.java | 46 +- .../compaction/utils/CompactionCheckerUtils.java | 14 +- .../utils/CompactionFileGeneratorUtils.java | 29 +- .../compaction/utils/CompactionTestFileWriter.java | 8 +- .../utils/MultiTsFileDeviceIteratorTest.java | 144 ++- .../dataregion/memtable/MemTableTestUtils.java | 6 +- .../dataregion/memtable/TsFileProcessorTest.java | 11 +- .../series/AbstractAlignedSeriesScanTest.java | 10 +- .../read/reader/series/AbstractSeriesScanTest.java | 6 +- .../AlignedSeriesScanLimitOffsetPushDownTest.java | 3 +- .../AlignedSeriesScanPredicatePushDownTest.java | 3 +- ...gleColumnSeriesScanLimitOffsetPushDownTest.java | 3 +- .../read/reader/series/SeriesReaderTestUtil.java | 5 +- .../dataregion/snapshot/IoTDBSnapshotTest.java | 16 +- .../dataregion/tsfile/TsFileResourceTest.java | 16 +- .../wal/allocation/ElasticStrategyTest.java | 12 + .../wal/allocation/FirstCreateStrategyTest.java | 12 + .../wal/allocation/RoundRobinStrategyTest.java | 12 + .../dataregion/wal/buffer/WALBufferCommonTest.java | 3 - .../dataregion/wal/io/WALFileTest.java | 5 - .../wal/node/ConsensusReqReaderTest.java | 2 - .../dataregion/wal/node/WALEntryHandlerTest.java | 4 - .../dataregion/wal/node/WALNodeTest.java | 23 +- .../wal/node/WalDeleteOutdatedNewTest.java | 16 +- .../wal/recover/WALRecoverManagerTest.java | 18 +- .../file/SealedTsFileRecoverPerformerTest.java | 6 +- .../wal/recover/file/TsFilePlanRedoerTest.java | 54 +- .../file/UnsealedTsFileRecoverPerformerTest.java | 98 +- .../wal/utils/WALInsertNodeCacheTest.java | 4 - .../rescon/memory/ResourceManagerTest.java | 6 +- .../org/apache/iotdb/db/tools/MLogParserTest.java | 8 + .../iotdb/db/tools/TsFileAndModSettleToolTest.java | 2 +- .../TsFileOverlapValidationAndRepairToolTest.java | 97 +- .../iotdb/db/tools/TsFileSelfCheckToolTest.java | 2 +- .../iotdb/db/tools/TsFileSketchToolTest.java | 13 +- .../db/utils/SchemaRegionSnapshotParserTest.java | 20 +- .../iotdb/db/utils/TypeInferenceUtilsTest.java | 37 +- .../apache/iotdb/db/utils/sort/SortUtilTest.java | 2 +- .../apache/iotdb/metrics/utils/SystemMetric.java | 2 +- .../resources/conf/iotdb-common.properties | 26 +- .../assembly/resources/sbin/start-standalone.bat | 3 + .../assembly/resources/sbin/start-standalone.sh | 2 + .../commons/auth/authorizer/BasicAuthorizer.java | 4 +- .../auth/authorizer/LocalFileAuthorizer.java | 2 +- .../iotdb/commons/auth/entity/PathPrivilege.java | 41 +- .../iotdb/commons/auth/entity/PrivilegeType.java | 11 + .../org/apache/iotdb/commons/auth/entity/Role.java | 63 +- .../commons/auth/role/LocalFileRoleAccessor.java | 8 +- .../iotdb/commons/auth/user/BasicUserManager.java | 8 +- .../commons/auth/user/LocalFileUserAccessor.java | 16 +- .../sync/SyncThriftClientWithErrorHandler.java | 1 + .../iotdb/commons/concurrent/ThreadName.java | 3 + .../apache/iotdb/commons/conf/CommonConfig.java | 107 ++ .../iotdb/commons/conf/CommonDescriptor.java | 52 + .../apache/iotdb/commons/conf/IoTDBConstant.java | 14 +- .../iotdb/commons/consensus/ConsensusGroupId.java | 2 +- .../commons/consensus/index/ProgressIndexType.java | 8 + .../index/impl/TimeWindowStateProgressIndex.java | 280 ++++ .../{ => subscription}/SubscriptionException.java | 2 +- .../SubscriptionPollTimeOutException.java} | 13 +- .../apache/iotdb/commons/path/MeasurementPath.java | 7 + .../org/apache/iotdb/commons/path/PartialPath.java | 10 + .../apache/iotdb/commons/path/PathPatternTree.java | 54 + .../iotdb/commons/path/PathPatternTreeUtils.java | 22 +- .../commons/pipe/agent/plugin/PipePluginAgent.java | 88 +- .../pipe/agent/plugin/PipePluginConstructor.java | 2 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 5 +- .../iotdb/commons/pipe/config/PipeConfig.java | 7 + .../config/constant/PipeConnectorConstant.java | 21 + .../config/constant/PipeExtractorConstant.java | 5 + .../config/constant/PipeProcessorConstant.java | 20 + .../env/PipeTaskProcessorRuntimeEnvironment.java | 12 +- .../pipe/connector/PipeReceiverStatusHandler.java | 62 +- .../pipe/connector/client/IoTDBClientManager.java | 8 + .../pipe/connector/client/IoTDBSyncClient.java | 15 +- .../connector/client/IoTDBSyncClientManager.java | 143 ++- .../payload/thrift/request/PipeRequestType.java | 10 +- ...SealReq.java => PipeTransferFileSealReqV1.java} | 8 +- .../thrift/request/PipeTransferFileSealReqV2.java | 165 +++ .../connector/protocol/IoTDBAirGapConnector.java | 186 ++- .../pipe/connector/protocol/IoTDBConnector.java | 57 +- .../connector/protocol/IoTDBSslSyncConnector.java | 88 +- .../options/PipeInclusionOptions.java | 19 +- .../queue/ConcurrentIterableLinkedQueue.java | 18 + .../listening/AbstractPipeListeningQueue.java | 84 +- .../iotdb/commons/pipe/event/EnrichedEvent.java | 16 +- .../commons/pipe/event/PipeSnapshotEvent.java | 69 +- .../commons/pipe/extractor/IoTDBExtractor.java | 2 +- .../extractor/IoTDBNonDataRegionExtractor.java | 62 +- .../pipe/metric/PipeEventCommitMetrics.java | 2 +- .../pipe/plugin/builtin/BuiltinPipePlugin.java | 15 +- .../connector/donothing/DoNothingConnector.java | 16 +- .../processor/aggregate/AggregateProcessor.java | 22 +- .../aggregate/StandardStatisticsProcessor.java | 22 +- .../aggregate/TumblingWindowingProcessor.java | 22 +- .../{committer => }/PipeEventCommitManager.java | 2 +- .../{committer => }/PipeEventCommitter.java | 2 +- .../commons/pipe/receiver/IoTDBFileReceiver.java | 287 +++-- .../iotdb/commons/pipe/receiver/IoTDBReceiver.java | 2 +- .../commons/pipe/receiver/IoTDBReceiverAgent.java | 25 +- .../pipe/resource/PipeSnapshotResourceManager.java | 62 +- .../commons/pipe/task/meta/PipeStaticMeta.java | 13 + .../iotdb/commons/pipe/task/meta/PipeType.java | 22 +- .../commons/pipe/task/stage/PipeTaskStage.java | 4 +- .../service/AbstractThriftServiceThread.java | 4 +- .../apache/iotdb/commons/service/ServiceType.java | 2 + .../iotdb/commons/service/StartupChecks.java | 43 +- .../subscription/config/SubscriptionConfig.java | 98 ++ .../meta/consumer/ConsumerGroupMeta.java | 19 +- .../meta/consumer/ConsumerGroupMetaKeeper.java | 76 +- .../subscription/meta/consumer/ConsumerMeta.java | 4 +- .../meta/subscription/SubscriptionMeta.java | 44 +- .../commons/subscription/meta/topic/TopicMeta.java | 90 +- .../org/apache/iotdb/commons/utils/AuthUtils.java | 85 ++ .../org/apache/iotdb/commons/utils/FileUtils.java | 61 +- .../utils/KillPoint/DataNodeKillPoints.java} | 17 +- .../IoTConsensusRemovePeerKillPoints.java | 9 +- .../iotdb/commons/utils/KillPoint/KillPoint.java | 80 ++ .../utils/KillPoint/NeverTriggeredKillPoint.java | 6 +- .../commons/utils/ThriftCommonsSerDeUtils.java | 1 + .../commons/pipe/task/meta/PipeMetaDeSerTest.java | 20 + .../consumer/ConsumerGroupDeSerTest.java | 4 +- .../commons/subscription/topic/TopicDeSerTest.java | 2 +- .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 4 +- .../iotdb/tsfile/compress/IUnCompressor.java | 2 +- .../tsfile/encoding/encoder/DictionaryEncoder.java | 17 +- .../iotdb/tsfile/file/IMetadataIndexEntry.java} | 23 +- .../iotdb/tsfile/file/header/ChunkGroupHeader.java | 24 +- .../tsfile/file/metadata/ChunkGroupMetadata.java | 6 +- ...dexEntry.java => DeviceMetadataIndexEntry.java} | 55 +- .../iotdb/tsfile/file/metadata}/IDeviceID.java | 33 +- ...try.java => MeasurementMetadataIndexEntry.java} | 36 +- .../file/metadata/MetadataIndexConstructor.java | 37 +- .../tsfile/file/metadata/MetadataIndexNode.java | 51 +- .../iotdb/tsfile/file/metadata}/PlainDeviceID.java | 38 +- .../iotdb/tsfile/file/metadata/TsFileMetadata.java | 2 +- .../iotdb/tsfile/read/TsFileDeviceIterator.java | 17 +- .../org/apache/iotdb/tsfile/read/TsFileReader.java | 2 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 322 ++--- ...leSequenceReaderTimeseriesMetadataIterator.java | 41 +- .../org/apache/iotdb/tsfile/read/common/Path.java | 16 + .../read/controller/MetadataQuerierByFileImpl.java | 25 +- .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 48 +- .../apache/iotdb/tsfile/write/TsFileWriter.java | 46 +- .../write/chunk/AlignedChunkGroupWriterImpl.java | 8 +- .../chunk/NonAlignedChunkGroupWriterImpl.java | 8 +- .../apache/iotdb/tsfile/write/record/TSRecord.java | 7 + .../apache/iotdb/tsfile/write/record/Tablet.java | 70 +- .../write/writer/ForceAppendTsFileWriter.java | 5 +- .../write/writer/RestorableTsFileIOWriter.java | 10 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 31 +- .../write/writer/TsFileIOWriterEndFileTest.java | 3 +- .../write/writer/tsmiterator/TSMIterator.java | 19 +- .../org/apache/iotdb/tsfile/compress/LZ4Test.java | 21 + .../file/metadata/MetadataIndexNodeTest.java | 13 +- .../tsfile/file/metadata/utils/TestHelper.java | 4 +- .../iotdb/tsfile/read/GetAllDevicesTest.java | 8 +- ...easurementChunkMetadataListMapIteratorTest.java | 18 +- .../tsfile/read/TimeSeriesMetadataReadTest.java | 4 +- .../apache/iotdb/tsfile/read/TsFileReaderTest.java | 15 +- .../tsfile/read/TsFileSequenceReaderTest.java | 9 +- .../tsfile/write/MetadataIndexConstructorTest.java | 75 +- .../iotdb/tsfile/write/TsFileIOWriterTest.java | 15 +- .../tsfile/write/TsFileIntegrityCheckingTool.java | 10 +- .../iotdb/tsfile/write/TsFileWriteApiTest.java | 7 +- .../write/writer/RestorableTsFileIOWriterTest.java | 3 +- .../writer/TsFileIOWriterMemoryControlTest.java | 96 +- .../thrift-commons/src/main/thrift/common.thrift | 10 +- .../src/main/thrift/confignode.thrift | 10 +- .../thrift-datanode/src/main/thrift/client.thrift | 2 +- .../src/main/thrift/datanode.thrift | 82 +- .../library/frequency/UDFEnvelopeAnalysis.java | 6 +- pom.xml | 9 +- 826 files changed, 33315 insertions(+), 10312 deletions(-)
