This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch native_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from b8ea8514632 change back ratis write log buffer size
add 8c044fcfb7d Introducing new features to IoTDB JVM GC options (#12088)
add 8b5626c8922 Pipe: filtered empty tsFiles which should not be parsed or
reported (#12216)
add 56e21e4fee1 support config snapshot parser (#12211)
add fa9f21e23cd Pipe Subscription: initialize the subscription agent on DN
(#12138)
add 0a65556c6fa Fix NPE problem in EXPLAIN ANALYZE
add 9feffb1510d Allow series data type not consistent in compaction
add 1e129ec8096 Remove duplicate lookups in dictionary encoder (#12224)
add f04e020df77 fix: consider more general type of nodes above TopK-Node
as parent
add 945112d642d Pipe: Introduced aggregate processor as the universal
aggregation framework (#12210)
add 8c8f66668bd Fix procedure concurrency problem caused by non-atomic
procedure Id generation (#12229)
add 58ade690a9c fix cn parser and code style. (#12231)
add d96556cfc7e Fix envelope function execution error without frequency
parameter (#12233)
add 960891dc0c9 Log: add logs for notifyLeaderReady, notifyLeaderChanged
and procedure workers (#12235)
add 8e78fcae1f1 Use heartbeat to update DataNode's ConfigNodeList (#12232)
add 806582343b8 remove JavaUtils dep (#12236)
add 6bee4cc6b7f Warn user if they use GraalVM (#12238)
add a5df27c75cf Region migration improvement (#12165)
add 16f4a97f568 fix: remove -XX:UseBiasedLock (#12239)
add 4da4fd80a72 feat: Refactored the visibility of some methods to make it
possible to add BasicAuthorizer implementations in other packages. (#12195)
add 39118785846 Optimize SeriesScanUtil by memorizing the order time and
satisfied information for each Seq and Unseq Resource (#12227)
add 563a71492b2 Subscription: fix some issues on CN (#12217)
add ebf52326b68 change cli classpath (#12237)
add 77f17daf4d5 Fix C++ client build problem (#12241)
add 8d802e60b88 [RatisConsensus] Bump ratis snapshot version (52e8db6) to
support force snapshot (#12242)
add bd61c224742 Optimize String value type infer logic (#12223)
add 1333ea7c4d7 IoTConsensus: Skip retry sending batch caused by
TApplicationException (#12243)
add 1b67601e976 Bump jackson.version from 2.15.3 to 2.15.4 (#12190)
add ba25d64aec9 Pipe: fix deadlock caused by
PipeResourceManagerHolder.<clinit> and PipePeriodicalJobExecutor (#12245)
add 4223cff7861 Merge branch 'master' into native_raft
No new revisions were added by this update.
Summary of changes:
.../apache/iotdb/opcua/ClientExampleRunner.java | 13 +-
.../java/org/apache/iotdb/opcua/ClientTest.java | 19 +-
.../iotdb/opcua/IoTDBKeyStoreLoaderClient.java | 14 +-
.../apache/iotdb/SubscriptionSessionExample.java | 99 +++
.../iotdb/it/env/cluster/env/AbstractEnv.java | 29 +-
.../it/env/cluster/node/AbstractNodeWrapper.java | 4 +-
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 6 +
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 3 +
.../apache/iotdb/cli/it/StartClientScriptIT.java | 11 +-
.../it/IoTDBRegionMigrateReliabilityIT.java | 428 ++++++++++++
.../confignode/it/procedure/IoTDBProcedureIT.java | 2 +
.../db/it/IOTDBInsertWithTimeAtAnyIndexIT.java | 6 +-
.../java/org/apache/iotdb/db/it/IoTDBFilterIT.java | 4 +-
.../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 2 +-
.../db/it/aligned/IoTDBInsertAlignedValuesIT.java | 30 +-
.../it/last/IoTDBLastQueryWithLimitOffsetIT.java | 6 +-
.../iotdb/db/it/last/IoTDBLastWithTTLIT.java | 8 +-
.../iotdb/db/it/query/IoTDBCaseWhenThenIT.java | 4 +-
.../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 16 +-
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 4 +-
.../iotdb/db/it/udf/IoTDBUDFManagementIT.java | 2 +-
.../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 9 +-
.../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 28 +-
.../iotdb/pipe/it/single/IoTDBPipeAggregateIT.java | 121 ++++
.../it/{ => single}/IoTDBPipeSingleEnvDemoIT.java | 20 +-
.../session/it/IoTDBSessionAlignedInsertIT.java | 4 +-
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 14 +-
.../subscription/it/IoTDBSubscriptionBasicIT.java | 245 +++++++
.../it/IoTDBSubscriptionConsumerGroupIT.java | 342 ++++++++++
.../subscription/it/IoTDBSubscriptionTopicIT.java | 250 +++++++
.../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 8 +-
.../api/customizer/parameter/PipeParameters.java | 21 +-
.../cli/src/assembly/resources/sbin/start-cli.sh | 4 +-
iotdb-client/client-py/tests/test_dataframe.py | 2 +-
.../java/org/apache/iotdb/isession/ISession.java | 19 +
iotdb-client/service-rpc/pom.xml | 5 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 3 +-
.../payload}/config/ConsumerConfig.java | 32 +-
.../payload/config/ConsumerConstant.java | 13 +-
.../subscription/payload/config/TopicConfig.java | 94 +++
.../payload}/config/TopicConstant.java | 6 +-
.../payload/request/ConsumerConfig.java | 85 ---
.../payload/request/PipeSubscribeCommitReq.java | 35 +-
.../payload/request/PipeSubscribeHandshakeReq.java | 3 +-
.../payload/request/PipeSubscribePollReq.java | 18 +-
.../payload/request/PipeSubscribeSubscribeReq.java | 16 +-
.../request/PipeSubscribeUnsubscribeReq.java | 16 +-
.../payload/response/EnrichedTablets.java | 47 +-
.../response/PipeSubscribeHandshakeResp.java | 21 +-
.../payload/response/PipeSubscribePollResp.java | 55 +-
.../java/org/apache/iotdb/session/Session.java | 37 ++
.../apache/iotdb/session/SessionConnection.java | 67 ++
.../flink/sql/client/IoTDBWebSocketClient.java | 9 +-
.../confignode/client/DataNodeRequestType.java | 6 +-
.../client/async/AsyncDataNodeClientPool.java | 35 +-
.../client/async/handlers/AsyncClientHandler.java | 5 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 4 +
.../ConsumerGroupPushMetaRPCHandler.java | 4 +-
.../client/sync/SyncDataNodeClientPool.java | 5 +-
.../consensus/request/ConfigPhysicalPlan.java | 30 +-
.../consensus/request/ConfigPhysicalPlanType.java | 8 +-
.../consensus/request/auth/AuthorPlan.java | 20 +-
...ocationPlan.java => AddRegionLocationPlan.java} | 40 +-
...tionPlan.java => RemoveRegionLocationPlan.java} | 46 +-
.../write/partition/UpdateRegionLocationPlan.java | 1 +
.../pipe/runtime/PipeHandleMetaChangePlan.java | 18 +
.../pipe/task/OperateMultiplePipesPlanV2.java | 125 ++++
.../runtime/ConsumerGroupHandleMetaChangePlan.java | 85 +++
.../topic/AlterMultipleTopicsPlan.java | 101 +++
.../topic/runtime/TopicHandleMetaChangePlan.java} | 50 +-
.../response/pipe/task/PipeTableResp.java | 3 +
.../subscription/SubscriptionTableResp.java | 16 +-
.../statemachine/ConfigRegionStateMachine.java | 24 +-
.../iotdb/confignode/manager/ConfigManager.java | 10 -
.../apache/iotdb/confignode/manager/IManager.java | 9 -
.../confignode/manager/PermissionManager.java | 1 +
.../iotdb/confignode/manager/ProcedureManager.java | 388 ++++++-----
.../manager/consensus/ConsensusManager.java | 2 +-
.../confignode/manager/load/cache/LoadCache.java | 14 +
.../manager/load/service/HeartbeatService.java | 25 +
.../manager/load/service/StatisticsService.java | 26 +-
.../iotdb/confignode/manager/node/NodeManager.java | 8 +-
.../manager/partition/PartitionManager.java | 46 +-
.../pipe/coordinator/runtime/PipeMetaSyncer.java | 2 +-
.../agent/plugin/PipeConfigNodePluginAgent.java | 10 +-
.../PipeConfigRegionConnectorConstructor.java | 2 +-
.../PipeConfigRegionExtractorConstructor.java | 2 +-
.../PipeConfigRegionProcessorConstructor.java | 2 +-
.../transfer/execution/PipeConfigNodeSubtask.java | 21 +-
.../subscription/SubscriptionCoordinator.java | 28 +-
.../subscription/SubscriptionMetaSyncer.java | 117 ++++
.../confignode/persistence/ProcedureInfo.java | 16 +-
.../persistence/executor/ConfigPlanExecutor.java | 19 +
.../partition/DatabasePartitionTable.java | 16 +-
.../persistence/partition/PartitionInfo.java | 41 +-
.../confignode/persistence/pipe/PipeInfo.java | 11 +
.../persistence/pipe/PipePluginInfo.java | 4 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 55 +-
.../schema/CNPhysicalPlanGenerator.java | 390 +++++++++++
.../persistence/schema/CNSnapshotFileType.java} | 16 +-
.../schema/ConfignodeSnapshotParser.java | 167 +++++
.../persistence/subscription/SubscriptionInfo.java | 198 +++++-
.../confignode/procedure/ProcedureExecutor.java | 9 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 185 ++++--
...moveHandler.java => RegionMaintainHandler.java} | 181 ++++--
.../procedure/impl/StateMachineProcedure.java | 14 +-
.../impl/node/AddConfigNodeProcedure.java | 1 -
.../impl/node/RemoveConfigNodeProcedure.java | 1 -
.../impl/node/RemoveDataNodeProcedure.java | 22 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 20 +-
.../impl/pipe/runtime/PipeMetaSyncProcedure.java | 2 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 31 +-
.../impl/pipe/task/DropPipeProcedureV2.java | 16 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 4 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 4 +-
.../impl/region/AddRegionPeerProcedure.java | 203 ++++++
.../impl/region/RegionMigrateProcedure.java | 182 ++----
.../impl/region/RemoveRegionPeerProcedure.java | 188 ++++++
.../AbstractOperateSubscriptionProcedure.java | 67 +-
.../impl/subscription/SubscriptionOperation.java | 2 +
.../consumer/AlterConsumerGroupProcedure.java | 11 +
.../consumer/CreateConsumerProcedure.java | 6 +-
.../runtime/ConsumerGroupMetaSyncProcedure.java} | 95 ++-
...bstractOperateSubscriptionAndPipeProcedure.java | 159 +++++
.../subscription/CreateSubscriptionProcedure.java | 224 +++++--
.../subscription/DropSubscriptionProcedure.java | 244 +++++--
.../subscription/topic/AlterTopicProcedure.java | 20 +
.../topic/runtime/TopicMetaSyncProcedure.java} | 94 ++-
.../impl/sync/AuthOperationProcedure.java | 21 +-
...ransitionState.java => AddRegionPeerState.java} | 10 +-
.../procedure/state/RegionTransitionState.java | 5 +-
...sitionState.java => RemoveRegionPeerState.java} | 8 +-
.../procedure/store/ProcedureFactory.java | 24 +
.../confignode/procedure/store/ProcedureType.java | 4 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 6 -
.../request/ConfigPhysicalPlanSerDeTest.java | 140 +++-
.../persistence/CNPhysicalPlanGeneratorTest.java | 446 +++++++++++++
...reTest.java => AddRegionPeerProcedureTest.java} | 16 +-
.../impl/region/RegionMigrateProcedureTest.java | 16 +-
...est.java => RemoveRegionPeerProcedureTest.java} | 16 +-
.../ConsumerGroupMetaSyncProcedureTest.java | 55 ++
.../CreateSubscriptionProcedureTest.java | 10 +-
.../DropSubscriptionProcedureTest.java | 3 +-
.../topic/runtime/TopicMetaSyncProcedureTest.java | 55 ++
.../org/apache/iotdb/consensus/IConsensus.java | 15 +-
.../org/apache/iotdb/consensus/common/Peer.java | 21 +
.../apache/iotdb/consensus/iot/IoTConsensus.java | 26 +-
.../consensus/iot/IoTConsensusServerImpl.java | 102 ++-
.../consensus/iot/client/DispatchLogHandler.java | 23 +-
.../apache/iotdb/consensus/ratis/DiskGuardian.java | 2 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 34 +-
.../iotdb/consensus/ratis/utils/Retriable.java | 11 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 1 +
.../iotdb/consensus/simple/SimpleConsensus.java | 7 +-
.../apache/iotdb/consensus/iot/StabilityTest.java | 10 +-
.../iotdb/consensus/ratis/DiskGuardianTest.java | 4 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 2 +-
.../apache/iotdb/consensus/ratis/TestUtils.java | 6 +-
.../consensus/simple/SimpleConsensusTest.java | 2 +-
.../src/assembly/resources/conf/datanode-env.bat | 36 +-
.../src/assembly/resources/conf/datanode-env.sh | 38 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 38 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +-
.../schemaregion/SchemaRegionStateMachine.java | 23 +
.../PipeDataRegionConnectorConstructor.java | 2 +
.../dataregion/PipeDataRegionPluginAgent.java | 10 +-
.../PipeDataRegionProcessorConstructor.java | 11 +
.../PipeSchemaRegionConnectorConstructor.java | 2 +-
.../PipeSchemaRegionExtractorConstructor.java | 2 +-
.../schemaregion/PipeSchemaRegionPluginAgent.java | 10 +-
.../PipeSchemaRegionProcessorConstructor.java | 2 +-
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 6 +
.../agent/runtime/PipePeriodicalJobExecutor.java | 9 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 6 +-
.../connector/protocol/opcua/OpcUaConnector.java | 16 +-
.../protocol/opcua/OpcUaKeyStoreLoader.java | 20 +-
.../protocol/opcua/OpcUaServerBuilder.java | 44 +-
.../async/IoTDBDataRegionAsyncConnector.java | 3 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 6 +-
.../protocol/writeback/WriteBackConnector.java | 2 +-
.../pipe/event/common/row/PipeResetTabletRow.java | 54 ++
.../db/pipe/event/common/row/PipeRowCollector.java | 9 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 14 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 23 +-
.../tablet/TabletInsertionDataContainer.java | 2 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 38 +-
.../executor/PipeConnectorSubtaskExecutor.java | 4 +
.../executor/PipeSubtaskExecutorManager.java | 8 +
.../PipeHistoricalDataRegionTsFileExtractor.java | 29 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 6 +-
.../schemaregion/IoTDBSchemaRegionExtractor.java | 4 +-
.../aggregate/AbstractFormalProcessor.java | 54 ++
.../processor/aggregate/AggregateProcessor.java | 719 +++++++++++++++++++++
.../aggregate/TimeSeriesRuntimeState.java | 396 ++++++++++++
.../aggregatedresult/AggregatedResultOperator.java | 62 ++
.../standardstatistics/AverageOperator.java | 56 ++
.../ClearanceFactorOperator.java | 60 ++
.../standardstatistics/CrestFactorOperator.java | 59 ++
.../standardstatistics/FormFactorOperator.java | 57 ++
.../standardstatistics/KurtosisOperator.java | 67 ++
.../standardstatistics/PeakOperator.java | 52 ++
.../standardstatistics/PulseFactorOperator.java | 57 ++
.../standardstatistics/RootMeanSquareOperator.java | 57 ++
.../standardstatistics/SkewnessOperator.java | 63 ++
.../standardstatistics/VarianceOperator.java | 59 ++
.../CustomizedReadableIntermediateResults.java | 158 +++++
.../IntermediateResultOperator.java | 113 ++++
.../sametype/numeric/AbsoluteMaxOperator.java | 71 ++
.../numeric/AbstractSameTypeNumericOperator.java | 152 +++++
.../numeric/IntegralPoweredSumOperator.java | 77 +++
.../doubletype/FractionPoweredSumOperator.java | 137 ++++
.../specifictype/integertype/CountOperator.java | 125 ++++
.../processor/AbstractOperatorProcessor.java | 73 +++
.../StandardStatisticsOperatorProcessor.java | 76 +++
.../window/datastructure/TimeSeriesWindow.java | 495 ++++++++++++++
.../window/datastructure/WindowOutput.java | 71 ++
.../window/datastructure/WindowState.java | 75 +++
.../AbstractSimpleTimeWindowingProcessor.java | 102 +++
.../processor/AbstractWindowingProcessor.java | 119 ++++
.../processor/TumblingWindowingProcessor.java | 110 ++++
.../sdt/SwingingDoorTrendingFilter.java | 8 +-
.../sdt/SwingingDoorTrendingSamplingProcessor.java | 6 +-
.../SimpleConsensusProgressIndexAssigner.java | 6 +
.../receiver/thrift/IoTDBDataNodeReceiver.java | 2 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 2 +-
.../pipe/task/builder/PipeDataNodeTaskBuilder.java | 46 +-
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 19 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 47 +-
.../subtask/connector/PipeConnectorSubtask.java | 4 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 4 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 7 -
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 34 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 5 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 213 +++++-
.../db/queryengine/common/DataNodeEndPoints.java | 26 +
.../queryengine/execution/driver/DataDriver.java | 2 +
.../fragment/FragmentInstanceContext.java | 1 +
.../fragment/FragmentInstanceManager.java | 12 +-
.../execution/operator/source/SeriesScanUtil.java | 88 ++-
.../config/executor/ClusterConfigTaskExecutor.java | 189 ++++--
.../OrderByExpressionWithLimitChangeToTopK.java | 68 +-
.../SimpleFragmentParallelPlanner.java | 5 +-
.../planner/plan/node/write/InsertTabletNode.java | 3 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 3 +
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 2 +-
.../iotdb/db/service/RegionMigrateService.java | 169 +++--
.../buffer/TimeSeriesMetadataCache.java | 6 +-
.../impl/ReadChunkCompactionPerformer.java | 47 ++
.../impl/ReadPointCompactionPerformer.java | 4 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 16 +-
.../fast/AlignedSeriesCompactionExecutor.java | 22 +-
.../ReadChunkAlignedSeriesCompactionExecutor.java | 13 +-
.../compaction/io/CompactionTsFileReader.java | 51 ++
.../dataregion/memtable/TsFileProcessor.java | 2 +-
.../dataregion/read/QueryDataSource.java | 120 +++-
.../dataregion/tsfile/TsFileManager.java | 6 +-
.../dataregion/tsfile/TsFileResource.java | 2 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 2 +-
.../db/subscription/agent/SubscriptionAgent.java | 32 +-
.../agent/SubscriptionAgentLauncher.java | 170 +++++
.../agent/SubscriptionBrokerAgent.java | 135 ++++
.../agent/SubscriptionConsumerAgent.java | 203 ++++++
.../agent/SubscriptionReceiverAgent.java | 108 ++++
.../agent/SubscriptionRuntimeAgent.java | 98 +++
...TopicAgent.java => SubscriptionTopicAgent.java} | 72 ++-
.../broker/EnrichedTabletsBinaryCache.java | 62 ++
.../broker/SerializedEnrichedEvent.java | 114 ++++
.../db/subscription/broker/SubscriptionBroker.java | 119 ++++
.../broker/SubscriptionPrefetchingQueue.java | 263 ++++++++
.../executor/SubscriptionSubtaskExecutor.java} | 14 +-
.../receiver/SubscriptionReceiver.java} | 17 +-
.../receiver/SubscriptionReceiverV1.java | 553 ++++++++++++++++
.../stage/SubscriptionTaskConnectorStage.java} | 40 +-
.../task/subtask/SubscriptionConnectorSubtask.java | 75 +++
.../SubscriptionConnectorSubtaskLifeCycle.java} | 56 +-
.../SubscriptionConnectorSubtaskManager.java | 198 ++++++
.../db/tools/schema/SRStatementGenerator.java | 13 +-
.../tools/schema/SchemaRegionSnapshotParser.java | 11 +-
.../java/org/apache/iotdb/db/utils/MemUtils.java | 9 +-
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 40 +-
.../plan/planner/FragmentInstanceSerdeTest.java | 1 +
.../compaction/CompactionDataTypeNotMatchTest.java | 212 ++++++
...nkCompactionPerformerWithAlignedSeriesTest.java | 2 +-
.../db/utils/SchemaRegionSnapshotParserTest.java | 2 +-
.../iotdb/db/utils/TypeInferenceUtilsTest.java | 37 +-
.../resources/conf/iotdb-common.properties | 17 +-
.../assembly/resources/sbin/start-standalone.bat | 3 +
.../assembly/resources/sbin/start-standalone.sh | 2 +
.../commons/auth/authorizer/BasicAuthorizer.java | 4 +-
.../auth/authorizer/LocalFileAuthorizer.java | 2 +-
.../iotdb/commons/auth/entity/PathPrivilege.java | 41 +-
.../iotdb/commons/auth/entity/PrivilegeType.java | 11 +
.../org/apache/iotdb/commons/auth/entity/Role.java | 63 +-
.../commons/auth/role/LocalFileRoleAccessor.java | 2 +
.../iotdb/commons/auth/user/BasicUserManager.java | 8 +-
.../commons/auth/user/LocalFileUserAccessor.java | 8 +-
.../iotdb/commons/concurrent/ThreadName.java | 3 +
.../apache/iotdb/commons/conf/CommonConfig.java | 83 +++
.../iotdb/commons/conf/CommonDescriptor.java | 47 ++
.../commons/consensus/index/ProgressIndexType.java | 8 +
.../index/impl/TimeWindowStateProgressIndex.java | 280 ++++++++
.../commons/pipe/agent/plugin/PipePluginAgent.java | 88 ++-
.../pipe/agent/plugin/PipePluginConstructor.java | 2 +-
.../config/constant/PipeConnectorConstant.java | 3 +
.../config/constant/PipeProcessorConstant.java | 20 +
.../env/PipeTaskProcessorRuntimeEnvironment.java | 12 +-
.../pipe/connector/client/IoTDBSyncClient.java | 2 +-
.../queue/ConcurrentIterableLinkedQueue.java | 18 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 14 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 15 +-
.../processor/aggregate/AggregateProcessor.java} | 15 +-
.../aggregate/StandardStatisticsProcessor.java} | 15 +-
.../aggregate/TumblingWindowingProcessor.java} | 15 +-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 2 +-
.../commons/pipe/task/meta/PipeStaticMeta.java | 13 +
.../task/meta/PipeType.java} | 21 +-
.../apache/iotdb/commons/service/ServiceType.java | 1 +
.../iotdb/commons/service/StartupChecks.java | 43 +-
.../subscription/config/SubscriptionConfig.java | 100 +++
.../meta/consumer/ConsumerGroupMeta.java | 17 +
.../meta/consumer/ConsumerGroupMetaKeeper.java | 76 ++-
.../subscription/meta/consumer/ConsumerMeta.java | 4 +-
.../commons/subscription/meta/topic/TopicMeta.java | 90 ++-
.../org/apache/iotdb/commons/utils/AuthUtils.java | 85 +++
.../DataNodeKillPoints.java} | 14 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 10 +
.../commons/pipe/task/meta/PipeMetaDeSerTest.java | 20 +
.../consumer/ConsumerGroupDeSerTest.java | 2 +
.../commons/subscription/topic/TopicDeSerTest.java | 2 +-
.../tsfile/encoding/encoder/DictionaryEncoder.java | 17 +-
.../iotdb/tsfile/file/metadata/PlainDeviceID.java | 2 +-
.../org/apache/iotdb/tsfile/read/TsFileReader.java | 2 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 48 +-
.../apache/iotdb/tsfile/write/record/Tablet.java | 70 +-
.../thrift-commons/src/main/thrift/common.thrift | 10 +-
.../src/main/thrift/confignode.thrift | 9 -
.../thrift-datanode/src/main/thrift/client.thrift | 2 +-
.../src/main/thrift/datanode.thrift | 82 ++-
.../library/frequency/UDFEnvelopeAnalysis.java | 6 +-
pom.xml | 4 +-
341 files changed, 15185 insertions(+), 2260 deletions(-)
create mode 100644
example/session/src/main/java/org/apache/iotdb/SubscriptionSessionExample.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBRegionMigrateReliabilityIT.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionAgent.java
=>
integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
(60%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java
rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ =>
single}/IoTDBPipeSingleEnvDemoIT.java (81%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionBasicIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionConsumerGroupIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionTopicIT.java
copy
{iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription =>
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload}/config/ConsumerConfig.java
(53%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/config/TopicConfig.java
=>
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/config/ConsumerConstant.java
(73%)
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/config/TopicConfig.java
rename
{iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription =>
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload}/config/TopicConstant.java
(87%)
delete mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/ConsumerConfig.java
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/partition/{UpdateRegionLocationPlan.java
=> AddRegionLocationPlan.java} (62%)
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/partition/{UpdateRegionLocationPlan.java
=> RemoveRegionLocationPlan.java} (63%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/OperateMultiplePipesPlanV2.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/subscription/consumer/runtime/ConsumerGroupHandleMetaChangePlan.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/subscription/topic/AlterMultipleTopicsPlan.java
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/{pipe/runtime/PipeHandleMetaChangePlan.java
=> subscription/topic/runtime/TopicHandleMetaChangePlan.java} (53%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionMetaSyncer.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNPhysicalPlanGenerator.java
rename
iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons/subscription/config/ConsumerConfig.java
=>
confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNSnapshotFileType.java}
(74%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfignodeSnapshotParser.java
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/{DataNodeRemoveHandler.java
=> RegionMaintainHandler.java} (82%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/AddRegionPeerProcedure.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/RemoveRegionPeerProcedure.java
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/{pipe/runtime/PipeMetaSyncProcedure.java
=> subscription/consumer/runtime/ConsumerGroupMetaSyncProcedure.java} (54%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/AbstractOperateSubscriptionAndPipeProcedure.java
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/{pipe/runtime/PipeMetaSyncProcedure.java
=> subscription/topic/runtime/TopicMetaSyncProcedure.java} (54%)
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/{RegionTransitionState.java
=> AddRegionPeerState.java} (82%)
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/{RegionTransitionState.java
=> RemoveRegionPeerState.java} (84%)
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CNPhysicalPlanGeneratorTest.java
copy
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/{RegionMigrateProcedureTest.java
=> AddRegionPeerProcedureTest.java} (87%)
copy
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/{RegionMigrateProcedureTest.java
=> RemoveRegionPeerProcedureTest.java} (87%)
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/consumer/runtime/ConsumerGroupMetaSyncProcedureTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/runtime/TopicMetaSyncProcedureTest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeResetTabletRow.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AbstractFormalProcessor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/AggregatedResultOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/AverageOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/ClearanceFactorOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/CrestFactorOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/FormFactorOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/KurtosisOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PeakOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/PulseFactorOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/RootMeanSquareOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/SkewnessOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/aggregatedresult/standardstatistics/VarianceOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbsoluteMaxOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/IntegralPoweredSumOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/processor/AbstractOperatorProcessor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/processor/StandardStatisticsOperatorProcessor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/WindowOutput.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/WindowState.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractSimpleTimeWindowingProcessor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractWindowingProcessor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionAgentLauncher.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionRuntimeAgent.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/{topic/TopicAgent.java
=> SubscriptionTopicAgent.java} (52%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/EnrichedTabletsBinaryCache.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SerializedEnrichedEvent.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/execution/executor/PipeConnectorSubtaskExecutor.java
=> subscription/execution/executor/SubscriptionSubtaskExecutor.java} (65%)
copy
iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons/subscription/config/TopicConfig.java
=>
datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java}
(65%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/task/stage/PipeTaskConnectorStage.java
=> subscription/task/stage/SubscriptionTaskConnectorStage.java} (63%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
=> subscription/task/subtask/SubscriptionConnectorSubtaskLifeCycle.java} (65%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskManager.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeNotMatchTest.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/consensus/index/impl/TimeWindowStateProgressIndex.java
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/{config/plugin/env/PipeTaskProcessorRuntimeEnvironment.java
=> plugin/builtin/processor/aggregate/AggregateProcessor.java} (59%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/{config/plugin/env/PipeTaskProcessorRuntimeEnvironment.java
=> plugin/builtin/processor/aggregate/StandardStatisticsProcessor.java} (58%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/{config/plugin/env/PipeTaskProcessorRuntimeEnvironment.java
=> plugin/builtin/processor/aggregate/TumblingWindowingProcessor.java} (58%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/{subscription/config/TopicConfig.java
=> pipe/task/meta/PipeType.java} (68%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/config/SubscriptionConfig.java
rename
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/{subscription/config/TopicConfig.java
=> utils/DataNodeKillPoints.java} (74%)
