This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 25ecfcc1c7f [To dev/1.3] Pipe: Add a tool for validating and repairing
isGeneratedByPipe mark in tsfile resources (#15934)
new bf809bae068 [To dev/1.3] Fixed multiple bugs of pipe
new 6405a06885f [TIMECHODB] Continue to revert the local persisting logic
new 2e339af15f6 Pipe: Added rate limiter for tsFile sending (#15765)
(#15947)
new 097f725a901 Pipe: Fixed the ping-pong of certain realtime tsFiles
(#15949)
new 1fbbdb3bece [To dev/1.3] Pipe: Fix DataNodeShutdownHook waiting report
logic and add capture history file log (#15952) (#15951)
new 1ee10537dda Revert "Pipe: Added rate limiter for tsFile sending
(#15765) (#15947)"
new ebdf275a350 [To dev/1.3] Pipe: Added progressIndex reset function to
the resource resetting script (#15957)
new 7e403f4f5d6 [to dev/1.3] Set global time filter before analyzing last
query
new 968d65cba04 [To dev/1.3] Optimize partition cache getRegionReplicaSet
interface performance by batching
new 9da931acd60 [To dev/1.3] Pipe: Fixed the issue that frequent logins on
the receiving end caused indicator leaks #16076 (#16085)
new fedfa10fc8d [To dev/1.3] Pipe: Reduced the progress index report
interval & Added some logs (#15905) (#15908)
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 2 +
.../pipe/it/autocreate/AbstractPipeDualAutoIT.java | 6 +-
.../it/autocreate/IoTDBPipeAutoConflictIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeClusterIT.java | 6 +-
.../IoTDBPipeConnectorCompressionIT.java | 7 +-
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 1 -
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeWithLoadIT.java | 6 +-
.../pipe/it/manual/AbstractPipeDualManualIT.java | 6 +-
.../manual/IoTDBPipeTypeConversionISessionIT.java | 12 +-
.../pipe/it/manual/IoTDBPipeTypeConversionIT.java | 2 +-
.../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 1 +
.../it/cluster/IoTDBSubscriptionRestartIT.java | 3 +-
.../it/dual/AbstractSubscriptionDualIT.java | 3 +
.../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 3 +
.../it/dual/IoTDBSubscriptionTimePrecisionIT.java | 2 +
.../it/dual/IoTDBSubscriptionTopicIT.java | 3 +
.../it/local/AbstractSubscriptionLocalIT.java | 7 +-
.../it/triple/AbstractSubscriptionTripleIT.java | 4 +
.../apache/iotdb/tools/it/ExportTsFileTestIT.java | 6 +-
.../event/dml/insertion/TsFileInsertionEvent.java | 10 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../agent/task/PipeConfigNodeSubtaskExecutor.java | 4 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 28 +-
.../runtime/heartbeat/PipeHeartbeatParser.java | 2 -
.../confignode/persistence/pipe/PipeTaskInfo.java | 7 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 18 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 10 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 14 +-
.../confignode/service/ConfigNodeShutdownHook.java | 3 -
.../request/ConfigPhysicalPlanSerDeTest.java | 12 +-
.../consensus/response/pipe/PipeTableRespTest.java | 6 +-
.../agent/PipeConfigNodeSubtaskExecutorTest.java | 3 +-
.../connector/PipeConfigNodeThriftRequestTest.java | 4 -
.../iotdb/confignode/persistence/PipeInfoTest.java | 4 +-
...pipe.bat => reset-resource-pipe-statistics.bat} | 4 +-
...y-pipe.sh => reset-resource-pipe-statistics.sh} | 4 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 30 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +-
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 2 -
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 10 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 585 ++++++++++----------
.../task/builder/PipeDataNodeTaskBuilder.java | 23 +-
.../execution/PipeConnectorSubtaskExecutor.java | 8 +-
.../execution/PipeProcessorSubtaskExecutor.java | 2 +-
.../task/execution/PipeSubtaskExecutorManager.java | 18 +-
.../agent/task/stage/PipeTaskConnectorStage.java | 6 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 5 +
.../connector/PipeConnectorSubtaskManager.java | 19 +-
.../PipeRealtimePriorityBlockingQueue.java | 151 +++++-
.../client/IoTDBDataNodeAsyncClientManager.java | 36 ++
.../evolvable/batch/PipeTabletEventBatch.java | 39 +-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 10 +-
.../batch/PipeTabletEventTsFileBatch.java | 7 +
.../evolvable/request/PipeTransferPlanNodeReq.java | 1 -
.../request/PipeTransferTabletBatchReq.java | 1 -
.../request/PipeTransferTabletBinaryReq.java | 1 -
.../request/PipeTransferTabletInsertNodeReq.java | 1 -
.../request/PipeTransferTabletRawReq.java | 1 -
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 3 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 2 +-
.../PipeConsensusTransferBatchReqBuilder.java | 3 +-
.../async/IoTDBDataRegionAsyncConnector.java | 91 ++--
.../PipeTransferTabletInsertionEventHandler.java | 2 +-
.../async/handler/PipeTransferTsFileHandler.java | 45 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 3 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../consensus/PipeConsensusSubtaskExecutor.java | 6 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 124 ++---
.../event/common/terminate/PipeTerminateEvent.java | 44 --
.../tsfile/PipeCompactedTsFileInsertionEvent.java | 252 +++++++++
.../common/tsfile/PipeTsFileInsertionEvent.java | 129 ++---
.../TsFileInsertionDataContainerProvider.java | 7 +-
.../event/realtime/PipeRealtimeEventFactory.java | 16 +-
.../dataregion/DataRegionWatermarkInjector.java | 4 -
.../dataregion/IoTDBDataRegionExtractor.java | 107 ++--
.../PipeHistoricalDataRegionTsFileExtractor.java | 162 ++----
.../realtime/PipeRealtimeDataRegionExtractor.java | 35 ++
.../PipeRealtimeDataRegionHybridExtractor.java | 180 ++-----
.../PipeRealtimeDataRegionLogExtractor.java | 5 +-
.../PipeRealtimeDataRegionTsFileExtractor.java | 7 +
.../realtime/assigner/PipeDataRegionAssigner.java | 50 +-
.../PipeTsFileEpochProgressIndexKeeper.java | 30 +-
.../dataregion/realtime/epoch/TsFileEpoch.java | 17 +-
.../realtime/epoch/TsFileEpochManager.java | 4 +-
.../listener/PipeInsertionDataNodeListener.java | 14 +-
.../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 3 -
.../PipeDataNodeRemainingEventAndTimeOperator.java | 22 +-
.../overview/PipeDataNodeSinglePipeMetrics.java | 84 ++-
.../pipe/metric/overview/PipeResourceMetrics.java | 39 +-
.../overview/PipeWALInsertNodeCacheMetrics.java | 86 ---
.../sink/PipeDataRegionConnectorMetrics.java | 34 ++
.../twostage/exchange/payload/CombineRequest.java | 1 -
.../payload/FetchCombineResultRequest.java | 1 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 40 +-
.../pipe/resource/PipeDataNodeResourceManager.java | 25 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 33 +-
.../pipe/resource/memory/PipeMemoryWeightUtil.java | 12 +-
.../resource/tsfile/PipeCompactionManager.java | 68 +++
...Resource.java => PipeTsFilePublicResource.java} | 158 +-----
.../pipe/resource/tsfile/PipeTsFileResource.java | 253 +--------
.../resource/tsfile/PipeTsFileResourceManager.java | 329 ++++++------
.../db/pipe/resource/wal/PipeWALResource.java | 169 ------
.../pipe/resource/wal/PipeWALResourceManager.java | 142 -----
.../wal/hardlink/PipeWALHardlinkResource.java | 47 --
.../hardlink/PipeWALHardlinkResourceManager.java | 152 ------
.../wal/selfhost/PipeWALSelfHostResource.java | 41 --
.../selfhost/PipeWALSelfHostResourceManager.java | 38 --
.../impl/DataNodeInternalRPCServiceImpl.java | 95 +++-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 1 +
.../plan/analyze/ClusterPartitionFetcher.java | 54 +-
.../plan/analyze/LoadTsFileAnalyzer.java | 34 +-
.../analyze/cache/partition/PartitionCache.java | 347 ++++++------
.../analyze/lock/DataNodeSchemaLockManager.java | 1 +
.../scheduler/load/LoadTsFileDispatcherImpl.java | 1 +
.../plan/scheduler/load/LoadTsFileScheduler.java | 2 +-
.../iotdb/db/service/DataNodeShutdownHook.java | 5 +-
.../iotdb/db/storageengine/StorageEngine.java | 8 -
.../db/storageengine/dataregion/DataRegion.java | 5 +-
.../task/InsertionCrossSpaceCompactionTask.java | 4 +-
.../compaction/execute/utils/CompactionUtils.java | 7 +-
.../dataregion/flush/NotifyFlushMemTable.java | 7 -
.../dataregion/memtable/AbstractMemTable.java | 13 -
.../dataregion/memtable/IMemTable.java | 4 -
.../dataregion/memtable/TsFileProcessor.java | 38 +-
.../dataregion/tsfile/TsFileManager.java | 13 +
.../dataregion/tsfile/TsFileResource.java | 39 +-
.../dataregion/wal/buffer/WALBuffer.java | 6 -
.../dataregion/wal/buffer/WALEntry.java | 4 +-
.../wal/checkpoint/CheckpointManager.java | 70 +--
.../dataregion/wal/checkpoint/MemTableInfo.java | 19 +-
.../dataregion/wal/io/WALByteBufReader.java | 13 -
.../dataregion/wal/node/WALFakeNode.java | 4 +-
.../storageengine/dataregion/wal/node/WALNode.java | 96 +---
.../dataregion/wal/utils/WALEntryHandler.java | 233 --------
.../dataregion/wal/utils/WALEntryPosition.java | 229 --------
.../dataregion/wal/utils/WALInsertNodeCache.java | 335 ------------
.../wal/utils/listener/WALFlushListener.java | 12 +-
.../db/storageengine/load/LoadTsFileManager.java | 1 +
...eeStatementDataTypeConvertExecutionVisitor.java | 169 ++----
.../converter/LoadTsFileDataTypeConverter.java | 54 +-
.../execution/SubscriptionSubtaskExecutor.java | 9 +-
.../task/stage/SubscriptionTaskConnectorStage.java | 4 +-
...va => TsFileResourcePipeStatisticsSetTool.java} | 125 +++--
.../connector/PipeDataNodeThriftRequestTest.java | 12 -
.../event/TsFileInsertionDataContainerTest.java | 16 +-
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 101 ++--
.../resource/PipeTsFileResourceManagerTest.java | 96 ++--
.../PipeWALHardlinkResourceManagerTest.java | 114 ----
.../plan/analyze/cache/PartitionCacheTest.java | 5 +-
.../TsFileResourceProgressIndexTest.java | 23 +-
.../wal/checkpoint/CheckpointManagerTest.java | 172 ------
.../dataregion/wal/node/WALEntryHandlerTest.java | 586 ---------------------
.../dataregion/wal/node/WALNodeTest.java | 17 -
.../wal/node/WalDeleteOutdatedNewTest.java | 65 ---
.../wal/utils/WALInsertNodeCacheTest.java | 190 -------
.../commons/concurrent/IoTDBThreadPoolFactory.java | 15 +
.../iotdb/commons/concurrent/ThreadName.java | 10 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 407 +++++---------
.../commons/consensus/index/ProgressIndex.java | 2 +-
.../consensus/index/impl/IoTProgressIndex.java | 27 +-
.../agent/runtime/PipePeriodicalJobExecutor.java | 33 --
.../commons/pipe/agent/task/PipeTaskAgent.java | 61 ++-
.../agent/task/execution/PipeSubtaskExecutor.java | 45 +-
.../commons/pipe/agent/task/meta/PipeMeta.java | 6 +-
.../pipe/agent/task/meta/PipeRuntimeMeta.java | 36 +-
.../commons/pipe/agent/task/meta/PipeTaskMeta.java | 141 +----
.../task/progress/PipeEventCommitManager.java | 25 +-
.../agent/task/progress/PipeEventCommitter.java | 12 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 5 -
.../agent/task/subtask/PipeReportableSubtask.java | 24 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 160 ++----
.../iotdb/commons/pipe/config/PipeDescriptor.java | 129 ++---
.../config/constant/PipeConnectorConstant.java | 2 +-
.../thrift/request/PipeTransferFilePieceReq.java | 1 -
.../thrift/request/PipeTransferFileSealReqV1.java | 1 -
.../thrift/request/PipeTransferFileSealReqV2.java | 1 -
.../thrift/request/PipeTransferHandshakeV1Req.java | 1 -
.../thrift/request/PipeTransferHandshakeV2Req.java | 1 -
.../thrift/request/PipeTransferSliceReq.java | 1 -
.../pipe/connector/protocol/IoTDBConnector.java | 12 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 13 +
.../commons/pipe/resource/log/PipeLogManager.java | 2 +-
.../commons/pipe/resource/log/PipeLogStatus.java | 6 +-
.../iotdb/commons/service/metric/enums/Metric.java | 9 +-
.../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 22 +-
194 files changed, 2994 insertions(+), 6033 deletions(-)
rename
iotdb-core/datanode/src/assembly/resources/tools/tsfile/{mark-is-generated-by-pipe.bat
=> reset-resource-pipe-statistics.bat} (92%)
rename
iotdb-core/datanode/src/assembly/resources/tools/tsfile/{mark-is-generated-by-pipe.sh
=> reset-resource-pipe-statistics.sh} (90%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/{PipeTsFileResource.java
=> PipeTsFilePublicResource.java} (63%)
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/{TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool.java
=> TsFileResourcePipeStatisticsSetTool.java} (60%)
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeWALHardlinkResourceManagerTest.java
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManagerTest.java
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALEntryHandlerTest.java
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCacheTest.java
