This is an automated email from the ASF dual-hosted git repository.

Caideyipi pushed a commit to branch fix/pipe-compaction-queue-replacement
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 5667d06f96cfd543958e66bfd155dfeee28b9cc2
Merge: 0ed84f65128 f12fe701c8c
Author: Caideyipi <[email protected]>
AuthorDate: Mon Nov 16 16:09:53 2026 +0800

    Merge master and address compaction replacement review comments

 .github/workflows/cluster-it-1c1d1a.yml            |  29 +
 AGENTS.md                                          |  10 +
 LICENSE-binary                                     | 142 ++--
 NOTICE-binary                                      | 112 ++-
 THREAT_MODEL.md                                    |  20 +-
 dependencies.json                                  |   1 -
 external-service-impl/rest/pom.xml                 |  10 +
 .../rest/protocol/v1/handler/ExceptionHandler.java |   3 +-
 .../rest/protocol/v2/handler/ExceptionHandler.java |   3 +-
 .../protocol/handler/ExceptionHandlerTest.java     | 162 +++++
 integration-test/AGENTS.md                         |  17 +
 integration-test/pom.xml                           |   7 +-
 .../cluster/env/IntegrationTestEnvMessages.java    |  35 +
 .../cluster/env/IntegrationTestEnvMessages.java    |  35 +
 .../org/apache/iotdb/it/env/cluster/EnvUtils.java  |   5 +-
 .../it/env/cluster/config/MppDataNodeConfig.java   |   6 +
 .../iotdb/it/env/cluster/config/MppJVMConfig.java  |  26 +
 .../iotdb/it/env/cluster/env/AbstractEnv.java      | 149 ++--
 .../it/env/cluster/node/AbstractNodeWrapper.java   |  18 +-
 .../it/env/cluster/node/ConfigNodeWrapper.java     |  23 +
 .../it/env/remote/config/RemoteDataNodeConfig.java |   5 +
 .../apache/iotdb/itbase/env/DataNodeConfig.java    |   2 +
 .../it/cluster/IoTDBClusterNodeErrorStartUpIT.java | 168 +++--
 .../it/cluster/IoTDBClusterNodeGetterIT.java       |  56 +-
 .../it/database/IoTDBDatabaseSetAndDeleteIT.java   |  34 +-
 ...ataPartitionTableIntegrityCheckProcedureIT.java |  67 +-
 .../it/partition/IoTDBPartitionCreationIT.java     |  24 +-
 .../it/partition/IoTDBPartitionDurableIT.java      |   4 +-
 .../partition/IoTDBPartitionTableAutoCleanIT.java  |  82 ++-
 .../IoTDBPartitionTableAutoCleanUSIT.java          |  92 +--
 .../pass/commit/RegionMigrateFileAssertions.java   |   3 +
 .../IoTDBRegionMigrateNormalForIoTV2StreamIT.java  |   3 +
 .../IoTDBRegionMigrateWithCompressionRatioIT.java  |  12 +-
 .../IoTDBRegionMigrateWithLastEmptyDeletionIT.java |   3 +
 .../iotdb/db/it/IoTDBCustomizedClusterIT.java      |  49 +-
 .../org/apache/iotdb/db/it/IoTDBDeletionIT.java    |   9 +-
 .../iotdb/db/it/IoTDBDiskFullFolderRecoveryIT.java | 158 +++++
 .../iotdb/db/it/IoTDBIPv6ExternalServiceIT.java    |   4 +-
 .../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java |  31 +-
 .../apache/iotdb/db/it/IoTDBLoadLastCacheIT.java   |  66 +-
 .../iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java  | 438 ++++++++++++
 .../iotdb/db/it/IoTDBLoadTsFileClusterIT.java      | 245 ++++++-
 .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java  | 213 +++++-
 .../iotdb/db/it/IoTDBLoadTsFileInstallIT.java      | 451 +++++++++++++
 .../org/apache/iotdb/db/it/IoTDBRecoverIT.java     | 109 +--
 .../apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java | 103 ++-
 .../org/apache/iotdb/db/it/IoTDBRestartIT.java     |  49 +-
 .../apache/iotdb/db/it/IoTDBRestartRatisIT.java    |  62 +-
 .../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java |  66 +-
 .../java/org/apache/iotdb/db/it/IoTDBTtlIT.java    |  29 +-
 .../iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java   |  46 +-
 .../db/it/aggregation/IoTDBAggregationIT.java      | 209 ++----
 .../db/it/aligned/IoTDBAlignedDataDeletionIT.java  |  66 +-
 .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java   | 161 ++++-
 .../scalar/IoTDBReplaceFunctionIT.java             |   2 +-
 .../scalar/IoTDBSubStringFunctionIT.java           |   2 +-
 .../org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java   | 390 ++++++++---
 .../apache/iotdb/db/it/cq/IoTDBCQExecInNsIT.java   |  95 ++-
 .../apache/iotdb/db/it/cq/IoTDBCQExecInUsIT.java   |  95 ++-
 .../java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java  |  53 ++
 .../iotdb/db/it/mqtt/IoTDBMQTTServiceJsonIT.java   | 121 ++--
 .../it/query/IoTDBAggregationFirstInGroupIT.java   |   2 +-
 .../iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java      |  58 +-
 .../iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java   | 157 +++--
 .../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java      |  14 +-
 .../db/it/udf/IoTDBUDFOtherWindowQueryIT.java      | 120 ++--
 .../iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java     |  17 +-
 .../apache/iotdb/db/it/utils/IoTDBTestUtilsIT.java | 173 +++++
 .../org/apache/iotdb/db/it/utils/NodeDebugger.java | 232 +++++++
 .../org/apache/iotdb/db/it/utils/TestUtils.java    | 357 ++++++----
 .../iotdb/it/utils/InternalSessionITUtils.java     | 158 +++++
 .../iotdb/libudf/it/dprofile/DProfileIT.java       |  44 +-
 .../iotdb/libudf/it/dquality/DQualityIT.java       |  42 +-
 .../iotdb/libudf/it/frequency/FrequencyIT.java     |  29 +-
 .../apache/iotdb/pipe/it/dual/PipeTestUtils.java   | 359 ++++++++++
 .../pipe/it/dual/tablemodel/TableModelUtils.java   | 201 +++++-
 .../manual/AbstractPipeTableModelDualManualIT.java |  19 +-
 .../AbstractPipeTableModelSharedClusterIT.java     | 211 ++++++
 .../tablemodel/manual/basic/IoTDBPipeAlterIT.java  | 655 +++++++++---------
 .../manual/basic/IoTDBPipeDataSinkIT.java          | 640 +++++++++++++-----
 .../manual/basic/IoTDBPipeIsolationIT.java         |   4 +-
 .../manual/basic/IoTDBPipeLifeCycleIT.java         |  29 +-
 .../manual/basic/IoTDBPipePermissionIT.java        |  72 +-
 .../manual/basic/IoTDBPipeProtocolIT.java          |  74 +-
 .../basic/IoTDBPipeRelationalSecurityIT.java       |  21 +-
 .../tablemodel/manual/basic/IoTDBPipeSourceIT.java | 484 +++++++------
 .../manual/basic/IoTDBPipeSwitchStatusIT.java      |   4 +-
 .../tablemodel/manual/basic/IoTDBPipeSyntaxIT.java | 106 ++-
 .../IoTDBPipeTsFileDecompositionWithModsIT.java    | 205 ++----
 .../manual/basic/IoTDBPipeWithLoadIT.java          | 176 +++--
 .../manual/basic/IoTDBTablePatternFormatIT.java    | 281 ++++----
 .../manual/enhanced/IoTDBPipeAutoConflictIT.java   | 180 +++--
 .../manual/enhanced/IoTDBPipeAutoDropIT.java       |  82 +--
 .../manual/enhanced/IoTDBPipeClusterIT.java        | 141 ++--
 .../manual/enhanced/IoTDBPipeDoubleLivingIT.java   | 334 ++++-----
 .../manual/enhanced/IoTDBPipeIdempotentIT.java     |  71 +-
 .../manual/enhanced/IoTDBPipeMetaIT.java           |  79 ++-
 .../manual/enhanced/IoTDBPipeNullValueIT.java      | 118 ++--
 .../enhanced/IoTDBPipeSinkCompressionIT.java       | 329 +++++----
 .../manual/enhanced/IoTDBPipeSinkParallelIT.java   |  31 +-
 .../IoTDBPipeTypeConversionISessionIT.java         |  63 +-
 .../IoTDBPipeTypeConversionSemanticIT.java         |  25 +-
 .../auto/AbstractPipeDualTreeModelAutoIT.java      |  23 +-
 .../auto/AbstractPipeTreeModelSharedClusterIT.java | 164 +++++
 .../treemodel/auto/basic/IoTDBPipeAlterIT.java     | 750 ++++++++++-----------
 .../treemodel/auto/basic/IoTDBPipeAutoSplitIT.java |   6 +-
 .../treemodel/auto/basic/IoTDBPipeDataSinkIT.java  | 374 ++++++----
 .../dual/treemodel/auto/basic/IoTDBPipeIPv6IT.java |   4 +-
 .../treemodel/auto/basic/IoTDBPipeLifeCycleIT.java | 251 ++++---
 .../treemodel/auto/basic/IoTDBPipeMutualSSLIT.java |  40 +-
 .../treemodel/auto/basic/IoTDBPipeProtocolIT.java  |  34 +-
 .../IoTDBPipeReceiverAutoCreateDisabledIT.java     |  20 +-
 .../treemodel/auto/basic/IoTDBPipeSourceIT.java    | 733 +++++++++++---------
 .../auto/basic/IoTDBPipeSwitchStatusIT.java        |   4 +-
 .../treemodel/auto/basic/IoTDBPipeSyntaxIT.java    | 356 ++++++----
 .../auto/basic/IoTDBShowReceiversLifecycleIT.java  |  32 +-
 .../auto/basic/IoTDBTreePatternFormatIT.java       | 249 ++++---
 .../auto/enhanced/IoTDBPipeAutoConflictIT.java     | 316 +++++----
 .../auto/enhanced/IoTDBPipeAutoDropIT.java         | 116 +++-
 .../auto/enhanced/IoTDBPipeClusterIT.java          |  94 +--
 .../enhanced/IoTDBPipeConditionalOperationsIT.java |  94 +--
 .../auto/enhanced/IoTDBPipeIdempotentIT.java       | 185 +++--
 .../auto/enhanced/IoTDBPipeNullValueIT.java        | 158 ++---
 .../auto/enhanced/IoTDBPipeSinkCompressionIT.java  | 247 ++++---
 .../IoTDBPipeTypeConversionSemanticIT.java         |  25 +-
 .../auto/enhanced/IoTDBPipeWithLoadIT.java         |  28 +-
 .../treemodel/auto/enhanced/PipeNowFunctionIT.java | 158 ++---
 .../manual/AbstractPipeDualTreeModelManualIT.java  |  72 +-
 ...AbstractPipeTreeModelManualSharedClusterIT.java | 194 ++++++
 .../treemodel/manual/IoTDBPipeInclusionIT.java     |  25 +-
 .../manual/IoTDBPipeMetaHistoricalIT.java          |  38 +-
 .../manual/IoTDBPipeMetaLeaderChangeIT.java        | 169 +++--
 .../treemodel/manual/IoTDBPipeMetaRestartIT.java   |  95 +--
 .../treemodel/manual/IoTDBPipePermissionIT.java    | 235 ++++---
 .../treemodel/manual/IoTDBPipeReqAutoSliceIT.java  |   3 +-
 .../IoTDBPipeTsFileDecompositionWithModsIT.java    | 475 ++++++-------
 .../manual/IoTDBPipeTypeConversionISessionIT.java  | 114 +++-
 .../treemodel/manual/IoTDBPipeWriteBackSinkIT.java |  27 +-
 .../iotdb/pipe/it/single/AbstractPipeSingleIT.java |   5 +
 .../it/single/IoTDBPipeOperationDiagnosticsIT.java |  79 ++-
 .../pipe/it/single/IoTDBPipePermissionIT.java      | 151 ++++-
 .../iotdb/pipe/it/single/IoTDBShowReceiversIT.java | 215 ++++--
 .../pipe/it/triple/AbstractPipeTripleManualIT.java |  13 +-
 .../iotdb/pipe/it/triple/IoTDBPipeForwardIT.java   |   2 +
 .../pipe/it/triple/IoTDBShowReceiversTripleIT.java |  60 +-
 .../relational/it/db/it/IoTDBDeletionTableIT.java  | 109 ++-
 .../relational/it/db/it/IoTDBInsertTableIT.java    |  80 ++-
 .../it/IoTDBLoadStorageTsFileWithCompactionIT.java | 269 ++++++++
 .../it/db/it/IoTDBLoadTableSchemaCompletionIT.java | 265 ++++++++
 .../relational/it/db/it/IoTDBLoadTsFileIT.java     | 210 +++++-
 .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java |   6 +
 .../db/it/IoTDBRegionReadyAfterRestartTableIT.java |  97 +++
 .../relational/it/db/it/IoTDBRestartTableIT.java   |  44 +-
 .../it/db/it/IoTDBSetSystemStatusTableIT.java      |  35 +
 .../it/db/it/IoTDBTablePreparedStatementIT.java    |  35 +
 .../db/it/IoTDBTablePreparedStatementJDBCIT.java   | 233 +++++++
 .../relational/it/db/it/IoTDBWindowTVFIT.java      |  94 +++
 .../it/db/it/IoTDBWindowTVFRegionsIT.java          | 135 ++++
 .../relational/it/db/it/udf/IoTDBLocalIT.java      |  92 +++
 .../it/db/it/udf/IoTDBSQLFunctionManagementIT.java | 233 +++++++
 .../it/insertquery/IoTDBInsertQueryIT.java         |  93 +--
 .../relational/it/mqtt/IoTDBMQTTServiceIT.java     |  76 ++-
 .../it/query/old/IoTDBSimpleQueryTableIT.java      |  20 +-
 ...oTDBNoSelectExpressionAfterAnalyzedTableIT.java |   7 +-
 .../it/query/recent/IoTDBComplexQueryIT.java       | 109 +++
 .../relational/it/query/recent/IoTDBCteIT.java     |  28 +-
 .../it/query/recent/IoTDBDistinctTagIT.java        |   9 +-
 .../it/query/recent/IoTDBGapFillTableIT.java       |  40 ++
 .../it/query/recent/IoTDBTableAggregationIT.java   |  90 ++-
 .../view/old/IoTDBSimpleQueryTableViewIT.java      |  29 +-
 .../view/recent/IoTDBComplexQueryTableViewIT.java  |   2 -
 .../query/view/recent/IoTDBTableViewQueryIT.java   |  53 +-
 ...oTDBTableViewQueryWithNotMatchedDataTypeIT.java |  44 +-
 .../it/schema/IoTDBAlterColumnTypeIT.java          |   1 -
 .../it/schema/IoTDBAttributeUpdateOrderingIT.java  |  82 +++
 .../relational/it/schema/IoTDBDatabaseIT.java      | 128 +++-
 .../schema/IoTDBDatabaseMaxRegionGroupNumIT.java   |  31 +-
 .../it/schema/IoTDBNullableAttributeCacheIT.java   | 261 +++++++
 .../iotdb/relational/it/schema/IoTDBTableIT.java   | 130 +++-
 .../it/schema/IoTDBTableTagSchemaMigrationIT.java  | 386 +++++++++++
 .../schema/TableMetadataPropagationTestBase.java   | 450 +++++++++++++
 .../it/schema/fault/MetadataFaultAgent.java        | 600 +++++++++++++++++
 .../it/schema/fault/MetadataFaultController.java   | 244 +++++++
 .../it/session/InternalTableSessionIT.java         | 174 +++++
 .../it/session/InternalTableSessionRedirectIT.java | 141 ++++
 .../it/session/IoTDBSessionRelationalIT.java       | 239 ++++---
 .../it/session/IoTDBTableModelSessionIT.java       |  25 +-
 .../apache/iotdb/session/it/InternalSessionIT.java | 225 +++++++
 .../session/it/InternalSessionRedirectIT.java      | 140 ++++
 .../iotdb/session/it/IoTDBConnectionsIT.java       |  74 +-
 .../session/it/IoTDBSessionAlignedInsertIT.java    | 260 +++----
 .../iotdb/session/it/IoTDBSessionComplexIT.java    |  38 +-
 .../iotdb/session/it/IoTDBSessionInsertNullIT.java | 192 +++---
 .../iotdb/session/it/IoTDBSessionSimpleIT.java     |  23 +-
 .../session/it/IoTDBSessionSyntaxConventionIT.java | 121 ++--
 .../org/apache/iotdb/session/it/SessionIT.java     |  72 +-
 .../session/it/SessionSeedNodeFallbackIT.java      |  40 +-
 .../iotdb/session/it/pool/SessionPoolIT.java       |  76 ++-
 .../tablemodel/InternalSubscriptionTableIT.java    | 226 +++++++
 .../org/apache/iotdb/pipe/api/type/Binary.java     |   5 +-
 .../org/apache/iotdb/pipe/api/type/BinaryTest.java |  61 ++
 iotdb-api/udf-api/pom.xml                          |   5 +
 .../java/org/apache/iotdb/udf/api/type/Binary.java |   5 +-
 .../org/apache/iotdb/udf/api/type/BinaryTest.java  |  61 ++
 .../en/org/apache/iotdb/cli/i18n/CliMessages.java  |   4 +-
 .../zh/org/apache/iotdb/cli/i18n/CliMessages.java  |   4 +-
 .../src/main/java/org/apache/iotdb/cli/Cli.java    |  11 +-
 .../apache/iotdb/tool/data/AbstractDataTool.java   |  47 +-
 .../apache/iotdb/tool/schema/ImportSchemaTree.java |   2 +-
 .../test/java/org/apache/iotdb/cli/CliTest.java    |  85 +++
 .../iotdb/tool/data/DataToolRegressionTest.java    | 248 +++++++
 iotdb-client/client-py/iotdb/utils/rpc_utils.py    |  17 +-
 .../client-py/tests/unit/test_rpc_utils.py         |  32 +
 .../org/apache/iotdb/isession/INodeSupplier.java   |  18 +
 .../org/apache/iotdb/jdbc/IoTDBConnection.java     |  53 +-
 .../iotdb/jdbc/IoTDBTablePreparedStatement.java    |   7 +-
 .../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 150 +++++
 .../jdbc/IoTDBTablePreparedStatementTest.java      | 182 +++++
 .../en/org/apache/iotdb/rpc/i18n/RpcMessages.java  |  21 +
 .../zh/org/apache/iotdb/rpc/i18n/RpcMessages.java  |  21 +
 .../org/apache/iotdb/rpc/AutoResizingBuffer.java   |  36 +-
 .../iotdb/rpc/AutoScalingBufferReadTransport.java  |  34 +-
 .../iotdb/rpc/AutoScalingBufferWriteTransport.java |  94 ++-
 .../iotdb/rpc/DeepCopyRpcTransportFactory.java     |  18 +-
 .../org/apache/iotdb/rpc/RpcServiceConstants.java  |  11 +-
 .../rpc/TCompressedElasticFramedTransport.java     |  52 +-
 .../apache/iotdb/rpc/TElasticFramedTransport.java  |  73 +-
 .../java/org/apache/iotdb/rpc/TSStatusCode.java    |   1 +
 .../iotdb/rpc/TSnappyElasticFramedTransport.java   |  30 +-
 .../rpc/TimeoutChangeableTFastFramedTransport.java |  29 +-
 .../TimeoutChangeableTSnappyFramedTransport.java   |  29 +-
 .../apache/iotdb/rpc/AutoResizingBufferTest.java   | 131 ++++
 .../iotdb/rpc/DeepCopyRpcTransportFactoryTest.java |  41 ++
 .../rpc/TCompressedElasticFramedTransportTest.java | 295 ++++++++
 .../iotdb/rpc/TElasticFramedTransportTest.java     | 252 +++++++
 .../apache/iotdb/session/i18n/SessionMessages.java |   9 +
 .../apache/iotdb/session/i18n/SessionMessages.java |   9 +
 .../iotdb/session/AbstractSessionBuilder.java      |   1 +
 .../org/apache/iotdb/session/NodesSupplier.java    |  57 +-
 .../java/org/apache/iotdb/session/Session.java     | 122 +++-
 .../apache/iotdb/session/SessionConnection.java    |  18 +-
 .../apache/iotdb/session/TableSessionBuilder.java  |  20 +-
 .../org/apache/iotdb/session/ThriftConnection.java |  19 +-
 .../session/pool/AbstractSessionPoolBuilder.java   |   2 +
 .../org/apache/iotdb/session/pool/SessionPool.java | 107 ++-
 .../iotdb/session/pool/TableSessionPool.java       |   2 +-
 .../session/pool/TableSessionPoolBuilder.java      |  22 +-
 .../iotdb/session/pool/TableSessionWrapper.java    |  57 +-
 .../iotdb/session/rpccompress/TabletEncoder.java   | 104 ++-
 .../iotdb/session/util/SessionTypeServices.java    | 141 ++++
 .../apache/iotdb/session/util/SessionUtils.java    |  17 +
 .../iotdb/session/SessionSeedNodeFallbackTest.java | 149 ++++
 .../java/org/apache/iotdb/session/SessionTest.java |  75 +++
 .../iotdb/session/pool/SessionPoolCloseTest.java   | 381 +++++++++++
 .../rpccompress/TabletEncoderCompressionTest.java  | 230 +++++++
 .../rpccompress/TabletEncoderPlainTest.java        | 289 ++++++++
 .../subscription/i18n/SubscriptionMessages.java    |   1 -
 .../subscription/i18n/SubscriptionMessages.java    |   1 -
 .../rpc/subscription/config/ConsumerConstant.java  |   7 +
 .../subscription/SubscriptionSessionWrapper.java   |  28 +-
 .../subscription/SubscriptionTableSession.java     |   4 +
 .../SubscriptionTableSessionBuilder.java           |  36 +-
 .../SubscriptionTreeSessionBuilder.java            |  30 +
 .../base/AbstractSubscriptionConsumer.java         |  51 +-
 .../base/AbstractSubscriptionConsumerBuilder.java  |  37 +
 .../base/AbstractSubscriptionProvider.java         |  24 +-
 .../base/AbstractSubscriptionProviders.java        |   2 +-
 .../SubscriptionTablePullConsumerBuilder.java      |  40 ++
 .../SubscriptionTablePushConsumerBuilder.java      |  36 +
 .../tree/SubscriptionTreePullConsumerBuilder.java  |  36 +
 .../tree/SubscriptionTreePushConsumerBuilder.java  |  36 +
 .../SubscriptionConsumerMultiProviderPollTest.java |  73 ++
 .../SubscriptionProviderConnectionTimeoutTest.java |  25 +-
 .../base/SubscriptionProviderStatusTest.java       |  12 +
 iotdb-core/ainode/README.md                        |  24 +-
 iotdb-core/ainode/pyproject.toml                   |  17 +-
 .../ainode/tests/test_dependency_security.py       | 115 ++++
 .../AbstractApproxMostFrequentAccumulator.java     |  19 +-
 .../ApproxCountDistinctAccumulator.java            |  12 +-
 .../BlobApproxMostFrequentAccumulator.java         |   4 +
 .../approximate/HyperLogLogStateFactory.java       |   2 +-
 .../aggregation/approximate/SpaceSaving.java       |   4 +
 ...stractGroupedApproxMostFrequentAccumulator.java |  18 +-
 .../BlobGroupedApproxMostFrequentAccumulator.java  |   9 +-
 .../GroupedApproxCountDistinctAccumulator.java     |  12 +-
 .../ir/GlobalTimePredicateExtractVisitor.java      |  18 +-
 .../transformation/dag/util/CastFunctionUtils.java |   2 +-
 .../dag/util/CastFunctionUtilsTest.java            |  31 +
 .../iotdb/confignode/i18n/ConfigNodeMessages.java  |  11 +
 .../iotdb/confignode/i18n/ManagerMessages.java     |   7 +-
 .../iotdb/confignode/i18n/ConfigNodeMessages.java  |  10 +
 .../iotdb/confignode/i18n/ManagerMessages.java     |   6 +-
 .../async/DataNodeTransportFailureUtils.java       |  92 +++
 .../handlers/rpc/DataNodeTSStatusRPCHandler.java   |   8 +-
 .../async/handlers/rpc/PipePushMetaRPCHandler.java |  10 +-
 .../ConsumerGroupPushMetaRPCHandler.java           |  11 +-
 .../iotdb/confignode/conf/ConfigNodeConfig.java    |   9 +-
 .../confignode/conf/ConfigNodeDescriptor.java      |  20 +-
 .../confignode/conf/ConfigNodeStartupCheck.java    |  25 +
 .../iotdb/confignode/manager/ConfigManager.java    |  68 +-
 .../manager/load/balancer/RouteBalancer.java       |   4 +-
 .../iotdb/confignode/manager/node/NodeManager.java |  37 +
 .../manager/partition/PartitionManager.java        |  11 +
 .../manager/pipe/agent/PipeConfigNodeAgent.java    |  34 +
 .../pipe/agent/task/PipeConfigNodeTaskAgent.java   |   5 +
 .../pipe/coordinator/runtime/PipeMetaSyncer.java   |  25 +-
 .../pipe/coordinator/task/PipeTaskCoordinator.java |  50 +-
 .../manager/schema/ClusterSchemaManager.java       |  34 +-
 .../subscription/SubscriptionCoordinator.java      |   7 +-
 .../runtime/SubscriptionLeaderChangeHandler.java   |   9 +-
 .../confignode/persistence/ProcedureInfo.java      |   4 +-
 .../partition/DatabasePartitionTable.java          |  16 +
 .../persistence/partition/PartitionInfo.java       |  17 +
 .../persistence/partition/RegionGroup.java         |   7 +
 .../confignode/persistence/pipe/PipeTaskInfo.java  |  46 +-
 .../confignode/procedure/ProcedureExecutor.java    |   6 +-
 .../procedure/TimeoutExecutorThread.java           |  18 +-
 .../procedure/env/ConfigNodeProcedureEnv.java      |  23 +-
 .../procedure/env/RegionMaintainHandler.java       |  37 +-
 .../impl/region/AddRegionPeerProcedure.java        |  17 +
 .../impl/schema/DataNodeRegionGroupUtil.java       |   7 +-
 .../schema/table/SetTablePropertiesProcedure.java  |  16 +-
 .../AbstractOperateSubscriptionProcedure.java      |  14 +
 .../consumer/AlterConsumerGroupProcedure.java      |  50 +-
 .../subscription/DropSubscriptionProcedure.java    |  23 +-
 .../confignode/service/ConfigNodeShutdownHook.java |   3 +-
 .../confignode/conf/ConfigNodeConfigTest.java      |  25 +
 .../conf/ConfigNodeStartupCheckTest.java           |  87 +++
 .../manager/ClusterSchemaManagerTest.java          | 187 +++++
 .../ConfigManagerRegionGroupPolicyVersionTest.java | 280 ++++++++
 .../manager/lease/ClusterCachePropagatorTest.java  |  42 ++
 .../manager/node/NodeManagerStatusReasonTest.java  |  23 +-
 .../node/NodeManagerVersionConsistencyTest.java    | 178 +++++
 .../agent/task/PipeConfigNodeTaskAgentTest.java    | 111 +++
 .../coordinator/runtime/PipeMetaSyncerTest.java    |  79 +++
 .../coordinator/task/PipeTaskCoordinatorTest.java  | 157 +++++
 .../confignode/persistence/ProcedureInfoTest.java  |  38 ++
 .../persistence/pipe/PipeTaskInfoRatisTest.java    | 149 ++++
 .../procedure/ProcedureExecutorStopTest.java       |  57 ++
 .../procedure/env/ConfigNodeProcedureEnvTest.java  |  59 ++
 .../RegionMaintainHandlerConsensusPipeTest.java    |  83 +++
 .../impl/region/AddRegionPeerProcedureTest.java    | 122 ++++
 .../impl/schema/DataNodeRegionGroupUtilTest.java   |  54 ++
 .../table/SetTablePropertiesProcedureTest.java     | 148 ++++
 .../DropSubscriptionProcedureTest.java             | 382 ++++++++++-
 iotdb-core/datanode/pom.xml                        |  10 +-
 .../apache/iotdb/db/i18n/DataNodeMiscMessages.java |  16 +-
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |  10 +
 .../iotdb/db/i18n/DataNodeQueryMessages.java       |  64 ++
 .../iotdb/db/i18n/StorageEngineMessages.java       |   4 +-
 .../apache/iotdb/db/i18n/DataNodeMiscMessages.java |  16 +-
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |  10 +
 .../iotdb/db/i18n/DataNodeQueryMessages.java       |  65 ++
 .../iotdb/db/i18n/StorageEngineMessages.java       |   4 +-
 .../org/apache/iotdb/db/auth/AuthorityChecker.java |   2 +-
 .../apache/iotdb/db/conf/DataNodeMemoryConfig.java |   6 +-
 .../consensus/statemachine/BaseStateMachine.java   |  26 +-
 .../dataregion/DataRegionStateMachine.java         |   6 +-
 .../schemaregion/SchemaRegionStateMachine.java     |   6 +-
 .../load/LoadAnalyzeMissingDatabaseException.java} |   9 +-
 .../iotdb/db/pipe/agent/PipeDataNodeAgent.java     |  34 +
 .../db/pipe/agent/task/PipeDataNodeTaskAgent.java  |   6 +
 .../agent/task/connection/PipeEventCollector.java  |   6 +-
 .../execution/PipeProcessorSubtaskExecutor.java    |  10 +
 .../task/execution/PipeSubtaskExecutorManager.java |  31 +-
 .../subtask/processor/PipeProcessorSubtask.java    |  22 +-
 .../processor/PipeProcessorSubtaskWorker.java      | 108 ++-
 .../PipeProcessorSubtaskWorkerManager.java         |  27 +-
 .../sink/PipeRealtimePriorityBlockingQueue.java    |   8 +-
 .../common/tablet/PipeRawTabletInsertionEvent.java |  23 +-
 .../common/tsfile/PipeTsFileInsertionEvent.java    |  59 +-
 .../pipe/metric/overview/PipeResourceMetrics.java  |  48 +-
 .../legacy/IoTDBLegacyPipeReceiverAgent.java       |  14 +
 .../protocol/thrift/IoTDBDataNodeReceiver.java     |  48 ++
 .../db/pipe/resource/memory/PipeMemoryManager.java | 257 +++++--
 .../builder/PipeTableModelTsFileBuilderV2.java     |   4 +
 ...istoricalDataRegionTsFileAndDeletionSource.java |  34 +-
 .../realtime/PipeRealtimeDataRegionSource.java     |   1 +
 .../PipeRealtimeDataRegionTsFileSource.java        |   3 +-
 .../realtime/epoch/TsFileEpochManager.java         |   5 +-
 .../iotdb/db/protocol/session/IClientSession.java  |  13 +
 .../iotdb/db/protocol/session/SessionManager.java  | 134 +++-
 .../thrift/InternalClientRPCServiceProcessor.java  |  70 ++
 .../handler/InternalServiceThriftHandler.java      |  43 +-
 .../protocol/thrift/impl/ClientRPCServiceImpl.java |  66 +-
 .../impl/DataNodeInternalRPCServiceImpl.java       | 162 +++--
 .../thrift/impl/InternalClientRPCServiceImpl.java} |  11 +-
 .../db/queryengine/common/DataNodeEndPoints.java   |   7 +
 .../db/queryengine/common/MPPQueryContext.java     |  55 +-
 .../db/queryengine/common/QueryAttemptId.java      |  59 ++
 .../queryengine/execution/QueryStateMachine.java   |   3 +-
 .../execution/fragment/DataNodeQueryContext.java   |   9 +-
 .../fragment/FragmentInstanceContext.java          |  87 ++-
 .../fragment/FragmentInstanceExecution.java        |   5 +
 .../fragment/FragmentInstanceManager.java          | 135 +++-
 .../fragment/FragmentInstanceStateMachine.java     |  74 +-
 .../execution/fragment/QueryAttemptRegistry.java   | 389 +++++++++++
 .../operator/process/TransformOperator.java        |  37 +-
 .../iotdb/db/queryengine/plan/Coordinator.java     |  39 +-
 .../db/queryengine/plan/analyze/Analysis.java      |  20 +-
 .../db/queryengine/plan/analyze/AnalyzeUtils.java  |  48 +-
 .../queryengine/plan/analyze/AnalyzeVisitor.java   |  24 +-
 .../db/queryengine/plan/analyze/Analyzer.java      |   1 +
 .../db/queryengine/plan/analyze/IAnalysis.java     |   5 +-
 .../plan/analyze/load/LoadTsFileAnalyzer.java      | 122 ++--
 .../analyze/load/LoadTsFileTableSchemaCache.java   | 111 ++-
 .../load/TreeSchemaAutoCreatorAndVerifier.java     |  30 +-
 .../plan/analyze/schema/NormalSchemaFetcher.java   |  28 +-
 .../plan/analyze/schema/SchemaValidator.java       |   8 +-
 .../plan/analyze/schema/TemplateSchemaFetcher.java |  13 +-
 .../queryengine/plan/execution/QueryExecution.java | 169 ++++-
 .../config/executor/ClusterConfigTaskExecutor.java |  64 +-
 .../config/metadata/ShowFunctionsTask.java         |  16 +-
 .../plan/parser/StatementGenerator.java            |   4 +
 .../planner/DataNodeTableOperatorGenerator.java    |   6 +-
 .../db/queryengine/plan/planner/IPlanner.java      |   3 +-
 .../plan/planner/LocalExecutionPlanner.java        |  73 +-
 .../queryengine/plan/planner/TreeModelPlanner.java |   3 +-
 .../NotThreadSafeMemoryReservationManager.java     |  15 +-
 .../plan/planner/plan/FragmentInstance.java        |  51 +-
 .../plan/node/load/LoadSingleTsFileNode.java       |  37 +-
 .../planner/plan/node/load/LoadTsFileNode.java     |   6 +-
 .../planner/plan/node/write/InsertRowNode.java     |   3 +-
 .../planner/plan/node/write/InsertRowsNode.java    |   6 +-
 .../plan/node/write/InsertRowsOfOneDeviceNode.java |   3 +-
 .../planner/plan/node/write/InsertTabletNode.java  |   6 +-
 .../plan/node/write/RelationalInsertRowsNode.java  |  18 +-
 .../node/write/RelationalInsertTabletNode.java     |  19 +-
 .../plan/relational/analyzer/Analysis.java         |  21 +-
 .../plan/relational/analyzer/Analyzer.java         |   1 +
 .../relational/analyzer/ExpressionAnalyzer.java    |  11 +
 .../relational/analyzer/StatementAnalyzer.java     |  46 ++
 .../ConvertPredicateToTimeFilterVisitor.java       |  12 +-
 .../plan/relational/metadata/Metadata.java         |   3 +-
 .../relational/metadata/TableMetadataImpl.java     |   4 +-
 .../fetcher/DeviceInCacheFilterVisitor.java        |   8 +-
 .../fetcher/TableDeviceCacheAttributeGuard.java    |   3 +-
 .../metadata/fetcher/TableDeviceSchemaFetcher.java |   2 +-
 .../fetcher/TableHeaderSchemaValidator.java        |   8 +-
 .../planner/IrExpressionInterpreter.java           | 326 ++++++---
 .../plan/relational/planner/QueryPlanner.java      |  12 +-
 .../plan/relational/planner/TableModelPlanner.java |   3 +-
 .../plan/relational/planner/TagColumnUtils.java    |  67 ++
 .../plan/relational/planner/TranslationMap.java    |   5 +-
 .../distribute/TableDistributedPlanGenerator.java  | 175 ++++-
 .../ir/GapFillStartAndEndTimeExtractVisitor.java   |  46 +-
 .../planner/optimizations/ParallelizeGrouping.java |  59 +-
 .../optimizations/TransformSortToStreamSort.java   |  18 +-
 .../security/TreeAccessCheckContext.java           |  10 +-
 .../security/TreeAccessCheckVisitor.java           |   8 +-
 .../plan/relational/sql/ast/InsertRows.java        |  45 ++
 .../plan/relational/sql/ast/InsertValues.java      | 346 ++++++++++
 .../plan/relational/sql/parser/AstBuilder.java     | 234 +------
 .../plan/relational/sql/parser/SqlParser.java      |  13 +-
 .../plan/relational/sql/util/AstUtil.java          |  83 +++
 .../plan/scheduler/ClusterScheduler.java           |  61 +-
 .../scheduler/FragmentInstanceDispatcherImpl.java  | 159 ++++-
 .../plan/scheduler/IQueryTerminator.java           |   8 +
 .../db/queryengine/plan/scheduler/IScheduler.java  |   1 +
 .../plan/scheduler/SimpleQueryTerminator.java      |  44 +-
 .../plan/scheduler/load/LoadFallbackHandler.java   |  10 +-
 .../plan/scheduler/load/LoadTsFileScheduler.java   |  18 +
 .../plan/statement/crud/InsertBaseStatement.java   |  22 +-
 .../crud/InsertMultiTabletsStatement.java          |  46 +-
 .../crud/InsertRowsOfOneDeviceStatement.java       |  46 +-
 .../plan/statement/crud/InsertRowsStatement.java   |  55 +-
 .../db/queryengine/plan/udf/TreeUDFUtils.java      |  17 +-
 .../transformation/dag/udf/UDTFContext.java        |   2 -
 .../mtree/impl/pbtree/schemafile/SchemaFile.java   | 115 +++-
 .../mtree/impl/pbtree/schemafile/SchemaPage.java   |   3 +-
 .../pbtree/schemafile/pagemgr/PageIOChannel.java   |  79 ++-
 .../pbtree/schemafile/pagemgr/PageManager.java     |  27 +-
 .../reader/impl/TimeseriesReaderWithViewFetch.java |  44 ++
 .../java/org/apache/iotdb/db/service/DataNode.java |   6 +-
 .../db/service/DataNodeInternalRPCService.java     |  40 +-
 .../metrics/IoTDBInternalLocalReporter.java        |   2 +-
 .../db/storageengine/dataregion/DataRegion.java    | 179 +++--
 .../impl/ReadChunkCompactionPerformer.java         |   4 +
 .../compaction/io/CompactionTsFileWriter.java      |  11 +
 .../dataregion/flush/MemTableFlushTask.java        |  39 +-
 .../dataregion/memtable/TsFileProcessor.java       |  17 +
 .../dataregion/snapshot/SnapshotLoader.java        |  28 +-
 .../dataregion/snapshot/SnapshotTaker.java         |  13 +-
 .../dataregion/tsfile/TsFileResource.java          |  24 +-
 .../utils/TsFileTableSchemaRegistrationUtils.java  | 110 +++
 .../tableDiskUsageIndex/TableDiskUsageIndex.java   |  18 +
 .../dataregion/wal/buffer/WALBuffer.java           |  22 +-
 .../storageengine/dataregion/wal/node/WALNode.java |  11 +-
 .../file/UnsealedTsFileRecoverPerformer.java       |  25 +-
 .../db/storageengine/load/LoadStagingDirs.java     |  16 +-
 .../db/storageengine/load/LoadTsFileProgress.java  |  73 +-
 .../db/storageengine/load/LoadTsFileSnapshot.java  |  59 +-
 .../load/active/ActiveLoadPendingQueue.java        |  18 +-
 .../load/active/ActiveLoadTsFileLoader.java        | 278 +++++++-
 .../converter/LoadTsFileDataTypeConverter.java     |  71 +-
 .../splitter/BatchedAlignedValueChunkData.java     | 107 +--
 .../iotdb/db/storageengine/load/util/LoadUtil.java |  18 +-
 .../db/storageengine/rescon/disk/TierManager.java  | 207 ++++--
 .../agent/SubscriptionReceiverAgent.java           |  42 +-
 .../agent/SubscriptionRuntimeAgent.java            |   4 +-
 .../broker/ConsensusSubscriptionBroker.java        |  28 +-
 .../consensus/ConsensusPrefetchingQueue.java       | 179 +++--
 .../ConsensusSubscriptionCommitManager.java        | 393 +++++++++--
 .../ConsensusSubscriptionSetupHandler.java         |   7 +-
 .../broker/consensus/ProgressWALIterator.java      |  98 ++-
 .../consensus/SubscriptionConsensusProgress.java   |  29 +-
 .../event/batch/SubscriptionPipeEventBatches.java  |   8 +-
 .../receiver/SubscriptionReceiver.java             |   5 +
 .../receiver/SubscriptionReceiverV1.java           | 117 +++-
 ...nsensusSubscriptionPrefetchExecutorManager.java |   8 +-
 .../org/apache/iotdb/db/utils/CommonUtils.java     |  62 --
 .../apache/iotdb/db/utils/ErrorHandlingUtils.java  |  91 +--
 .../org/apache/iotdb/db/auth/TreeAccessTest.java   |   2 +-
 .../db/conf/DataNodeMemoryProportionTest.java      | 106 +++
 .../iotdb/db/conf/DiskFullHotReloadTest.java       | 143 ++++
 .../dataregion/IoTConsensusShutdownTest.java       |  10 +-
 .../metadata/mtree/schemafile/SchemaFileTest.java  | 261 +++++++
 .../metadata/mtree/schemafile/SchemaPageTest.java  | 110 +++
 .../pipe/agent/task/PipeDataNodeTaskAgentTest.java | 143 +++-
 .../task/PipeProcessorSubtaskExecutorTest.java     |  33 +
 .../agent/task/PipeSinkSubtaskExecutorTest.java    | 183 +++++
 .../pipe/agent/task/PipeSubtaskExecutorTest.java   |  25 +
 .../task/connection/PipeEventCollectorTest.java    |  31 +
 .../execution/PipeSubtaskExecutorManagerTest.java  |  66 ++
 .../PipeProcessorSubtaskExecutionGuardTest.java    | 264 +++++++-
 .../processor/PipeProcessorSubtaskWorkerTest.java  | 494 +++++++++++++-
 .../PipeRealtimePriorityBlockingQueueTest.java     |  33 +-
 .../task/subtask/sink/PipeSinkSubtaskTest.java     | 150 +++--
 .../pipe/event/PipeTsFileInsertionEventTest.java   |  32 +
 .../metric/overview/PipeResourceMetricsTest.java   |  81 +++
 .../metric/sink/PipeDataRegionSinkMetricsTest.java |  38 +-
 .../airgap/IoTDBAirGapReceiverAgentTest.java       |  14 +-
 .../legacy/IoTDBLegacyPipeReceiverAgentTest.java   |  72 ++
 .../protocol/thrift/IoTDBDataNodeReceiverTest.java | 202 ++++++
 .../resource/memory/PipeMemoryManagerTest.java     | 490 +++++++++++++-
 .../airgap/IoTDBDataRegionAirGapSinkTest.java      |   6 +-
 .../handler/PipeTransferTrackableHandlerTest.java  |  16 +-
 .../db/pipe/source/PipeRealtimeExtractTest.java    | 415 +++++++++++-
 ...ricalDataRegionTsFileAndDeletionSourceTest.java | 268 ++++++++
 .../realtime/disruptor/DisruptorShutdownTest.java  |  81 ++-
 .../realtime/epoch/TsFileEpochManagerTest.java     |  11 -
 .../db/protocol/client/ConfigNodeClientTest.java   | 107 +++
 .../handler/InternalServiceThriftHandlerTest.java  |  57 ++
 .../impl/DataNodeInternalRPCServiceImplCQTest.java | 163 +++++
 .../DataNodeInternalRPCServiceImplDiskTest.java    |  46 ++
 .../db/queryengine/common/MPPQueryContextTest.java |  38 ++
 .../fragment/FragmentInstanceManagerTest.java      | 100 +++
 .../fragment/FragmentInstanceStateMachineTest.java |  56 ++
 .../fragment/QueryAttemptAdmissionTest.java        | 184 +++++
 .../fragment/QueryAttemptRegistryTest.java         | 379 +++++++++++
 .../execution/fragment/QueryAttemptRpcTest.java    | 140 ++++
 ...nAlignedTreeDeviceViewScanOperatorTreeTest.java |   2 +-
 .../sink/SinkOperatorDownStreamNodeIdTest.java     |   2 +-
 .../iotdb/db/queryengine/plan/CoordinatorTest.java | 149 ++++
 .../plan/analyze/QueryPartitionMemoryTest.java     | 300 +++++++++
 .../plan/analyze/QueryTimePartitionTest.java       |  47 ++
 .../load/LoadTsFileAnalyzerDiagnosticsTest.java    | 541 ++++++++++++++-
 .../plan/analyze/load/LoadTsFileAnalyzerTest.java  | 411 +++++++++++
 .../load/LoadTsFileTableSchemaRecoveryTest.java    | 185 +++++
 .../schema/TreeSchemaAutoCreationPolicyTest.java   | 207 ++++++
 .../execution/QueryExecutionLifecycleTest.java     | 171 +++++
 .../plan/execution/QueryExecutionRetryTest.java    |  91 ++-
 .../plan/planner/FragmentInstanceSerdeTest.java    |  41 +-
 .../plan/planner/LastQueryCacheTest.java           |   7 +-
 .../planner/LocalExecutionPlannerBudgetTest.java   |   4 +-
 .../LocalExecutionPlannerOperatorsMemoryTest.java  |  56 ++
 .../plan/planner/PipelineBuilderTest.java          |   4 +-
 .../plan/planner/node/load/LoadTsFileNodeTest.java |  38 +-
 .../planner/node/write/WritePlanNodeSplitTest.java |  29 +
 .../relational/analyzer/TableFunctionTest.java     | 241 +++++++
 .../TableDeviceCacheAttributeGuardTest.java        |  64 ++
 .../relational/planner/TableModelPlannerTest.java  |  13 +-
 .../planner/WindowFunctionOptimizationTest.java    |  19 +
 .../planner/assertions/MergeSortMatcher.java       |  67 ++
 .../planner/assertions/PlanMatchPattern.java       |   4 +
 .../planner/distribute/RateOrderedInputTest.java   | 284 ++++++++
 .../planner/distribute/RateOrderedPlannerTest.java | 152 +++++
 .../relational/sql/AstMemoryEstimatorTest.java     | 185 +++++
 .../plan/relational/sql/ast/InsertValuesTest.java  | 204 ++++++
 .../scheduler/ClusterSchedulerLifecycleTest.java   | 130 ++++
 .../FragmentInstanceDispatcherLifecycleTest.java   | 263 ++++++++
 .../plan/scheduler/QueryAttemptProtocolTest.java   |  95 +++
 .../plan/scheduler/QueryTerminalCleanupTest.java   | 152 +++++
 .../plan/scheduler/SimpleQueryTerminatorTest.java  | 123 ++++
 .../scheduler/load/LoadFallbackHandlerTest.java    | 139 ++++
 .../scheduler/load/LoadTsFileSchedulerTest.java    |  21 +-
 .../crud/InsertStatementPartialInsertTest.java     | 117 ++++
 .../builder/EvaluationDAGBuilderTest.java          |   2 +-
 .../storageengine/dataregion/DataRegionTest.java   | 281 ++++++++
 .../dataregion/flush/FlushMemoryEstimateTest.java  |  53 ++
 .../dataregion/memtable/MemTableFlushTaskTest.java | 208 ++++++
 .../dataregion/snapshot/IoTDBSnapshotTest.java     |  30 +
 .../snapshot/SnapshotLinkFallbackTest.java         |  72 ++
 .../snapshot/SnapshotLoaderDiskFullReloadTest.java | 126 ++++
 .../dataregion/tsfile/TsFileResourceTest.java      |   5 +
 .../TsFileTableSchemaRegistrationUtilsTest.java    | 204 ++++++
 .../wal/buffer/WALBufferRollRecoveryTest.java      |  65 ++
 .../dataregion/wal/node/WALNodeTest.java           | 118 +++-
 .../wal/node/WALNodeWaitForRollFileTest.java       | 101 +--
 .../wal/recover/WALRecoverManagerTest.java         |   6 +-
 ...ncryptedUnsealedTsFileRecoverPerformerTest.java |  10 +-
 .../file/UnsealedTsFileRecoverPerformerTest.java   |  13 +-
 .../storageengine/load/LoadTsFileProgressTest.java | 149 ++++
 .../storageengine/load/LoadTsFileSnapshotTest.java | 420 ++++++++++--
 .../db/storageengine/load/TsFileSplitterTest.java  | 209 ++++++
 .../load/active/ActiveLoadTsFileLoaderTest.java    | 147 +++-
 .../LoadTsFileDataTypeConverterSchemaTest.java     | 349 ++++++++++
 .../disk/FolderManagerDiskFullRecoveryTest.java    | 160 +++++
 .../rescon/disk/TierManagerDiskFullReloadTest.java | 120 ++++
 .../storageengine/rescon/disk/TierManagerTest.java |  76 ++-
 .../agent/SubscriptionReceiverAgentTest.java       |  54 ++
 ...onsensusSubscriptionBrokerPayloadLimitTest.java |  37 +-
 ...ConsensusSubscriptionBrokerPollTimeoutTest.java | 198 ++++++
 .../ConsensusPrefetchingQueuePollTimeoutTest.java  | 180 +++++
 .../ConsensusPrefetchingQueueReplicationTest.java  | 319 +++++++++
 .../ConsensusPrefetchingQueueSeekTest.java         | 242 +++++++
 ...nsensusPrefetchingQueueWalBackpressureTest.java | 195 ++++++
 .../ConsensusSubscriptionCommitStateTest.java      | 358 +++++++++-
 .../broker/consensus/ProgressWALIteratorTest.java  | 325 ++++++++-
 ...ubscriptionPipeEventBatchesConcurrencyTest.java | 144 ++++
 .../receiver/SubscriptionReceiverV1Test.java       | 274 ++++++++
 .../apache/iotdb/db/utils/DateTimeUtilsTest.java   |  99 ++-
 .../db/utils/FileChannelWriteArchitectureTest.java |  91 +++
 iotdb-core/node-commons/pom.xml                    |  10 +
 .../resources/conf/edge/iotdb-system.properties    |   7 +
 .../apache/iotdb/commons/i18n/PipeMessages.java    |  21 +
 .../apache/iotdb/commons/i18n/QueryMessages.java   |   9 +
 .../apache/iotdb/commons/i18n/UtilMessages.java    |   2 +
 .../apache/iotdb/commons/i18n/PipeMessages.java    |  20 +
 .../apache/iotdb/commons/i18n/QueryMessages.java   |   9 +
 .../apache/iotdb/commons/i18n/UtilMessages.java    |   2 +
 .../apache/iotdb/commons/audit/IAuditEntity.java   |   8 +-
 .../commons/client/AbstractConfigNodeClient.java   |   7 +
 .../client/internal/session/InternalSession.java   |  61 ++
 .../internal/session/InternalSessionBuilder.java   |  59 ++
 .../internal/session/InternalSessionPool.java}     |  17 +-
 .../session/InternalSessionPoolBuilder.java        |  55 ++
 .../session/InternalTableSessionBuilder.java       |  60 ++
 .../session/InternalTableSessionPool.java}         |  35 +-
 .../session/InternalTableSessionPoolBuilder.java   |  56 ++
 .../InternalSubscriptionSessionWrapper.java        |  78 +++
 .../InternalSubscriptionTablePullConsumer.java     |  25 +-
 ...ternalSubscriptionTablePullConsumerBuilder.java |  43 ++
 .../InternalSubscriptionTableSession.java          |  15 +-
 .../InternalSubscriptionTableSessionBuilder.java   |  55 ++
 .../apache/iotdb/commons/conf/CommonConfig.java    |  79 ++-
 .../apache/iotdb/commons/disk/FolderManager.java   |  37 +-
 .../commons/disk/strategy/DirectoryStrategy.java   |  22 +-
 .../commons/enums/HandleSystemErrorStrategy.java   |  31 +-
 .../commons/pipe/agent/task/PipeTaskAgent.java     |  47 +-
 .../agent/task/execution/PipeSubtaskExecutor.java  |  52 +-
 .../task/subtask/PipeAbstractSinkSubtask.java      |  33 +-
 .../options/PipeInclusionOptions.java              |  11 +
 .../commons/queryengine/common/SessionInfo.java    |  70 +-
 .../queryengine/plan/udf/UDFManagementService.java | 129 +++-
 .../commons/queryengine/utils/DateTimeUtils.java   |   5 +-
 .../AbstractPeriodicalServiceWithAdvance.java      |  14 +-
 .../iotdb/commons/service/ThriftService.java       |  10 +-
 .../iotdb/commons/service/metric/enums/Metric.java |   1 +
 .../org/apache/iotdb/commons/udf/UDFTable.java     |  81 ++-
 .../iotdb/commons/udf/service/UDFClassLoader.java  |  18 +-
 .../commons/udf/service/UDFClassLoaderManager.java | 179 ++++-
 .../org/apache/iotdb/commons/utils/IOUtils.java    |  36 +
 .../apache/iotdb/commons/utils/JVMCommonUtils.java |  26 +-
 .../session/InternalSessionProtocolTest.java       | 167 +++++
 .../iotdb/commons/conf/CommonConfigTest.java       | 205 +++++-
 .../options/PipeInclusionOptionsTest.java          |  41 ++
 .../plan/udf/UDFManagementServiceTest.java         | 211 ++++++
 .../AbstractPeriodicalServiceWithAdvanceTest.java  |  95 +++
 .../udf/service/UDFClassLoaderManagerTest.java     |  95 +++
 .../apache/iotdb/commons/utils/IOUtilsTest.java    | 152 +++++
 .../iotdb/commons/utils/JVMCommonUtilsTest.java    |  35 +
 .../src/main/thrift/confignode.thrift              |   2 +
 .../src/main/thrift/datanode.thrift                |  16 +
 .../iotdb/library/i18n/LibraryUdfMessages.java     |   6 +
 .../iotdb/library/i18n/LibraryUdfMessages.java     |   5 +
 .../org/apache/iotdb/library/dprofile/UDAFMad.java |  10 +-
 .../dprofile/util/ExactOrderStatistics.java        |  16 +-
 .../apache/iotdb/library/dprofile/UDAFMadTest.java | 114 ++++
 .../dprofile/util/ExactOrderStatisticsTest.java    |  73 ++
 licenses/CDDL License 1.1                          |  90 ---
 licenses/Creative Commons CC0 1.0 Universal        | 121 ++++
 .../Eclipse Distribution License 1.0 (EDL-1.0)     |  30 +
 licenses/Eclipse Public License 1.0 (EPL-1.0)      |  83 ---
 ...(Boost Software License 1.0, MIT, BSD 2-Clause) |  91 +++
 licenses/MIT No Attribution (Reactive Streams)     |   7 +
 ...essbar) => The MIT License (Checker Framework)} |  13 +-
 ... (progressbar) => The MIT License (ClassGraph)} |   4 +-
 licenses/The MIT License (FastDoubleParser)        |  61 ++
 licenses/The MIT License (Java-WebSocket)          |  24 +
 ...e (progressbar) => The MIT License (Schubfach)} |  12 +-
 pom.xml                                            |  25 +-
 692 files changed, 49703 insertions(+), 9045 deletions(-)

diff --cc 
integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java
index 6f140578282,3db7bff4f8b..8566835ac5a
--- 
a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java
+++ 
b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java
@@@ -25,10 -25,9 +25,11 @@@ import org.apache.iotdb.confignode.rpc.
  import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo;
  import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
  import org.apache.iotdb.consensus.ConsensusFactory;
++import org.apache.iotdb.db.i18n.DataNodePipeMessages;
  import org.apache.iotdb.db.it.utils.TestUtils;
  import org.apache.iotdb.isession.ISession;
  import org.apache.iotdb.isession.SessionConfig;
- import org.apache.iotdb.it.env.MultiEnvFactory;
++import org.apache.iotdb.it.env.cluster.config.MppDataNodeConfig;
  import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
  import org.apache.iotdb.it.framework.IoTDBTestRunner;
  import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic;
@@@ -46,29 -45,30 +47,49 @@@ import org.junit.Test
  import org.junit.experimental.categories.Category;
  import org.junit.runner.RunWith;
  
++import java.nio.file.Files;
++import java.nio.file.Path;
  import java.sql.Connection;
  import java.sql.SQLException;
  import java.sql.Statement;
  import java.util.Arrays;
++import java.util.Base64;
  import java.util.Collections;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
++import java.util.Properties;
++import java.util.Random;
++import java.util.Set;
++import java.util.concurrent.TimeUnit;
  import java.util.function.Consumer;
++import java.util.regex.Pattern;
++import java.util.stream.Collectors;
++import java.util.stream.Stream;
  
++import static org.awaitility.Awaitility.await;
  import static org.junit.Assert.fail;
  
  @RunWith(IoTDBTestRunner.class)
  @Category({MultiClusterIT2DualTreeAutoBasic.class})
- public class IoTDBPipeSourceIT extends AbstractPipeDualTreeModelAutoIT {
- 
-   @Before
-   public void setUp() {
-     MultiEnvFactory.createEnv(2);
-     senderEnv = MultiEnvFactory.getEnv(0);
-     receiverEnv = MultiEnvFactory.getEnv(1);
+ public class IoTDBPipeSourceIT extends AbstractPipeTreeModelSharedClusterIT {
  
+   @Override
+   protected void setupConfig() {
+     super.setupConfig();
+     senderEnv
+         .getConfig()
+         .getDataNodeConfig()
 -        .setMetricReporterType(Collections.singletonList("JMX"));
++        .setMetricReporterType(Collections.singletonList("JMX"))
++        .setCompactionScheduleInterval(100);
++    // Keep a backlog at the oldest end while one file is being sent. 
Compaction stays disabled
++    // for the other source tests; only the concurrent-compaction scenario 
enables it temporarily.
++    final Properties queueProperties = new Properties();
++    
queueProperties.setProperty("pipe_realtime_queue_max_waiting_tsfile_size", "1");
++    
queueProperties.setProperty("pipe_realtime_queue_poll_historical_tsfile_threshold",
 "1000000");
++    ((MppDataNodeConfig) senderEnv.getConfig().getDataNodeConfig())
++        .updateProperties(queueProperties);
      senderEnv
          .getConfig()
          .getCommonConfig()
@@@ -80,6 -80,6 +101,7 @@@
          .setEnableSeqSpaceCompaction(false)
          .setEnableUnseqSpaceCompaction(false)
          .setEnableCrossSpaceCompaction(false)
++        .setTimePartitionInterval(TimeUnit.DAYS.toMillis(7))
          .setPipeMemoryManagementEnabled(false)
          .setIsPipeEnableMemoryCheck(false)
          .setPipeAutoSplitFullEnabled(false);
@@@ -1150,6 -1114,6 +1136,135 @@@
      }
    }
  
++  @Test
++  public void testConcurrentSendingAndCompaction() throws Exception {
++    final String pipeName = "compaction_correctness";
++    final String device = "root.compaction_correctness.d1";
++    final String query = "select s1 from " + device;
++    final String header = "Time," + device + ".s1,";
++    final Set<String> expectedRows = new HashSet<>();
++    final DataNodeWrapper receiver = receiverEnv.getDataNodeWrapper(0);
++    try (final Connection connection = senderEnv.getConnection();
++        final Statement statement = connection.createStatement();
++        final SyncConfigNodeIServiceClient client =
++            (SyncConfigNodeIServiceClient) 
senderEnv.getLeaderConfigNodeConnection()) {
++      statement.execute("create timeseries " + device + ".s1 with 
datatype=INT64,encoding=RLE");
++      for (int file = 0; file < 24; file++) {
++        final StringBuilder insert =
++            new StringBuilder("insert into " + device + "(time,s1) values ");
++        for (int point = 0; point < 64; point++) {
++          final int value = file * 64 + point;
++          if (point > 0) {
++            insert.append(',');
++          }
++          
insert.append('(').append(value).append(',').append(value).append(')');
++          expectedRows.add(value + "," + value + ",");
++        }
++        statement.execute(insert.toString());
++        statement.execute("flush");
++      }
++
++      // The newest files occupy the send slot in a separate time partition. 
This leaves every
++      // source of the older compaction queued, including while the first 
send finishes.
++      final long blockerTime = TimeUnit.DAYS.toMillis(7);
++      statement.execute(
++          "create timeseries " + device + ".payload with 
datatype=TEXT,encoding=PLAIN");
++      final byte[] payload = new byte[65536];
++      final Random random = new Random(0);
++      for (int blocker = 0; blocker < 2; blocker++) {
++        random.nextBytes(payload);
++        final long time = blockerTime + blocker;
++        final int value = 1536 + blocker;
++        statement.execute(
++            "insert into "
++                + device
++                + "(time,s1,payload) values ("
++                + time
++                + ","
++                + value
++                + ",'"
++                + Base64.getEncoder().encodeToString(payload)
++                + "')");
++        statement.execute("flush");
++        expectedRows.add(time + "," + value + ",");
++      }
++
++      final Map<String, String> sinkAttributes = new HashMap<>();
++      sinkAttributes.put("sink", "iotdb-thrift-sink");
++      sinkAttributes.put("sink.ip", receiver.getIp());
++      sinkAttributes.put("sink.port", Integer.toString(receiver.getPort()));
++      sinkAttributes.put("sink.enable-compaction", "true");
++      // Keep a send in flight while the remaining historical files are 
compacted.
++      sinkAttributes.put("sink.rate-limit-bytes-per-second", "8192");
++      Assert.assertEquals(
++          TSStatusCode.SUCCESS_STATUS.getStatusCode(),
++          client
++              .createPipe(
++                  new TCreatePipeReq(pipeName, sinkAttributes)
++                      .setExtractorAttributes(
++                          Map.of(
++                              "source.path", "root.compaction_correctness.**",
++                              "source.inclusion", "data.insert",
++                              "source.realtime.mode", "batch")))
++              .getCode());
++      Assert.assertEquals(
++          TSStatusCode.SUCCESS_STATUS.getStatusCode(), 
client.startPipe(pipeName).getCode());
++      TestUtils.assertDataEventuallyOnEnv(
++          receiverEnv,
++          query + " where time = " + (blockerTime + 1),
++          header,
++          Collections.singleton((blockerTime + 1) + ",1537,"));
++
++      try {
++        statement.execute("set configuration 
'enable_seq_space_compaction'='true'");
++        final Pattern replacementMessage =
++            Pattern.compile(
++                
Pattern.quote(DataNodePipeMessages.REGION_REPLACED_TSFILEINSERTIONEVENTS_WITH)
++                    .replace("{}", "\\E.*\\Q"));
++        await()
++            .alias("A real queued compaction replacement for " + pipeName)
++            .atMost(60, TimeUnit.SECONDS)
++            .until(() -> hasCompactionReplacement(pipeName, 
replacementMessage));
++
++        PipeTestUtils.awaitDataPipeCompletion(senderEnv, pipeName);
++        TestUtils.assertDataEventuallyOnEnv(senderEnv, query, header, 
expectedRows);
++        TestUtils.assertDataEventuallyOnEnv(receiverEnv, query, header, 
expectedRows);
++
++        // A later overwrite must also arrive and commit after the compacted 
backlog is drained.
++        statement.execute("insert into " + device + "(time,s1) values 
(0,-1),(1536,1536)");
++        statement.execute("flush");
++        expectedRows.remove("0,0,");
++        expectedRows.add("0,-1,");
++        expectedRows.add("1536,1536,");
++        PipeTestUtils.awaitDataPipeCompletion(senderEnv, pipeName);
++        TestUtils.assertDataEventuallyOnEnv(senderEnv, query, header, 
expectedRows);
++        TestUtils.assertDataEventuallyOnEnv(receiverEnv, query, header, 
expectedRows);
++      } finally {
++        statement.execute("set configuration 
'enable_seq_space_compaction'='false'");
++      }
++    }
++  }
++
++  private boolean hasCompactionReplacement(final String pipeName, final 
Pattern replacementMessage)
++      throws Exception {
++    for (final DataNodeWrapper node : senderEnv.getDataNodeWrapperList()) {
++      try (final Stream<Path> logs = 
Files.walk(Path.of(node.getLogDirPath()))) {
++        for (final Path log :
++            logs.filter(Files::isRegularFile)
++                .filter(path -> path.toString().endsWith(".log"))
++                .collect(Collectors.toList())) {
++          try (final Stream<String> lines = Files.lines(log)) {
++            if (lines.anyMatch(
++                line -> line.contains(pipeName) && 
replacementMessage.matcher(line).find())) {
++              return true;
++            }
++          }
++        }
++      }
++    }
++    return false;
++  }
++
    private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int 
count) {
      TestUtils.assertDataEventuallyOnEnv(
          receiverEnv,
diff --cc 
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java
index d461dcb903b,83c8666eb3c..d8e9bd91de2
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java
@@@ -55,6 -49,6 +55,7 @@@ import java.util.concurrent.TimeUnit
  import java.util.concurrent.atomic.AtomicInteger;
  import java.util.concurrent.atomic.AtomicLong;
  import java.util.function.Consumer;
++import java.util.function.Predicate;
  import java.util.stream.Collectors;
  
  public class PipeRealtimePriorityBlockingQueue extends 
UnboundedBlockingPendingQueue<Event> {
@@@ -102,13 -83,11 +103,15 @@@
        return true;
      }
  
 +    if (!checkBeforeOffer(event)) {
 +      return false;
 +    }
 +
+     final Event lastEvent = super.peekLast();
      if (event instanceof PipeHeartbeatEvent
          && !((PipeHeartbeatEvent) event).isCompletionBarrier()
-         && super.peekLast() instanceof PipeHeartbeatEvent) {
+         && lastEvent instanceof PipeHeartbeatEvent
+         && !((PipeHeartbeatEvent) lastEvent).isCompletionBarrier()) {
        // We can NOT keep too many PipeHeartbeatEvent in bufferQueue because 
they may cause OOM.
        ((EnrichedEvent) 
event).decreaseReferenceCount(PipeEventCollector.class.getName(), false);
        return false;
@@@ -514,38 -398,4 +517,37 @@@
    public synchronized void setOfferTsFileCounter(AtomicInteger 
offerTsFileCounter) {
      this.offerTsFileCounter = offerTsFileCounter;
    }
 +
 +  public void enableTsFileStartupReservation() {
 +    reserveTsFileStartupOnPoll = true;
 +  }
 +
 +  public boolean consumeTsFileStartupReservation(final TsFileInsertionEvent 
event) {
 +    synchronized (offerTsFileCounter) {
 +      return reservedTsFileStartupEvents.remove(event);
 +    }
 +  }
 +
 +  public boolean cancelTsFileStartupReservation(final TsFileInsertionEvent 
event) {
 +    synchronized (offerTsFileCounter) {
 +      if (!reservedTsFileStartupEvents.remove(event)) {
 +        return false;
 +      }
 +      offerTsFileCounter.decrementAndGet();
 +      offerTsFileCounter.notifyAll();
 +      return true;
 +    }
 +  }
 +
-   private void clearTsFileStartupReservations(
-       final java.util.function.Predicate<TsFileInsertionEvent> predicate) {
++  private void clearTsFileStartupReservations(final 
Predicate<TsFileInsertionEvent> predicate) {
 +    synchronized (offerTsFileCounter) {
 +      final int sizeBefore = reservedTsFileStartupEvents.size();
 +      reservedTsFileStartupEvents.removeIf(predicate);
 +      final int canceledReservations = sizeBefore - 
reservedTsFileStartupEvents.size();
 +      if (canceledReservations > 0) {
 +        offerTsFileCounter.addAndGet(-canceledReservations);
 +        offerTsFileCounter.notifyAll();
 +      }
 +    }
 +  }
  }
diff --cc 
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueueTest.java
index 722aa9c0f0d,00000000000..252544cd3a0
mode 100644,000000..100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueueTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueueTest.java
@@@ -1,675 -1,0 +1,706 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +
 +package org.apache.iotdb.db.pipe.agent.task.subtask.sink;
 +
 +import org.apache.iotdb.commons.conf.CommonDescriptor;
 +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
 +import org.apache.iotdb.commons.pipe.config.PipeConfig;
 +import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
 +import 
org.apache.iotdb.db.pipe.event.common.tsfile.PipeCompactedTsFileInsertionEvent;
 +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
 +import org.apache.iotdb.db.pipe.resource.tsfile.PipeCompactionManager;
 +import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
 +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 +import org.apache.iotdb.pipe.api.event.Event;
 +import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
 +
 +import org.junit.Assert;
 +import org.junit.Rule;
 +import org.junit.Test;
 +import org.junit.rules.TemporaryFolder;
 +import org.mockito.Mockito;
 +
 +import java.io.File;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.concurrent.CountDownLatch;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
 +import java.util.concurrent.Future;
 +import java.util.concurrent.FutureTask;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.atomic.AtomicInteger;
 +
 +import static org.awaitility.Awaitility.await;
 +import static org.mockito.ArgumentMatchers.anyBoolean;
 +import static org.mockito.ArgumentMatchers.anyString;
 +import static org.mockito.Mockito.doAnswer;
 +import static org.mockito.Mockito.doReturn;
 +import static org.mockito.Mockito.never;
 +import static org.mockito.Mockito.verify;
 +
 +public class PipeRealtimePriorityBlockingQueueTest {
 +
 +  @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
 +
 +  @Test
 +  public void testCompletionBarrierIsNotDroppedAfterHeartbeat() {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue();
 +    final PipeHeartbeatEvent heartbeatEvent = new PipeHeartbeatEvent(1, 
false);
 +    final PipeHeartbeatEvent completionBarrier = new PipeHeartbeatEvent(1, 
false, true);
 +
 +    Assert.assertTrue(queue.offer(heartbeatEvent));
 +    Assert.assertTrue(queue.offer(completionBarrier));
 +    Assert.assertSame(heartbeatEvent, queue.directPoll());
 +    Assert.assertSame(completionBarrier, queue.directPoll());
 +  }
 +
++  @Test
++  public void testHeartbeatAfterCompletionBarrierIsNotDropped() {
++    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue();
++    final PipeHeartbeatEvent completionBarrier = new PipeHeartbeatEvent(1, 
false, true);
++    final PipeHeartbeatEvent heartbeat = new PipeHeartbeatEvent(1, false);
++
++    Assert.assertTrue(queue.offer(completionBarrier));
++    Assert.assertTrue(queue.offer(heartbeat));
++    Assert.assertFalse(queue.offer(new PipeHeartbeatEvent(1, false)));
++    Assert.assertSame(completionBarrier, queue.directPoll());
++    Assert.assertSame(heartbeat, queue.directPoll());
++    Assert.assertNull(queue.directPoll());
++  }
++
 +  @Test
 +  public void testWaitedPollRetainsTsFilesUntilAsyncSendCapacityIsAvailable() 
throws Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final int maxWaitingTsFiles =
 +        PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize();
 +    final AtomicInteger waitingTsFiles = new AtomicInteger(maxWaitingTsFiles);
 +    queue.setOfferTsFileCounter(waitingTsFiles);
 +    final PipeTsFileInsertionEvent event =
 +        createEvent(newTsFile("waiting.tsfile"), new CommitterKey("pipe", 1L, 
1, 0), 1L);
 +    Assert.assertTrue(queue.offer(event));
 +
 +    for (final int waiting : new int[] {maxWaitingTsFiles, maxWaitingTsFiles 
+ 1}) {
 +      waitingTsFiles.set(waiting);
 +      for (int i = 0; i < 3; i++) {
 +        Assert.assertNull(queue.waitedPoll());
 +        Assert.assertNull(queue.directPoll());
 +        Assert.assertEquals(1, queue.getTsFileInsertionEventCount());
 +        Assert.assertSame(event, queue.peek());
 +      }
 +    }
 +
 +    waitingTsFiles.set(maxWaitingTsFiles - 1);
 +    Assert.assertSame(event, queue.waitedPoll());
 +    Assert.assertEquals(0, queue.getTsFileInsertionEventCount());
 +    verify(event, never()).decreaseReferenceCount(anyString(), anyBoolean());
 +  }
 +
 +  @Test
 +  public void testConcurrentPollReservesOnlyAvailableTsFileStartupSlot() 
throws Exception {
 +    final int previousMaxWaitingTsFiles =
 +        PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize();
 +    final AtomicInteger startupCount = new AtomicInteger(0);
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final ExecutorService pollers = Executors.newFixedThreadPool(16);
 +    final CountDownLatch startPollers = new CountDownLatch(1);
 +    try {
 +      
CommonDescriptor.getInstance().getConfig().setPipeRealTimeQueueMaxWaitingTsFileSize(1);
 +      queue.setOfferTsFileCounter(startupCount);
 +      queue.enableTsFileStartupReservation();
 +      for (int i = 0; i < 16; i++) {
 +        Assert.assertTrue(
 +            queue.offer(
 +                createEvent(
 +                    newTsFile("parallel-" + i + ".tsfile"),
 +                    new CommitterKey("pipe", 1L, 1, 0),
 +                    i)));
 +      }
 +
 +      final List<Future<Event>> polls = new ArrayList<>();
 +      for (int i = 0; i < 16; i++) {
 +        polls.add(
 +            pollers.submit(
 +                () -> {
 +                  startPollers.await();
 +                  return queue.directPoll();
 +                }));
 +      }
 +      startPollers.countDown();
 +
 +      TsFileInsertionEvent polledEvent = null;
 +      int polledEventCount = 0;
 +      for (final Future<Event> poll : polls) {
 +        final Event event = poll.get(5, TimeUnit.SECONDS);
 +        if (event != null) {
 +          polledEvent = (TsFileInsertionEvent) event;
 +          polledEventCount++;
 +        }
 +      }
 +
 +      Assert.assertEquals(1, polledEventCount);
 +      Assert.assertEquals(1, startupCount.get());
 +      Assert.assertEquals(15, queue.getTsFileInsertionEventCount());
 +      Assert.assertTrue(queue.cancelTsFileStartupReservation(polledEvent));
 +      Assert.assertEquals(0, startupCount.get());
 +    } finally {
 +      pollers.shutdownNow();
 +      pollers.awaitTermination(5, TimeUnit.SECONDS);
 +      queue.discardAllEvents();
 +      CommonDescriptor.getInstance()
 +          .getConfig()
 +          
.setPipeRealTimeQueueMaxWaitingTsFileSize(previousMaxWaitingTsFiles);
 +    }
 +  }
 +
 +  @Test
 +  public void 
testDiscardedPipeKeepsPolledStartupReservationUntilSinkCancelsIt() throws 
Exception {
 +    final int previousMaxWaitingTsFiles =
 +        PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize();
 +    final AtomicInteger startupCount = new AtomicInteger(0);
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final PipeTsFileInsertionEvent event =
 +        createEvent(newTsFile("discarded.tsfile"), committerKey, 1L);
 +    try {
 +      
CommonDescriptor.getInstance().getConfig().setPipeRealTimeQueueMaxWaitingTsFileSize(1);
 +      queue.setOfferTsFileCounter(startupCount);
 +      queue.enableTsFileStartupReservation();
 +      Assert.assertTrue(queue.offer(event));
 +      Assert.assertSame(event, queue.directPoll());
 +      Assert.assertEquals(1, startupCount.get());
 +
 +      queue.discardEventsOfPipe(committerKey);
 +
 +      Assert.assertEquals(1, startupCount.get());
 +      Assert.assertNull(queue.directPoll());
 +      Assert.assertTrue(queue.cancelTsFileStartupReservation(event));
 +      Assert.assertEquals(0, startupCount.get());
 +    } finally {
 +      queue.discardAllEvents();
 +      CommonDescriptor.getInstance()
 +          .getConfig()
 +          
.setPipeRealTimeQueueMaxWaitingTsFileSize(previousMaxWaitingTsFiles);
 +    }
 +  }
 +
 +  @Test
 +  public void testWaitedPollDeliversHeartbeatsWhileTsFileSendQueueIsFull() 
throws Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    queue.setOfferTsFileCounter(
 +        new 
AtomicInteger(PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize()));
 +    final PipeTsFileInsertionEvent event =
 +        createEvent(newTsFile("waiting.tsfile"), new CommitterKey("pipe", 1L, 
1, 0), 1L);
 +    final PipeHeartbeatEvent heartbeat = new PipeHeartbeatEvent(1, false);
 +    final PipeHeartbeatEvent completionBarrier = new PipeHeartbeatEvent(1, 
false, true);
 +    Assert.assertTrue(queue.offer(event));
 +    Assert.assertTrue(queue.offer(heartbeat));
 +    Assert.assertTrue(queue.offer(completionBarrier));
 +
 +    Assert.assertSame(heartbeat, queue.waitedPoll());
 +    Assert.assertSame(completionBarrier, queue.waitedPoll());
 +    Assert.assertNull(queue.waitedPoll());
 +    Assert.assertSame(event, queue.peek());
 +    Assert.assertEquals(1, queue.getTsFileInsertionEventCount());
 +  }
 +
 +  @Test
 +  public void testHeartbeatWakesBackpressuredPoll() throws Exception {
 +    final long previousBlockingTimeMs =
 +        
PipeConfig.getInstance().getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs();
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    queue.setOfferTsFileCounter(
 +        new 
AtomicInteger(PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize()));
 +    final PipeTsFileInsertionEvent event =
 +        createEvent(newTsFile("waiting.tsfile"), new CommitterKey("pipe", 1L, 
1, 0), 1L);
 +    Assert.assertTrue(queue.offer(event));
 +    final FutureTask<Event> poll = new FutureTask<>(queue::waitedPoll);
 +    final Thread pollingThread = new Thread(poll);
 +    try {
 +      CommonDescriptor.getInstance()
 +          .getConfig()
 +          .setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(10_000);
 +      pollingThread.start();
 +      await()
 +          .atMost(5, TimeUnit.SECONDS)
 +          .until(() -> pollingThread.getState() == 
Thread.State.TIMED_WAITING);
 +
 +      final PipeHeartbeatEvent heartbeat = new PipeHeartbeatEvent(1, false);
 +      Assert.assertTrue(queue.offer(heartbeat));
 +      Assert.assertSame(heartbeat, poll.get(1, TimeUnit.SECONDS));
 +      Assert.assertSame(event, queue.peek());
 +    } finally {
 +      pollingThread.interrupt();
 +      pollingThread.join(5_000);
 +      CommonDescriptor.getInstance()
 +          .getConfig()
 +          
.setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(previousBlockingTimeMs);
 +    }
 +  }
 +
 +  @Test
 +  public void testWaitedPollRechecksCapacityAfterWaitingForAnEvent() throws 
Exception {
 +    final long previousBlockingTimeMs =
 +        
PipeConfig.getInstance().getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs();
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final AtomicInteger waitingTsFiles = new AtomicInteger(0);
 +    queue.setOfferTsFileCounter(waitingTsFiles);
 +    final PipeTsFileInsertionEvent event =
 +        createEvent(newTsFile("waiting.tsfile"), new CommitterKey("pipe", 1L, 
1, 0), 1L);
 +    final FutureTask<Event> poll = new FutureTask<>(queue::waitedPoll);
 +    final Thread pollingThread = new Thread(poll);
 +    try {
 +      CommonDescriptor.getInstance()
 +          .getConfig()
 +          .setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(1_000);
 +      pollingThread.start();
 +      await()
 +          .pollInterval(5, TimeUnit.MILLISECONDS)
 +          .atMost(5, TimeUnit.SECONDS)
 +          .until(() -> pollingThread.getState() == 
Thread.State.TIMED_WAITING);
 +
 +      
waitingTsFiles.set(PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize());
 +      Assert.assertTrue(queue.offer(event));
 +      Assert.assertNull(poll.get(2, TimeUnit.SECONDS));
 +      Assert.assertSame(event, queue.peek());
 +      Assert.assertEquals(1, queue.getTsFileInsertionEventCount());
 +    } finally {
 +      pollingThread.interrupt();
 +      pollingThread.join(5_000);
 +      CommonDescriptor.getInstance()
 +          .getConfig()
 +          
.setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(previousBlockingTimeMs);
 +    }
 +  }
 +
 +  @Test
 +  public void testWaitedPollPreservesPendingFilesForCompaction() throws 
Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final int maxWaitingTsFiles =
 +        PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize();
 +    final AtomicInteger waitingTsFiles = new AtomicInteger(maxWaitingTsFiles);
 +    queue.setOfferTsFileCounter(waitingTsFiles);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final File sourceFile0 = newTsFile("source0.tsfile");
 +    final File sourceFile1 = newTsFile("source1.tsfile");
 +    final File targetFile = newTsFile("target.tsfile");
 +    final PipeTsFileInsertionEvent sourceEvent0 = createEvent(sourceFile0, 
committerKey, 1L);
 +    final PipeTsFileInsertionEvent sourceEvent1 = createEvent(sourceFile1, 
committerKey, 2L);
 +    Assert.assertTrue(queue.offer(sourceEvent0));
 +    Assert.assertTrue(queue.offer(sourceEvent1));
 +    Assert.assertNull(queue.waitedPoll());
 +
 +    queue.replace(
 +        "1",
 +        new HashSet<>(
 +            Arrays.asList(new TsFileResource(sourceFile0), new 
TsFileResource(sourceFile1))),
 +        Collections.singletonList(new TsFileResource(targetFile)));
 +
 +    Assert.assertEquals(1, queue.getTsFileInsertionEventCount());
 +    Assert.assertNull(queue.waitedPoll());
 +    waitingTsFiles.decrementAndGet();
 +    final Event event = queue.waitedPoll();
 +    Assert.assertTrue(event instanceof PipeCompactedTsFileInsertionEvent);
 +    final PipeCompactedTsFileInsertionEvent compactedEvent =
 +        (PipeCompactedTsFileInsertionEvent) event;
 +    try {
 +      Assert.assertEquals(
 +          targetFile.getAbsoluteFile(), 
compactedEvent.getSourceTsFile().getAbsoluteFile());
 +      Assert.assertEquals(
 +          new HashSet<>(Arrays.asList(1L, 2L)), new 
HashSet<>(compactedEvent.getCommitIds()));
 +      Assert.assertEquals(0, queue.getTsFileInsertionEventCount());
 +      verify(sourceEvent0)
 +          
.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
 +      verify(sourceEvent1)
 +          
.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
 +    } finally {
 +      compactedEvent.decreaseReferenceCount(
 +          PipeRealtimePriorityBlockingQueue.class.getName(), false);
 +    }
 +  }
 +
 +  @Test
 +  public void testCompactedHistoricalEventPinsTargetStorageFile() throws 
Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final File sourceFile = newTsFile("source.tsfile");
 +    final File targetFile = newTsFile("target.tsfile");
 +    final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, 
committerKey, 1L);
 +    doReturn(true).when(sourceEvent).isGeneratedByHistoricalExtractor();
 +    doReturn(true).when(sourceEvent).isWithMod();
 +
 +    Assert.assertTrue(queue.offer(sourceEvent));
 +    queue.replace(
 +        "1",
 +        Collections.singleton(new TsFileResource(sourceFile)),
 +        Collections.singletonList(new TsFileResource(targetFile)));
 +
 +    final Event event = queue.directPoll();
 +    Assert.assertTrue(event instanceof PipeCompactedTsFileInsertionEvent);
 +    final PipeCompactedTsFileInsertionEvent compactedEvent =
 +        (PipeCompactedTsFileInsertionEvent) event;
 +    try {
 +      Assert.assertTrue(compactedEvent.isGeneratedByHistoricalExtractor());
 +      Assert.assertFalse(compactedEvent.isWithMod());
 +      Assert.assertTrue(
 +          
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(targetFile, "pipe_1")
 +              .exists());
 +    } finally {
 +      compactedEvent.decreaseReferenceCount(
 +          PipeRealtimePriorityBlockingQueue.class.getName(), false);
 +    }
 +  }
 +
 +  @Test
 +  public void testEmitCompactionResultSkipsDeletedTargets() throws Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final PipeCompactionManager compactionManager = 
createCompactionManager(queue);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final File sourceFile0 = newTsFile("source0.tsfile");
 +    final File sourceFile1 = newTsFile("source1.tsfile");
 +    final File targetFile0 = newTsFile("target0.tsfile");
 +    final File targetFile1 = newTsFile("target1.tsfile");
 +    final TsFileResource deletedTarget0 =
 +        new TsFileResource(new File(targetFile0.getParentFile(), 
"unused0.inner"));
 +    final TsFileResource deletedTarget1 =
 +        new TsFileResource(new File(targetFile0.getParentFile(), 
"unused1.inner"));
 +    deletedTarget0.forceMarkDeleted();
 +    deletedTarget1.forceMarkDeleted();
 +    final PipeTsFileInsertionEvent sourceEvent0 = createEvent(sourceFile0, 
committerKey, 1L);
 +    final PipeTsFileInsertionEvent sourceEvent1 = createEvent(sourceFile1, 
committerKey, 2L);
 +    queue.offer(sourceEvent0);
 +    queue.offer(sourceEvent1);
 +
 +    compactionManager.emitResult(
 +        "root.db",
 +        "1",
 +        0,
 +        Arrays.asList(new TsFileResource(sourceFile0), new 
TsFileResource(sourceFile1)),
 +        Collections.emptyList(),
 +        Arrays.asList(
 +            deletedTarget0,
 +            new TsFileResource(targetFile0),
 +            new TsFileResource(targetFile1),
 +            deletedTarget1));
 +
 +    final List<Event> queuedEvents = new ArrayList<>();
 +    queue.forEach(queuedEvents::add);
 +    Assert.assertEquals(2, queuedEvents.size());
 +    Assert.assertTrue(queuedEvents.get(0) instanceof 
PipeCompactedTsFileInsertionEvent);
 +    Assert.assertTrue(queuedEvents.get(1) instanceof 
PipeCompactedTsFileInsertionEvent);
 +    final PipeCompactedTsFileInsertionEvent compactedEvent0 =
 +        (PipeCompactedTsFileInsertionEvent) queuedEvents.get(0);
 +    final PipeCompactedTsFileInsertionEvent compactedEvent1 =
 +        (PipeCompactedTsFileInsertionEvent) queuedEvents.get(1);
 +    try {
 +      Assert.assertEquals(
 +          targetFile0.getAbsoluteFile(), 
compactedEvent0.getSourceTsFile().getAbsoluteFile());
 +      Assert.assertEquals(
 +          targetFile1.getAbsoluteFile(), 
compactedEvent1.getSourceTsFile().getAbsoluteFile());
 +      Assert.assertTrue(
 +          
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(targetFile0, 
"pipe_1")
 +              .exists());
 +      Assert.assertTrue(
 +          
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(targetFile1, 
"pipe_1")
 +              .exists());
 +      Assert.assertEquals(
 +          new HashSet<>(Arrays.asList(1L, 2L)), new 
HashSet<>(compactedEvent1.getCommitIds()));
 +      Assert.assertFalse(compactedEvent0.isShouldReportOnCommit());
 +      Assert.assertTrue(compactedEvent1.isShouldReportOnCommit());
 +      verify(sourceEvent0)
 +          
.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
 +      verify(sourceEvent1)
 +          
.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
 +    } finally {
 +      compactedEvent0.decreaseReferenceCount(
 +          PipeRealtimePriorityBlockingQueue.class.getName(), false);
 +      compactedEvent1.decreaseReferenceCount(
 +          PipeRealtimePriorityBlockingQueue.class.getName(), false);
 +      queue.clear();
 +    }
 +  }
 +
 +  @Test
 +  public void testEmitCompactionResultKeepsSourcesWithoutSurvivingTargets() 
throws Exception {
 +    final File sourceFile = newTsFile("source.tsfile");
 +    final TsFileResource deletedTarget =
 +        new TsFileResource(new File(sourceFile.getParentFile(), 
"unused.inner"));
 +    deletedTarget.forceMarkDeleted();
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    for (final List<TsFileResource> targets :
 +        Arrays.asList(
 +            Collections.<TsFileResource>emptyList(), 
Collections.singletonList(deletedTarget))) {
 +      final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +      final PipeCompactionManager compactionManager = 
createCompactionManager(queue);
 +      final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, 
committerKey, 1L);
 +      queue.offer(sourceEvent);
 +
 +      compactionManager.emitResult(
 +          "root.db",
 +          "1",
 +          0,
 +          Collections.singletonList(new TsFileResource(sourceFile)),
 +          Collections.emptyList(),
 +          targets);
 +
 +      Assert.assertSame(sourceEvent, queue.directPoll());
 +      Assert.assertNull(queue.directPoll());
 +      Assert.assertTrue(sourceEvent.isShouldReportOnCommit());
 +      verify(sourceEvent, never()).decreaseReferenceCount(anyString(), 
anyBoolean());
 +    }
 +  }
 +
 +  @Test
 +  public void testReplaceOnlyRemovesEventsForCompactionSourceFiles() throws 
Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final File sourceFile = newTsFile("source.tsfile");
 +    final File unrelatedFile = newTsFile("unrelated.tsfile");
 +    final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, 
committerKey, 1L);
 +    final PipeTsFileInsertionEvent unrelatedEvent = 
createEvent(unrelatedFile, committerKey, 2L);
 +
 +    queue.offer(sourceEvent);
 +    queue.offer(unrelatedEvent);
 +    queue.replace(
 +        "1", Collections.singleton(new TsFileResource(sourceFile)), 
Collections.emptyList());
 +
 +    Assert.assertSame(unrelatedEvent, queue.directPoll());
 +    Assert.assertNull(queue.directPoll());
 +    verify(sourceEvent)
 +        
.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
 +    verify(unrelatedEvent, never()).decreaseReferenceCount(anyString(), 
anyBoolean());
 +  }
 +
 +  @Test
 +  public void testReplaceDoesNotMatchUnrelatedEventsByCount() throws 
Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final File sourceFile = newTsFile("source.tsfile");
 +    final PipeTsFileInsertionEvent unrelatedEvent =
 +        createEvent(newTsFile("unrelated.tsfile"), committerKey, 1L);
 +
 +    queue.offer(unrelatedEvent);
 +    queue.replace(
 +        "1", Collections.singleton(new TsFileResource(sourceFile)), 
Collections.emptyList());
 +
 +    Assert.assertSame(unrelatedEvent, queue.directPoll());
 +    verify(unrelatedEvent, never()).decreaseReferenceCount(anyString(), 
anyBoolean());
 +  }
 +
 +  @Test
 +  public void testReplaceKeepsCompactedEventsAtTheOldestQueuePosition() 
throws Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final File sourceFile0 = newTsFile("source0.tsfile");
 +    final File sourceFile1 = newTsFile("source1.tsfile");
 +    final File laterFile = newTsFile("later.tsfile");
 +    final File targetFile0 = newTsFile("target0.tsfile");
 +    final File targetFile1 = newTsFile("target1.tsfile");
 +    final PipeTsFileInsertionEvent sourceEvent0 = createEvent(sourceFile0, 
committerKey, 1L);
 +    final PipeTsFileInsertionEvent sourceEvent1 = createEvent(sourceFile1, 
committerKey, 2L);
 +    final PipeTsFileInsertionEvent laterEvent = createEvent(laterFile, 
committerKey, 3L);
 +
 +    queue.offer(sourceEvent0);
 +    queue.offer(sourceEvent1);
 +    queue.offer(laterEvent);
 +    queue.replace(
 +        "1",
 +        new HashSet<>(
 +            Arrays.asList(new TsFileResource(sourceFile0), new 
TsFileResource(sourceFile1))),
 +        Arrays.asList(new TsFileResource(targetFile0), new 
TsFileResource(targetFile1)));
 +
 +    final List<Event> queuedEvents = new ArrayList<>();
 +    queue.forEach(queuedEvents::add);
 +    Assert.assertEquals(3, queuedEvents.size());
 +    Assert.assertSame(laterEvent, queuedEvents.get(2));
 +    final PipeTsFileInsertionEvent compactedEvent0 = 
(PipeTsFileInsertionEvent) queuedEvents.get(0);
 +    final PipeTsFileInsertionEvent compactedEvent1 = 
(PipeTsFileInsertionEvent) queuedEvents.get(1);
 +    Assert.assertTrue(compactedEvent1 instanceof 
PipeCompactedTsFileInsertionEvent);
 +    Assert.assertTrue(compactedEvent0 instanceof 
PipeCompactedTsFileInsertionEvent);
 +    Assert.assertEquals(
 +        targetFile1.getAbsoluteFile(), 
compactedEvent1.getSourceTsFile().getAbsoluteFile());
 +    Assert.assertEquals(
 +        targetFile0.getAbsoluteFile(), 
compactedEvent0.getSourceTsFile().getAbsoluteFile());
 +    compactedEvent1.decreaseReferenceCount(
 +        PipeRealtimePriorityBlockingQueue.class.getName(), false);
 +    compactedEvent0.decreaseReferenceCount(
 +        PipeRealtimePriorityBlockingQueue.class.getName(), false);
 +  }
 +
 +  @Test
 +  public void testReplaceLeavesNonHeadSourceEventsQueuedToPreserveOrder() 
throws Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final PipeTsFileInsertionEvent olderEvent =
 +        createEvent(newTsFile("older.tsfile"), committerKey, 1L);
 +    final File sourceFile = newTsFile("source.tsfile");
 +    final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, 
committerKey, 2L);
 +    final PipeTsFileInsertionEvent laterEvent =
 +        createEvent(newTsFile("later.tsfile"), committerKey, 3L);
 +
 +    queue.offer(olderEvent);
 +    queue.offer(sourceEvent);
 +    queue.offer(laterEvent);
 +    queue.replace(
 +        "1",
 +        Collections.singleton(new TsFileResource(sourceFile)),
 +        Collections.singletonList(new 
TsFileResource(newTsFile("target.tsfile"))));
 +
 +    final List<Event> queuedEvents = new ArrayList<>();
 +    queue.forEach(queuedEvents::add);
 +    Assert.assertEquals(3, queuedEvents.size());
 +    Assert.assertSame(olderEvent, queuedEvents.get(0));
 +    Assert.assertSame(sourceEvent, queuedEvents.get(1));
 +    Assert.assertSame(laterEvent, queuedEvents.get(2));
 +    verify(sourceEvent, never())
 +        
.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
 +  }
 +
 +  @Test
 +  public void testEarlierEventFromAnotherPipeDoesNotBlockReplacement() throws 
Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final PipeTsFileInsertionEvent otherPipeEvent =
 +        createEvent(newTsFile("other-pipe.tsfile"), new 
CommitterKey("otherPipe", 1L, 1, 0), 1L);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final File sourceFile = newTsFile("source.tsfile");
 +    final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, 
committerKey, 2L);
 +
 +    queue.offer(otherPipeEvent);
 +    queue.offer(sourceEvent);
 +    queue.replace(
 +        "1",
 +        Collections.singleton(new TsFileResource(sourceFile)),
 +        Collections.singletonList(new 
TsFileResource(newTsFile("target.tsfile"))));
 +
 +    final List<Event> queuedEvents = new ArrayList<>();
 +    queue.forEach(queuedEvents::add);
 +    Assert.assertEquals(2, queuedEvents.size());
 +    Assert.assertTrue(queuedEvents.get(0) instanceof 
PipeCompactedTsFileInsertionEvent);
 +    Assert.assertSame(otherPipeEvent, queuedEvents.get(1));
 +    verify(sourceEvent)
 +        
.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
 +  }
 +
 +  @Test
 +  public void testPollCanRaceWithReplacementWithoutCorruptingQueue() throws 
Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue(true);
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final File sourceFile = newTsFile("source.tsfile");
 +    final PipeTsFileInsertionEvent event = createEvent(sourceFile, 
committerKey, 1L);
 +    final CountDownLatch sourcePathRead = new CountDownLatch(1);
 +    final CountDownLatch allowReplacementToContinue = new CountDownLatch(1);
 +    final ExecutorService executor = Executors.newFixedThreadPool(2);
++    final File targetFile = newTsFile("target.tsfile");
++    PipeCompactedTsFileInsertionEvent replacement = null;
 +
 +    try {
 +      doAnswer(
 +              invocation -> {
 +                sourcePathRead.countDown();
 +                Assert.assertTrue(allowReplacementToContinue.await(5, 
TimeUnit.SECONDS));
 +                return sourceFile;
 +              })
 +          .when(event)
 +          .getSourceTsFile();
 +      queue.offer(event);
 +
 +      final Future<?> replacementFuture =
 +          executor.submit(
 +              () ->
 +                  queue.replace(
 +                      "1",
 +                      Collections.singleton(new TsFileResource(sourceFile)),
-                       Collections.emptyList()));
++                      Collections.singletonList(new 
TsFileResource(targetFile))));
 +      Assert.assertTrue(sourcePathRead.await(5, TimeUnit.SECONDS));
 +      final Future<Event> pollFuture = executor.submit(queue::directPoll);
 +      Assert.assertSame(event, pollFuture.get(5, TimeUnit.SECONDS));
 +
 +      allowReplacementToContinue.countDown();
 +      replacementFuture.get(5, TimeUnit.SECONDS);
 +      verify(event, never())
 +          
.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
++      Assert.assertEquals(1, queue.getTsFileInsertionEventCount());
++      final Event queuedEvent = queue.directPoll();
++      Assert.assertTrue(queuedEvent instanceof 
PipeCompactedTsFileInsertionEvent);
++      replacement = (PipeCompactedTsFileInsertionEvent) queuedEvent;
++      Assert.assertEquals(
++          targetFile.getAbsoluteFile(), 
replacement.getSourceTsFile().getAbsoluteFile());
++      Assert.assertEquals(Collections.singletonList(1L), 
replacement.getCommitIds());
++      Assert.assertEquals(1L, replacement.getCommitId());
++      Assert.assertNull(queue.directPoll());
 +    } finally {
 +      allowReplacementToContinue.countDown();
 +      executor.shutdownNow();
++      executor.awaitTermination(5, TimeUnit.SECONDS);
++      if (replacement != null) {
++        replacement.decreaseReferenceCount(
++            PipeRealtimePriorityBlockingQueue.class.getName(), false);
++      }
++      queue.discardAllEvents();
 +    }
 +  }
 +
 +  @Test
 +  public void testReplaceIsDisabledByDefault() throws Exception {
 +    final PipeRealtimePriorityBlockingQueue queue = new 
PipeRealtimePriorityBlockingQueue();
 +    final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0);
 +    final File sourceFile = newTsFile("source.tsfile");
 +    final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, 
committerKey, 1L);
 +
 +    queue.offer(sourceEvent);
 +    queue.replace(
 +        "1", Collections.singleton(new TsFileResource(sourceFile)), 
Collections.emptyList());
 +
 +    Assert.assertSame(sourceEvent, queue.directPoll());
 +    Assert.assertNull(queue.directPoll());
 +    verify(sourceEvent, never())
 +        
.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
 +  }
 +
 +  private PipeCompactionManager createCompactionManager(
 +      final PipeRealtimePriorityBlockingQueue queue) {
 +    final PipeSinkSubtaskLifeCycle lifeCycle = 
Mockito.mock(PipeSinkSubtaskLifeCycle.class);
 +    doReturn(queue).when(lifeCycle).getPendingQueue();
 +    final PipeCompactionManager compactionManager = new 
PipeCompactionManager();
 +    compactionManager.registerPipeConnectorSubtaskLifeCycle(lifeCycle);
 +    return compactionManager;
 +  }
 +
 +  private PipeTsFileInsertionEvent createEvent(
 +      final File file, final CommitterKey committerKey, final long commitId) {
 +    final PipeTsFileInsertionEvent event =
 +        Mockito.spy(
 +            new PipeTsFileInsertionEvent(false, "root.db", new 
TsFileResource(file), false));
 +    event.setCommitterKeyAndCommitId(committerKey, commitId);
 +    doReturn(true).when(event).decreaseReferenceCount(anyString(), 
anyBoolean());
 +    return event;
 +  }
 +
 +  private File newTsFile(final String name) throws Exception {
 +    final File sequenceDir = new File(temporaryFolder.getRoot(), 
"sequence/1/0");
 +    Assert.assertTrue(sequenceDir.mkdirs() || sequenceDir.isDirectory());
 +    final File tsFile = new File(sequenceDir, name);
 +    Assert.assertTrue(tsFile.createNewFile());
 +    return tsFile;
 +  }
 +}
diff --cc 
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java
index 1a260212546,f15cfa4fad9..fea89773adc
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java
@@@ -55,8 -48,8 +55,9 @@@ import java.util.concurrent.ExecutionEx
  import java.util.concurrent.FutureTask;
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicBoolean;
 +import java.util.concurrent.atomic.AtomicInteger;
  
+ import static org.awaitility.Awaitility.await;
  import static org.mockito.ArgumentMatchers.any;
  import static org.mockito.Mockito.doAnswer;
  import static org.mockito.Mockito.doThrow;

Reply via email to