This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch fix/pipe-compaction-queue-replacement in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5667d06f96cfd543958e66bfd155dfeee28b9cc2 Merge: 0ed84f65128 f12fe701c8c Author: Caideyipi <[email protected]> AuthorDate: Mon Nov 16 16:09:53 2026 +0800 Merge master and address compaction replacement review comments .github/workflows/cluster-it-1c1d1a.yml | 29 + AGENTS.md | 10 + LICENSE-binary | 142 ++-- NOTICE-binary | 112 ++- THREAT_MODEL.md | 20 +- dependencies.json | 1 - external-service-impl/rest/pom.xml | 10 + .../rest/protocol/v1/handler/ExceptionHandler.java | 3 +- .../rest/protocol/v2/handler/ExceptionHandler.java | 3 +- .../protocol/handler/ExceptionHandlerTest.java | 162 +++++ integration-test/AGENTS.md | 17 + integration-test/pom.xml | 7 +- .../cluster/env/IntegrationTestEnvMessages.java | 35 + .../cluster/env/IntegrationTestEnvMessages.java | 35 + .../org/apache/iotdb/it/env/cluster/EnvUtils.java | 5 +- .../it/env/cluster/config/MppDataNodeConfig.java | 6 + .../iotdb/it/env/cluster/config/MppJVMConfig.java | 26 + .../iotdb/it/env/cluster/env/AbstractEnv.java | 149 ++-- .../it/env/cluster/node/AbstractNodeWrapper.java | 18 +- .../it/env/cluster/node/ConfigNodeWrapper.java | 23 + .../it/env/remote/config/RemoteDataNodeConfig.java | 5 + .../apache/iotdb/itbase/env/DataNodeConfig.java | 2 + .../it/cluster/IoTDBClusterNodeErrorStartUpIT.java | 168 +++-- .../it/cluster/IoTDBClusterNodeGetterIT.java | 56 +- .../it/database/IoTDBDatabaseSetAndDeleteIT.java | 34 +- ...ataPartitionTableIntegrityCheckProcedureIT.java | 67 +- .../it/partition/IoTDBPartitionCreationIT.java | 24 +- .../it/partition/IoTDBPartitionDurableIT.java | 4 +- .../partition/IoTDBPartitionTableAutoCleanIT.java | 82 ++- .../IoTDBPartitionTableAutoCleanUSIT.java | 92 +-- .../pass/commit/RegionMigrateFileAssertions.java | 3 + .../IoTDBRegionMigrateNormalForIoTV2StreamIT.java | 3 + .../IoTDBRegionMigrateWithCompressionRatioIT.java | 12 +- .../IoTDBRegionMigrateWithLastEmptyDeletionIT.java | 3 + .../iotdb/db/it/IoTDBCustomizedClusterIT.java | 49 +- .../org/apache/iotdb/db/it/IoTDBDeletionIT.java | 9 +- .../iotdb/db/it/IoTDBDiskFullFolderRecoveryIT.java | 158 +++++ .../iotdb/db/it/IoTDBIPv6ExternalServiceIT.java | 4 +- .../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java | 31 +- .../apache/iotdb/db/it/IoTDBLoadLastCacheIT.java | 66 +- .../iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java | 438 ++++++++++++ .../iotdb/db/it/IoTDBLoadTsFileClusterIT.java | 245 ++++++- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 213 +++++- .../iotdb/db/it/IoTDBLoadTsFileInstallIT.java | 451 +++++++++++++ .../org/apache/iotdb/db/it/IoTDBRecoverIT.java | 109 +-- .../apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java | 103 ++- .../org/apache/iotdb/db/it/IoTDBRestartIT.java | 49 +- .../apache/iotdb/db/it/IoTDBRestartRatisIT.java | 62 +- .../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 66 +- .../java/org/apache/iotdb/db/it/IoTDBTtlIT.java | 29 +- .../iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java | 46 +- .../db/it/aggregation/IoTDBAggregationIT.java | 209 ++---- .../db/it/aligned/IoTDBAlignedDataDeletionIT.java | 66 +- .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 161 ++++- .../scalar/IoTDBReplaceFunctionIT.java | 2 +- .../scalar/IoTDBSubStringFunctionIT.java | 2 +- .../org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java | 390 ++++++++--- .../apache/iotdb/db/it/cq/IoTDBCQExecInNsIT.java | 95 ++- .../apache/iotdb/db/it/cq/IoTDBCQExecInUsIT.java | 95 ++- .../java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java | 53 ++ .../iotdb/db/it/mqtt/IoTDBMQTTServiceJsonIT.java | 121 ++-- .../it/query/IoTDBAggregationFirstInGroupIT.java | 2 +- .../iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java | 58 +- .../iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java | 157 +++-- .../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java | 14 +- .../db/it/udf/IoTDBUDFOtherWindowQueryIT.java | 120 ++-- .../iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java | 17 +- .../apache/iotdb/db/it/utils/IoTDBTestUtilsIT.java | 173 +++++ .../org/apache/iotdb/db/it/utils/NodeDebugger.java | 232 +++++++ .../org/apache/iotdb/db/it/utils/TestUtils.java | 357 ++++++---- .../iotdb/it/utils/InternalSessionITUtils.java | 158 +++++ .../iotdb/libudf/it/dprofile/DProfileIT.java | 44 +- .../iotdb/libudf/it/dquality/DQualityIT.java | 42 +- .../iotdb/libudf/it/frequency/FrequencyIT.java | 29 +- .../apache/iotdb/pipe/it/dual/PipeTestUtils.java | 359 ++++++++++ .../pipe/it/dual/tablemodel/TableModelUtils.java | 201 +++++- .../manual/AbstractPipeTableModelDualManualIT.java | 19 +- .../AbstractPipeTableModelSharedClusterIT.java | 211 ++++++ .../tablemodel/manual/basic/IoTDBPipeAlterIT.java | 655 +++++++++--------- .../manual/basic/IoTDBPipeDataSinkIT.java | 640 +++++++++++++----- .../manual/basic/IoTDBPipeIsolationIT.java | 4 +- .../manual/basic/IoTDBPipeLifeCycleIT.java | 29 +- .../manual/basic/IoTDBPipePermissionIT.java | 72 +- .../manual/basic/IoTDBPipeProtocolIT.java | 74 +- .../basic/IoTDBPipeRelationalSecurityIT.java | 21 +- .../tablemodel/manual/basic/IoTDBPipeSourceIT.java | 484 +++++++------ .../manual/basic/IoTDBPipeSwitchStatusIT.java | 4 +- .../tablemodel/manual/basic/IoTDBPipeSyntaxIT.java | 106 ++- .../IoTDBPipeTsFileDecompositionWithModsIT.java | 205 ++---- .../manual/basic/IoTDBPipeWithLoadIT.java | 176 +++-- .../manual/basic/IoTDBTablePatternFormatIT.java | 281 ++++---- .../manual/enhanced/IoTDBPipeAutoConflictIT.java | 180 +++-- .../manual/enhanced/IoTDBPipeAutoDropIT.java | 82 +-- .../manual/enhanced/IoTDBPipeClusterIT.java | 141 ++-- .../manual/enhanced/IoTDBPipeDoubleLivingIT.java | 334 ++++----- .../manual/enhanced/IoTDBPipeIdempotentIT.java | 71 +- .../manual/enhanced/IoTDBPipeMetaIT.java | 79 ++- .../manual/enhanced/IoTDBPipeNullValueIT.java | 118 ++-- .../enhanced/IoTDBPipeSinkCompressionIT.java | 329 +++++---- .../manual/enhanced/IoTDBPipeSinkParallelIT.java | 31 +- .../IoTDBPipeTypeConversionISessionIT.java | 63 +- .../IoTDBPipeTypeConversionSemanticIT.java | 25 +- .../auto/AbstractPipeDualTreeModelAutoIT.java | 23 +- .../auto/AbstractPipeTreeModelSharedClusterIT.java | 164 +++++ .../treemodel/auto/basic/IoTDBPipeAlterIT.java | 750 ++++++++++----------- .../treemodel/auto/basic/IoTDBPipeAutoSplitIT.java | 6 +- .../treemodel/auto/basic/IoTDBPipeDataSinkIT.java | 374 ++++++---- .../dual/treemodel/auto/basic/IoTDBPipeIPv6IT.java | 4 +- .../treemodel/auto/basic/IoTDBPipeLifeCycleIT.java | 251 ++++--- .../treemodel/auto/basic/IoTDBPipeMutualSSLIT.java | 40 +- .../treemodel/auto/basic/IoTDBPipeProtocolIT.java | 34 +- .../IoTDBPipeReceiverAutoCreateDisabledIT.java | 20 +- .../treemodel/auto/basic/IoTDBPipeSourceIT.java | 733 +++++++++++--------- .../auto/basic/IoTDBPipeSwitchStatusIT.java | 4 +- .../treemodel/auto/basic/IoTDBPipeSyntaxIT.java | 356 ++++++---- .../auto/basic/IoTDBShowReceiversLifecycleIT.java | 32 +- .../auto/basic/IoTDBTreePatternFormatIT.java | 249 ++++--- .../auto/enhanced/IoTDBPipeAutoConflictIT.java | 316 +++++---- .../auto/enhanced/IoTDBPipeAutoDropIT.java | 116 +++- .../auto/enhanced/IoTDBPipeClusterIT.java | 94 +-- .../enhanced/IoTDBPipeConditionalOperationsIT.java | 94 +-- .../auto/enhanced/IoTDBPipeIdempotentIT.java | 185 +++-- .../auto/enhanced/IoTDBPipeNullValueIT.java | 158 ++--- .../auto/enhanced/IoTDBPipeSinkCompressionIT.java | 247 ++++--- .../IoTDBPipeTypeConversionSemanticIT.java | 25 +- .../auto/enhanced/IoTDBPipeWithLoadIT.java | 28 +- .../treemodel/auto/enhanced/PipeNowFunctionIT.java | 158 ++--- .../manual/AbstractPipeDualTreeModelManualIT.java | 72 +- ...AbstractPipeTreeModelManualSharedClusterIT.java | 194 ++++++ .../treemodel/manual/IoTDBPipeInclusionIT.java | 25 +- .../manual/IoTDBPipeMetaHistoricalIT.java | 38 +- .../manual/IoTDBPipeMetaLeaderChangeIT.java | 169 +++-- .../treemodel/manual/IoTDBPipeMetaRestartIT.java | 95 +-- .../treemodel/manual/IoTDBPipePermissionIT.java | 235 ++++--- .../treemodel/manual/IoTDBPipeReqAutoSliceIT.java | 3 +- .../IoTDBPipeTsFileDecompositionWithModsIT.java | 475 ++++++------- .../manual/IoTDBPipeTypeConversionISessionIT.java | 114 +++- .../treemodel/manual/IoTDBPipeWriteBackSinkIT.java | 27 +- .../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 5 + .../it/single/IoTDBPipeOperationDiagnosticsIT.java | 79 ++- .../pipe/it/single/IoTDBPipePermissionIT.java | 151 ++++- .../iotdb/pipe/it/single/IoTDBShowReceiversIT.java | 215 ++++-- .../pipe/it/triple/AbstractPipeTripleManualIT.java | 13 +- .../iotdb/pipe/it/triple/IoTDBPipeForwardIT.java | 2 + .../pipe/it/triple/IoTDBShowReceiversTripleIT.java | 60 +- .../relational/it/db/it/IoTDBDeletionTableIT.java | 109 ++- .../relational/it/db/it/IoTDBInsertTableIT.java | 80 ++- .../it/IoTDBLoadStorageTsFileWithCompactionIT.java | 269 ++++++++ .../it/db/it/IoTDBLoadTableSchemaCompletionIT.java | 265 ++++++++ .../relational/it/db/it/IoTDBLoadTsFileIT.java | 210 +++++- .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 6 + .../db/it/IoTDBRegionReadyAfterRestartTableIT.java | 97 +++ .../relational/it/db/it/IoTDBRestartTableIT.java | 44 +- .../it/db/it/IoTDBSetSystemStatusTableIT.java | 35 + .../it/db/it/IoTDBTablePreparedStatementIT.java | 35 + .../db/it/IoTDBTablePreparedStatementJDBCIT.java | 233 +++++++ .../relational/it/db/it/IoTDBWindowTVFIT.java | 94 +++ .../it/db/it/IoTDBWindowTVFRegionsIT.java | 135 ++++ .../relational/it/db/it/udf/IoTDBLocalIT.java | 92 +++ .../it/db/it/udf/IoTDBSQLFunctionManagementIT.java | 233 +++++++ .../it/insertquery/IoTDBInsertQueryIT.java | 93 +-- .../relational/it/mqtt/IoTDBMQTTServiceIT.java | 76 ++- .../it/query/old/IoTDBSimpleQueryTableIT.java | 20 +- ...oTDBNoSelectExpressionAfterAnalyzedTableIT.java | 7 +- .../it/query/recent/IoTDBComplexQueryIT.java | 109 +++ .../relational/it/query/recent/IoTDBCteIT.java | 28 +- .../it/query/recent/IoTDBDistinctTagIT.java | 9 +- .../it/query/recent/IoTDBGapFillTableIT.java | 40 ++ .../it/query/recent/IoTDBTableAggregationIT.java | 90 ++- .../view/old/IoTDBSimpleQueryTableViewIT.java | 29 +- .../view/recent/IoTDBComplexQueryTableViewIT.java | 2 - .../query/view/recent/IoTDBTableViewQueryIT.java | 53 +- ...oTDBTableViewQueryWithNotMatchedDataTypeIT.java | 44 +- .../it/schema/IoTDBAlterColumnTypeIT.java | 1 - .../it/schema/IoTDBAttributeUpdateOrderingIT.java | 82 +++ .../relational/it/schema/IoTDBDatabaseIT.java | 128 +++- .../schema/IoTDBDatabaseMaxRegionGroupNumIT.java | 31 +- .../it/schema/IoTDBNullableAttributeCacheIT.java | 261 +++++++ .../iotdb/relational/it/schema/IoTDBTableIT.java | 130 +++- .../it/schema/IoTDBTableTagSchemaMigrationIT.java | 386 +++++++++++ .../schema/TableMetadataPropagationTestBase.java | 450 +++++++++++++ .../it/schema/fault/MetadataFaultAgent.java | 600 +++++++++++++++++ .../it/schema/fault/MetadataFaultController.java | 244 +++++++ .../it/session/InternalTableSessionIT.java | 174 +++++ .../it/session/InternalTableSessionRedirectIT.java | 141 ++++ .../it/session/IoTDBSessionRelationalIT.java | 239 ++++--- .../it/session/IoTDBTableModelSessionIT.java | 25 +- .../apache/iotdb/session/it/InternalSessionIT.java | 225 +++++++ .../session/it/InternalSessionRedirectIT.java | 140 ++++ .../iotdb/session/it/IoTDBConnectionsIT.java | 74 +- .../session/it/IoTDBSessionAlignedInsertIT.java | 260 +++---- .../iotdb/session/it/IoTDBSessionComplexIT.java | 38 +- .../iotdb/session/it/IoTDBSessionInsertNullIT.java | 192 +++--- .../iotdb/session/it/IoTDBSessionSimpleIT.java | 23 +- .../session/it/IoTDBSessionSyntaxConventionIT.java | 121 ++-- .../org/apache/iotdb/session/it/SessionIT.java | 72 +- .../session/it/SessionSeedNodeFallbackIT.java | 40 +- .../iotdb/session/it/pool/SessionPoolIT.java | 76 ++- .../tablemodel/InternalSubscriptionTableIT.java | 226 +++++++ .../org/apache/iotdb/pipe/api/type/Binary.java | 5 +- .../org/apache/iotdb/pipe/api/type/BinaryTest.java | 61 ++ iotdb-api/udf-api/pom.xml | 5 + .../java/org/apache/iotdb/udf/api/type/Binary.java | 5 +- .../org/apache/iotdb/udf/api/type/BinaryTest.java | 61 ++ .../en/org/apache/iotdb/cli/i18n/CliMessages.java | 4 +- .../zh/org/apache/iotdb/cli/i18n/CliMessages.java | 4 +- .../src/main/java/org/apache/iotdb/cli/Cli.java | 11 +- .../apache/iotdb/tool/data/AbstractDataTool.java | 47 +- .../apache/iotdb/tool/schema/ImportSchemaTree.java | 2 +- .../test/java/org/apache/iotdb/cli/CliTest.java | 85 +++ .../iotdb/tool/data/DataToolRegressionTest.java | 248 +++++++ iotdb-client/client-py/iotdb/utils/rpc_utils.py | 17 +- .../client-py/tests/unit/test_rpc_utils.py | 32 + .../org/apache/iotdb/isession/INodeSupplier.java | 18 + .../org/apache/iotdb/jdbc/IoTDBConnection.java | 53 +- .../iotdb/jdbc/IoTDBTablePreparedStatement.java | 7 +- .../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 150 +++++ .../jdbc/IoTDBTablePreparedStatementTest.java | 182 +++++ .../en/org/apache/iotdb/rpc/i18n/RpcMessages.java | 21 + .../zh/org/apache/iotdb/rpc/i18n/RpcMessages.java | 21 + .../org/apache/iotdb/rpc/AutoResizingBuffer.java | 36 +- .../iotdb/rpc/AutoScalingBufferReadTransport.java | 34 +- .../iotdb/rpc/AutoScalingBufferWriteTransport.java | 94 ++- .../iotdb/rpc/DeepCopyRpcTransportFactory.java | 18 +- .../org/apache/iotdb/rpc/RpcServiceConstants.java | 11 +- .../rpc/TCompressedElasticFramedTransport.java | 52 +- .../apache/iotdb/rpc/TElasticFramedTransport.java | 73 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 + .../iotdb/rpc/TSnappyElasticFramedTransport.java | 30 +- .../rpc/TimeoutChangeableTFastFramedTransport.java | 29 +- .../TimeoutChangeableTSnappyFramedTransport.java | 29 +- .../apache/iotdb/rpc/AutoResizingBufferTest.java | 131 ++++ .../iotdb/rpc/DeepCopyRpcTransportFactoryTest.java | 41 ++ .../rpc/TCompressedElasticFramedTransportTest.java | 295 ++++++++ .../iotdb/rpc/TElasticFramedTransportTest.java | 252 +++++++ .../apache/iotdb/session/i18n/SessionMessages.java | 9 + .../apache/iotdb/session/i18n/SessionMessages.java | 9 + .../iotdb/session/AbstractSessionBuilder.java | 1 + .../org/apache/iotdb/session/NodesSupplier.java | 57 +- .../java/org/apache/iotdb/session/Session.java | 122 +++- .../apache/iotdb/session/SessionConnection.java | 18 +- .../apache/iotdb/session/TableSessionBuilder.java | 20 +- .../org/apache/iotdb/session/ThriftConnection.java | 19 +- .../session/pool/AbstractSessionPoolBuilder.java | 2 + .../org/apache/iotdb/session/pool/SessionPool.java | 107 ++- .../iotdb/session/pool/TableSessionPool.java | 2 +- .../session/pool/TableSessionPoolBuilder.java | 22 +- .../iotdb/session/pool/TableSessionWrapper.java | 57 +- .../iotdb/session/rpccompress/TabletEncoder.java | 104 ++- .../iotdb/session/util/SessionTypeServices.java | 141 ++++ .../apache/iotdb/session/util/SessionUtils.java | 17 + .../iotdb/session/SessionSeedNodeFallbackTest.java | 149 ++++ .../java/org/apache/iotdb/session/SessionTest.java | 75 +++ .../iotdb/session/pool/SessionPoolCloseTest.java | 381 +++++++++++ .../rpccompress/TabletEncoderCompressionTest.java | 230 +++++++ .../rpccompress/TabletEncoderPlainTest.java | 289 ++++++++ .../subscription/i18n/SubscriptionMessages.java | 1 - .../subscription/i18n/SubscriptionMessages.java | 1 - .../rpc/subscription/config/ConsumerConstant.java | 7 + .../subscription/SubscriptionSessionWrapper.java | 28 +- .../subscription/SubscriptionTableSession.java | 4 + .../SubscriptionTableSessionBuilder.java | 36 +- .../SubscriptionTreeSessionBuilder.java | 30 + .../base/AbstractSubscriptionConsumer.java | 51 +- .../base/AbstractSubscriptionConsumerBuilder.java | 37 + .../base/AbstractSubscriptionProvider.java | 24 +- .../base/AbstractSubscriptionProviders.java | 2 +- .../SubscriptionTablePullConsumerBuilder.java | 40 ++ .../SubscriptionTablePushConsumerBuilder.java | 36 + .../tree/SubscriptionTreePullConsumerBuilder.java | 36 + .../tree/SubscriptionTreePushConsumerBuilder.java | 36 + .../SubscriptionConsumerMultiProviderPollTest.java | 73 ++ .../SubscriptionProviderConnectionTimeoutTest.java | 25 +- .../base/SubscriptionProviderStatusTest.java | 12 + iotdb-core/ainode/README.md | 24 +- iotdb-core/ainode/pyproject.toml | 17 +- .../ainode/tests/test_dependency_security.py | 115 ++++ .../AbstractApproxMostFrequentAccumulator.java | 19 +- .../ApproxCountDistinctAccumulator.java | 12 +- .../BlobApproxMostFrequentAccumulator.java | 4 + .../approximate/HyperLogLogStateFactory.java | 2 +- .../aggregation/approximate/SpaceSaving.java | 4 + ...stractGroupedApproxMostFrequentAccumulator.java | 18 +- .../BlobGroupedApproxMostFrequentAccumulator.java | 9 +- .../GroupedApproxCountDistinctAccumulator.java | 12 +- .../ir/GlobalTimePredicateExtractVisitor.java | 18 +- .../transformation/dag/util/CastFunctionUtils.java | 2 +- .../dag/util/CastFunctionUtilsTest.java | 31 + .../iotdb/confignode/i18n/ConfigNodeMessages.java | 11 + .../iotdb/confignode/i18n/ManagerMessages.java | 7 +- .../iotdb/confignode/i18n/ConfigNodeMessages.java | 10 + .../iotdb/confignode/i18n/ManagerMessages.java | 6 +- .../async/DataNodeTransportFailureUtils.java | 92 +++ .../handlers/rpc/DataNodeTSStatusRPCHandler.java | 8 +- .../async/handlers/rpc/PipePushMetaRPCHandler.java | 10 +- .../ConsumerGroupPushMetaRPCHandler.java | 11 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 9 +- .../confignode/conf/ConfigNodeDescriptor.java | 20 +- .../confignode/conf/ConfigNodeStartupCheck.java | 25 + .../iotdb/confignode/manager/ConfigManager.java | 68 +- .../manager/load/balancer/RouteBalancer.java | 4 +- .../iotdb/confignode/manager/node/NodeManager.java | 37 + .../manager/partition/PartitionManager.java | 11 + .../manager/pipe/agent/PipeConfigNodeAgent.java | 34 + .../pipe/agent/task/PipeConfigNodeTaskAgent.java | 5 + .../pipe/coordinator/runtime/PipeMetaSyncer.java | 25 +- .../pipe/coordinator/task/PipeTaskCoordinator.java | 50 +- .../manager/schema/ClusterSchemaManager.java | 34 +- .../subscription/SubscriptionCoordinator.java | 7 +- .../runtime/SubscriptionLeaderChangeHandler.java | 9 +- .../confignode/persistence/ProcedureInfo.java | 4 +- .../partition/DatabasePartitionTable.java | 16 + .../persistence/partition/PartitionInfo.java | 17 + .../persistence/partition/RegionGroup.java | 7 + .../confignode/persistence/pipe/PipeTaskInfo.java | 46 +- .../confignode/procedure/ProcedureExecutor.java | 6 +- .../procedure/TimeoutExecutorThread.java | 18 +- .../procedure/env/ConfigNodeProcedureEnv.java | 23 +- .../procedure/env/RegionMaintainHandler.java | 37 +- .../impl/region/AddRegionPeerProcedure.java | 17 + .../impl/schema/DataNodeRegionGroupUtil.java | 7 +- .../schema/table/SetTablePropertiesProcedure.java | 16 +- .../AbstractOperateSubscriptionProcedure.java | 14 + .../consumer/AlterConsumerGroupProcedure.java | 50 +- .../subscription/DropSubscriptionProcedure.java | 23 +- .../confignode/service/ConfigNodeShutdownHook.java | 3 +- .../confignode/conf/ConfigNodeConfigTest.java | 25 + .../conf/ConfigNodeStartupCheckTest.java | 87 +++ .../manager/ClusterSchemaManagerTest.java | 187 +++++ .../ConfigManagerRegionGroupPolicyVersionTest.java | 280 ++++++++ .../manager/lease/ClusterCachePropagatorTest.java | 42 ++ .../manager/node/NodeManagerStatusReasonTest.java | 23 +- .../node/NodeManagerVersionConsistencyTest.java | 178 +++++ .../agent/task/PipeConfigNodeTaskAgentTest.java | 111 +++ .../coordinator/runtime/PipeMetaSyncerTest.java | 79 +++ .../coordinator/task/PipeTaskCoordinatorTest.java | 157 +++++ .../confignode/persistence/ProcedureInfoTest.java | 38 ++ .../persistence/pipe/PipeTaskInfoRatisTest.java | 149 ++++ .../procedure/ProcedureExecutorStopTest.java | 57 ++ .../procedure/env/ConfigNodeProcedureEnvTest.java | 59 ++ .../RegionMaintainHandlerConsensusPipeTest.java | 83 +++ .../impl/region/AddRegionPeerProcedureTest.java | 122 ++++ .../impl/schema/DataNodeRegionGroupUtilTest.java | 54 ++ .../table/SetTablePropertiesProcedureTest.java | 148 ++++ .../DropSubscriptionProcedureTest.java | 382 ++++++++++- iotdb-core/datanode/pom.xml | 10 +- .../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 16 +- .../apache/iotdb/db/i18n/DataNodePipeMessages.java | 10 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 64 ++ .../iotdb/db/i18n/StorageEngineMessages.java | 4 +- .../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 16 +- .../apache/iotdb/db/i18n/DataNodePipeMessages.java | 10 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 65 ++ .../iotdb/db/i18n/StorageEngineMessages.java | 4 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 2 +- .../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 6 +- .../consensus/statemachine/BaseStateMachine.java | 26 +- .../dataregion/DataRegionStateMachine.java | 6 +- .../schemaregion/SchemaRegionStateMachine.java | 6 +- .../load/LoadAnalyzeMissingDatabaseException.java} | 9 +- .../iotdb/db/pipe/agent/PipeDataNodeAgent.java | 34 + .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 6 + .../agent/task/connection/PipeEventCollector.java | 6 +- .../execution/PipeProcessorSubtaskExecutor.java | 10 + .../task/execution/PipeSubtaskExecutorManager.java | 31 +- .../subtask/processor/PipeProcessorSubtask.java | 22 +- .../processor/PipeProcessorSubtaskWorker.java | 108 ++- .../PipeProcessorSubtaskWorkerManager.java | 27 +- .../sink/PipeRealtimePriorityBlockingQueue.java | 8 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 23 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 59 +- .../pipe/metric/overview/PipeResourceMetrics.java | 48 +- .../legacy/IoTDBLegacyPipeReceiverAgent.java | 14 + .../protocol/thrift/IoTDBDataNodeReceiver.java | 48 ++ .../db/pipe/resource/memory/PipeMemoryManager.java | 257 +++++-- .../builder/PipeTableModelTsFileBuilderV2.java | 4 + ...istoricalDataRegionTsFileAndDeletionSource.java | 34 +- .../realtime/PipeRealtimeDataRegionSource.java | 1 + .../PipeRealtimeDataRegionTsFileSource.java | 3 +- .../realtime/epoch/TsFileEpochManager.java | 5 +- .../iotdb/db/protocol/session/IClientSession.java | 13 + .../iotdb/db/protocol/session/SessionManager.java | 134 +++- .../thrift/InternalClientRPCServiceProcessor.java | 70 ++ .../handler/InternalServiceThriftHandler.java | 43 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 66 +- .../impl/DataNodeInternalRPCServiceImpl.java | 162 +++-- .../thrift/impl/InternalClientRPCServiceImpl.java} | 11 +- .../db/queryengine/common/DataNodeEndPoints.java | 7 + .../db/queryengine/common/MPPQueryContext.java | 55 +- .../db/queryengine/common/QueryAttemptId.java | 59 ++ .../queryengine/execution/QueryStateMachine.java | 3 +- .../execution/fragment/DataNodeQueryContext.java | 9 +- .../fragment/FragmentInstanceContext.java | 87 ++- .../fragment/FragmentInstanceExecution.java | 5 + .../fragment/FragmentInstanceManager.java | 135 +++- .../fragment/FragmentInstanceStateMachine.java | 74 +- .../execution/fragment/QueryAttemptRegistry.java | 389 +++++++++++ .../operator/process/TransformOperator.java | 37 +- .../iotdb/db/queryengine/plan/Coordinator.java | 39 +- .../db/queryengine/plan/analyze/Analysis.java | 20 +- .../db/queryengine/plan/analyze/AnalyzeUtils.java | 48 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 24 +- .../db/queryengine/plan/analyze/Analyzer.java | 1 + .../db/queryengine/plan/analyze/IAnalysis.java | 5 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 122 ++-- .../analyze/load/LoadTsFileTableSchemaCache.java | 111 ++- .../load/TreeSchemaAutoCreatorAndVerifier.java | 30 +- .../plan/analyze/schema/NormalSchemaFetcher.java | 28 +- .../plan/analyze/schema/SchemaValidator.java | 8 +- .../plan/analyze/schema/TemplateSchemaFetcher.java | 13 +- .../queryengine/plan/execution/QueryExecution.java | 169 ++++- .../config/executor/ClusterConfigTaskExecutor.java | 64 +- .../config/metadata/ShowFunctionsTask.java | 16 +- .../plan/parser/StatementGenerator.java | 4 + .../planner/DataNodeTableOperatorGenerator.java | 6 +- .../db/queryengine/plan/planner/IPlanner.java | 3 +- .../plan/planner/LocalExecutionPlanner.java | 73 +- .../queryengine/plan/planner/TreeModelPlanner.java | 3 +- .../NotThreadSafeMemoryReservationManager.java | 15 +- .../plan/planner/plan/FragmentInstance.java | 51 +- .../plan/node/load/LoadSingleTsFileNode.java | 37 +- .../planner/plan/node/load/LoadTsFileNode.java | 6 +- .../planner/plan/node/write/InsertRowNode.java | 3 +- .../planner/plan/node/write/InsertRowsNode.java | 6 +- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 3 +- .../planner/plan/node/write/InsertTabletNode.java | 6 +- .../plan/node/write/RelationalInsertRowsNode.java | 18 +- .../node/write/RelationalInsertTabletNode.java | 19 +- .../plan/relational/analyzer/Analysis.java | 21 +- .../plan/relational/analyzer/Analyzer.java | 1 + .../relational/analyzer/ExpressionAnalyzer.java | 11 + .../relational/analyzer/StatementAnalyzer.java | 46 ++ .../ConvertPredicateToTimeFilterVisitor.java | 12 +- .../plan/relational/metadata/Metadata.java | 3 +- .../relational/metadata/TableMetadataImpl.java | 4 +- .../fetcher/DeviceInCacheFilterVisitor.java | 8 +- .../fetcher/TableDeviceCacheAttributeGuard.java | 3 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 2 +- .../fetcher/TableHeaderSchemaValidator.java | 8 +- .../planner/IrExpressionInterpreter.java | 326 ++++++--- .../plan/relational/planner/QueryPlanner.java | 12 +- .../plan/relational/planner/TableModelPlanner.java | 3 +- .../plan/relational/planner/TagColumnUtils.java | 67 ++ .../plan/relational/planner/TranslationMap.java | 5 +- .../distribute/TableDistributedPlanGenerator.java | 175 ++++- .../ir/GapFillStartAndEndTimeExtractVisitor.java | 46 +- .../planner/optimizations/ParallelizeGrouping.java | 59 +- .../optimizations/TransformSortToStreamSort.java | 18 +- .../security/TreeAccessCheckContext.java | 10 +- .../security/TreeAccessCheckVisitor.java | 8 +- .../plan/relational/sql/ast/InsertRows.java | 45 ++ .../plan/relational/sql/ast/InsertValues.java | 346 ++++++++++ .../plan/relational/sql/parser/AstBuilder.java | 234 +------ .../plan/relational/sql/parser/SqlParser.java | 13 +- .../plan/relational/sql/util/AstUtil.java | 83 +++ .../plan/scheduler/ClusterScheduler.java | 61 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 159 ++++- .../plan/scheduler/IQueryTerminator.java | 8 + .../db/queryengine/plan/scheduler/IScheduler.java | 1 + .../plan/scheduler/SimpleQueryTerminator.java | 44 +- .../plan/scheduler/load/LoadFallbackHandler.java | 10 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 18 + .../plan/statement/crud/InsertBaseStatement.java | 22 +- .../crud/InsertMultiTabletsStatement.java | 46 +- .../crud/InsertRowsOfOneDeviceStatement.java | 46 +- .../plan/statement/crud/InsertRowsStatement.java | 55 +- .../db/queryengine/plan/udf/TreeUDFUtils.java | 17 +- .../transformation/dag/udf/UDTFContext.java | 2 - .../mtree/impl/pbtree/schemafile/SchemaFile.java | 115 +++- .../mtree/impl/pbtree/schemafile/SchemaPage.java | 3 +- .../pbtree/schemafile/pagemgr/PageIOChannel.java | 79 ++- .../pbtree/schemafile/pagemgr/PageManager.java | 27 +- .../reader/impl/TimeseriesReaderWithViewFetch.java | 44 ++ .../java/org/apache/iotdb/db/service/DataNode.java | 6 +- .../db/service/DataNodeInternalRPCService.java | 40 +- .../metrics/IoTDBInternalLocalReporter.java | 2 +- .../db/storageengine/dataregion/DataRegion.java | 179 +++-- .../impl/ReadChunkCompactionPerformer.java | 4 + .../compaction/io/CompactionTsFileWriter.java | 11 + .../dataregion/flush/MemTableFlushTask.java | 39 +- .../dataregion/memtable/TsFileProcessor.java | 17 + .../dataregion/snapshot/SnapshotLoader.java | 28 +- .../dataregion/snapshot/SnapshotTaker.java | 13 +- .../dataregion/tsfile/TsFileResource.java | 24 +- .../utils/TsFileTableSchemaRegistrationUtils.java | 110 +++ .../tableDiskUsageIndex/TableDiskUsageIndex.java | 18 + .../dataregion/wal/buffer/WALBuffer.java | 22 +- .../storageengine/dataregion/wal/node/WALNode.java | 11 +- .../file/UnsealedTsFileRecoverPerformer.java | 25 +- .../db/storageengine/load/LoadStagingDirs.java | 16 +- .../db/storageengine/load/LoadTsFileProgress.java | 73 +- .../db/storageengine/load/LoadTsFileSnapshot.java | 59 +- .../load/active/ActiveLoadPendingQueue.java | 18 +- .../load/active/ActiveLoadTsFileLoader.java | 278 +++++++- .../converter/LoadTsFileDataTypeConverter.java | 71 +- .../splitter/BatchedAlignedValueChunkData.java | 107 +-- .../iotdb/db/storageengine/load/util/LoadUtil.java | 18 +- .../db/storageengine/rescon/disk/TierManager.java | 207 ++++-- .../agent/SubscriptionReceiverAgent.java | 42 +- .../agent/SubscriptionRuntimeAgent.java | 4 +- .../broker/ConsensusSubscriptionBroker.java | 28 +- .../consensus/ConsensusPrefetchingQueue.java | 179 +++-- .../ConsensusSubscriptionCommitManager.java | 393 +++++++++-- .../ConsensusSubscriptionSetupHandler.java | 7 +- .../broker/consensus/ProgressWALIterator.java | 98 ++- .../consensus/SubscriptionConsensusProgress.java | 29 +- .../event/batch/SubscriptionPipeEventBatches.java | 8 +- .../receiver/SubscriptionReceiver.java | 5 + .../receiver/SubscriptionReceiverV1.java | 117 +++- ...nsensusSubscriptionPrefetchExecutorManager.java | 8 +- .../org/apache/iotdb/db/utils/CommonUtils.java | 62 -- .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 91 +-- .../org/apache/iotdb/db/auth/TreeAccessTest.java | 2 +- .../db/conf/DataNodeMemoryProportionTest.java | 106 +++ .../iotdb/db/conf/DiskFullHotReloadTest.java | 143 ++++ .../dataregion/IoTConsensusShutdownTest.java | 10 +- .../metadata/mtree/schemafile/SchemaFileTest.java | 261 +++++++ .../metadata/mtree/schemafile/SchemaPageTest.java | 110 +++ .../pipe/agent/task/PipeDataNodeTaskAgentTest.java | 143 +++- .../task/PipeProcessorSubtaskExecutorTest.java | 33 + .../agent/task/PipeSinkSubtaskExecutorTest.java | 183 +++++ .../pipe/agent/task/PipeSubtaskExecutorTest.java | 25 + .../task/connection/PipeEventCollectorTest.java | 31 + .../execution/PipeSubtaskExecutorManagerTest.java | 66 ++ .../PipeProcessorSubtaskExecutionGuardTest.java | 264 +++++++- .../processor/PipeProcessorSubtaskWorkerTest.java | 494 +++++++++++++- .../PipeRealtimePriorityBlockingQueueTest.java | 33 +- .../task/subtask/sink/PipeSinkSubtaskTest.java | 150 +++-- .../pipe/event/PipeTsFileInsertionEventTest.java | 32 + .../metric/overview/PipeResourceMetricsTest.java | 81 +++ .../metric/sink/PipeDataRegionSinkMetricsTest.java | 38 +- .../airgap/IoTDBAirGapReceiverAgentTest.java | 14 +- .../legacy/IoTDBLegacyPipeReceiverAgentTest.java | 72 ++ .../protocol/thrift/IoTDBDataNodeReceiverTest.java | 202 ++++++ .../resource/memory/PipeMemoryManagerTest.java | 490 +++++++++++++- .../airgap/IoTDBDataRegionAirGapSinkTest.java | 6 +- .../handler/PipeTransferTrackableHandlerTest.java | 16 +- .../db/pipe/source/PipeRealtimeExtractTest.java | 415 +++++++++++- ...ricalDataRegionTsFileAndDeletionSourceTest.java | 268 ++++++++ .../realtime/disruptor/DisruptorShutdownTest.java | 81 ++- .../realtime/epoch/TsFileEpochManagerTest.java | 11 - .../db/protocol/client/ConfigNodeClientTest.java | 107 +++ .../handler/InternalServiceThriftHandlerTest.java | 57 ++ .../impl/DataNodeInternalRPCServiceImplCQTest.java | 163 +++++ .../DataNodeInternalRPCServiceImplDiskTest.java | 46 ++ .../db/queryengine/common/MPPQueryContextTest.java | 38 ++ .../fragment/FragmentInstanceManagerTest.java | 100 +++ .../fragment/FragmentInstanceStateMachineTest.java | 56 ++ .../fragment/QueryAttemptAdmissionTest.java | 184 +++++ .../fragment/QueryAttemptRegistryTest.java | 379 +++++++++++ .../execution/fragment/QueryAttemptRpcTest.java | 140 ++++ ...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 2 +- .../sink/SinkOperatorDownStreamNodeIdTest.java | 2 +- .../iotdb/db/queryengine/plan/CoordinatorTest.java | 149 ++++ .../plan/analyze/QueryPartitionMemoryTest.java | 300 +++++++++ .../plan/analyze/QueryTimePartitionTest.java | 47 ++ .../load/LoadTsFileAnalyzerDiagnosticsTest.java | 541 ++++++++++++++- .../plan/analyze/load/LoadTsFileAnalyzerTest.java | 411 +++++++++++ .../load/LoadTsFileTableSchemaRecoveryTest.java | 185 +++++ .../schema/TreeSchemaAutoCreationPolicyTest.java | 207 ++++++ .../execution/QueryExecutionLifecycleTest.java | 171 +++++ .../plan/execution/QueryExecutionRetryTest.java | 91 ++- .../plan/planner/FragmentInstanceSerdeTest.java | 41 +- .../plan/planner/LastQueryCacheTest.java | 7 +- .../planner/LocalExecutionPlannerBudgetTest.java | 4 +- .../LocalExecutionPlannerOperatorsMemoryTest.java | 56 ++ .../plan/planner/PipelineBuilderTest.java | 4 +- .../plan/planner/node/load/LoadTsFileNodeTest.java | 38 +- .../planner/node/write/WritePlanNodeSplitTest.java | 29 + .../relational/analyzer/TableFunctionTest.java | 241 +++++++ .../TableDeviceCacheAttributeGuardTest.java | 64 ++ .../relational/planner/TableModelPlannerTest.java | 13 +- .../planner/WindowFunctionOptimizationTest.java | 19 + .../planner/assertions/MergeSortMatcher.java | 67 ++ .../planner/assertions/PlanMatchPattern.java | 4 + .../planner/distribute/RateOrderedInputTest.java | 284 ++++++++ .../planner/distribute/RateOrderedPlannerTest.java | 152 +++++ .../relational/sql/AstMemoryEstimatorTest.java | 185 +++++ .../plan/relational/sql/ast/InsertValuesTest.java | 204 ++++++ .../scheduler/ClusterSchedulerLifecycleTest.java | 130 ++++ .../FragmentInstanceDispatcherLifecycleTest.java | 263 ++++++++ .../plan/scheduler/QueryAttemptProtocolTest.java | 95 +++ .../plan/scheduler/QueryTerminalCleanupTest.java | 152 +++++ .../plan/scheduler/SimpleQueryTerminatorTest.java | 123 ++++ .../scheduler/load/LoadFallbackHandlerTest.java | 139 ++++ .../scheduler/load/LoadTsFileSchedulerTest.java | 21 +- .../crud/InsertStatementPartialInsertTest.java | 117 ++++ .../builder/EvaluationDAGBuilderTest.java | 2 +- .../storageengine/dataregion/DataRegionTest.java | 281 ++++++++ .../dataregion/flush/FlushMemoryEstimateTest.java | 53 ++ .../dataregion/memtable/MemTableFlushTaskTest.java | 208 ++++++ .../dataregion/snapshot/IoTDBSnapshotTest.java | 30 + .../snapshot/SnapshotLinkFallbackTest.java | 72 ++ .../snapshot/SnapshotLoaderDiskFullReloadTest.java | 126 ++++ .../dataregion/tsfile/TsFileResourceTest.java | 5 + .../TsFileTableSchemaRegistrationUtilsTest.java | 204 ++++++ .../wal/buffer/WALBufferRollRecoveryTest.java | 65 ++ .../dataregion/wal/node/WALNodeTest.java | 118 +++- .../wal/node/WALNodeWaitForRollFileTest.java | 101 +-- .../wal/recover/WALRecoverManagerTest.java | 6 +- ...ncryptedUnsealedTsFileRecoverPerformerTest.java | 10 +- .../file/UnsealedTsFileRecoverPerformerTest.java | 13 +- .../storageengine/load/LoadTsFileProgressTest.java | 149 ++++ .../storageengine/load/LoadTsFileSnapshotTest.java | 420 ++++++++++-- .../db/storageengine/load/TsFileSplitterTest.java | 209 ++++++ .../load/active/ActiveLoadTsFileLoaderTest.java | 147 +++- .../LoadTsFileDataTypeConverterSchemaTest.java | 349 ++++++++++ .../disk/FolderManagerDiskFullRecoveryTest.java | 160 +++++ .../rescon/disk/TierManagerDiskFullReloadTest.java | 120 ++++ .../storageengine/rescon/disk/TierManagerTest.java | 76 ++- .../agent/SubscriptionReceiverAgentTest.java | 54 ++ ...onsensusSubscriptionBrokerPayloadLimitTest.java | 37 +- ...ConsensusSubscriptionBrokerPollTimeoutTest.java | 198 ++++++ .../ConsensusPrefetchingQueuePollTimeoutTest.java | 180 +++++ .../ConsensusPrefetchingQueueReplicationTest.java | 319 +++++++++ .../ConsensusPrefetchingQueueSeekTest.java | 242 +++++++ ...nsensusPrefetchingQueueWalBackpressureTest.java | 195 ++++++ .../ConsensusSubscriptionCommitStateTest.java | 358 +++++++++- .../broker/consensus/ProgressWALIteratorTest.java | 325 ++++++++- ...ubscriptionPipeEventBatchesConcurrencyTest.java | 144 ++++ .../receiver/SubscriptionReceiverV1Test.java | 274 ++++++++ .../apache/iotdb/db/utils/DateTimeUtilsTest.java | 99 ++- .../db/utils/FileChannelWriteArchitectureTest.java | 91 +++ iotdb-core/node-commons/pom.xml | 10 + .../resources/conf/edge/iotdb-system.properties | 7 + .../apache/iotdb/commons/i18n/PipeMessages.java | 21 + .../apache/iotdb/commons/i18n/QueryMessages.java | 9 + .../apache/iotdb/commons/i18n/UtilMessages.java | 2 + .../apache/iotdb/commons/i18n/PipeMessages.java | 20 + .../apache/iotdb/commons/i18n/QueryMessages.java | 9 + .../apache/iotdb/commons/i18n/UtilMessages.java | 2 + .../apache/iotdb/commons/audit/IAuditEntity.java | 8 +- .../commons/client/AbstractConfigNodeClient.java | 7 + .../client/internal/session/InternalSession.java | 61 ++ .../internal/session/InternalSessionBuilder.java | 59 ++ .../internal/session/InternalSessionPool.java} | 17 +- .../session/InternalSessionPoolBuilder.java | 55 ++ .../session/InternalTableSessionBuilder.java | 60 ++ .../session/InternalTableSessionPool.java} | 35 +- .../session/InternalTableSessionPoolBuilder.java | 56 ++ .../InternalSubscriptionSessionWrapper.java | 78 +++ .../InternalSubscriptionTablePullConsumer.java | 25 +- ...ternalSubscriptionTablePullConsumerBuilder.java | 43 ++ .../InternalSubscriptionTableSession.java | 15 +- .../InternalSubscriptionTableSessionBuilder.java | 55 ++ .../apache/iotdb/commons/conf/CommonConfig.java | 79 ++- .../apache/iotdb/commons/disk/FolderManager.java | 37 +- .../commons/disk/strategy/DirectoryStrategy.java | 22 +- .../commons/enums/HandleSystemErrorStrategy.java | 31 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 47 +- .../agent/task/execution/PipeSubtaskExecutor.java | 52 +- .../task/subtask/PipeAbstractSinkSubtask.java | 33 +- .../options/PipeInclusionOptions.java | 11 + .../commons/queryengine/common/SessionInfo.java | 70 +- .../queryengine/plan/udf/UDFManagementService.java | 129 +++- .../commons/queryengine/utils/DateTimeUtils.java | 5 +- .../AbstractPeriodicalServiceWithAdvance.java | 14 +- .../iotdb/commons/service/ThriftService.java | 10 +- .../iotdb/commons/service/metric/enums/Metric.java | 1 + .../org/apache/iotdb/commons/udf/UDFTable.java | 81 ++- .../iotdb/commons/udf/service/UDFClassLoader.java | 18 +- .../commons/udf/service/UDFClassLoaderManager.java | 179 ++++- .../org/apache/iotdb/commons/utils/IOUtils.java | 36 + .../apache/iotdb/commons/utils/JVMCommonUtils.java | 26 +- .../session/InternalSessionProtocolTest.java | 167 +++++ .../iotdb/commons/conf/CommonConfigTest.java | 205 +++++- .../options/PipeInclusionOptionsTest.java | 41 ++ .../plan/udf/UDFManagementServiceTest.java | 211 ++++++ .../AbstractPeriodicalServiceWithAdvanceTest.java | 95 +++ .../udf/service/UDFClassLoaderManagerTest.java | 95 +++ .../apache/iotdb/commons/utils/IOUtilsTest.java | 152 +++++ .../iotdb/commons/utils/JVMCommonUtilsTest.java | 35 + .../src/main/thrift/confignode.thrift | 2 + .../src/main/thrift/datanode.thrift | 16 + .../iotdb/library/i18n/LibraryUdfMessages.java | 6 + .../iotdb/library/i18n/LibraryUdfMessages.java | 5 + .../org/apache/iotdb/library/dprofile/UDAFMad.java | 10 +- .../dprofile/util/ExactOrderStatistics.java | 16 +- .../apache/iotdb/library/dprofile/UDAFMadTest.java | 114 ++++ .../dprofile/util/ExactOrderStatisticsTest.java | 73 ++ licenses/CDDL License 1.1 | 90 --- licenses/Creative Commons CC0 1.0 Universal | 121 ++++ .../Eclipse Distribution License 1.0 (EDL-1.0) | 30 + licenses/Eclipse Public License 1.0 (EPL-1.0) | 83 --- ...(Boost Software License 1.0, MIT, BSD 2-Clause) | 91 +++ licenses/MIT No Attribution (Reactive Streams) | 7 + ...essbar) => The MIT License (Checker Framework)} | 13 +- ... (progressbar) => The MIT License (ClassGraph)} | 4 +- licenses/The MIT License (FastDoubleParser) | 61 ++ licenses/The MIT License (Java-WebSocket) | 24 + ...e (progressbar) => The MIT License (Schubfach)} | 12 +- pom.xml | 25 +- 692 files changed, 49703 insertions(+), 9045 deletions(-) diff --cc integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java index 6f140578282,3db7bff4f8b..8566835ac5a --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java @@@ -25,10 -25,9 +25,11 @@@ import org.apache.iotdb.confignode.rpc. import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.consensus.ConsensusFactory; ++import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionConfig; - import org.apache.iotdb.it.env.MultiEnvFactory; ++import org.apache.iotdb.it.env.cluster.config.MppDataNodeConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; @@@ -46,29 -45,30 +47,49 @@@ import org.junit.Test import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; ++import java.nio.file.Files; ++import java.nio.file.Path; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; ++import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; ++import java.util.Properties; ++import java.util.Random; ++import java.util.Set; ++import java.util.concurrent.TimeUnit; import java.util.function.Consumer; ++import java.util.regex.Pattern; ++import java.util.stream.Collectors; ++import java.util.stream.Stream; ++import static org.awaitility.Awaitility.await; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) - public class IoTDBPipeSourceIT extends AbstractPipeDualTreeModelAutoIT { - - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); + public class IoTDBPipeSourceIT extends AbstractPipeTreeModelSharedClusterIT { + @Override + protected void setupConfig() { + super.setupConfig(); + senderEnv + .getConfig() + .getDataNodeConfig() - .setMetricReporterType(Collections.singletonList("JMX")); ++ .setMetricReporterType(Collections.singletonList("JMX")) ++ .setCompactionScheduleInterval(100); ++ // Keep a backlog at the oldest end while one file is being sent. Compaction stays disabled ++ // for the other source tests; only the concurrent-compaction scenario enables it temporarily. ++ final Properties queueProperties = new Properties(); ++ queueProperties.setProperty("pipe_realtime_queue_max_waiting_tsfile_size", "1"); ++ queueProperties.setProperty("pipe_realtime_queue_poll_historical_tsfile_threshold", "1000000"); ++ ((MppDataNodeConfig) senderEnv.getConfig().getDataNodeConfig()) ++ .updateProperties(queueProperties); senderEnv .getConfig() .getCommonConfig() @@@ -80,6 -80,6 +101,7 @@@ .setEnableSeqSpaceCompaction(false) .setEnableUnseqSpaceCompaction(false) .setEnableCrossSpaceCompaction(false) ++ .setTimePartitionInterval(TimeUnit.DAYS.toMillis(7)) .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) .setPipeAutoSplitFullEnabled(false); @@@ -1150,6 -1114,6 +1136,135 @@@ } } ++ @Test ++ public void testConcurrentSendingAndCompaction() throws Exception { ++ final String pipeName = "compaction_correctness"; ++ final String device = "root.compaction_correctness.d1"; ++ final String query = "select s1 from " + device; ++ final String header = "Time," + device + ".s1,"; ++ final Set<String> expectedRows = new HashSet<>(); ++ final DataNodeWrapper receiver = receiverEnv.getDataNodeWrapper(0); ++ try (final Connection connection = senderEnv.getConnection(); ++ final Statement statement = connection.createStatement(); ++ final SyncConfigNodeIServiceClient client = ++ (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { ++ statement.execute("create timeseries " + device + ".s1 with datatype=INT64,encoding=RLE"); ++ for (int file = 0; file < 24; file++) { ++ final StringBuilder insert = ++ new StringBuilder("insert into " + device + "(time,s1) values "); ++ for (int point = 0; point < 64; point++) { ++ final int value = file * 64 + point; ++ if (point > 0) { ++ insert.append(','); ++ } ++ insert.append('(').append(value).append(',').append(value).append(')'); ++ expectedRows.add(value + "," + value + ","); ++ } ++ statement.execute(insert.toString()); ++ statement.execute("flush"); ++ } ++ ++ // The newest files occupy the send slot in a separate time partition. This leaves every ++ // source of the older compaction queued, including while the first send finishes. ++ final long blockerTime = TimeUnit.DAYS.toMillis(7); ++ statement.execute( ++ "create timeseries " + device + ".payload with datatype=TEXT,encoding=PLAIN"); ++ final byte[] payload = new byte[65536]; ++ final Random random = new Random(0); ++ for (int blocker = 0; blocker < 2; blocker++) { ++ random.nextBytes(payload); ++ final long time = blockerTime + blocker; ++ final int value = 1536 + blocker; ++ statement.execute( ++ "insert into " ++ + device ++ + "(time,s1,payload) values (" ++ + time ++ + "," ++ + value ++ + ",'" ++ + Base64.getEncoder().encodeToString(payload) ++ + "')"); ++ statement.execute("flush"); ++ expectedRows.add(time + "," + value + ","); ++ } ++ ++ final Map<String, String> sinkAttributes = new HashMap<>(); ++ sinkAttributes.put("sink", "iotdb-thrift-sink"); ++ sinkAttributes.put("sink.ip", receiver.getIp()); ++ sinkAttributes.put("sink.port", Integer.toString(receiver.getPort())); ++ sinkAttributes.put("sink.enable-compaction", "true"); ++ // Keep a send in flight while the remaining historical files are compacted. ++ sinkAttributes.put("sink.rate-limit-bytes-per-second", "8192"); ++ Assert.assertEquals( ++ TSStatusCode.SUCCESS_STATUS.getStatusCode(), ++ client ++ .createPipe( ++ new TCreatePipeReq(pipeName, sinkAttributes) ++ .setExtractorAttributes( ++ Map.of( ++ "source.path", "root.compaction_correctness.**", ++ "source.inclusion", "data.insert", ++ "source.realtime.mode", "batch"))) ++ .getCode()); ++ Assert.assertEquals( ++ TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe(pipeName).getCode()); ++ TestUtils.assertDataEventuallyOnEnv( ++ receiverEnv, ++ query + " where time = " + (blockerTime + 1), ++ header, ++ Collections.singleton((blockerTime + 1) + ",1537,")); ++ ++ try { ++ statement.execute("set configuration 'enable_seq_space_compaction'='true'"); ++ final Pattern replacementMessage = ++ Pattern.compile( ++ Pattern.quote(DataNodePipeMessages.REGION_REPLACED_TSFILEINSERTIONEVENTS_WITH) ++ .replace("{}", "\\E.*\\Q")); ++ await() ++ .alias("A real queued compaction replacement for " + pipeName) ++ .atMost(60, TimeUnit.SECONDS) ++ .until(() -> hasCompactionReplacement(pipeName, replacementMessage)); ++ ++ PipeTestUtils.awaitDataPipeCompletion(senderEnv, pipeName); ++ TestUtils.assertDataEventuallyOnEnv(senderEnv, query, header, expectedRows); ++ TestUtils.assertDataEventuallyOnEnv(receiverEnv, query, header, expectedRows); ++ ++ // A later overwrite must also arrive and commit after the compacted backlog is drained. ++ statement.execute("insert into " + device + "(time,s1) values (0,-1),(1536,1536)"); ++ statement.execute("flush"); ++ expectedRows.remove("0,0,"); ++ expectedRows.add("0,-1,"); ++ expectedRows.add("1536,1536,"); ++ PipeTestUtils.awaitDataPipeCompletion(senderEnv, pipeName); ++ TestUtils.assertDataEventuallyOnEnv(senderEnv, query, header, expectedRows); ++ TestUtils.assertDataEventuallyOnEnv(receiverEnv, query, header, expectedRows); ++ } finally { ++ statement.execute("set configuration 'enable_seq_space_compaction'='false'"); ++ } ++ } ++ } ++ ++ private boolean hasCompactionReplacement(final String pipeName, final Pattern replacementMessage) ++ throws Exception { ++ for (final DataNodeWrapper node : senderEnv.getDataNodeWrapperList()) { ++ try (final Stream<Path> logs = Files.walk(Path.of(node.getLogDirPath()))) { ++ for (final Path log : ++ logs.filter(Files::isRegularFile) ++ .filter(path -> path.toString().endsWith(".log")) ++ .collect(Collectors.toList())) { ++ try (final Stream<String> lines = Files.lines(log)) { ++ if (lines.anyMatch( ++ line -> line.contains(pipeName) && replacementMessage.matcher(line).find())) { ++ return true; ++ } ++ } ++ } ++ } ++ } ++ return false; ++ } ++ private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int count) { TestUtils.assertDataEventuallyOnEnv( receiverEnv, diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java index d461dcb903b,83c8666eb3c..d8e9bd91de2 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java @@@ -55,6 -49,6 +55,7 @@@ import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; ++import java.util.function.Predicate; import java.util.stream.Collectors; public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQueue<Event> { @@@ -102,13 -83,11 +103,15 @@@ return true; } + if (!checkBeforeOffer(event)) { + return false; + } + + final Event lastEvent = super.peekLast(); if (event instanceof PipeHeartbeatEvent && !((PipeHeartbeatEvent) event).isCompletionBarrier() - && super.peekLast() instanceof PipeHeartbeatEvent) { + && lastEvent instanceof PipeHeartbeatEvent + && !((PipeHeartbeatEvent) lastEvent).isCompletionBarrier()) { // We can NOT keep too many PipeHeartbeatEvent in bufferQueue because they may cause OOM. ((EnrichedEvent) event).decreaseReferenceCount(PipeEventCollector.class.getName(), false); return false; @@@ -514,38 -398,4 +517,37 @@@ public synchronized void setOfferTsFileCounter(AtomicInteger offerTsFileCounter) { this.offerTsFileCounter = offerTsFileCounter; } + + public void enableTsFileStartupReservation() { + reserveTsFileStartupOnPoll = true; + } + + public boolean consumeTsFileStartupReservation(final TsFileInsertionEvent event) { + synchronized (offerTsFileCounter) { + return reservedTsFileStartupEvents.remove(event); + } + } + + public boolean cancelTsFileStartupReservation(final TsFileInsertionEvent event) { + synchronized (offerTsFileCounter) { + if (!reservedTsFileStartupEvents.remove(event)) { + return false; + } + offerTsFileCounter.decrementAndGet(); + offerTsFileCounter.notifyAll(); + return true; + } + } + - private void clearTsFileStartupReservations( - final java.util.function.Predicate<TsFileInsertionEvent> predicate) { ++ private void clearTsFileStartupReservations(final Predicate<TsFileInsertionEvent> predicate) { + synchronized (offerTsFileCounter) { + final int sizeBefore = reservedTsFileStartupEvents.size(); + reservedTsFileStartupEvents.removeIf(predicate); + final int canceledReservations = sizeBefore - reservedTsFileStartupEvents.size(); + if (canceledReservations > 0) { + offerTsFileCounter.addAndGet(-canceledReservations); + offerTsFileCounter.notifyAll(); + } + } + } } diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueueTest.java index 722aa9c0f0d,00000000000..252544cd3a0 mode 100644,000000..100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueueTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueueTest.java @@@ -1,675 -1,0 +1,706 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.agent.task.subtask.sink; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; +import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeCompactedTsFileInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.pipe.resource.tsfile.PipeCompactionManager; +import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +public class PipeRealtimePriorityBlockingQueueTest { + + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testCompletionBarrierIsNotDroppedAfterHeartbeat() { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(); + final PipeHeartbeatEvent heartbeatEvent = new PipeHeartbeatEvent(1, false); + final PipeHeartbeatEvent completionBarrier = new PipeHeartbeatEvent(1, false, true); + + Assert.assertTrue(queue.offer(heartbeatEvent)); + Assert.assertTrue(queue.offer(completionBarrier)); + Assert.assertSame(heartbeatEvent, queue.directPoll()); + Assert.assertSame(completionBarrier, queue.directPoll()); + } + ++ @Test ++ public void testHeartbeatAfterCompletionBarrierIsNotDropped() { ++ final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(); ++ final PipeHeartbeatEvent completionBarrier = new PipeHeartbeatEvent(1, false, true); ++ final PipeHeartbeatEvent heartbeat = new PipeHeartbeatEvent(1, false); ++ ++ Assert.assertTrue(queue.offer(completionBarrier)); ++ Assert.assertTrue(queue.offer(heartbeat)); ++ Assert.assertFalse(queue.offer(new PipeHeartbeatEvent(1, false))); ++ Assert.assertSame(completionBarrier, queue.directPoll()); ++ Assert.assertSame(heartbeat, queue.directPoll()); ++ Assert.assertNull(queue.directPoll()); ++ } ++ + @Test + public void testWaitedPollRetainsTsFilesUntilAsyncSendCapacityIsAvailable() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final int maxWaitingTsFiles = + PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize(); + final AtomicInteger waitingTsFiles = new AtomicInteger(maxWaitingTsFiles); + queue.setOfferTsFileCounter(waitingTsFiles); + final PipeTsFileInsertionEvent event = + createEvent(newTsFile("waiting.tsfile"), new CommitterKey("pipe", 1L, 1, 0), 1L); + Assert.assertTrue(queue.offer(event)); + + for (final int waiting : new int[] {maxWaitingTsFiles, maxWaitingTsFiles + 1}) { + waitingTsFiles.set(waiting); + for (int i = 0; i < 3; i++) { + Assert.assertNull(queue.waitedPoll()); + Assert.assertNull(queue.directPoll()); + Assert.assertEquals(1, queue.getTsFileInsertionEventCount()); + Assert.assertSame(event, queue.peek()); + } + } + + waitingTsFiles.set(maxWaitingTsFiles - 1); + Assert.assertSame(event, queue.waitedPoll()); + Assert.assertEquals(0, queue.getTsFileInsertionEventCount()); + verify(event, never()).decreaseReferenceCount(anyString(), anyBoolean()); + } + + @Test + public void testConcurrentPollReservesOnlyAvailableTsFileStartupSlot() throws Exception { + final int previousMaxWaitingTsFiles = + PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize(); + final AtomicInteger startupCount = new AtomicInteger(0); + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final ExecutorService pollers = Executors.newFixedThreadPool(16); + final CountDownLatch startPollers = new CountDownLatch(1); + try { + CommonDescriptor.getInstance().getConfig().setPipeRealTimeQueueMaxWaitingTsFileSize(1); + queue.setOfferTsFileCounter(startupCount); + queue.enableTsFileStartupReservation(); + for (int i = 0; i < 16; i++) { + Assert.assertTrue( + queue.offer( + createEvent( + newTsFile("parallel-" + i + ".tsfile"), + new CommitterKey("pipe", 1L, 1, 0), + i))); + } + + final List<Future<Event>> polls = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + polls.add( + pollers.submit( + () -> { + startPollers.await(); + return queue.directPoll(); + })); + } + startPollers.countDown(); + + TsFileInsertionEvent polledEvent = null; + int polledEventCount = 0; + for (final Future<Event> poll : polls) { + final Event event = poll.get(5, TimeUnit.SECONDS); + if (event != null) { + polledEvent = (TsFileInsertionEvent) event; + polledEventCount++; + } + } + + Assert.assertEquals(1, polledEventCount); + Assert.assertEquals(1, startupCount.get()); + Assert.assertEquals(15, queue.getTsFileInsertionEventCount()); + Assert.assertTrue(queue.cancelTsFileStartupReservation(polledEvent)); + Assert.assertEquals(0, startupCount.get()); + } finally { + pollers.shutdownNow(); + pollers.awaitTermination(5, TimeUnit.SECONDS); + queue.discardAllEvents(); + CommonDescriptor.getInstance() + .getConfig() + .setPipeRealTimeQueueMaxWaitingTsFileSize(previousMaxWaitingTsFiles); + } + } + + @Test + public void testDiscardedPipeKeepsPolledStartupReservationUntilSinkCancelsIt() throws Exception { + final int previousMaxWaitingTsFiles = + PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize(); + final AtomicInteger startupCount = new AtomicInteger(0); + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final PipeTsFileInsertionEvent event = + createEvent(newTsFile("discarded.tsfile"), committerKey, 1L); + try { + CommonDescriptor.getInstance().getConfig().setPipeRealTimeQueueMaxWaitingTsFileSize(1); + queue.setOfferTsFileCounter(startupCount); + queue.enableTsFileStartupReservation(); + Assert.assertTrue(queue.offer(event)); + Assert.assertSame(event, queue.directPoll()); + Assert.assertEquals(1, startupCount.get()); + + queue.discardEventsOfPipe(committerKey); + + Assert.assertEquals(1, startupCount.get()); + Assert.assertNull(queue.directPoll()); + Assert.assertTrue(queue.cancelTsFileStartupReservation(event)); + Assert.assertEquals(0, startupCount.get()); + } finally { + queue.discardAllEvents(); + CommonDescriptor.getInstance() + .getConfig() + .setPipeRealTimeQueueMaxWaitingTsFileSize(previousMaxWaitingTsFiles); + } + } + + @Test + public void testWaitedPollDeliversHeartbeatsWhileTsFileSendQueueIsFull() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + queue.setOfferTsFileCounter( + new AtomicInteger(PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize())); + final PipeTsFileInsertionEvent event = + createEvent(newTsFile("waiting.tsfile"), new CommitterKey("pipe", 1L, 1, 0), 1L); + final PipeHeartbeatEvent heartbeat = new PipeHeartbeatEvent(1, false); + final PipeHeartbeatEvent completionBarrier = new PipeHeartbeatEvent(1, false, true); + Assert.assertTrue(queue.offer(event)); + Assert.assertTrue(queue.offer(heartbeat)); + Assert.assertTrue(queue.offer(completionBarrier)); + + Assert.assertSame(heartbeat, queue.waitedPoll()); + Assert.assertSame(completionBarrier, queue.waitedPoll()); + Assert.assertNull(queue.waitedPoll()); + Assert.assertSame(event, queue.peek()); + Assert.assertEquals(1, queue.getTsFileInsertionEventCount()); + } + + @Test + public void testHeartbeatWakesBackpressuredPoll() throws Exception { + final long previousBlockingTimeMs = + PipeConfig.getInstance().getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(); + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + queue.setOfferTsFileCounter( + new AtomicInteger(PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize())); + final PipeTsFileInsertionEvent event = + createEvent(newTsFile("waiting.tsfile"), new CommitterKey("pipe", 1L, 1, 0), 1L); + Assert.assertTrue(queue.offer(event)); + final FutureTask<Event> poll = new FutureTask<>(queue::waitedPoll); + final Thread pollingThread = new Thread(poll); + try { + CommonDescriptor.getInstance() + .getConfig() + .setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(10_000); + pollingThread.start(); + await() + .atMost(5, TimeUnit.SECONDS) + .until(() -> pollingThread.getState() == Thread.State.TIMED_WAITING); + + final PipeHeartbeatEvent heartbeat = new PipeHeartbeatEvent(1, false); + Assert.assertTrue(queue.offer(heartbeat)); + Assert.assertSame(heartbeat, poll.get(1, TimeUnit.SECONDS)); + Assert.assertSame(event, queue.peek()); + } finally { + pollingThread.interrupt(); + pollingThread.join(5_000); + CommonDescriptor.getInstance() + .getConfig() + .setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(previousBlockingTimeMs); + } + } + + @Test + public void testWaitedPollRechecksCapacityAfterWaitingForAnEvent() throws Exception { + final long previousBlockingTimeMs = + PipeConfig.getInstance().getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(); + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final AtomicInteger waitingTsFiles = new AtomicInteger(0); + queue.setOfferTsFileCounter(waitingTsFiles); + final PipeTsFileInsertionEvent event = + createEvent(newTsFile("waiting.tsfile"), new CommitterKey("pipe", 1L, 1, 0), 1L); + final FutureTask<Event> poll = new FutureTask<>(queue::waitedPoll); + final Thread pollingThread = new Thread(poll); + try { + CommonDescriptor.getInstance() + .getConfig() + .setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(1_000); + pollingThread.start(); + await() + .pollInterval(5, TimeUnit.MILLISECONDS) + .atMost(5, TimeUnit.SECONDS) + .until(() -> pollingThread.getState() == Thread.State.TIMED_WAITING); + + waitingTsFiles.set(PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize()); + Assert.assertTrue(queue.offer(event)); + Assert.assertNull(poll.get(2, TimeUnit.SECONDS)); + Assert.assertSame(event, queue.peek()); + Assert.assertEquals(1, queue.getTsFileInsertionEventCount()); + } finally { + pollingThread.interrupt(); + pollingThread.join(5_000); + CommonDescriptor.getInstance() + .getConfig() + .setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(previousBlockingTimeMs); + } + } + + @Test + public void testWaitedPollPreservesPendingFilesForCompaction() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final int maxWaitingTsFiles = + PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize(); + final AtomicInteger waitingTsFiles = new AtomicInteger(maxWaitingTsFiles); + queue.setOfferTsFileCounter(waitingTsFiles); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final File sourceFile0 = newTsFile("source0.tsfile"); + final File sourceFile1 = newTsFile("source1.tsfile"); + final File targetFile = newTsFile("target.tsfile"); + final PipeTsFileInsertionEvent sourceEvent0 = createEvent(sourceFile0, committerKey, 1L); + final PipeTsFileInsertionEvent sourceEvent1 = createEvent(sourceFile1, committerKey, 2L); + Assert.assertTrue(queue.offer(sourceEvent0)); + Assert.assertTrue(queue.offer(sourceEvent1)); + Assert.assertNull(queue.waitedPoll()); + + queue.replace( + "1", + new HashSet<>( + Arrays.asList(new TsFileResource(sourceFile0), new TsFileResource(sourceFile1))), + Collections.singletonList(new TsFileResource(targetFile))); + + Assert.assertEquals(1, queue.getTsFileInsertionEventCount()); + Assert.assertNull(queue.waitedPoll()); + waitingTsFiles.decrementAndGet(); + final Event event = queue.waitedPoll(); + Assert.assertTrue(event instanceof PipeCompactedTsFileInsertionEvent); + final PipeCompactedTsFileInsertionEvent compactedEvent = + (PipeCompactedTsFileInsertionEvent) event; + try { + Assert.assertEquals( + targetFile.getAbsoluteFile(), compactedEvent.getSourceTsFile().getAbsoluteFile()); + Assert.assertEquals( + new HashSet<>(Arrays.asList(1L, 2L)), new HashSet<>(compactedEvent.getCommitIds())); + Assert.assertEquals(0, queue.getTsFileInsertionEventCount()); + verify(sourceEvent0) + .decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false); + verify(sourceEvent1) + .decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false); + } finally { + compactedEvent.decreaseReferenceCount( + PipeRealtimePriorityBlockingQueue.class.getName(), false); + } + } + + @Test + public void testCompactedHistoricalEventPinsTargetStorageFile() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final File sourceFile = newTsFile("source.tsfile"); + final File targetFile = newTsFile("target.tsfile"); + final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, committerKey, 1L); + doReturn(true).when(sourceEvent).isGeneratedByHistoricalExtractor(); + doReturn(true).when(sourceEvent).isWithMod(); + + Assert.assertTrue(queue.offer(sourceEvent)); + queue.replace( + "1", + Collections.singleton(new TsFileResource(sourceFile)), + Collections.singletonList(new TsFileResource(targetFile))); + + final Event event = queue.directPoll(); + Assert.assertTrue(event instanceof PipeCompactedTsFileInsertionEvent); + final PipeCompactedTsFileInsertionEvent compactedEvent = + (PipeCompactedTsFileInsertionEvent) event; + try { + Assert.assertTrue(compactedEvent.isGeneratedByHistoricalExtractor()); + Assert.assertFalse(compactedEvent.isWithMod()); + Assert.assertTrue( + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(targetFile, "pipe_1") + .exists()); + } finally { + compactedEvent.decreaseReferenceCount( + PipeRealtimePriorityBlockingQueue.class.getName(), false); + } + } + + @Test + public void testEmitCompactionResultSkipsDeletedTargets() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final PipeCompactionManager compactionManager = createCompactionManager(queue); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final File sourceFile0 = newTsFile("source0.tsfile"); + final File sourceFile1 = newTsFile("source1.tsfile"); + final File targetFile0 = newTsFile("target0.tsfile"); + final File targetFile1 = newTsFile("target1.tsfile"); + final TsFileResource deletedTarget0 = + new TsFileResource(new File(targetFile0.getParentFile(), "unused0.inner")); + final TsFileResource deletedTarget1 = + new TsFileResource(new File(targetFile0.getParentFile(), "unused1.inner")); + deletedTarget0.forceMarkDeleted(); + deletedTarget1.forceMarkDeleted(); + final PipeTsFileInsertionEvent sourceEvent0 = createEvent(sourceFile0, committerKey, 1L); + final PipeTsFileInsertionEvent sourceEvent1 = createEvent(sourceFile1, committerKey, 2L); + queue.offer(sourceEvent0); + queue.offer(sourceEvent1); + + compactionManager.emitResult( + "root.db", + "1", + 0, + Arrays.asList(new TsFileResource(sourceFile0), new TsFileResource(sourceFile1)), + Collections.emptyList(), + Arrays.asList( + deletedTarget0, + new TsFileResource(targetFile0), + new TsFileResource(targetFile1), + deletedTarget1)); + + final List<Event> queuedEvents = new ArrayList<>(); + queue.forEach(queuedEvents::add); + Assert.assertEquals(2, queuedEvents.size()); + Assert.assertTrue(queuedEvents.get(0) instanceof PipeCompactedTsFileInsertionEvent); + Assert.assertTrue(queuedEvents.get(1) instanceof PipeCompactedTsFileInsertionEvent); + final PipeCompactedTsFileInsertionEvent compactedEvent0 = + (PipeCompactedTsFileInsertionEvent) queuedEvents.get(0); + final PipeCompactedTsFileInsertionEvent compactedEvent1 = + (PipeCompactedTsFileInsertionEvent) queuedEvents.get(1); + try { + Assert.assertEquals( + targetFile0.getAbsoluteFile(), compactedEvent0.getSourceTsFile().getAbsoluteFile()); + Assert.assertEquals( + targetFile1.getAbsoluteFile(), compactedEvent1.getSourceTsFile().getAbsoluteFile()); + Assert.assertTrue( + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(targetFile0, "pipe_1") + .exists()); + Assert.assertTrue( + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(targetFile1, "pipe_1") + .exists()); + Assert.assertEquals( + new HashSet<>(Arrays.asList(1L, 2L)), new HashSet<>(compactedEvent1.getCommitIds())); + Assert.assertFalse(compactedEvent0.isShouldReportOnCommit()); + Assert.assertTrue(compactedEvent1.isShouldReportOnCommit()); + verify(sourceEvent0) + .decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false); + verify(sourceEvent1) + .decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false); + } finally { + compactedEvent0.decreaseReferenceCount( + PipeRealtimePriorityBlockingQueue.class.getName(), false); + compactedEvent1.decreaseReferenceCount( + PipeRealtimePriorityBlockingQueue.class.getName(), false); + queue.clear(); + } + } + + @Test + public void testEmitCompactionResultKeepsSourcesWithoutSurvivingTargets() throws Exception { + final File sourceFile = newTsFile("source.tsfile"); + final TsFileResource deletedTarget = + new TsFileResource(new File(sourceFile.getParentFile(), "unused.inner")); + deletedTarget.forceMarkDeleted(); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + for (final List<TsFileResource> targets : + Arrays.asList( + Collections.<TsFileResource>emptyList(), Collections.singletonList(deletedTarget))) { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final PipeCompactionManager compactionManager = createCompactionManager(queue); + final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, committerKey, 1L); + queue.offer(sourceEvent); + + compactionManager.emitResult( + "root.db", + "1", + 0, + Collections.singletonList(new TsFileResource(sourceFile)), + Collections.emptyList(), + targets); + + Assert.assertSame(sourceEvent, queue.directPoll()); + Assert.assertNull(queue.directPoll()); + Assert.assertTrue(sourceEvent.isShouldReportOnCommit()); + verify(sourceEvent, never()).decreaseReferenceCount(anyString(), anyBoolean()); + } + } + + @Test + public void testReplaceOnlyRemovesEventsForCompactionSourceFiles() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final File sourceFile = newTsFile("source.tsfile"); + final File unrelatedFile = newTsFile("unrelated.tsfile"); + final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, committerKey, 1L); + final PipeTsFileInsertionEvent unrelatedEvent = createEvent(unrelatedFile, committerKey, 2L); + + queue.offer(sourceEvent); + queue.offer(unrelatedEvent); + queue.replace( + "1", Collections.singleton(new TsFileResource(sourceFile)), Collections.emptyList()); + + Assert.assertSame(unrelatedEvent, queue.directPoll()); + Assert.assertNull(queue.directPoll()); + verify(sourceEvent) + .decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false); + verify(unrelatedEvent, never()).decreaseReferenceCount(anyString(), anyBoolean()); + } + + @Test + public void testReplaceDoesNotMatchUnrelatedEventsByCount() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final File sourceFile = newTsFile("source.tsfile"); + final PipeTsFileInsertionEvent unrelatedEvent = + createEvent(newTsFile("unrelated.tsfile"), committerKey, 1L); + + queue.offer(unrelatedEvent); + queue.replace( + "1", Collections.singleton(new TsFileResource(sourceFile)), Collections.emptyList()); + + Assert.assertSame(unrelatedEvent, queue.directPoll()); + verify(unrelatedEvent, never()).decreaseReferenceCount(anyString(), anyBoolean()); + } + + @Test + public void testReplaceKeepsCompactedEventsAtTheOldestQueuePosition() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final File sourceFile0 = newTsFile("source0.tsfile"); + final File sourceFile1 = newTsFile("source1.tsfile"); + final File laterFile = newTsFile("later.tsfile"); + final File targetFile0 = newTsFile("target0.tsfile"); + final File targetFile1 = newTsFile("target1.tsfile"); + final PipeTsFileInsertionEvent sourceEvent0 = createEvent(sourceFile0, committerKey, 1L); + final PipeTsFileInsertionEvent sourceEvent1 = createEvent(sourceFile1, committerKey, 2L); + final PipeTsFileInsertionEvent laterEvent = createEvent(laterFile, committerKey, 3L); + + queue.offer(sourceEvent0); + queue.offer(sourceEvent1); + queue.offer(laterEvent); + queue.replace( + "1", + new HashSet<>( + Arrays.asList(new TsFileResource(sourceFile0), new TsFileResource(sourceFile1))), + Arrays.asList(new TsFileResource(targetFile0), new TsFileResource(targetFile1))); + + final List<Event> queuedEvents = new ArrayList<>(); + queue.forEach(queuedEvents::add); + Assert.assertEquals(3, queuedEvents.size()); + Assert.assertSame(laterEvent, queuedEvents.get(2)); + final PipeTsFileInsertionEvent compactedEvent0 = (PipeTsFileInsertionEvent) queuedEvents.get(0); + final PipeTsFileInsertionEvent compactedEvent1 = (PipeTsFileInsertionEvent) queuedEvents.get(1); + Assert.assertTrue(compactedEvent1 instanceof PipeCompactedTsFileInsertionEvent); + Assert.assertTrue(compactedEvent0 instanceof PipeCompactedTsFileInsertionEvent); + Assert.assertEquals( + targetFile1.getAbsoluteFile(), compactedEvent1.getSourceTsFile().getAbsoluteFile()); + Assert.assertEquals( + targetFile0.getAbsoluteFile(), compactedEvent0.getSourceTsFile().getAbsoluteFile()); + compactedEvent1.decreaseReferenceCount( + PipeRealtimePriorityBlockingQueue.class.getName(), false); + compactedEvent0.decreaseReferenceCount( + PipeRealtimePriorityBlockingQueue.class.getName(), false); + } + + @Test + public void testReplaceLeavesNonHeadSourceEventsQueuedToPreserveOrder() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final PipeTsFileInsertionEvent olderEvent = + createEvent(newTsFile("older.tsfile"), committerKey, 1L); + final File sourceFile = newTsFile("source.tsfile"); + final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, committerKey, 2L); + final PipeTsFileInsertionEvent laterEvent = + createEvent(newTsFile("later.tsfile"), committerKey, 3L); + + queue.offer(olderEvent); + queue.offer(sourceEvent); + queue.offer(laterEvent); + queue.replace( + "1", + Collections.singleton(new TsFileResource(sourceFile)), + Collections.singletonList(new TsFileResource(newTsFile("target.tsfile")))); + + final List<Event> queuedEvents = new ArrayList<>(); + queue.forEach(queuedEvents::add); + Assert.assertEquals(3, queuedEvents.size()); + Assert.assertSame(olderEvent, queuedEvents.get(0)); + Assert.assertSame(sourceEvent, queuedEvents.get(1)); + Assert.assertSame(laterEvent, queuedEvents.get(2)); + verify(sourceEvent, never()) + .decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false); + } + + @Test + public void testEarlierEventFromAnotherPipeDoesNotBlockReplacement() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final PipeTsFileInsertionEvent otherPipeEvent = + createEvent(newTsFile("other-pipe.tsfile"), new CommitterKey("otherPipe", 1L, 1, 0), 1L); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final File sourceFile = newTsFile("source.tsfile"); + final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, committerKey, 2L); + + queue.offer(otherPipeEvent); + queue.offer(sourceEvent); + queue.replace( + "1", + Collections.singleton(new TsFileResource(sourceFile)), + Collections.singletonList(new TsFileResource(newTsFile("target.tsfile")))); + + final List<Event> queuedEvents = new ArrayList<>(); + queue.forEach(queuedEvents::add); + Assert.assertEquals(2, queuedEvents.size()); + Assert.assertTrue(queuedEvents.get(0) instanceof PipeCompactedTsFileInsertionEvent); + Assert.assertSame(otherPipeEvent, queuedEvents.get(1)); + verify(sourceEvent) + .decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false); + } + + @Test + public void testPollCanRaceWithReplacementWithoutCorruptingQueue() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(true); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final File sourceFile = newTsFile("source.tsfile"); + final PipeTsFileInsertionEvent event = createEvent(sourceFile, committerKey, 1L); + final CountDownLatch sourcePathRead = new CountDownLatch(1); + final CountDownLatch allowReplacementToContinue = new CountDownLatch(1); + final ExecutorService executor = Executors.newFixedThreadPool(2); ++ final File targetFile = newTsFile("target.tsfile"); ++ PipeCompactedTsFileInsertionEvent replacement = null; + + try { + doAnswer( + invocation -> { + sourcePathRead.countDown(); + Assert.assertTrue(allowReplacementToContinue.await(5, TimeUnit.SECONDS)); + return sourceFile; + }) + .when(event) + .getSourceTsFile(); + queue.offer(event); + + final Future<?> replacementFuture = + executor.submit( + () -> + queue.replace( + "1", + Collections.singleton(new TsFileResource(sourceFile)), - Collections.emptyList())); ++ Collections.singletonList(new TsFileResource(targetFile)))); + Assert.assertTrue(sourcePathRead.await(5, TimeUnit.SECONDS)); + final Future<Event> pollFuture = executor.submit(queue::directPoll); + Assert.assertSame(event, pollFuture.get(5, TimeUnit.SECONDS)); + + allowReplacementToContinue.countDown(); + replacementFuture.get(5, TimeUnit.SECONDS); + verify(event, never()) + .decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false); ++ Assert.assertEquals(1, queue.getTsFileInsertionEventCount()); ++ final Event queuedEvent = queue.directPoll(); ++ Assert.assertTrue(queuedEvent instanceof PipeCompactedTsFileInsertionEvent); ++ replacement = (PipeCompactedTsFileInsertionEvent) queuedEvent; ++ Assert.assertEquals( ++ targetFile.getAbsoluteFile(), replacement.getSourceTsFile().getAbsoluteFile()); ++ Assert.assertEquals(Collections.singletonList(1L), replacement.getCommitIds()); ++ Assert.assertEquals(1L, replacement.getCommitId()); ++ Assert.assertNull(queue.directPoll()); + } finally { + allowReplacementToContinue.countDown(); + executor.shutdownNow(); ++ executor.awaitTermination(5, TimeUnit.SECONDS); ++ if (replacement != null) { ++ replacement.decreaseReferenceCount( ++ PipeRealtimePriorityBlockingQueue.class.getName(), false); ++ } ++ queue.discardAllEvents(); + } + } + + @Test + public void testReplaceIsDisabledByDefault() throws Exception { + final PipeRealtimePriorityBlockingQueue queue = new PipeRealtimePriorityBlockingQueue(); + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, 0); + final File sourceFile = newTsFile("source.tsfile"); + final PipeTsFileInsertionEvent sourceEvent = createEvent(sourceFile, committerKey, 1L); + + queue.offer(sourceEvent); + queue.replace( + "1", Collections.singleton(new TsFileResource(sourceFile)), Collections.emptyList()); + + Assert.assertSame(sourceEvent, queue.directPoll()); + Assert.assertNull(queue.directPoll()); + verify(sourceEvent, never()) + .decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false); + } + + private PipeCompactionManager createCompactionManager( + final PipeRealtimePriorityBlockingQueue queue) { + final PipeSinkSubtaskLifeCycle lifeCycle = Mockito.mock(PipeSinkSubtaskLifeCycle.class); + doReturn(queue).when(lifeCycle).getPendingQueue(); + final PipeCompactionManager compactionManager = new PipeCompactionManager(); + compactionManager.registerPipeConnectorSubtaskLifeCycle(lifeCycle); + return compactionManager; + } + + private PipeTsFileInsertionEvent createEvent( + final File file, final CommitterKey committerKey, final long commitId) { + final PipeTsFileInsertionEvent event = + Mockito.spy( + new PipeTsFileInsertionEvent(false, "root.db", new TsFileResource(file), false)); + event.setCommitterKeyAndCommitId(committerKey, commitId); + doReturn(true).when(event).decreaseReferenceCount(anyString(), anyBoolean()); + return event; + } + + private File newTsFile(final String name) throws Exception { + final File sequenceDir = new File(temporaryFolder.getRoot(), "sequence/1/0"); + Assert.assertTrue(sequenceDir.mkdirs() || sequenceDir.isDirectory()); + final File tsFile = new File(sequenceDir, name); + Assert.assertTrue(tsFile.createNewFile()); + return tsFile; + } +} diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java index 1a260212546,f15cfa4fad9..fea89773adc --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java @@@ -55,8 -48,8 +55,9 @@@ import java.util.concurrent.ExecutionEx import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow;
