This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new ef9e799f50a [Bugfix] Disable Subscription and Pipes containing data
insertion under Ratis (#19053)
ef9e799f50a is described below
commit ef9e799f50a522ff249f2f77a4a00fa413e46aa5
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 10:07:06 2026 +0800
[Bugfix] Disable Subscription and Pipes containing data insertion under
Ratis (#19053)
* [Bugfix] Replay Ratis historical TsFiles without Pipe progress
Treat MinimumProgressIndex as unknown for Ratis historical extraction so
successful writes remain eligible for replay after a leader change.
Preserve valid-index and other consensus-protocol coverage filtering.
Add a regression test for initial and resumed checkpoints and the
unaffected filtering behavior.
* Disable Pipe and Subscription with Ratis
* Restore ConfigNode descriptor imports
* Revert "Disable Pipe and Subscription with Ratis"
This reverts commit 7d4912d72d1a29c98f52b415d30e79724486714d.
* Refine Ratis Pipe and Subscription restrictions
* Fix Ratis pipe recovery and receiver validation gaps
* Fix Pipe coordinator drop races and Ratis PlanNode admission
* Remove redundant consensus pipe exemptions from Ratis checks
* Address receiver review feedback
---
.../iotdb/confignode/manager/ConfigManager.java | 36 +++-
.../manager/pipe/agent/PipeConfigNodeAgent.java | 34 ++++
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 5 +
.../pipe/coordinator/task/PipeTaskCoordinator.java | 46 ++++-
.../subscription/SubscriptionCoordinator.java | 7 +-
.../runtime/SubscriptionLeaderChangeHandler.java | 9 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 17 ++
.../agent/task/PipeConfigNodeTaskAgentTest.java | 111 +++++++++++
.../coordinator/task/PipeTaskCoordinatorTest.java | 157 ++++++++++++++++
.../persistence/pipe/PipeTaskInfoRatisTest.java | 149 +++++++++++++++
.../iotdb/db/pipe/agent/PipeDataNodeAgent.java | 34 ++++
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 6 +
.../task/execution/PipeSubtaskExecutorManager.java | 4 +-
.../legacy/IoTDBLegacyPipeReceiverAgent.java | 14 ++
.../protocol/thrift/IoTDBDataNodeReceiver.java | 48 +++++
...istoricalDataRegionTsFileAndDeletionSource.java | 9 +
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 13 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 25 ++-
.../config/executor/ClusterConfigTaskExecutor.java | 30 +++
.../java/org/apache/iotdb/db/service/DataNode.java | 6 +-
.../agent/SubscriptionReceiverAgent.java | 23 ++-
.../agent/SubscriptionRuntimeAgent.java | 4 +-
.../ConsensusSubscriptionSetupHandler.java | 7 +-
...nsensusSubscriptionPrefetchExecutorManager.java | 8 +-
.../pipe/agent/task/PipeDataNodeTaskAgentTest.java | 95 ++++++++++
.../legacy/IoTDBLegacyPipeReceiverAgentTest.java | 72 ++++++++
.../protocol/thrift/IoTDBDataNodeReceiverTest.java | 202 +++++++++++++++++++++
...ricalDataRegionTsFileAndDeletionSourceTest.java | 46 +++++
.../agent/SubscriptionReceiverAgentTest.java | 30 +++
.../apache/iotdb/commons/i18n/PipeMessages.java | 6 +
.../apache/iotdb/commons/i18n/PipeMessages.java | 6 +
.../commons/pipe/agent/task/PipeTaskAgent.java | 12 ++
.../options/PipeInclusionOptions.java | 11 ++
.../options/PipeInclusionOptionsTest.java | 41 +++++
34 files changed, 1291 insertions(+), 32 deletions(-)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index b85cc9dc295..8b0b2d4751f 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -51,6 +51,7 @@ import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.enums.RepairDataPartitionTableProgressState;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.i18n.PipeMessages;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathDeserializeUtil;
@@ -1807,6 +1808,31 @@ public class ConfigManager implements IManager {
: new TGetJarInListResp(status, Collections.emptyList());
}
+ private TSStatus confirmSubscriptionFeatureAvailability() {
+ final TSStatus status = confirmLeader();
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ || PipeConfigNodeAgent.isSubscriptionFeatureEnabled()) {
+ return status;
+ }
+ return RpcUtils.getStatus(
+ TSStatusCode.UNSUPPORTED_OPERATION,
+ PipeMessages
+
.MESSAGE_SUBSCRIPTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_194CE431);
+ }
+
+ private TSStatus confirmPipeDataInsertionAvailability(
+ final Map<String, String> sourceAttributes) {
+ final TSStatus status = confirmLeader();
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ || PipeConfigNodeAgent.isPipeDataInsertionEnabled(sourceAttributes)) {
+ return status;
+ }
+ return RpcUtils.getStatus(
+ TSStatusCode.UNSUPPORTED_OPERATION,
+ PipeMessages
+
.MESSAGE_PIPE_DATA_INSERTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_7D669ECC);
+ }
+
@Override
public TSStatus merge() {
TSStatus status = confirmLeader();
@@ -2617,7 +2643,7 @@ public class ConfigManager implements IManager {
@Override
public TSStatus createPipe(TCreatePipeReq req) {
- TSStatus status = confirmLeader();
+ TSStatus status =
confirmPipeDataInsertionAvailability(req.getExtractorAttributes());
return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
? pipeManager.getPipeTaskCoordinator().createPipe(req)
: status;
@@ -2673,7 +2699,7 @@ public class ConfigManager implements IManager {
@Override
public TSStatus createTopic(TCreateTopicReq req) {
- TSStatus status = confirmLeader();
+ TSStatus status = confirmSubscriptionFeatureAvailability();
return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
? subscriptionManager.getSubscriptionCoordinator().createTopic(req)
: status;
@@ -2681,7 +2707,7 @@ public class ConfigManager implements IManager {
@Override
public TSStatus alterTopic(TAlterTopicReq req) {
- TSStatus status = confirmLeader();
+ TSStatus status = confirmSubscriptionFeatureAvailability();
return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
? subscriptionManager.getSubscriptionCoordinator().alterTopic(req)
: status;
@@ -2713,7 +2739,7 @@ public class ConfigManager implements IManager {
@Override
public TSStatus createConsumer(TCreateConsumerReq req) {
- TSStatus status = confirmLeader();
+ TSStatus status = confirmSubscriptionFeatureAvailability();
return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
? subscriptionManager.getSubscriptionCoordinator().createConsumer(req)
: status;
@@ -2729,7 +2755,7 @@ public class ConfigManager implements IManager {
@Override
public TSStatus createSubscription(TSubscribeReq req) {
- TSStatus status = confirmLeader();
+ TSStatus status = confirmSubscriptionFeatureAvailability();
return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
?
subscriptionManager.getSubscriptionCoordinator().createSubscription(req)
: status;
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/PipeConfigNodeAgent.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/PipeConfigNodeAgent.java
index 834b4e0ce35..2bd1d65c016 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/PipeConfigNodeAgent.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/PipeConfigNodeAgent.java
@@ -19,15 +19,49 @@
package org.apache.iotdb.confignode.manager.pipe.agent;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import
org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import
org.apache.iotdb.confignode.manager.pipe.agent.plugin.PipeConfigNodePluginAgent;
import
org.apache.iotdb.confignode.manager.pipe.agent.receiver.IoTDBConfigNodeReceiverAgent;
import
org.apache.iotdb.confignode.manager.pipe.agent.runtime.PipeConfigNodeRuntimeAgent;
import
org.apache.iotdb.confignode.manager.pipe.agent.task.PipeConfigNodeTaskAgent;
import org.apache.iotdb.confignode.service.ConfigNode;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+
+import java.util.Collections;
+import java.util.Map;
/** {@link PipeConfigNodeAgent} is the entry point of the pipe module in
{@link ConfigNode}. */
public class PipeConfigNodeAgent {
+ public static boolean isRatisConsensus() {
+ return ConsensusFactory.RATIS_CONSENSUS.equals(
+
ConfigNodeDescriptor.getInstance().getConf().getDataRegionConsensusProtocolClass());
+ }
+
+ public static boolean isSubscriptionFeatureEnabled() {
+ return !isRatisConsensus();
+ }
+
+ public static boolean isPipeDataInsertionEnabled(final Map<String, String>
sourceAttributes) {
+ return isPipeDataInsertionEnabled(
+ new PipeParameters(sourceAttributes == null ? Collections.emptyMap() :
sourceAttributes));
+ }
+
+ public static boolean isPipeDataInsertionEnabled(final PipeParameters
sourceParameters) {
+ if (!isRatisConsensus()) {
+ return true;
+ }
+ try {
+ return !PipeInclusionOptions.isDataInsertionIncluded(sourceParameters);
+ } catch (final IllegalPathException e) {
+ // Let the normal pipe parameter validation report malformed inclusion
options.
+ return true;
+ }
+ }
+
private final PipeConfigNodeTaskAgent pipeConfigNodeTaskAgent;
private final PipeConfigNodePluginAgent pipeConfigNodePluginAgent;
private final PipeConfigNodeRuntimeAgent pipeConfigNodeRuntimeAgent;
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
index 323fdb8ca00..f640507fead 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
@@ -68,6 +68,11 @@ public class PipeConfigNodeTaskAgent extends PipeTaskAgent {
return PipeConfigNodeAgent.runtime().isShutdown();
}
+ @Override
+ protected boolean isPipeDataInsertionEnabled(final PipeStaticMeta
pipeStaticMeta) {
+ return
PipeConfigNodeAgent.isPipeDataInsertionEnabled(pipeStaticMeta.getSourceParameters());
+ }
+
@Override
protected void thawRate(final String pipeName, final long creationTime) {
PipeConfigNodeRemainingTimeMetrics.getInstance().thawRate(pipeName + "_" +
creationTime);
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java
index feead3eea18..feb78adb15c 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java
@@ -21,6 +21,7 @@ package
org.apache.iotdb.confignode.manager.pipe.coordinator.task;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.i18n.PipeMessages;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus;
import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant;
@@ -31,6 +32,7 @@ import
org.apache.iotdb.confignode.consensus.response.pipe.task.PipeTableResp;
import org.apache.iotdb.confignode.i18n.ConfigNodeMessages;
import org.apache.iotdb.confignode.i18n.ManagerMessages;
import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent;
import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo;
import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
@@ -106,6 +108,10 @@ public class PipeTaskCoordinator {
/** Caller should ensure that the method is called in the lock {@link
#lock()}. */
public TSStatus createPipe(TCreatePipeReq req) {
+ if
(!PipeConfigNodeAgent.isPipeDataInsertionEnabled(req.getExtractorAttributes()))
{
+ return pipeDataInsertionDisabledStatus();
+ }
+
final TSStatus status;
if (req.getPipeName().startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) {
status = configManager.getProcedureManager().createConsensusPipe(req);
@@ -235,7 +241,8 @@ public class PipeTaskCoordinator {
.EXCEPTION_FAILED_TO_ALTER_PIPE_ARG_THE_PIPE_IS_BEING_DROPPED_919F1E2B,
pipeName));
}
- if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) {
+ final PipeMeta currentPipeMeta =
pipeTaskInfo.getPipeMetaByPipeName(pipeName, isTableModel);
+ if (currentPipeMeta == null) {
return isSetIfExistsCondition
? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)
: RpcUtils.getStatus(
@@ -246,6 +253,10 @@ public class PipeTaskCoordinator {
pipeName,
pipeName));
}
+ if (!PipeConfigNodeAgent.isPipeDataInsertionEnabled(
+ getUpdatedSourceParameters(req, currentPipeMeta))) {
+ return pipeDataInsertionDisabledStatus();
+ }
req.setIsTableModel(isTableModel);
final TSStatus status = configManager.getProcedureManager().alterPipe(req);
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
@@ -285,7 +296,8 @@ public class PipeTaskCoordinator {
.EXCEPTION_FAILED_TO_START_PIPE_ARG_THE_PIPE_IS_BEING_DROPPED_B41F4638,
pipeName));
}
- if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) {
+ final PipeMeta pipeMeta = pipeTaskInfo.getPipeMetaByPipeName(pipeName,
isTableModel);
+ if (pipeMeta == null) {
return RpcUtils.getStatus(
TSStatusCode.PIPE_NOT_EXIST_ERROR,
String.format(
@@ -293,9 +305,39 @@ public class PipeTaskCoordinator {
pipeName,
pipeName));
}
+ if (!PipeConfigNodeAgent.isPipeDataInsertionEnabled(
+ pipeMeta.getStaticMeta().getSourceParameters())) {
+ return pipeDataInsertionDisabledStatus();
+ }
return startPipe(pipeName, isTableModel);
}
+ private PipeParameters getUpdatedSourceParameters(
+ final TAlterPipeReq req, final PipeMeta currentPipeMeta) {
+ if (req.isReplaceAllExtractorAttributes) {
+ return new PipeParameters(
+ req.getExtractorAttributes() == null
+ ? Collections.emptyMap()
+ : req.getExtractorAttributes());
+ }
+
+ final PipeParameters sourceParameters =
+ new PipeParameters(
+ new
HashMap<>(currentPipeMeta.getStaticMeta().getSourceParameters().getAttribute()));
+ if (req.getExtractorAttributes() != null &&
!req.getExtractorAttributes().isEmpty()) {
+ sourceParameters.addOrReplaceEquivalentAttributes(
+ new PipeParameters(req.getExtractorAttributes()));
+ }
+ return sourceParameters;
+ }
+
+ private TSStatus pipeDataInsertionDisabledStatus() {
+ return RpcUtils.getStatus(
+ TSStatusCode.UNSUPPORTED_OPERATION,
+ PipeMessages
+
.MESSAGE_PIPE_DATA_INSERTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_7D669ECC);
+ }
+
/** Caller should ensure that the method is called in the lock {@link
#lock()}. */
private TSStatus stopPipe(String pipeName) {
return stopPipe(pipeName, false);
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionCoordinator.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionCoordinator.java
index 38b0846488e..a068f711f0d 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionCoordinator.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionCoordinator.java
@@ -31,6 +31,7 @@ import
org.apache.iotdb.confignode.consensus.response.subscription.SubscriptionT
import
org.apache.iotdb.confignode.consensus.response.subscription.TopicTableResp;
import org.apache.iotdb.confignode.i18n.ManagerMessages;
import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent;
import
org.apache.iotdb.confignode.manager.pipe.coordinator.task.PipeTaskCoordinatorLock;
import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo;
import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq;
@@ -187,8 +188,10 @@ public class SubscriptionCoordinator {
/////////////////////////////// Meta sync ///////////////////////////////
public void startSubscriptionMetaSync() {
- subscriptionMetaSyncer.start();
- subscriptionOwnerLeaseSyncer.start();
+ if (PipeConfigNodeAgent.isSubscriptionFeatureEnabled()) {
+ subscriptionMetaSyncer.start();
+ subscriptionOwnerLeaseSyncer.start();
+ }
}
public void stopSubscriptionMetaSync() {
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionLeaderChangeHandler.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionLeaderChangeHandler.java
index 6b888e424aa..a54b0ae2cb8 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionLeaderChangeHandler.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionLeaderChangeHandler.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.confignode.manager.subscription.runtime;
import
org.apache.iotdb.confignode.manager.load.subscriber.ConsensusGroupStatisticsChangeEvent;
import
org.apache.iotdb.confignode.manager.load.subscriber.IClusterStatusSubscriber;
import
org.apache.iotdb.confignode.manager.load.subscriber.NodeStatisticsChangeEvent;
+import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent;
public class SubscriptionLeaderChangeHandler implements
IClusterStatusSubscriber {
@@ -33,11 +34,15 @@ public class SubscriptionLeaderChangeHandler implements
IClusterStatusSubscriber
@Override
public void onNodeStatisticsChanged(final NodeStatisticsChangeEvent event) {
- runtimeCoordinator.handleNodeStatisticsChange(event);
+ if (PipeConfigNodeAgent.isSubscriptionFeatureEnabled()) {
+ runtimeCoordinator.handleNodeStatisticsChange(event);
+ }
}
@Override
public void onConsensusGroupStatisticsChanged(final
ConsensusGroupStatisticsChangeEvent event) {
- runtimeCoordinator.handleLeaderChangeEvent(event);
+ if (PipeConfigNodeAgent.isSubscriptionFeatureEnabled()) {
+ runtimeCoordinator.handleLeaderChangeEvent(event);
+ }
}
}
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
index 28f29c2cb94..269be5b4dee 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeException;
+import org.apache.iotdb.commons.i18n.PipeMessages;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
@@ -57,6 +58,7 @@ import
org.apache.iotdb.confignode.consensus.response.pipe.task.PipeTableResp;
import org.apache.iotdb.confignode.i18n.ConfigNodeMessages;
import org.apache.iotdb.confignode.i18n.ManagerMessages;
import org.apache.iotdb.confignode.i18n.ProcedureMessages;
+import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent;
import
org.apache.iotdb.confignode.manager.pipe.resource.PipeConfigNodeResourceManager;
import
org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleMetaChangeProcedure;
import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
@@ -198,6 +200,7 @@ public class PipeTaskInfo implements SnapshotProcessor {
createPipeRequest.getExtractorAttributes(),
createPipeRequest.getProcessorAttributes(),
createPipeRequest.getConnectorAttributes());
+ checkPipeDataInsertionAvailability(pipeStaticMeta);
if (!pipeMetaKeeper.containsPipeMetaOverlapped(pipeStaticMeta)) {
return true;
}
@@ -315,6 +318,7 @@ public class PipeTaskInfo implements SnapshotProcessor {
alterPipeRequest.getExtractorAttributes(),
alterPipeRequest.getProcessorAttributes(),
alterPipeRequest.getConnectorAttributes());
+ checkPipeDataInsertionAvailability(updatedPipeStaticMeta);
if (pipeMetaKeeper.containsPipeMetaOverlapped(
updatedPipeStaticMeta, pipeStaticMetaFromCoordinator)) {
final String exceptionMessage =
@@ -381,6 +385,7 @@ public class PipeTaskInfo implements SnapshotProcessor {
throw new PipeException(exceptionMessage);
}
+
checkPipeDataInsertionAvailability(getPipeMetaByPipeName(pipeName).getStaticMeta());
final PipeStatus pipeStatus = getPipeStatus(pipeName);
if (pipeStatus == PipeStatus.DROPPED) {
final String exceptionMessage =
@@ -410,6 +415,8 @@ public class PipeTaskInfo implements SnapshotProcessor {
throw new PipeException(exceptionMessage);
}
+ checkPipeDataInsertionAvailability(
+ getPipeMetaByPipeName(pipeName, isTableModel).getStaticMeta());
final PipeStatus pipeStatus = getPipeStatus(pipeName, isTableModel);
if (pipeStatus == PipeStatus.DROPPED) {
final String exceptionMessage =
@@ -429,6 +436,16 @@ public class PipeTaskInfo implements SnapshotProcessor {
}
}
+ private void checkPipeDataInsertionAvailability(final PipeStaticMeta
staticMeta) {
+ // Procedures call these validators while holding the coordinator lock.
Recheck the merged
+ // source parameters here because concurrent ALTER requests can change the
RPC-time snapshot.
+ if
(!PipeConfigNodeAgent.isPipeDataInsertionEnabled(staticMeta.getSourceParameters()))
{
+ throw new PipeException(
+ PipeMessages
+
.MESSAGE_PIPE_DATA_INSERTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_7D669ECC);
+ }
+ }
+
public void checkBeforeStopPipe(final String pipeName) throws PipeException {
acquireReadLock();
try {
diff --git
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgentTest.java
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgentTest.java
new file mode 100644
index 00000000000..a6d7e0317dd
--- /dev/null
+++
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgentTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager.pipe.agent.task;
+
+import org.apache.iotdb.commons.pipe.agent.task.PipeTask;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent;
+import org.apache.iotdb.consensus.ConsensusFactory;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class PipeConfigNodeTaskAgentTest {
+ @Test
+ public void testRecoveryDropsDisallowedPipeAndKeepsMetadataPipes() {
+ final String originalConsensus =
+
ConfigNodeDescriptor.getInstance().getConf().getDataRegionConsensusProtocolClass();
+ final boolean leaderReady = PipeConfigNodeAgent.runtime().isLeaderReady();
+ final TestingTaskAgent agent = new TestingTaskAgent();
+ try {
+ PipeConfigNodeAgent.runtime().notifyLeaderReady();
+ ConfigNodeDescriptor.getInstance()
+ .getConf()
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ final PipeMeta insertion = meta("old", "all");
+ Assert.assertNull(agent.handleSinglePipeMetaChanges(insertion));
+ final PipeTask oldTask = agent.tasks.get("old");
+ verify(oldTask).start();
+ ConfigNodeDescriptor.getInstance()
+ .getConf()
+
.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ Assert.assertNull(agent.handleSinglePipeMetaChanges(insertion));
+ verify(oldTask).stop();
+ verify(oldTask).drop();
+ Assert.assertFalse(agent.contains("old"));
+ Assert.assertNull(agent.handleSinglePipeMetaChanges(meta("blocked",
"all")));
+ Assert.assertFalse(agent.contains("blocked"));
+ Assert.assertFalse(agent.tasks.containsKey("blocked"));
+ for (String inclusion : new String[] {"schema", "data.delete",
"schema,data.delete"}) {
+ Assert.assertNull(agent.handleSinglePipeMetaChanges(meta(inclusion,
inclusion)));
+ verify(agent.tasks.get(inclusion)).start();
+ Assert.assertTrue(agent.contains(inclusion));
+ }
+ } finally {
+ agent.dropAllPipeTasks();
+ ConfigNodeDescriptor.getInstance()
+ .getConf()
+ .setDataRegionConsensusProtocolClass(originalConsensus);
+ if (!leaderReady) {
+ PipeConfigNodeAgent.runtime().notifyLeaderUnavailable();
+ }
+ }
+ }
+
+ private PipeMeta meta(String name, String inclusion) {
+ final PipeMeta meta =
+ new PipeMeta(
+ new PipeStaticMeta(
+ name,
+ 1L,
+ new HashMap<>(Map.of("inclusion", inclusion)),
+ new HashMap<>(),
+ new HashMap<>()),
+ new PipeRuntimeMeta());
+ meta.getRuntimeMeta().getStatus().set(PipeStatus.RUNNING);
+ return meta;
+ }
+
+ private static class TestingTaskAgent extends PipeConfigNodeTaskAgent {
+ private final Map<String, PipeTask> tasks = new HashMap<>();
+
+ @Override
+ protected Map<Integer, PipeTask> buildPipeTasks(PipeMeta meta) {
+ final PipeTask task = mock(PipeTask.class);
+ tasks.put(meta.getStaticMeta().getPipeName(), task);
+ return Collections.singletonMap(1, task);
+ }
+
+ private boolean contains(String name) {
+ return pipeMetaKeeper.containsPipeMeta(name);
+ }
+ }
+}
diff --git
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorTest.java
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorTest.java
new file mode 100644
index 00000000000..bbe1706c57f
--- /dev/null
+++
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager.pipe.coordinator.task;
+
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
+import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import
org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2;
+import
org.apache.iotdb.confignode.consensus.request.write.pipe.task.DropPipePlanV2;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
+import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+@RunWith(Parameterized.class)
+public class PipeTaskCoordinatorTest {
+ @Parameterized.Parameters(name = "{0}, tableModel={1}")
+ public static Collection<Object[]> parameters() {
+ final Collection<Object[]> cases = new ArrayList<>();
+ for (String protocol :
+ new String[] {
+ ConsensusFactory.RATIS_CONSENSUS,
+ ConsensusFactory.IOT_CONSENSUS,
+ ConsensusFactory.IOT_CONSENSUS_V2,
+ ConsensusFactory.SIMPLE_CONSENSUS
+ }) {
+ cases.add(new Object[] {protocol, false});
+ cases.add(new Object[] {protocol, true});
+ }
+ return cases;
+ }
+
+ private final String protocol;
+ private final boolean isTableModel;
+ private String originalProtocol;
+ private final ConfigManager configManager = mock(ConfigManager.class);
+ private final PipeTaskInfo pipeTaskInfo =
+ new PipeTaskInfo() {
+ @Override
+ public PipeMeta getPipeMetaByPipeName(String name, boolean tableModel)
{
+ // Schedule a real metadata removal just before lookup, after the
RPC's earlier checks.
+ Assert.assertTrue(isPipeExisted(name, tableModel));
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ dropPipe(new DropPipePlanV2(name, tableModel)).getCode());
+ return super.getPipeMetaByPipeName(name, tableModel);
+ }
+ };
+ private final PipeTaskCoordinator coordinator =
+ new PipeTaskCoordinator(configManager, pipeTaskInfo);
+
+ public PipeTaskCoordinatorTest(String protocol, boolean isTableModel) {
+ this.protocol = protocol;
+ this.isTableModel = isTableModel;
+ }
+
+ @Before
+ public void setUp() {
+ originalProtocol =
+
ConfigNodeDescriptor.getInstance().getConf().getDataRegionConsensusProtocolClass();
+
ConfigNodeDescriptor.getInstance().getConf().setDataRegionConsensusProtocolClass(protocol);
+ final Map<String, String> source =
+ new HashMap<>(
+ Map.of(
+ "inclusion",
+ "data.insert",
+ SystemConstant.SQL_DIALECT_KEY,
+ isTableModel
+ ? SystemConstant.SQL_DIALECT_TABLE_VALUE
+ : SystemConstant.SQL_DIALECT_TREE_VALUE));
+ pipeTaskInfo.createPipe(
+ new CreatePipePlanV2(
+ new PipeStaticMeta("p", 1L, source, new HashMap<>(), new
HashMap<>()),
+ new PipeRuntimeMeta()));
+ Assert.assertTrue(pipeTaskInfo.isPipeExisted("p", isTableModel));
+ }
+
+ @After
+ public void tearDown() {
+ ConfigNodeDescriptor.getInstance()
+ .getConf()
+ .setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+
+ @Test
+ public void testAlterHandlesConcurrentDrop() {
+ Assert.assertEquals(
+ TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(),
+ coordinator.alterPipe(alterRequest(false)).getCode());
+ verifyNoMoreInteractions(configManager);
+ Assert.assertFalse(pipeTaskInfo.isPipeExisted("p", isTableModel));
+ }
+
+ @Test
+ public void testAlterIfExistsHandlesConcurrentDrop() {
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ coordinator.alterPipe(alterRequest(true)).getCode());
+ verifyNoMoreInteractions(configManager);
+ Assert.assertFalse(pipeTaskInfo.isPipeExisted("p", isTableModel));
+ }
+
+ @Test
+ public void testStartHandlesConcurrentDrop() {
+ Assert.assertEquals(
+ TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(),
+ coordinator
+ .startPipe(new
TStartPipeReq().setPipeName("p").setIsTableModel(isTableModel))
+ .getCode());
+ verifyNoMoreInteractions(configManager);
+ Assert.assertFalse(pipeTaskInfo.isPipeExisted("p", isTableModel));
+ }
+
+ private TAlterPipeReq alterRequest(boolean ifExists) {
+ return new TAlterPipeReq("p", new HashMap<>(), new HashMap<>(), false,
false)
+ .setExtractorAttributes(new HashMap<>(Map.of("inclusion", "schema")))
+ .setIsReplaceAllExtractorAttributes(false)
+ .setIfExistsCondition(ifExists)
+ .setIsTableModel(isTableModel);
+ }
+}
diff --git
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoRatisTest.java
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoRatisTest.java
new file mode 100644
index 00000000000..deb36405473
--- /dev/null
+++
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoRatisTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.persistence.pipe;
+
+import org.apache.iotdb.commons.i18n.PipeMessages;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import
org.apache.iotdb.confignode.consensus.request.write.pipe.task.AlterPipePlanV2;
+import
org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2;
+import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
+import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class PipeTaskInfoRatisTest {
+ private String originalConsensus;
+ private final PipeTaskInfo pipeTaskInfo = new PipeTaskInfo();
+
+ @Before
+ public void setUp() {
+ originalConsensus =
+
ConfigNodeDescriptor.getInstance().getConf().getDataRegionConsensusProtocolClass();
+ ConfigNodeDescriptor.getInstance()
+ .getConf()
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ }
+
+ @After
+ public void tearDown() {
+ ConfigNodeDescriptor.getInstance()
+ .getConf()
+ .setDataRegionConsensusProtocolClass(originalConsensus);
+ }
+
+ @Test
+ public void testCreateAndStartValidateUnderProcedureLock() {
+ final TCreatePipeReq insertion = createRequest("insertion", Map.of());
+ assertDisabled(() -> pipeTaskInfo.checkBeforeCreatePipe(insertion));
+ assertDisabled(
+ () ->
+ pipeTaskInfo.checkBeforeCreatePipe(
+ createRequest("suffix", Map.of("inclusion",
"data.insert.extra"))));
+ for (final String inclusion : new String[] {"schema", "data.delete",
"schema,data.delete"}) {
+ final String name = inclusion.replace('.', '_').replace(',', '_');
+ final TCreatePipeReq allowed = createRequest(name, Map.of("inclusion",
inclusion));
+ Assert.assertTrue(pipeTaskInfo.checkBeforeCreatePipe(allowed));
+ store(allowed);
+ pipeTaskInfo.checkBeforeStartPipe(name);
+ pipeTaskInfo.checkBeforeStartPipe(name, false);
+ }
+ // Persisted metadata can predate the restriction and must be checked on
START as well.
+ store(insertion);
+ assertDisabled(() -> pipeTaskInfo.checkBeforeStartPipe("insertion"));
+ assertDisabled(() -> pipeTaskInfo.checkBeforeStartPipe("insertion",
false));
+ }
+
+ @Test
+ public void testAlterRevalidatesAgainstLatestSerializedMetadata() {
+ final TCreatePipeReq initial = createRequest("p", Map.of("inclusion",
"schema"));
+ store(initial);
+ final Map<String, String> firstAttributes =
+ Map.of("source.inclusion", "all", "source.inclusion.exclusion",
"data.insert");
+ final Map<String, String> secondAttributes =
Map.of("extractor.inclusion.exclusion", "");
+ // Both partial updates are allowed against the original RPC-time snapshot.
+
Assert.assertTrue(PipeConfigNodeAgent.isPipeDataInsertionEnabled(firstAttributes));
+ Assert.assertTrue(
+ PipeConfigNodeAgent.isPipeDataInsertionEnabled(
+ Map.of("inclusion", "schema", "inclusion.exclusion", "")));
+ final TAlterPipeReq first = alterRequest(firstAttributes, false);
+ pipeTaskInfo.checkAndUpdateRequestBeforeAlterPipe(first);
+ final PipeStaticMeta oldMeta =
pipeTaskInfo.getPipeMetaByPipeName("p").getStaticMeta();
+ final PipeStaticMeta newMeta =
+ new PipeStaticMeta(
+ "p", 2L, first.getExtractorAttributes(), new HashMap<>(), new
HashMap<>());
+ pipeTaskInfo.alterPipe(new AlterPipePlanV2(oldMeta, newMeta, new
PipeRuntimeMeta()));
+ assertDisabled(
+ () ->
+ pipeTaskInfo.checkAndUpdateRequestBeforeAlterPipe(
+ alterRequest(secondAttributes, false)));
+ Assert.assertTrue(
+ PipeConfigNodeAgent.isPipeDataInsertionEnabled(
+
pipeTaskInfo.getPipeMetaByPipeName("p").getStaticMeta().getSourceParameters()));
+ assertDisabled(
+ () ->
pipeTaskInfo.checkAndUpdateRequestBeforeAlterPipe(alterRequest(Map.of(),
true)));
+ pipeTaskInfo.checkAndUpdateRequestBeforeAlterPipe(
+ alterRequest(Map.of("inclusion", "schema,data.delete"), true));
+ }
+
+ private TCreatePipeReq createRequest(String name, Map<String, String>
source) {
+ return new TCreatePipeReq()
+ .setPipeName(name)
+ .setExtractorAttributes(new HashMap<>(source))
+ .setProcessorAttributes(new HashMap<>())
+ .setConnectorAttributes(new HashMap<>());
+ }
+
+ private TAlterPipeReq alterRequest(Map<String, String> source, boolean
replace) {
+ return new TAlterPipeReq("p", new HashMap<>(), new HashMap<>(), false,
false)
+ .setExtractorAttributes(new HashMap<>(source))
+ .setIsReplaceAllExtractorAttributes(replace)
+ .setIsTableModel(false);
+ }
+
+ private void store(TCreatePipeReq request) {
+ pipeTaskInfo.createPipe(
+ new CreatePipePlanV2(
+ new PipeStaticMeta(
+ request.getPipeName(),
+ 1L,
+ request.getExtractorAttributes(),
+ request.getProcessorAttributes(),
+ request.getConnectorAttributes()),
+ new PipeRuntimeMeta()));
+ }
+
+ private void assertDisabled(Runnable action) {
+ Assert.assertEquals(
+ PipeMessages
+
.MESSAGE_PIPE_DATA_INSERTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_7D669ECC,
+ Assert.assertThrows(PipeException.class, action::run).getMessage());
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeDataNodeAgent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeDataNodeAgent.java
index e350f0a6f4b..838a41e85a1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeDataNodeAgent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeDataNodeAgent.java
@@ -19,15 +19,49 @@
package org.apache.iotdb.db.pipe.agent;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import
org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.plugin.PipeDataNodePluginAgent;
import org.apache.iotdb.db.pipe.agent.receiver.PipeDataNodeReceiverAgent;
import org.apache.iotdb.db.pipe.agent.runtime.PipeDataNodeRuntimeAgent;
import org.apache.iotdb.db.pipe.agent.task.PipeDataNodeTaskAgent;
import org.apache.iotdb.db.service.DataNode;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+
+import java.util.Collections;
+import java.util.Map;
/** {@link PipeDataNodeAgent} is the entry point of the pipe module in {@link
DataNode}. */
public class PipeDataNodeAgent {
+ public static boolean isRatisConsensus() {
+ return ConsensusFactory.RATIS_CONSENSUS.equals(
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass());
+ }
+
+ public static boolean isSubscriptionFeatureEnabled() {
+ return !isRatisConsensus();
+ }
+
+ public static boolean isPipeDataInsertionEnabled(final Map<String, String>
sourceAttributes) {
+ return isPipeDataInsertionEnabled(
+ new PipeParameters(sourceAttributes == null ? Collections.emptyMap() :
sourceAttributes));
+ }
+
+ public static boolean isPipeDataInsertionEnabled(final PipeParameters
sourceParameters) {
+ if (!isRatisConsensus()) {
+ return true;
+ }
+ try {
+ return !PipeInclusionOptions.isDataInsertionIncluded(sourceParameters);
+ } catch (final IllegalPathException e) {
+ // Let the normal pipe parameter validation report malformed inclusion
options.
+ return true;
+ }
+ }
+
private final PipeDataNodePluginAgent pipeDataNodePluginAgent;
private final PipeDataNodeTaskAgent pipeDataNodeTaskAgent;
private final PipeDataNodeRuntimeAgent pipeDataNodeRuntimeAgent;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
index 91116fe1942..4bd1fc9f762 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
@@ -146,6 +146,11 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
return PipeDataNodeAgent.runtime().isShutdown();
}
+ @Override
+ protected boolean isPipeDataInsertionEnabled(final PipeStaticMeta
pipeStaticMeta) {
+ return
PipeDataNodeAgent.isPipeDataInsertionEnabled(pipeStaticMeta.getSourceParameters());
+ }
+
@Override
protected Map<Integer, PipeTask> buildPipeTasks(final PipeMeta
pipeMetaFromConfigNode)
throws IllegalPathException {
@@ -307,6 +312,7 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
final PipeStaticMeta staticMeta = pipeMeta.getStaticMeta();
if (isShutdown()
|| !SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ || !PipeDataNodeAgent.isSubscriptionFeatureEnabled()
|| pipeMeta.getRuntimeMeta().getStatus().get() != PipeStatus.RUNNING
|| !PipeStaticMeta.isSubscriptionPipe(staticMeta.getPipeName())
|| !staticMeta
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
index 654726912b1..8d8a9dcd5d6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
@@ -23,6 +23,7 @@ import
org.apache.iotdb.commons.consensus.iotv2.container.IoTV2GlobalComponentCo
import org.apache.iotdb.commons.pipe.agent.task.execution.PipeSubtaskExecutor;
import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.consensus.IoTConsensusV2SubtaskExecutor;
import
org.apache.iotdb.db.subscription.task.execution.SubscriptionSubtaskExecutor;
@@ -86,7 +87,8 @@ public class PipeSubtaskExecutorManager {
}
public synchronized void ensureSubscriptionExecutors() {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ || !PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
return;
}
if (subscriptionExecutor == null || subscriptionExecutor.isShutdown()) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgent.java
index 3bad4b45398..dd670bdfa99 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgent.java
@@ -28,6 +28,7 @@ import
org.apache.iotdb.commons.queryengine.common.SessionInfo;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.sink.payload.legacy.PipeData;
import org.apache.iotdb.db.pipe.sink.payload.legacy.TsFilePipeData;
import org.apache.iotdb.db.protocol.session.SessionManager;
@@ -116,6 +117,13 @@ public class IoTDBLegacyPipeReceiverAgent {
return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "");
}
+ private TSStatus pipeDataInsertionDisabledStatus() {
+ return RpcUtils.getStatus(
+ TSStatusCode.UNSUPPORTED_OPERATION,
+ PipeMessages
+
.MESSAGE_PIPE_DATA_INSERTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_7D669ECC);
+ }
+
private boolean validatePipeName(final TSyncIdentityInfo info) {
return info.isSetPipeName()
&&
Objects.isNull(FileUtils.getIllegalError4Directory(info.getPipeName()));
@@ -159,6 +167,9 @@ public class IoTDBLegacyPipeReceiverAgent {
buff.get(byteArray);
pipeData = PipeData.createPipeData(byteArray);
if (pipeData instanceof TsFilePipeData) {
+ if (PipeDataNodeAgent.isRatisConsensus()) {
+ return pipeDataInsertionDisabledStatus();
+ }
TsFilePipeData tsFilePipeData = (TsFilePipeData) pipeData;
tsFilePipeData.setDatabase(identityInfo.getDatabase());
handleTsFilePipeData(tsFilePipeData, fileDir);
@@ -265,6 +276,9 @@ public class IoTDBLegacyPipeReceiverAgent {
if (identityInfo == null) {
throw new
TException(DataNodePipeMessages.THRIFT_CONNECTION_IS_NOT_ALIVE);
}
+ if (PipeDataNodeAgent.isRatisConsensus()) {
+ return pipeDataInsertionDisabledStatus();
+ }
LOGGER.debug(
DataNodePipeMessages.INVOKE_TRANSPORTDATA_METHOD_FROM_CLIENT_IP,
identityInfo.getRemoteAddress());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
index b6559b3c18e..57c5de46c7a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
@@ -30,6 +30,7 @@ import
org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
import
org.apache.iotdb.commons.exception.pipe.PipeRuntimeOutOfMemoryCriticalException;
+import org.apache.iotdb.commons.i18n.PipeMessages;
import org.apache.iotdb.commons.log.LoggerPeriodicalLogReducer;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
@@ -100,6 +101,7 @@ import
org.apache.iotdb.db.queryengine.plan.execution.config.metadata.DatabaseSc
import
org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask;
import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner;
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.AlterLogicalViewNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import
org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckContext;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched;
import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
@@ -278,6 +280,9 @@ public class IoTDBDataNodeReceiver extends
IoTDBFileReceiver {
if (Objects.nonNull(authResp)) {
return authResp;
}
+ if (PipeDataNodeAgent.isRatisConsensus() &&
isDataInsertionRequest(requestType)) {
+ return pipeDataInsertionDisabledResponse();
+ }
switch (requestType) {
case HANDSHAKE_DATANODE_V1:
{
@@ -580,6 +585,46 @@ public class IoTDBDataNodeReceiver extends
IoTDBFileReceiver {
: executeStatementAndAddRedirectInfo(statement));
}
+ private static TPipeTransferResp pipeDataInsertionDisabledResponse() {
+ return new TPipeTransferResp(
+ RpcUtils.getStatus(
+ TSStatusCode.UNSUPPORTED_OPERATION,
+ PipeMessages
+
.MESSAGE_PIPE_DATA_INSERTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_7D669ECC));
+ }
+
+ private static boolean isDataInsertionRequest(final PipeRequestType
requestType) {
+ return switch (requestType) {
+ case TRANSFER_TABLET_INSERT_NODE,
+ TRANSFER_TABLET_INSERT_NODE_V2,
+ TRANSFER_TABLET_RAW,
+ TRANSFER_TABLET_RAW_V2,
+ TRANSFER_TABLET_BINARY,
+ TRANSFER_TABLET_BINARY_V2,
+ TRANSFER_TABLET_BATCH,
+ TRANSFER_TABLET_BATCH_V2,
+ TRANSFER_TS_FILE_PIECE,
+ TRANSFER_TS_FILE_SEAL,
+ TRANSFER_TS_FILE_PIECE_WITH_MOD,
+ TRANSFER_TS_FILE_SEAL_WITH_MOD ->
+ true;
+ case HANDSHAKE_CONFIGNODE_V1,
+ HANDSHAKE_DATANODE_V1,
+ HANDSHAKE_CONFIGNODE_V2,
+ HANDSHAKE_DATANODE_V2,
+ TRANSFER_PLAN_NODE,
+ TRANSFER_SCHEMA_SNAPSHOT_PIECE,
+ TRANSFER_SCHEMA_SNAPSHOT_SEAL,
+ TRANSFER_CONFIG_PLAN,
+ TRANSFER_CONFIG_SNAPSHOT_PIECE,
+ TRANSFER_CONFIG_SNAPSHOT_SEAL,
+ TRANSFER_COMPRESSED,
+ TRANSFER_SLICE,
+ TRANSFER_PIPE_RECEIVER_RUNTIME_INFO_CLEANUP ->
+ false;
+ };
+ }
+
private TPipeTransferResp handleTransferTabletBinary(final
PipeTransferTabletBinaryReq req) {
final InsertBaseStatement statement = req.constructStatement();
return new TPipeTransferResp(
@@ -1094,6 +1139,9 @@ public class IoTDBDataNodeReceiver extends
IoTDBFileReceiver {
}
private TPipeTransferResp handleTransferSchemaPlan(final
PipeTransferPlanNodeReq req) {
+ if (PipeDataNodeAgent.isRatisConsensus() && req.getPlanNode() instanceof
InsertNode) {
+ return pipeDataInsertionDisabledResponse();
+ }
// We may be able to skip the alter logical view's exception parsing
because
// the "AlterLogicalViewNode" is itself idempotent
if (req.getPlanNode() instanceof AlterLogicalViewNode) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
index 00d22abcb7f..19c78fa993b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
@@ -42,6 +42,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils;
import org.apache.iotdb.commons.utils.PathUtils;
+import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.consensus.pipe.IoTConsensusV2;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
@@ -921,6 +922,14 @@ public class
PipeHistoricalDataRegionTsFileAndDeletionSource
private boolean isProgressIndexNotCoveredByStartIndex(
PersistentResource resource, ProgressIndex progressIndex) {
+ // Ratis writes do not carry Pipe progress indexes. Minimum therefore
means unknown here,
+ // rather than an empty file already covered by the initial checkpoint.
Replay conservatively
+ // after a leader change, or successful writes can disappear from the
subscription backlog.
+ if (progressIndex instanceof MinimumProgressIndex
+ && ConsensusFactory.RATIS_CONSENSUS.equals(
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass()))
{
+ return true;
+ }
final ProgressIndex innerStartIndex = getInnerProgressIndex(startIndex);
if (innerStartIndex.isEqualOrAfter(progressIndex)
|| isProgressIndexCoveredByTimePartitionProgressIndex(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
index fcb8fc23999..796957d1735 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
@@ -36,6 +36,7 @@ import org.apache.iotdb.commons.conf.ConfigurationFileUtils;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.commons.i18n.PipeMessages;
import org.apache.iotdb.commons.partition.DataPartition;
import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
import org.apache.iotdb.commons.path.AlignedFullPath;
@@ -3537,6 +3538,15 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
if (!SESSION_MANAGER.checkLogin(clientSession)) {
return getNotLoggedInPipeSubscribeResp();
}
+ if (!PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
+ return new TPipeSubscribeResp(
+ RpcUtils.getStatus(
+ TSStatusCode.UNSUPPORTED_OPERATION,
+ PipeMessages
+
.MESSAGE_SUBSCRIPTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_194CE431),
+ PipeSubscribeResponseVersion.VERSION_1.getVersion(),
+ PipeSubscribeResponseType.ACK.getType());
+ }
return SubscriptionAgent.receiver().handle(req,
clientSession.getUsername());
} finally {
@@ -3687,7 +3697,8 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
}
PipeDataNodeAgent.receiver().thrift().handleClientExit();
PipeDataNodeAgent.receiver().legacy().handleClientExit();
- if (COMMON_CONFIG.getSubscriptionEnabled()) {
+ if (COMMON_CONFIG.getSubscriptionEnabled()
+ && PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
SubscriptionAgent.receiver().handleClientExit();
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 4b1e05f4e85..b0ec681b09b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -1477,7 +1477,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
@Override
public TPushTopicMetaResp pushTopicMeta(TPushTopicMetaReq req) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!isSubscriptionFeatureEnabled()) {
return new TPushTopicMetaResp()
.setStatus(new
TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
@@ -1505,7 +1505,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
@Override
public TPushTopicMetaResp pushSingleTopicMeta(TPushSingleTopicMetaReq req) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!isSubscriptionFeatureEnabled()) {
return new TPushTopicMetaResp()
.setStatus(new
TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
@@ -1543,7 +1543,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
@Override
public TPushTopicMetaResp pushMultiTopicMeta(TPushMultiTopicMetaReq req) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!isSubscriptionFeatureEnabled()) {
return new TPushTopicMetaResp()
.setStatus(new
TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
@@ -1595,7 +1595,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
@Override
public TSStatus pushTopicOwnerLease(TPushTopicOwnerLeaseReq req) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!isSubscriptionFeatureEnabled()) {
return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
}
try {
@@ -1609,7 +1609,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
@Override
public TPushConsumerGroupMetaResp
pushConsumerGroupMeta(TPushConsumerGroupMetaReq req) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!isSubscriptionFeatureEnabled()) {
return new TPushConsumerGroupMetaResp()
.setStatus(new
TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
@@ -1638,7 +1638,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
@Override
public TPushConsumerGroupMetaResp pushSingleConsumerGroupMeta(
TPushSingleConsumerGroupMetaReq req) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!isSubscriptionFeatureEnabled()) {
return new TPushConsumerGroupMetaResp()
.setStatus(new
TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
@@ -1673,7 +1673,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
@Override
public TPullCommitProgressResp pullCommitProgress(TPullCommitProgressReq
req) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!isSubscriptionFeatureEnabled()) {
return new
TPullCommitProgressResp(RpcUtils.getStatus(TSStatusCode.UNSUPPORTED_OPERATION))
.setCommitRegionProgress(Collections.emptyMap())
.setSubscriptionProgress(Collections.emptyMap());
@@ -1698,7 +1698,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
@Override
public TSStatus syncSubscriptionProgress(TSyncSubscriptionProgressReq req) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!isSubscriptionFeatureEnabled()) {
return RpcUtils.getStatus(TSStatusCode.UNSUPPORTED_OPERATION);
}
@@ -1723,7 +1723,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
@Override
public TSStatus pushSubscriptionRuntime(TPushSubscriptionRuntimeReq req) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!isSubscriptionFeatureEnabled()) {
return RpcUtils.getStatus(TSStatusCode.UNSUPPORTED_OPERATION);
}
@@ -2535,7 +2535,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
return RpcUtils.getStatus(TSStatusCode.PARTITION_CACHE_UPDATE_ERROR);
}
- if (SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (isSubscriptionFeatureEnabled()) {
// Notify consensus subscription queues of any preferred-writer changes
try {
ConsensusSubscriptionSetupHandler.onRegionRouteChanged(
@@ -3405,6 +3405,11 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
return nodeLocation.getSchemaRegionConsensusEndPoint();
}
+ private boolean isSubscriptionFeatureEnabled() {
+ return SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ && PipeDataNodeAgent.isSubscriptionFeatureEnabled();
+ }
+
@Override
public TSStatus createPipePlugin(TCreatePipePluginInstanceReq req) {
try {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 1d181b99015..0575c6938c7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -436,6 +436,9 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
}
private static final SettableFuture<ConfigTaskResult>
SUBSCRIPTION_NOT_ENABLED_ERROR_FUTURE;
+ private static final SettableFuture<ConfigTaskResult>
SUBSCRIPTION_DISABLED_BY_RATIS_ERROR_FUTURE;
+ private static final SettableFuture<ConfigTaskResult>
+ PIPE_DATA_INSERTION_DISABLED_BY_RATIS_ERROR_FUTURE;
static {
SUBSCRIPTION_NOT_ENABLED_ERROR_FUTURE = SettableFuture.create();
@@ -443,6 +446,20 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
new IoTDBException(
DataNodeQueryMessages.QUERY_EXCEPTION_SUBSCRIPTION_IS_NOT_ENABLED_7F43DCBB,
TSStatusCode.SUBSCRIPTION_NOT_ENABLED_ERROR.getStatusCode()));
+
+ SUBSCRIPTION_DISABLED_BY_RATIS_ERROR_FUTURE = SettableFuture.create();
+ SUBSCRIPTION_DISABLED_BY_RATIS_ERROR_FUTURE.setException(
+ new IoTDBException(
+ PipeMessages
+
.MESSAGE_SUBSCRIPTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_194CE431,
+ TSStatusCode.UNSUPPORTED_OPERATION.getStatusCode()));
+
+ PIPE_DATA_INSERTION_DISABLED_BY_RATIS_ERROR_FUTURE =
SettableFuture.create();
+ PIPE_DATA_INSERTION_DISABLED_BY_RATIS_ERROR_FUTURE.setException(
+ new IoTDBException(
+ PipeMessages
+
.MESSAGE_PIPE_DATA_INSERTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_7D669ECC,
+ TSStatusCode.UNSUPPORTED_OPERATION.getStatusCode()));
}
public static ClusterConfigTaskExecutor getInstance() {
@@ -2248,6 +2265,10 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
@Override
public SettableFuture<ConfigTaskResult> createPipe(
final CreatePipeStatement createPipeStatement) {
+ if
(!PipeDataNodeAgent.isPipeDataInsertionEnabled(createPipeStatement.getSourceAttributes()))
{
+ return PIPE_DATA_INSERTION_DISABLED_BY_RATIS_ERROR_FUTURE;
+ }
+
final SettableFuture<ConfigTaskResult> future = SettableFuture.create();
// Verify that Pipe is disabled if TSFile encryption is enabled
@@ -2748,6 +2769,9 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
checkedSink.remove(
PipeParameters.KeyReducer.reduce(PipeSinkConstant.SINK_IOTDB_PASSWORD_KEY));
}
+ if (!PipeDataNodeAgent.isPipeDataInsertionEnabled(sourceAttributes)) {
+ return PIPE_DATA_INSERTION_DISABLED_BY_RATIS_ERROR_FUTURE;
+ }
PipeDataNodeAgent.plugin()
.validate(pipeName, checkedSource, processorAttributes, checkedSink);
} catch (final Exception e) {
@@ -3166,6 +3190,9 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
return SUBSCRIPTION_NOT_ENABLED_ERROR_FUTURE;
}
+ if (!PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
+ return SUBSCRIPTION_DISABLED_BY_RATIS_ERROR_FUTURE;
+ }
final SettableFuture<ConfigTaskResult> future = SettableFuture.create();
@@ -3240,6 +3267,9 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
return SUBSCRIPTION_NOT_ENABLED_ERROR_FUTURE;
}
+ if (!PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
+ return SUBSCRIPTION_DISABLED_BY_RATIS_ERROR_FUTURE;
+ }
final SettableFuture<ConfigTaskResult> future = SettableFuture.create();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
index 9f6baa7791e..09af77c4e38 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
@@ -303,7 +303,8 @@ public class DataNode extends ServerCommandLine implements
DataNodeMBean {
// TierManager need DataNodeId to do some operations so the reset method
need to be invoked
// after DataNode adding
TierManager.getInstance().resetFolders();
- if (SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ && PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
ConsensusSubscriptionSetupHandler.ensureNewRegionListenerRegistered();
}
// Active DataNode
@@ -945,7 +946,8 @@ public class DataNode extends ServerCommandLine implements
DataNodeMBean {
registerInternalRPCService();
// Register subscription agent before pipe agent
- if (SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ && PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
registerManager.register(SubscriptionAgent.runtime());
}
registerManager.register(PipeDataNodeAgent.runtime());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java
index 95b1f0a93cc..828b1887204 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java
@@ -22,9 +22,11 @@ package org.apache.iotdb.db.subscription.agent;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil;
+import org.apache.iotdb.commons.i18n.PipeMessages;
import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.subscription.receiver.SubscriptionReceiver;
import org.apache.iotdb.db.subscription.receiver.SubscriptionReceiverV1;
import org.apache.iotdb.rpc.RpcUtils;
@@ -63,6 +65,15 @@ public class SubscriptionReceiverAgent {
PipeSubscribeResponseVersion.VERSION_1.getVersion(),
PipeSubscribeResponseType.ACK.getType());
+ private static final TPipeSubscribeResp SUBSCRIPTION_DISABLED_BY_RATIS_RESP =
+ new TPipeSubscribeResp(
+ RpcUtils.getStatus(
+ TSStatusCode.UNSUPPORTED_OPERATION,
+ PipeMessages
+
.MESSAGE_SUBSCRIPTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_194CE431),
+ PipeSubscribeResponseVersion.VERSION_1.getVersion(),
+ PipeSubscribeResponseType.ACK.getType());
+
private static final TPipeSubscribeResp SUBSCRIPTION_CONSUMER_FENCED_RESP =
new TPipeSubscribeResp(
RpcUtils.getStatus(
@@ -90,8 +101,13 @@ public class SubscriptionReceiverAgent {
SubscriptionReceiverAgent() {
this(
SubscriptionReceiverV1::new,
- SubscriptionConfig.getInstance().getSubscriptionEnabled(),
- () -> SubscriptionConfig.getInstance().getSubscriptionEnabled());
+ isSubscriptionEnabled(),
+ SubscriptionReceiverAgent::isSubscriptionEnabled);
+ }
+
+ private static boolean isSubscriptionEnabled() {
+ return SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ && PipeDataNodeAgent.isSubscriptionFeatureEnabled();
}
SubscriptionReceiverAgent(
@@ -129,6 +145,9 @@ public class SubscriptionReceiverAgent {
PipeSubscribeResponseVersion.VERSION_1.getVersion(),
PipeSubscribeResponseType.ACK.getType());
}
+ if (!PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
+ return SUBSCRIPTION_DISABLED_BY_RATIS_RESP;
+ }
if (!subscriptionEnabledSupplier.getAsBoolean()) {
return SUBSCRIPTION_NOT_ENABLED_ERROR_RESP;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionRuntimeAgent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionRuntimeAgent.java
index 2962f525e70..ff8ac555605 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionRuntimeAgent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionRuntimeAgent.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.exception.StartupException;
import org.apache.iotdb.commons.service.IService;
import org.apache.iotdb.commons.service.ServiceType;
import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import
org.apache.iotdb.db.subscription.broker.consensus.ConsensusSubscriptionSetupHandler;
import
org.apache.iotdb.db.subscription.task.execution.ConsensusSubscriptionPrefetchExecutorManager;
@@ -64,7 +65,8 @@ public class SubscriptionRuntimeAgent implements IService {
@Override
public void start() throws StartupException {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ || !PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
return;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java
index 1ae30b123e1..654030eaa2b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java
@@ -38,6 +38,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
@@ -1138,7 +1139,8 @@ public class ConsensusSubscriptionSetupHandler {
public static void applyRuntimeState(
final TConsensusGroupId groupId, final ConsensusRegionRuntimeState
runtimeState) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ || !PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
return;
}
@@ -1173,7 +1175,8 @@ public class ConsensusSubscriptionSetupHandler {
public static void onRegionRouteChanged(
final Map<TConsensusGroupId, TRegionReplicaSet> newMap, final long
routingTimestamp) {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ || !PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
return;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/ConsensusSubscriptionPrefetchExecutorManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/ConsensusSubscriptionPrefetchExecutorManager.java
index 9362a38a58b..f442ac8a9b4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/ConsensusSubscriptionPrefetchExecutorManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/ConsensusSubscriptionPrefetchExecutorManager.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.subscription.task.execution;
import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
public class ConsensusSubscriptionPrefetchExecutorManager {
@@ -31,7 +32,8 @@ public class ConsensusSubscriptionPrefetchExecutorManager {
}
public synchronized void start() {
- if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ || !PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
started = false;
return;
}
@@ -42,7 +44,9 @@ public class ConsensusSubscriptionPrefetchExecutorManager {
}
public synchronized ConsensusSubscriptionPrefetchExecutor getExecutor() {
- if (!started ||
!SubscriptionConfig.getInstance().getSubscriptionEnabled()) {
+ if (!started
+ || !SubscriptionConfig.getInstance().getSubscriptionEnabled()
+ || !PipeDataNodeAgent.isSubscriptionFeatureEnabled()) {
return null;
}
if (executor == null || executor.isShutdown()) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgentTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgentTest.java
index dde64b4ef93..f86b185d07f 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgentTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgentTest.java
@@ -34,9 +34,15 @@ import
org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
+import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant;
+import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
+import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter;
+import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.pipe.api.exception.PipeException;
import org.awaitility.Awaitility;
@@ -50,6 +56,7 @@ import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -60,6 +67,94 @@ public class PipeDataNodeTaskAgentTest {
private static final int LOCAL_NODE_ID = 1;
private static final int REGION_ID = 7;
+ @Test
+ public void testRatisDetectsPipeDataInsertionOption() throws Exception {
+ final String originalConsensus =
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass();
+ try {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+
+ Assert.assertTrue(
+ PipeDataNodeAgent.isPipeDataInsertionEnabled(
+ Map.of(PipeSourceConstant.EXTRACTOR_INCLUSION_KEY, "schema")));
+ Assert.assertTrue(
+ PipeDataNodeAgent.isPipeDataInsertionEnabled(
+ Map.of(PipeSourceConstant.EXTRACTOR_INCLUSION_KEY,
"data.delete")));
+ Assert.assertFalse(
+ PipeDataNodeAgent.isPipeDataInsertionEnabled(
+ Map.of(PipeSourceConstant.EXTRACTOR_INCLUSION_KEY,
"data.insert")));
+ Assert.assertFalse(
+ PipeDataNodeAgent.isPipeDataInsertionEnabled(
+ Map.of(PipeSourceConstant.EXTRACTOR_INCLUSION_KEY,
"data.insert,data.delete")));
+ for (final Map<String, String> attributes :
+ List.of(
+ Map.<String, String>of(),
+ Map.of("inclusion", "schema"),
+ Map.of("inclusion", "data.delete"),
+ Map.of("source.inclusion", "data.insert.extra"),
+ Map.of("inclusion", "all", "inclusion.exclusion",
"data.insert.extra"))) {
+ Assert.assertEquals(
+ attributes.toString(),
+
!DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(
+ new PipeParameters(attributes))
+ .getLeft(),
+ PipeDataNodeAgent.isPipeDataInsertionEnabled(attributes));
+ }
+ } finally {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDataRegionConsensusProtocolClass(originalConsensus);
+ }
+ }
+
+ @Test
+ public void testRatisDoesNotRestoreCompletedSubscriptionBroker() {
+ final String originalConsensus =
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass();
+ final String group = "ratis-completed-snapshot";
+ final PipeDataNodeTaskAgent taskAgent =
+ new PipeDataNodeTaskAgent() {
+ @Override
+ protected boolean isShutdown() {
+ return false;
+ }
+ };
+ try {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ Assert.assertFalse(SubscriptionAgent.broker().isBrokerExist(group));
+ final PipeMeta meta =
+ new PipeMeta(
+ new PipeStaticMeta(
+ PipeStaticMeta.SUBSCRIPTION_PIPE_PREFIX + "topic." + group,
+ 1L,
+ new HashMap<>(
+
Map.of(SystemConstant.SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY, "true")),
+ new HashMap<>(),
+ new HashMap<>(
+ Map.of(
+ PipeSinkConstant.SINK_CONSUMER_GROUP_KEY,
+ group,
+ PipeSinkConstant.SINK_TOPIC_KEY,
+ "topic"))),
+ new PipeRuntimeMeta());
+ meta.getRuntimeMeta().getStatus().set(PipeStatus.RUNNING);
+
+ Assert.assertNull(taskAgent.handleSinglePipeMetaChanges(meta));
+ Assert.assertFalse(SubscriptionAgent.broker().isBrokerExist(group));
+
Assert.assertTrue(taskAgent.handlePipeMetaChanges(Collections.singletonList(meta)).isEmpty());
+ Assert.assertFalse(SubscriptionAgent.broker().isBrokerExist(group));
+ } finally {
+ SubscriptionAgent.broker().dropBroker(group);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDataRegionConsensusProtocolClass(originalConsensus);
+ }
+ }
+
@Test
public void testEmptyPartialMetadataKeepsExistingPipe() throws Exception {
final PipeDataNodeTaskAgent agent = newActiveTaskAgent();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgentTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgentTest.java
index 111350d7607..d3f661f9e9f 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgentTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgentTest.java
@@ -20,10 +20,16 @@
package org.apache.iotdb.db.pipe.receiver.protocol.legacy;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.i18n.PipeMessages;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.commons.queryengine.common.SessionInfo;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
import org.apache.iotdb.db.pipe.sink.payload.legacy.TsFilePipeData;
+import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSyncIdentityInfo;
import org.apache.iotdb.service.rpc.thrift.TSyncTransportMetaInfo;
@@ -31,17 +37,21 @@ import
org.apache.iotdb.service.rpc.thrift.TSyncTransportMetaInfo;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.time.ZoneId;
public class IoTDBLegacyPipeReceiverAgentTest {
@@ -80,6 +90,68 @@ public class IoTDBLegacyPipeReceiverAgentTest {
}
}
+ @Test
+ public void testRatisRejectsFileTransfersButAllowsDeletion() throws
Exception {
+ final String originalConsensus =
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass();
+ final NodeStatus originalStatus =
CommonDescriptor.getInstance().getConfig().getNodeStatus();
+ final String fileName = "ratis.tsfile";
+ try {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ agent
+ .handshake(
+ new TSyncIdentityInfo(PIPE_NAME, CREATE_TIME, "UNKNOWN", ""),
+ REMOTE_ADDRESS,
+ null,
+ null)
+ .getCode());
+ final ByteBuffer bytes = ByteBuffer.wrap(new byte[] {1, 2, 3});
+ Assert.assertEquals(
+ TSStatusCode.UNSUPPORTED_OPERATION.getStatusCode(),
+ agent.transportFile(new TSyncTransportMetaInfo(fileName, 0),
bytes).getCode());
+ Assert.assertEquals(0, bytes.position());
+ Assert.assertFalse(Files.exists(getFileDataDir().resolve(fileName +
".patch")));
+
+ final Path patch = getFileDataDir().resolve(fileName + ".patch");
+ Files.write(patch, new byte[] {4, 5, 6});
+ Assert.assertEquals(
+ TSStatusCode.UNSUPPORTED_OPERATION.getStatusCode(),
+ agent
+ .transportPipeData(ByteBuffer.wrap(new TsFilePipeData("",
fileName, 1).serialize()))
+ .getCode());
+ Assert.assertTrue(Files.exists(patch));
+ Assert.assertFalse(Files.exists(getFileDataDir().resolve(fileName)));
+
+ final ByteArrayOutputStream deletionBytes = new ByteArrayOutputStream();
+ try (final DataOutputStream output = new
DataOutputStream(deletionBytes)) {
+ output.writeByte(1);
+ output.writeLong(2L);
+ ReadWriteIOUtils.write("root.sg", output);
+ new Deletion(new MeasurementPath("root.sg.d.s"), 0, 0, 1)
+ .serializeWithoutFileOffset(output);
+ }
+ // The real deletion loader must still be reached under Ratis. Read-only
mode gives it a
+ // deterministic error without requiring a running cluster or executing
an actual deletion.
+
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
+ final TSStatus deletionStatus =
+ agent.transportPipeData(
+ ByteBuffer.wrap(deletionBytes.toByteArray()),
+ new SessionInfo(1L, "root", ZoneId.systemDefault()));
+ Assert.assertEquals(TSStatusCode.PIPESERVER_ERROR.getStatusCode(),
deletionStatus.getCode());
+ Assert.assertTrue(
+
deletionStatus.getMessage().contains(DataNodePipeMessages.STORAGE_ENGINE_READONLY));
+ } finally {
+ CommonDescriptor.getInstance().getConfig().setNodeStatus(originalStatus);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDataRegionConsensusProtocolClass(originalConsensus);
+ }
+ }
+
@Test
public void testTransportFileRejectsPathTraversal() throws Exception {
final String traversal =
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiverTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiverTest.java
index 4f91a311a57..f5fd9cf0695 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiverTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiverTest.java
@@ -21,11 +21,28 @@ package org.apache.iotdb.db.pipe.receiver.protocol.thrift;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import
org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry;
import
org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeSnapshot;
+import org.apache.iotdb.commons.pipe.sink.compressor.PipeCompressor;
+import org.apache.iotdb.commons.pipe.sink.compressor.PipeCompressorFactory;
+import
org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapPseudoTPipeTransferRequest;
+import
org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferCompressedReq;
+import
org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferSliceReq;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import
org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferPlanNodeReq;
import
org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTsFileSealWithModReq;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateTimeSeriesNode;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import
org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
@@ -35,13 +52,18 @@ import
org.apache.iotdb.db.storageengine.load.active.ActiveLoadPathHelper;
import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
import
org.apache.iotdb.db.storageengine.load.converter.PipeTsFileConversionTaskManager;
import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
@@ -66,6 +88,186 @@ public class IoTDBDataNodeReceiverTest {
registry.clear();
}
+ @Test
+ public void testRatisRejectsInsertionInPlanNodeRequests() throws Exception {
+ final String originalProtocol =
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass();
+ try {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ for (final PlanNode node : insertionPlans()) {
+ assertPlanRequestEnvelopes(node, true);
+ }
+ } finally {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+ }
+
+ @Test
+ public void testRatisAllowsMetadataAndDeletionPlanNodeRequests() throws
Exception {
+ final String originalProtocol =
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass();
+ try {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ assertPlanRequestEnvelopes(
+ new DeleteDataNode(
+ new PlanNodeId("delete"),
+ Collections.singletonList(new MeasurementPath("root.sg.d.s")),
+ 0,
+ 1),
+ false);
+ assertPlanRequestEnvelopes(
+ new CreateTimeSeriesNode(
+ new PlanNodeId("schema"),
+ new MeasurementPath("root.sg.d.s"),
+ TSDataType.INT32,
+ TSEncoding.PLAIN,
+ CompressionType.UNCOMPRESSED,
+ null,
+ null,
+ null,
+ null),
+ false);
+ } finally {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+ }
+
+ @Test
+ public void testNonRatisAllowsInsertionPlanNodeRequests() throws Exception {
+ final String originalProtocol =
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass();
+ try {
+ for (final String protocol :
+ new String[] {
+ ConsensusFactory.IOT_CONSENSUS,
+ ConsensusFactory.IOT_CONSENSUS_V2,
+ ConsensusFactory.SIMPLE_CONSENSUS
+ }) {
+
IoTDBDescriptor.getInstance().getConfig().setDataRegionConsensusProtocolClass(protocol);
+ for (final PlanNode node : insertionPlans()) {
+ assertPlanRequestEnvelopes(node, false);
+ }
+ }
+ } finally {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+ }
+
+ private List<PlanNode> insertionPlans() throws Exception {
+ final InsertRowNode row =
+ new InsertRowNode(
+ new PlanNodeId("row"),
+ new PartialPath("root.sg.d"),
+ false,
+ new String[] {"s"},
+ new TSDataType[] {TSDataType.INT32},
+ 1L,
+ new Object[] {1},
+ false);
+ return Arrays.asList(
+ row,
+ new InsertRowsNode(
+ new PlanNodeId("rows"), Collections.singletonList(0),
Collections.singletonList(row)),
+ new InsertTabletNode(
+ new PlanNodeId("tablet"),
+ new PartialPath("root.sg.d"),
+ false,
+ new String[] {"s"},
+ new TSDataType[] {TSDataType.INT32},
+ new long[] {1},
+ null,
+ new Object[] {new int[] {1}},
+ 1),
+ new RelationalInsertTabletNode(
+ new PlanNodeId("table"),
+ new PartialPath("t"),
+ false,
+ new String[] {"s"},
+ new TSDataType[] {TSDataType.INT32},
+ new long[] {1},
+ null,
+ new Object[] {new int[] {1}},
+ 1,
+ new TsTableColumnCategory[] {TsTableColumnCategory.FIELD}));
+ }
+
+ private void assertPlanRequestEnvelopes(final PlanNode node, final boolean
rejected)
+ throws Exception {
+ final AdmissionTestReceiver receiver = new AdmissionTestReceiver();
+ final int expectedCode =
+ rejected
+ ? TSStatusCode.UNSUPPORTED_OPERATION.getStatusCode()
+ : TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode();
+ Assert.assertEquals(
+ expectedCode,
+
receiver.receive(PipeTransferPlanNodeReq.toTPipeTransferReq(node)).getStatus().getCode());
+
+ final TPipeTransferReq plain =
PipeTransferPlanNodeReq.toTPipeTransferReq(node);
+ final TPipeTransferReq airGap =
+ new AirGapPseudoTPipeTransferRequest()
+ .setVersion(plain.getVersion())
+ .setType(plain.getType())
+ .setBody(plain.body.duplicate());
+ Assert.assertEquals(expectedCode,
receiver.receive(airGap).getStatus().getCode());
+ Assert.assertEquals(
+ expectedCode,
+ receiver
+ .receive(
+ PipeTransferCompressedReq.toTPipeTransferReq(
+ PipeTransferPlanNodeReq.toTPipeTransferReq(node),
+ Collections.singletonList(
+ PipeCompressorFactory.getCompressor(
+
PipeCompressor.PipeCompressionType.GZIP.getIndex()))))
+ .getStatus()
+ .getCode());
+
+ final TPipeTransferReq sliced =
PipeTransferPlanNodeReq.toTPipeTransferReq(node);
+ final ByteBuffer body = sliced.body.duplicate();
+ final int midpoint = body.limit() / 2;
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ receiver
+ .receive(
+ PipeTransferSliceReq.toTPipeTransferReq(
+ 1, sliced.getType(), 0, 2, body, 0, midpoint))
+ .getStatus()
+ .getCode());
+ Assert.assertEquals(
+ expectedCode,
+ receiver
+ .receive(
+ PipeTransferSliceReq.toTPipeTransferReq(
+ 1, sliced.getType(), 1, 2, body, midpoint, body.limit()))
+ .getStatus()
+ .getCode());
+ Assert.assertEquals(rejected ? 0 : 4, receiver.executionAttempts);
+ }
+
+ private static class AdmissionTestReceiver extends IoTDBDataNodeReceiver {
+ private int executionAttempts;
+
+ private AdmissionTestReceiver() {
+ hasPipeHandshakeCredential = true;
+ }
+
+ @Override
+ protected TSStatus loginIfNecessary() {
+ // Record arrival at statement execution without starting a cluster or
writing data.
+ executionAttempts++;
+ return new TSStatus(TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode());
+ }
+ }
+
@Test
public void testTableModelTabletRedirectDoesNotCachePerRowLeader() {
final InsertTabletStatement statement = new InsertTabletStatement();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSourceTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSourceTest.java
index 2ba204d4def..13eeafc2e90 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSourceTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSourceTest.java
@@ -36,6 +36,8 @@ import
org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern;
import org.apache.iotdb.commons.pipe.datastructure.resource.PersistentResource;
import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
import org.apache.iotdb.db.pipe.consensus.ReplicateProgressDataNodeManager;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
@@ -884,6 +886,50 @@ public class
PipeHistoricalDataRegionTsFileAndDeletionSourceTest {
}
}
+ @Test
+ public void testRatisFilesWithoutPipeProgressAreNotCoveredByCheckpoint()
throws Exception {
+ final String originalConsensus =
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass();
+ final File tempDir =
Files.createTempDirectory("pipeHistoricalRatisCoverage").toFile();
+ try {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ assertMayTsFileContainUnprocessedData(
+ tempDir,
+ "unknown-initial.tsfile",
+ MinimumProgressIndex.INSTANCE,
+ MinimumProgressIndex.INSTANCE,
+ true);
+ assertMayTsFileContainUnprocessedData(
+ tempDir,
+ "unknown-recovery.tsfile",
+ new SimpleProgressIndex(0, 100),
+ MinimumProgressIndex.INSTANCE,
+ true);
+ assertMayTsFileContainUnprocessedData(
+ tempDir,
+ "indexed-covered.tsfile",
+ new SimpleProgressIndex(0, 100),
+ new SimpleProgressIndex(0, 50),
+ false);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ assertMayTsFileContainUnprocessedData(
+ tempDir,
+ "other-consensus.tsfile",
+ MinimumProgressIndex.INSTANCE,
+ MinimumProgressIndex.INSTANCE,
+ false);
+ } finally {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDataRegionConsensusProtocolClass(originalConsensus);
+ FileUtils.deleteFileOrDirectory(tempDir);
+ }
+ }
+
@Test
public void
testMayTsFileContainUnprocessedDataUsesTimePartitionProgressCoverage()
throws Exception {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java
index 21ef6ecb101..fc68f97c6e8 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java
@@ -20,6 +20,8 @@
package org.apache.iotdb.db.subscription.agent;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.subscription.receiver.SubscriptionReceiver;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -54,6 +56,34 @@ import java.util.function.Supplier;
public class SubscriptionReceiverAgentTest {
+ @Test
+ public void testRatisRejectsSubscriptionBeforeCreatingReceiver() throws
IOException {
+ final String originalConsensus =
+
IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass();
+ try {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ final SubscriptionReceiverAgent agent =
+ new SubscriptionReceiverAgent(
+ () -> {
+ throw new AssertionError("Receiver must not be created when
Ratis is enabled");
+ },
+ false,
+ () -> {
+ throw new AssertionError("Subscription supplier must not be
called");
+ });
+
+ Assert.assertEquals(
+ TSStatusCode.UNSUPPORTED_OPERATION.getStatusCode(),
+ agent.handle(createHandshakeRequest("group", "consumer"),
"root").getStatus().getCode());
+ } finally {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDataRegionConsensusProtocolClass(originalConsensus);
+ }
+ }
+
@Test
public void testDisabledSubscriptionRejectsRequest() throws IOException {
final SubscriptionReceiverAgent agent =
diff --git
a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/PipeMessages.java
b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/PipeMessages.java
index 8f7706426c3..b3b33fe3b71 100644
---
a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/PipeMessages.java
+++
b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/PipeMessages.java
@@ -27,6 +27,12 @@ public final class PipeMessages {
public static final String
LOG_AIR_GAP_HANDSHAKE_REQUEST_SENT_ON_SOCKET_ARG_WITHOUT_WAITING_FOR_ACKNOWLEDGEMENT_E1ECD582
=
"Air gap handshake request sent on socket {} without waiting for
acknowledgement.";
+ public static final String
MESSAGE_SUBSCRIPTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_194CE431
=
+ "Subscription is disabled when data region consensus protocol is Ratis.";
+
+ public static final String
MESSAGE_PIPE_DATA_INSERTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_7D669ECC
=
+ "Pipe synchronization with data insertion is disabled when data region
consensus protocol is Ratis.";
+
// ===================== Common / Shared =====================
public static final String UTILITY_CLASS = "Utility class";
diff --git
a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/PipeMessages.java
b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/PipeMessages.java
index 04a88163523..0facda0fe0b 100644
---
a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/PipeMessages.java
+++
b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/PipeMessages.java
@@ -27,6 +27,12 @@ public final class PipeMessages {
public static final String
LOG_AIR_GAP_HANDSHAKE_REQUEST_SENT_ON_SOCKET_ARG_WITHOUT_WAITING_FOR_ACKNOWLEDGEMENT_E1ECD582
=
"已通过 socket {} 发送 air-gap 握手请求,不等待确认。";
+ public static final String
MESSAGE_SUBSCRIPTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_194CE431
=
+ "当数据区域共识协议为 Ratis 时,Subscription 功能不可用。";
+
+ public static final String
MESSAGE_PIPE_DATA_INSERTION_IS_DISABLED_WHEN_DATA_REGION_CONSENSUS_PROTOCOL_IS_RATIS_7D669ECC
=
+ "当数据区域共识协议为 Ratis 时,包含 data.insert 的 Pipe 同步不可用。";
+
// ===================== Common / Shared =====================
public static final String UTILITY_CLASS = "工具类";
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
index 6130464a83e..2edf20190c6 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
@@ -210,6 +210,10 @@ public abstract class PipeTaskAgent {
protected abstract boolean isShutdown();
+ protected boolean isPipeDataInsertionEnabled(final PipeStaticMeta
pipeStaticMeta) {
+ return true;
+ }
+
private void executeSinglePipeMetaChanges(final PipeMeta metaFromCoordinator)
throws IllegalPathException {
final String pipeName = metaFromCoordinator.getStaticMeta().getPipeName();
@@ -232,6 +236,14 @@ public abstract class PipeTaskAgent {
return;
}
+ if (!isPipeDataInsertionEnabled(metaFromCoordinator.getStaticMeta())) {
+ final PipeMeta metaInAgent =
pipeMetaKeeper.getPipeMeta(metaFromCoordinator.getStaticMeta());
+ if (metaInAgent != null) {
+ dropPipe(pipeName, metaInAgent.getStaticMeta().getCreationTime());
+ }
+ return;
+ }
+
final PipeMeta metaInAgent =
pipeMetaKeeper.getPipeMeta(metaFromCoordinator.getStaticMeta());
// If pipe meta does not exist on local agent, create a new pipe
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java
index 3bde97c1a12..7f3dae51583 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java
@@ -257,6 +257,17 @@ public class PipeInclusionOptions {
return options;
}
+ public static boolean isDataInsertionIncluded(final PipeParameters
parameters)
+ throws IllegalPathException {
+ // Match the data-region listening filter, including prefixes longer than
the concrete option
+ // that the existing option validator accepts.
+ final PartialPath insertion = new
PartialPath(EXTRACTOR_INCLUSION_DEFAULT_VALUE);
+ return parseOptions(getInclusionString(parameters)).stream()
+ .anyMatch(insertion::overlapWithFullPathPrefix)
+ && parseOptions(getExclusionString(parameters)).stream()
+ .noneMatch(insertion::overlapWithFullPathPrefix);
+ }
+
public static boolean areOptionsEnabled(final PipeParameters parameters,
final String... options)
throws IllegalPathException {
final Set<PartialPath> inclusionOptions =
parseOptions(getInclusionString(parameters));
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptionsTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptionsTest.java
index 63ab1c73993..7251a398050 100644
---
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptionsTest.java
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptionsTest.java
@@ -20,10 +20,14 @@
package org.apache.iotdb.commons.pipe.datastructure.options;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.junit.Assert;
import org.junit.Test;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Set;
public class PipeInclusionOptionsTest {
@@ -45,4 +49,41 @@ public class PipeInclusionOptionsTest {
Assert.assertFalse(options.contains(new
PartialPath("schema.timeseries.ordinary.delete")));
Assert.assertTrue(PipeInclusionOptions.optionsAreAllLegal("schema.delete",
true, false));
}
+
+ @Test
+ public void testDataInsertionOptionSelection() throws Exception {
+ Assert.assertTrue(
+ PipeInclusionOptions.areOptionsEnabled(
+ new PipeParameters(Collections.emptyMap()), "data.insert"));
+ Assert.assertTrue(
+ PipeInclusionOptions.areOptionsEnabled(
+ new PipeParameters(Map.of("extractor.inclusion",
"data.insert,data.delete")),
+ "data.insert"));
+ Assert.assertFalse(
+ PipeInclusionOptions.areOptionsEnabled(
+ new PipeParameters(Map.of("extractor.inclusion", "schema")),
"data.insert"));
+ Assert.assertFalse(
+ PipeInclusionOptions.areOptionsEnabled(
+ new PipeParameters(Map.of("extractor.inclusion", "data.delete")),
"data.insert"));
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("extractor.inclusion", "all");
+ attributes.put("extractor.inclusion.exclusion", "data.insert");
+ Assert.assertFalse(
+ PipeInclusionOptions.areOptionsEnabled(new PipeParameters(attributes),
"data.insert"));
+ }
+
+ @Test
+ public void testDataInsertionUsesListeningPrefixSemantics() throws Exception
{
+
Assert.assertTrue(PipeInclusionOptions.optionsAreAllLegal("data.insert.extra",
true, true));
+ Assert.assertTrue(
+ PipeInclusionOptions.isDataInsertionIncluded(
+ new PipeParameters(Map.of("source.inclusion",
"data.insert.extra"))));
+ Assert.assertFalse(
+ PipeInclusionOptions.isDataInsertionIncluded(
+ new PipeParameters(
+ Map.of(
+ "source.inclusion", "all",
+ "source.inclusion.exclusion", "data.insert.extra"))));
+ }
}