This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/topic-realtime-mode in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8abf8d956b8d5e43a7cb3e7263dac829be9c9d01 Author: Caideyipi <[email protected]> AuthorDate: Fri Nov 20 16:15:18 2026 +0800 Add realtime mode configuration for topics --- .../iotdb/rpc/subscription/config/TopicConfig.java | 42 +++++++- .../rpc/subscription/config/TopicConstant.java | 7 ++ .../rpc/subscription/config/TopicProperty.java | 1 + .../rpc/subscription/config/TopicConfigTest.java | 55 +++++++++++ .../iotdb/confignode/i18n/ManagerMessages.java | 6 ++ .../iotdb/confignode/i18n/ManagerMessages.java | 6 ++ .../persistence/subscription/SubscriptionInfo.java | 101 +++++++++++++++++-- .../subscription/topic/AlterTopicProcedure.java | 46 ++++++++- .../SubscriptionInfoTopicValidationTest.java | 106 ++++++++++++++++++++ .../CreateSubscriptionProcedureTest.java | 62 ++++++++++++ .../topic/AlterTopicProcedureTest.java | 108 +++++++++++++++++++++ .../db/queryengine/plan/parser/ASTVisitor.java | 14 ++- .../plan/relational/sql/parser/AstBuilder.java | 14 ++- .../db/pipe/source/IoTDBDataRegionSourceTest.java | 38 ++++++++ .../plan/parser/StatementGeneratorTest.java | 13 +++ .../relational/sql/parser/TopicStatementTest.java | 32 ++++++ .../commons/subscription/topic/TopicMetaTest.java | 12 ++- 17 files changed, 642 insertions(+), 21 deletions(-) diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java index 3980f0909f6..17c32c16052 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java @@ -46,6 +46,7 @@ public class TopicConfig extends PipeParameters { private long filterVersion = NEXT_FILTER_VERSION.incrementAndGet(); private static final Set<String> MODE_VALUE_SET; + private static final Set<String> REALTIME_MODE_VALUE_SET; private static final Set<String> ORDER_MODE_VALUE_SET; static { @@ -55,6 +56,11 @@ public class TopicConfig extends PipeParameters { modes.add(TopicConstant.MODE_INCREMENTAL_VALUE); MODE_VALUE_SET = Collections.unmodifiableSet(modes); + final Set<String> realtimeModes = new HashSet<>(2); + realtimeModes.add(TopicConstant.REALTIME_MODE_STREAM_VALUE); + realtimeModes.add(TopicConstant.REALTIME_MODE_BATCH_VALUE); + REALTIME_MODE_VALUE_SET = Collections.unmodifiableSet(realtimeModes); + final Set<String> orderModes = new HashSet<>(3); orderModes.add(TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE); orderModes.add(TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE); @@ -108,9 +114,11 @@ public class TopicConfig extends PipeParameters { private static final String SQL_DIALECT_TABLE_VALUE = "table"; private static final Map<String, String> REALTIME_BATCH_MODE_CONFIG = - Collections.singletonMap("realtime.mode", "batch"); + Collections.singletonMap( + TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE); private static final Map<String, String> REALTIME_STREAM_MODE_CONFIG = - Collections.singletonMap("realtime.mode", "stream"); + Collections.singletonMap( + TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_STREAM_VALUE); private static final Map<String, String> SINK_TABLET_FORMAT_CONFIG = Collections.singletonMap("format", "tablet"); @@ -218,6 +226,29 @@ public class TopicConfig extends PipeParameters { } } + public String getRealtimeMode() { + return normalizeRealtimeMode( + getStringIgnoreCase( + TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_DEFAULT_VALUE)); + } + + public String getEffectiveRealtimeMode() { + final String realtimeMode = getRealtimeMode(); + return isValidRealtimeMode(realtimeMode) + ? realtimeMode + : TopicConstant.REALTIME_MODE_DEFAULT_VALUE; + } + + public static boolean isValidRealtimeMode(final String realtimeMode) { + return REALTIME_MODE_VALUE_SET.contains(normalizeRealtimeMode(realtimeMode)); + } + + public static String normalizeRealtimeMode(final String realtimeMode) { + return realtimeMode == null + ? TopicConstant.REALTIME_MODE_DEFAULT_VALUE + : realtimeMode.trim().toLowerCase(Locale.ROOT); + } + public String getOrderMode() { return normalizeOrderMode( attributes.getOrDefault( @@ -324,7 +355,9 @@ public class TopicConfig extends PipeParameters { } public Map<String, String> getAttributesWithSourceRealtimeMode() { - return REALTIME_STREAM_MODE_CONFIG; // default to stream (hybrid) + return TopicConstant.REALTIME_MODE_BATCH_VALUE.equals(getRealtimeMode()) + ? REALTIME_BATCH_MODE_CONFIG + : REALTIME_STREAM_MODE_CONFIG; } public Map<String, String> getAttributesWithSourceMode() { @@ -357,7 +390,8 @@ public class TopicConfig extends PipeParameters { final Map<String, String> attributesWithProcessorPrefix = new HashMap<>(); attributes.forEach( (key, value) -> { - if (key.toLowerCase().startsWith("source")) { + if (key.toLowerCase(Locale.ROOT).startsWith("source") + && !TopicConstant.SOURCE_REALTIME_MODE_KEY.equalsIgnoreCase(key.trim())) { attributesWithProcessorPrefix.put(key, value); } }); diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java index f6f3b5926d3..84105c25ae9 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java @@ -52,6 +52,13 @@ public class TopicConstant { public static final String MODE_INCREMENTAL_VALUE = "incremental"; public static final String MODE_DEFAULT_VALUE = MODE_INITIAL_VALUE; + public static final String REALTIME_MODE_KEY = "realtime.mode"; + public static final String REALTIME_MODE_STREAM_VALUE = "stream"; + public static final String REALTIME_MODE_BATCH_VALUE = "batch"; + public static final String REALTIME_MODE_DEFAULT_VALUE = REALTIME_MODE_STREAM_VALUE; + public static final String EXTRACTOR_REALTIME_MODE_KEY = "extractor.realtime.mode"; + public static final String SOURCE_REALTIME_MODE_KEY = "source.realtime.mode"; + static final String LEGACY_MODE_LIVE_VALUE = "live"; static final String LEGACY_MODE_CONSENSUS_VALUE = "consensus"; diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicProperty.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicProperty.java index c5b5915deaa..73b93c16e0b 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicProperty.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicProperty.java @@ -43,6 +43,7 @@ public enum TopicProperty { RETENTION_MS(TopicConstant.RETENTION_MS_KEY, false), RETAIN_PROGRESS_AFTER_UNSUBSCRIBE(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, false), MODE(TopicConstant.MODE_KEY, false), + REALTIME_MODE(TopicConstant.REALTIME_MODE_KEY, false), ORDER_MODE(TopicConstant.ORDER_MODE_KEY, false), FORMAT(TopicConstant.FORMAT_KEY, false), RECYCLE_UNCOMMITTED_EVENT_INTERVAL_MS( diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java index d3524b65a94..61ce57d08b2 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java @@ -22,6 +22,9 @@ package org.apache.iotdb.rpc.subscription.config; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -37,6 +40,7 @@ public class TopicConfigTest { attributes.put(TopicConstant.OWNER_EPOCH_KEY, "1"); attributes.put(TopicConstant.RETENTION_MS_KEY, "60000"); attributes.put(TopicConstant.RECYCLE_UNCOMMITTED_EVENT_INTERVAL_MS_KEY, "1000"); + attributes.put(TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE); attributes.put("future.non-filtering-property", "value"); final TopicConfig updated = new TopicConfig(attributes); updated.inheritFilterVersion(original); @@ -102,6 +106,57 @@ public class TopicConfigTest { Assert.assertTrue(topicConfigWithMode(" INCREMENTAL ").isIncrementalMode()); } + @Test + public void testRealtimeModeDefaultsToStreamAndSupportsBatch() { + final TopicConfig defaultConfig = new TopicConfig(); + Assert.assertEquals( + TopicConstant.REALTIME_MODE_STREAM_VALUE, + defaultConfig.getAttributesWithSourceRealtimeMode().get(TopicConstant.REALTIME_MODE_KEY)); + + final TopicConfig batchConfig = + new TopicConfig(Collections.singletonMap("Realtime.Mode", " BATCH ")); + Assert.assertEquals(TopicConstant.REALTIME_MODE_BATCH_VALUE, batchConfig.getRealtimeMode()); + Assert.assertTrue(TopicConfig.isValidRealtimeMode(batchConfig.getRealtimeMode())); + Assert.assertEquals( + TopicConstant.REALTIME_MODE_BATCH_VALUE, + batchConfig.getAttributesWithSourceRealtimeMode().get(TopicConstant.REALTIME_MODE_KEY)); + Assert.assertFalse(TopicConfig.isValidRealtimeMode("hybrid")); + } + + @Test + public void testRealtimeModeAliasesDoNotOverrideCanonicalTopicMode() { + final Map<String, String> attributes = new HashMap<>(); + attributes.put(TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE); + attributes.put(TopicConstant.SOURCE_REALTIME_MODE_KEY, " BATCH "); + attributes.put(TopicConstant.EXTRACTOR_REALTIME_MODE_KEY, "BATCH"); + final TopicConfig config = new TopicConfig(attributes); + + Assert.assertEquals(TopicConstant.REALTIME_MODE_BATCH_VALUE, config.getRealtimeMode()); + Assert.assertFalse( + config.getAttributesWithSourcePrefix().containsKey(TopicConstant.SOURCE_REALTIME_MODE_KEY)); + Assert.assertEquals( + TopicConstant.REALTIME_MODE_BATCH_VALUE, + config.getAttributesWithSourceRealtimeMode().get(TopicConstant.REALTIME_MODE_KEY)); + Assert.assertEquals( + TopicConstant.REALTIME_MODE_DEFAULT_VALUE, + new TopicConfig(Collections.singletonMap(TopicConstant.REALTIME_MODE_KEY, "hybrid")) + .getEffectiveRealtimeMode()); + } + + @Test + public void testRealtimeModeSurvivesSerialization() throws Exception { + final TopicConfig original = + new TopicConfig( + Collections.singletonMap( + TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE)); + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + original.serialize(new DataOutputStream(output)); + + final TopicConfig restored = TopicConfig.deserialize(ByteBuffer.wrap(output.toByteArray())); + + Assert.assertEquals(TopicConstant.REALTIME_MODE_BATCH_VALUE, restored.getRealtimeMode()); + } + @Test public void testRecycleUncommittedEventInterval() { Assert.assertNull(new TopicConfig().getRecycleUncommittedEventIntervalMs()); diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 00efedc5d9e..1f6f0b42994 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -20,6 +20,12 @@ package org.apache.iotdb.confignode.i18n; public final class ManagerMessages { + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_REALTIME_MODE_ARG_CONFLICTS_WITH_ARG_ARG_D08CF59B = + "Failed to create or alter topic, realtime.mode=%s conflicts with %s=%s"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_UNSUPPORTED_REALTIME_MODE_ARG_EXPECTED_ONE_OF_STREAM_BATCH_79399284 = + "Failed to create or alter topic, unsupported realtime.mode=%s, expected one of [stream, batch]"; public static final String ACTIVATEDATAALLOTTABLE_ACTIVATE_SERIESPARTITIONSLOT = "[ActivateDataAllotTable] Activate SeriesPartitionSlot {} "; diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index de0c89da1f0..611bdb4a186 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -20,6 +20,12 @@ package org.apache.iotdb.confignode.i18n; public final class ManagerMessages { + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_REALTIME_MODE_ARG_CONFLICTS_WITH_ARG_ARG_D08CF59B = + "\u521b\u5efa\u6216\u4fee\u6539\u4e3b\u9898\u5931\u8d25\uff0crealtime.mode=%s \u4e0e %s=%s \u51b2\u7a81"; + public static final String + EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_UNSUPPORTED_REALTIME_MODE_ARG_EXPECTED_ONE_OF_STREAM_BATCH_79399284 = + "创建或修改主题失败,不支持 realtime.mode=%s,预期值为 [stream, batch] 之一"; public static final String ACTIVATEDATAALLOTTABLE_ACTIVATE_SERIESPARTITIONSLOT = "[ActivateDataAllotTable] 激活 SeriesPartitionSlot {} "; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java index fa0870d23b9..de692d1d7c4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java @@ -60,6 +60,7 @@ import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.subscription.config.TopicConfig; import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.rpc.subscription.config.TopicProperty; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.thrift.annotation.Nullable; @@ -116,6 +117,9 @@ public class SubscriptionInfo implements SnapshotProcessor { TopicConstant.START_TIME_KEY, TopicConstant.END_TIME_KEY, TopicConstant.MODE_KEY, + TopicConstant.REALTIME_MODE_KEY, + PipeSourceConstant.EXTRACTOR_REALTIME_MODE_KEY, + PipeSourceConstant.SOURCE_REALTIME_MODE_KEY, TopicConstant.ORDER_MODE_KEY, TopicConstant.FORMAT_KEY, TopicConstant.LOOSE_RANGE_KEY, @@ -235,8 +239,12 @@ public class SubscriptionInfo implements SnapshotProcessor { private boolean checkBeforeCreateTopicInternal(TCreateTopicReq createTopicReq) throws SubscriptionException { - final TopicConfig topicConfig = - new TopicConfig(safeTopicAttributes(createTopicReq.getTopicAttributes())); + final Map<String, String> topicAttributes = + new HashMap<>(safeTopicAttributes(createTopicReq.getTopicAttributes())); + final TopicConfig topicConfig = new TopicConfig(topicAttributes); + validateDuplicateTopicAttributes(topicConfig); + canonicalizeTopicAttributes(topicAttributes); + createTopicReq.setTopicAttributes(topicAttributes); validateTopicConfig(topicConfig); if (!isTopicExisted(createTopicReq.getTopicName(), topicConfig.isTableTopic())) { @@ -329,9 +337,14 @@ public class SubscriptionInfo implements SnapshotProcessor { } public void validateBeforeAlteringTopic(TopicMeta topicMeta) throws SubscriptionException { + validateBeforeAlteringTopic(topicMeta, false); + } + + public void validateBeforeAlteringTopic( + TopicMeta topicMeta, final boolean hasExistingSubscriptionPipe) throws SubscriptionException { acquireReadLock(); try { - checkBeforeAlteringTopicInternal(topicMeta); + checkBeforeAlteringTopicInternal(topicMeta, hasExistingSubscriptionPipe); } finally { releaseReadLock(); } @@ -342,18 +355,23 @@ public class SubscriptionInfo implements SnapshotProcessor { validateDuplicateTopicAttributes(new TopicConfig(safeTopicAttributes(updatedAttributes))); } - private void checkBeforeAlteringTopicInternal(TopicMeta topicMeta) throws SubscriptionException { - validateTopicConfig(topicMeta.getConfig()); - + private void checkBeforeAlteringTopicInternal( + TopicMeta topicMeta, final boolean hasExistingSubscriptionPipe) throws SubscriptionException { final boolean isTableModel = topicMeta.visibleUnderTableModel(); if (isTopicExisted(topicMeta.getTopicName(), isTableModel)) { final TopicMeta existedTopicMeta = topicMetaKeeper.getTopicMeta(topicMeta.getTopicName(), isTableModel); + validateTopicConfig(topicMeta.getConfig()); validateUnsupportedHotUpdatedTopicConfig( - topicMeta.getTopicName(), existedTopicMeta.getConfig(), topicMeta.getConfig()); + topicMeta.getTopicName(), + existedTopicMeta.getConfig(), + topicMeta.getConfig(), + hasExistingSubscriptionPipe); return; } + validateTopicConfig(topicMeta.getConfig()); + final String exceptionMessage = String.format( ManagerMessages.FAILED_TO_ALTER_TOPIC_THE_TOPIC_IS_NOT_EXISTED, @@ -366,6 +384,14 @@ public class SubscriptionInfo implements SnapshotProcessor { return Objects.nonNull(attributes) ? attributes : Collections.emptyMap(); } + private void canonicalizeTopicAttributes(final Map<String, String> attributes) { + final Map<String, String> canonicalAttributes = new HashMap<>(); + attributes.forEach( + (key, value) -> canonicalAttributes.put(TopicProperty.canonicalizeKey(key), value)); + attributes.clear(); + attributes.putAll(canonicalAttributes); + } + private void validateTopicConfig(final TopicConfig topicConfig) throws SubscriptionException { validateDuplicateTopicAttributes(topicConfig); validateTopicDoesNotOnlySelectAuditDatabase(topicConfig); @@ -387,6 +413,7 @@ public class SubscriptionInfo implements SnapshotProcessor { validateIncrementalTopicAttributes(topicConfig); validateIncrementalProtocolSupport(topicConfig); + validateRealtimeMode(topicConfig); if (topicConfig.isIncrementalMode() && !topicConfig.isRecordFormat()) { final String exceptionMessage = @@ -437,6 +464,48 @@ public class SubscriptionInfo implements SnapshotProcessor { } } + private void validateRealtimeMode(final TopicConfig topicConfig) throws SubscriptionException { + final String realtimeMode = topicConfig.getRealtimeMode(); + validateRealtimeModeValue(realtimeMode); + + for (final Map.Entry<String, String> entry : topicConfig.getAttribute().entrySet()) { + final String key = entry.getKey(); + if (Objects.isNull(key) + || (!PipeSourceConstant.EXTRACTOR_REALTIME_MODE_KEY.equalsIgnoreCase(key.trim()) + && !PipeSourceConstant.SOURCE_REALTIME_MODE_KEY.equalsIgnoreCase(key.trim()))) { + continue; + } + + final String aliasMode = TopicConfig.normalizeRealtimeMode(entry.getValue()); + validateRealtimeModeValue(aliasMode); + if (!Objects.equals(realtimeMode, aliasMode)) { + final String exceptionMessage = + String.format( + ManagerMessages + .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_REALTIME_MODE_ARG_CONFLICTS_WITH_ARG_ARG_D08CF59B, + realtimeMode, + key.trim(), + aliasMode); + LOGGER.warn(exceptionMessage); + throw new SubscriptionException(exceptionMessage); + } + } + } + + private void validateRealtimeModeValue(final String realtimeMode) throws SubscriptionException { + if (TopicConfig.isValidRealtimeMode(realtimeMode)) { + return; + } + + final String exceptionMessage = + String.format( + ManagerMessages + .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_UNSUPPORTED_REALTIME_MODE_ARG_EXPECTED_ONE_OF_STREAM_BATCH_79399284, + realtimeMode); + LOGGER.warn(exceptionMessage); + throw new SubscriptionException(exceptionMessage); + } + private void validateTopicDoesNotOnlySelectAuditDatabase(final TopicConfig topicConfig) throws SubscriptionException { final boolean onlySelectsAuditDatabase = @@ -753,7 +822,10 @@ public class SubscriptionInfo implements SnapshotProcessor { } private void validateUnsupportedHotUpdatedTopicConfig( - final String topicName, final TopicConfig existedConfig, final TopicConfig updatedConfig) + final String topicName, + final TopicConfig existedConfig, + final TopicConfig updatedConfig, + final boolean hasExistingSubscriptionPipe) throws SubscriptionException { final String existedMode = existedConfig.getMode(); final String updatedMode = updatedConfig.getMode(); @@ -768,6 +840,19 @@ public class SubscriptionInfo implements SnapshotProcessor { throw new SubscriptionException(exceptionMessage); } + if (hasExistingSubscriptionPipe + && !Objects.equals( + existedConfig.getEffectiveRealtimeMode(), updatedConfig.getEffectiveRealtimeMode())) { + final String exceptionMessage = + String.format( + ManagerMessages + .EXCEPTION_FAILED_TO_ALTER_TOPIC_ARG_CHANGING_ARG_IS_NOT_SUPPORTED_BECAUSE_EXISTING_SUBSCRIPTION_RUNTIMES_DO_NOT_HOT_REFRESH_SOURCE_MODE_F27410E6, + topicName, + TopicConstant.REALTIME_MODE_KEY); + LOGGER.warn(exceptionMessage); + throw new SubscriptionException(exceptionMessage); + } + validateUnsupportedHotUpdatedRetentionConfig( topicName, existedConfig, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java index 58f1c66ccee..95c61fa526c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java @@ -22,6 +22,7 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.topic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.subscription.meta.consumer.CommitProgressKeeper; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; @@ -45,6 +46,8 @@ import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -171,7 +174,11 @@ public class AlterTopicProcedure extends AbstractOperateSubscriptionAndPipeProce updatedTopicMeta = existedTopicMeta.deepCopyWithUpdatedAttributes(updatedTopicAttributes); } - subscriptionInfo.get().validateBeforeAlteringTopic(updatedTopicMeta); + migrateLegacyRealtimeMode(); + + subscriptionInfo + .get() + .validateBeforeAlteringTopic(updatedTopicMeta, hasExistingSubscriptionPipe()); if (Objects.nonNull(existedTopicMeta) && existedTopicMeta.getConfig().isProgressRetainedAfterUnsubscribe() @@ -182,6 +189,43 @@ public class AlterTopicProcedure extends AbstractOperateSubscriptionAndPipeProce return true; } + private void migrateLegacyRealtimeMode() { + if (Objects.isNull(existedTopicMeta) + || TopicConfig.isValidRealtimeMode(existedTopicMeta.getConfig().getRealtimeMode()) + || (Objects.nonNull(updatedTopicAttributes) + && updatedTopicAttributes.keySet().stream() + .anyMatch( + key -> + Objects.nonNull(key) + && TopicConstant.REALTIME_MODE_KEY.equalsIgnoreCase(key.trim())))) { + return; + } + + updatedTopicMeta + .getConfig() + .getAttribute() + .put(TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_DEFAULT_VALUE); + } + + private boolean hasExistingSubscriptionPipe() { + if (Objects.isNull(pipeTaskInfo) || Objects.isNull(pipeTaskInfo.get())) { + return false; + } + + final boolean isTableModel = updatedTopicMeta.visibleUnderTableModel(); + for (final PipeMeta pipeMeta : pipeTaskInfo.get().getPipeMetaList()) { + final PipeStaticMeta staticMeta = pipeMeta.getStaticMeta(); + if (PipeStaticMeta.isSubscriptionPipe(staticMeta.getPipeName()) + && staticMeta.visibleUnder(isTableModel) + && Objects.equals( + updatedTopicMeta.getTopicName(), + staticMeta.getSinkParameters().getString(PipeSinkConstant.SINK_TOPIC_KEY))) { + return true; + } + } + return false; + } + private void prepareRetainedProgressCleanup(final ConfigNodeProcedureEnv env) { final String topicName = updatedTopicMeta.getTopicName(); final boolean isTableModel = updatedTopicMeta.visibleUnderTableModel(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java index 6ede9a970bd..3c07bd46f4e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java @@ -533,6 +533,112 @@ public class SubscriptionInfoTopicValidationTest { assertCreateRejected(subscriptionInfo, attributes, "unsupported mode"); } + @Test + public void testValidateRealtimeMode() throws Exception { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map<String, String> attributes = newInitialTableTopicAttributes(); + attributes.put(TopicConstant.REALTIME_MODE_KEY, " BATCH "); + + Assert.assertTrue( + subscriptionInfo.validateBeforeCreatingTopic( + new TCreateTopicReq("batch_topic").setTopicAttributes(attributes))); + } + + @Test + public void testRejectInvalidRealtimeMode() { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map<String, String> attributes = newInitialTableTopicAttributes(); + attributes.put(" realtime.mode ", "hybrid"); + + assertCreateRejected(subscriptionInfo, attributes, "hybrid"); + } + + @Test + public void testCanonicalizeCreateAttributesBeforeValidation() { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map<String, String> attributes = new HashMap<>(); + attributes.put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); + attributes.put(" MODE ", TopicConstant.MODE_INCREMENTAL_VALUE); + attributes.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_RECORD_HANDLER_VALUE); + attributes.put(" realtime.mode ", TopicConstant.REALTIME_MODE_BATCH_VALUE); + final TCreateTopicReq request = + new TCreateTopicReq("table_topic").setTopicAttributes(attributes); + + try { + subscriptionInfo.validateBeforeCreatingTopic(request); + Assert.fail("Expected incremental topic to reject realtime.mode"); + } catch (final SubscriptionException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(TopicConstant.REALTIME_MODE_KEY)); + } + + Assert.assertEquals( + TopicConstant.MODE_INCREMENTAL_VALUE, + request.getTopicAttributes().get(TopicConstant.MODE_KEY)); + Assert.assertEquals( + TopicConstant.REALTIME_MODE_BATCH_VALUE, + request.getTopicAttributes().get(TopicConstant.REALTIME_MODE_KEY)); + Assert.assertFalse(request.getTopicAttributes().containsKey(" MODE ")); + } + + @Test + public void testRejectConflictingRealtimeModeAliases() { + for (final String alias : + new String[] { + PipeSourceConstant.SOURCE_REALTIME_MODE_KEY, + PipeSourceConstant.EXTRACTOR_REALTIME_MODE_KEY + }) { + final Map<String, String> attributes = newInitialTableTopicAttributes(); + attributes.put(TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE); + attributes.put(alias, TopicConstant.REALTIME_MODE_STREAM_VALUE); + assertCreateRejected(new SubscriptionInfo(), attributes, alias); + } + } + + @Test + public void testRejectCaseInsensitiveDuplicateRealtimeModeAliases() { + final Map<String, String> attributes = newInitialTableTopicAttributes(); + attributes.put(PipeSourceConstant.SOURCE_REALTIME_MODE_KEY, "batch"); + attributes.put("SOURCE.REALTIME.MODE", "batch"); + + assertCreateRejected( + new SubscriptionInfo(), attributes, PipeSourceConstant.SOURCE_REALTIME_MODE_KEY); + } + + @Test + public void testRejectAlteringRealtimeMode() throws Exception { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map<String, String> originalAttributes = newInitialTableTopicAttributes(); + originalAttributes.put( + TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_STREAM_VALUE); + subscriptionInfo.createTopic( + new CreateTopicPlan(new TopicMeta("table_topic", 1L, originalAttributes))); + + final Map<String, String> updatedAttributes = new HashMap<>(originalAttributes); + updatedAttributes.put(TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE); + try { + subscriptionInfo.validateBeforeAlteringTopic( + new TopicMeta("table_topic", 2L, updatedAttributes), true); + Assert.fail("Expected altering realtime.mode to be rejected"); + } catch (final SubscriptionException e) { + Assert.assertTrue(e.getMessage().contains(TopicConstant.REALTIME_MODE_KEY)); + } + } + + @Test + public void testAllowAlteringRealtimeModeWithoutSubscriptionPipe() throws Exception { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map<String, String> originalAttributes = newInitialTableTopicAttributes(); + originalAttributes.put( + TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_STREAM_VALUE); + subscriptionInfo.createTopic( + new CreateTopicPlan(new TopicMeta("table_topic", 1L, originalAttributes))); + + final Map<String, String> updatedAttributes = new HashMap<>(originalAttributes); + updatedAttributes.put(TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE); + subscriptionInfo.validateBeforeAlteringTopic( + new TopicMeta("table_topic", 2L, updatedAttributes)); + } + @SuppressWarnings("deprecation") @Test public void testAcceptLegacyModeValues() throws Exception { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java index d4087f712a5..094b26b2271 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; @@ -71,6 +72,67 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; public class CreateSubscriptionProcedureTest { + @Test + public void realtimeModeIsPreservedInCreatedSubscriptionPipe() throws Exception { + for (final String[] modeAndExpectedValue : + new String[][] { + {null, TopicConstant.REALTIME_MODE_STREAM_VALUE}, + {TopicConstant.REALTIME_MODE_BATCH_VALUE, TopicConstant.REALTIME_MODE_BATCH_VALUE} + }) { + final Map<String, String> topicAttributes = new HashMap<>(); + topicAttributes.put(TopicConstant.PATH_KEY, "root.**"); + if (modeAndExpectedValue[0] != null) { + topicAttributes.put(TopicConstant.REALTIME_MODE_KEY, modeAndExpectedValue[0]); + } + final TopicMeta topicMeta = new TopicMeta("test_topic", 1L, topicAttributes); + final Map<String, String> consumerAttributes = new HashMap<>(); + consumerAttributes.put("username", "user"); + consumerAttributes.put("password", "password"); + final ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta( + "test_consumer_group", 1L, new ConsumerMeta("old_consumer", 1L, consumerAttributes)); + + final SubscriptionInfo subscriptionInfo = Mockito.mock(SubscriptionInfo.class); + Mockito.when(subscriptionInfo.getConsumerGroupMeta("test_consumer_group")) + .thenReturn(consumerGroupMeta); + Mockito.when(subscriptionInfo.deepCopyConsumerGroupMeta("test_consumer_group")) + .thenAnswer(invocation -> consumerGroupMeta.deepCopy()); + Mockito.when(subscriptionInfo.deepCopyTopicMeta("test_topic", false)).thenReturn(topicMeta); + + final PipeTaskInfo pipeTaskInfo = Mockito.mock(PipeTaskInfo.class); + Mockito.when(pipeTaskInfo.checkBeforeCreatePipe(Mockito.any(TCreatePipeReq.class))) + .thenReturn(true); + + final CreateSubscriptionProcedure procedure = + new CreateSubscriptionProcedure( + new TSubscribeReq( + "old_consumer", "test_consumer_group", Collections.singleton("test_topic"))); + setField(procedure, "subscriptionInfo", new AtomicReference<>(subscriptionInfo)); + setField(procedure, "pipeTaskInfo", new AtomicReference<>(pipeTaskInfo)); + procedure.executeFromValidate(mockCreateSubscriptionValidationEnv()); + + Assert.assertEquals(1, procedure.getCreatePipeProcedures().size()); + Assert.assertEquals( + modeAndExpectedValue[1], + procedure + .getCreatePipeProcedures() + .get(0) + .getPipeStaticMeta() + .getSourceParameters() + .getStringByKeys( + PipeSourceConstant.EXTRACTOR_REALTIME_MODE_KEY, + PipeSourceConstant.SOURCE_REALTIME_MODE_KEY)); + Assert.assertFalse( + procedure + .getCreatePipeProcedures() + .get(0) + .getPipeStaticMeta() + .getSourceParameters() + .getAttribute() + .containsKey(PipeSourceConstant.SOURCE_REALTIME_MODE_KEY)); + } + } + @Test public void serializeDeserializeTest() { PublicBAOS byteArrayOutputStream = new PublicBAOS(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java index e15d7cd7c3c..4117075e161 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java @@ -19,9 +19,13 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.topic; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.AlterTopicPlan; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.CreateTopicPlan; +import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.rpc.TSStatusCode; @@ -32,6 +36,7 @@ import org.apache.tsfile.utils.PublicBAOS; import org.junit.Test; import java.io.DataOutputStream; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; @@ -146,4 +151,107 @@ public class AlterTopicProcedureTest { assertTrue(e.getMessage(), e.getMessage().contains("duplicate tag-filter")); } } + + @Test + public void testRealtimeModeCanChangeBeforeSubscriptionPipeCreation() throws Exception { + final String topicName = "test_topic"; + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map<String, String> originalAttributes = new HashMap<>(); + originalAttributes.put(TopicConstant.PATH_KEY, "root.**"); + originalAttributes.put( + TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_STREAM_VALUE); + subscriptionInfo.createTopic( + new CreateTopicPlan(new TopicMeta(topicName, 1L, originalAttributes))); + + final Map<String, String> updatedAttributes = + Map.of(TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE); + final AlterTopicProcedure procedure = + new AlterTopicProcedure( + subscriptionInfo.deepCopyTopicMetaWithUpdatedAttributes(topicName, updatedAttributes), + updatedAttributes, + new AtomicReference<>(subscriptionInfo)); + procedure.executeFromValidate(null); + + assertEquals( + TopicConstant.REALTIME_MODE_BATCH_VALUE, + procedure.getUpdatedTopicMeta().getConfig().getRealtimeMode()); + } + + @Test + public void testRealtimeModeCannotChangeWhileSubscriptionPipeExists() throws Exception { + final String topicName = "test_topic"; + final String consumerGroupId = "test_group"; + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map<String, String> originalAttributes = new HashMap<>(); + originalAttributes.put(TopicConstant.PATH_KEY, "root.**"); + originalAttributes.put( + TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_STREAM_VALUE); + final TopicMeta topicMeta = new TopicMeta(topicName, 1L, originalAttributes); + subscriptionInfo.createTopic(new CreateTopicPlan(topicMeta)); + + final PipeStaticMeta pipeStaticMeta = + new PipeStaticMeta( + PipeStaticMeta.generateSubscriptionPipeName(topicName, consumerGroupId), + 1L, + topicMeta.generateExtractorAttributes("user"), + topicMeta.generateProcessorAttributes(), + topicMeta.generateConnectorAttributes(consumerGroupId)); + final PipeTaskInfo pipeTaskInfo = new PipeTaskInfo(); + pipeTaskInfo.createPipe(new CreatePipePlanV2(pipeStaticMeta, new PipeRuntimeMeta())); + + final Map<String, String> updatedAttributes = + Map.of(TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE); + final AlterTopicProcedure procedure = + new AlterTopicProcedure( + subscriptionInfo.deepCopyTopicMetaWithUpdatedAttributes(topicName, updatedAttributes), + updatedAttributes, + new AtomicReference<>(subscriptionInfo)); + setField(procedure, "pipeTaskInfo", new AtomicReference<>(pipeTaskInfo)); + + try { + procedure.executeFromValidate(null); + fail("Expected mode change with an existing subscription Pipe to be rejected"); + } catch (final SubscriptionException e) { + assertTrue(e.getMessage(), e.getMessage().contains(TopicConstant.REALTIME_MODE_KEY)); + } + } + + @Test + public void testUnrelatedAlterMigratesLegacyRealtimeModeToStream() throws Exception { + final String topicName = "test_topic"; + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map<String, String> originalAttributes = new HashMap<>(); + originalAttributes.put(TopicConstant.PATH_KEY, "root.**"); + originalAttributes.put(TopicConstant.REALTIME_MODE_KEY, "hybrid"); + subscriptionInfo.createTopic( + new CreateTopicPlan(new TopicMeta(topicName, 1L, originalAttributes))); + + final Map<String, String> updatedAttributes = Map.of("processor", "test_processor"); + final AlterTopicProcedure procedure = + new AlterTopicProcedure( + subscriptionInfo.deepCopyTopicMetaWithUpdatedAttributes(topicName, updatedAttributes), + updatedAttributes, + new AtomicReference<>(subscriptionInfo)); + procedure.executeFromValidate(null); + + assertEquals( + TopicConstant.REALTIME_MODE_STREAM_VALUE, + procedure.getUpdatedTopicMeta().getConfig().getRealtimeMode()); + } + + private static void setField(final Object target, final String name, final Object value) + throws Exception { + Class<?> type = target.getClass(); + while (type != null) { + try { + final Field field = type.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + return; + } catch (final NoSuchFieldException ignored) { + type = type.getSuperclass(); + } + } + throw new NoSuchFieldException(name); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index c0be20613b9..6927aaec8cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -266,6 +266,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuot import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator; import org.apache.iotdb.db.utils.DataNodeDateTimeUtils; +import org.apache.iotdb.rpc.subscription.config.TopicProperty; import org.apache.iotdb.trigger.api.enums.TriggerEvent; import org.apache.iotdb.trigger.api.enums.TriggerType; @@ -4556,9 +4557,16 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { List<IoTDBSqlParser.TopicAttributeClauseContext> contexts) { final Map<String, String> collectorMap = new HashMap<>(); for (IoTDBSqlParser.TopicAttributeClauseContext context : contexts) { - collectorMap.put( - parseStringLiteral(context.topicKey.getText()), - parseStringLiteral(context.topicValue.getText())); + final String key = + TopicProperty.canonicalizeKey(parseStringLiteral(context.topicKey.getText())); + if (collectorMap.containsKey(key)) { + throw new SemanticException( + String.format( + DataNodeQueryMessages + .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_DUPLICATE_ARG_ATTRIBUTES_ARE_NOT_ALLOWED_27315578, + key)); + } + collectorMap.put(key, parseStringLiteral(context.topicValue.getText())); } return collectorMap; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 189589365b0..26c364170ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -273,6 +273,7 @@ import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlLexer; import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlParser; import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator; import org.apache.iotdb.db.utils.DataNodeDateTimeUtils; +import org.apache.iotdb.rpc.subscription.config.TopicProperty; import com.google.common.collect.ImmutableList; import org.antlr.v4.runtime.ParserRuleContext; @@ -1205,9 +1206,16 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> { List<RelationalSqlParser.TopicAttributeClauseContext> contexts) { final Map<String, String> tppicMap = new HashMap<>(); for (RelationalSqlParser.TopicAttributeClauseContext context : contexts) { - tppicMap.put( - ((StringLiteral) visit(context.topicKey)).getValue(), - ((StringLiteral) visit(context.topicValue)).getValue()); + final String key = + TopicProperty.canonicalizeKey(((StringLiteral) visit(context.topicKey)).getValue()); + if (tppicMap.containsKey(key)) { + throw new SemanticException( + String.format( + DataNodeQueryMessages + .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_DUPLICATE_ARG_ATTRIBUTES_ARE_NOT_ALLOWED_27315578, + key)); + } + tppicMap.put(key, ((StringLiteral) visit(context.topicValue)).getValue()); } return tppicMap; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java index f660423b0b0..15516dd3f53 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/IoTDBDataRegionSourceTest.java @@ -20,17 +20,55 @@ package org.apache.iotdb.db.pipe.source; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; import org.apache.iotdb.db.pipe.source.dataregion.IoTDBDataRegionSource; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionHybridSource; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionTsFileSource; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.junit.Assert; import org.junit.Test; import java.util.HashMap; +import java.util.Map; public class IoTDBDataRegionSourceTest { + @Test + public void testSubscriptionRealtimeModeSelectsExpectedSource() throws Exception { + for (final String[] modeAndExpectedSource : + new String[][] { + { + TopicConstant.REALTIME_MODE_STREAM_VALUE, + PipeRealtimeDataRegionHybridSource.class.getName() + }, + { + TopicConstant.REALTIME_MODE_BATCH_VALUE, + PipeRealtimeDataRegionTsFileSource.class.getName() + } + }) { + final TopicMeta topicMeta = + new TopicMeta( + "topic", + 1L, + Map.of( + TopicConstant.PATH_KEY, + "root.**", + TopicConstant.REALTIME_MODE_KEY, + modeAndExpectedSource[0])); + try (final IoTDBDataRegionSource extractor = new IoTDBDataRegionSource()) { + extractor.validate( + new PipeParameterValidator( + new PipeParameters(topicMeta.generateExtractorAttributes("user")))); + Assert.assertEquals( + modeAndExpectedSource[1], + extractor.getRealtimeSourceForCompletion().getClass().getName()); + } + } + } + @Test public void testIoTDBDataRegionExtractor() { try (final IoTDBDataRegionSource extractor = new IoTDBDataRegionSource()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index 952e825bc27..0c3dc1a423a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@ -136,6 +136,19 @@ public class StatementGeneratorTest { } } + @Test + public void testDuplicateRealtimeModeAttributesAreRejected() { + for (final String statement : + new String[] { + "CREATE TOPIC topic1 WITH ('realtime.mode'='hybrid','REALTIME.MODE'='batch')", + "ALTER TOPIC topic1 WITH ('realtime.mode'='hybrid','REALTIME.MODE'='batch')" + }) { + Assert.assertThrows( + SemanticException.class, + () -> StatementGenerator.createStatement(statement, ZonedDateTime.now().getOffset())); + } + } + @Test public void testAlterTopicStatement() { final Statement statement = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/TopicStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/TopicStatementTest.java index 5c8686c484c..e365853fba8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/TopicStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/TopicStatementTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.parser; +import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.subscription.ShowCreateTopicTask; @@ -27,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTopic; import org.apache.iotdb.db.queryengine.plan.relational.sql.util.DataNodeSqlFormatter; import org.apache.iotdb.db.subscription.tagfilter.TagFilterEvaluator; import org.apache.iotdb.db.subscription.tagfilter.TagFilterParser; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.junit.Assert; import org.junit.Test; @@ -38,6 +40,19 @@ public class TopicStatementTest { private final SqlParser sqlParser = new SqlParser(); + @Test + public void testDuplicateRealtimeModeAttributesAreRejected() { + for (final String command : + new String[] { + "CREATE TOPIC topic1 WITH ('realtime.mode'='hybrid','REALTIME.MODE'='batch')", + "ALTER TOPIC topic1 WITH ('realtime.mode'='hybrid','REALTIME.MODE'='batch')" + }) { + Assert.assertThrows( + SemanticException.class, + () -> sqlParser.createStatement(command, ZoneId.systemDefault(), null)); + } + } + @Test public void testTagFilterQuotesInCreateAndAlterTopic() throws Exception { final String attributes = @@ -89,6 +104,23 @@ public class TopicStatementTest { Assert.assertEquals(filter, recreated.getTopicAttributes().get("tag-filter")); } + @Test + public void testShowCreateTopicPreservesRealtimeMode() { + final TopicMeta topicMeta = + new TopicMeta( + "topic-1", + 0L, + Map.of(TopicConstant.REALTIME_MODE_KEY, TopicConstant.REALTIME_MODE_BATCH_VALUE)); + final CreateTopic recreated = + (CreateTopic) + sqlParser.createStatement( + ShowCreateTopicTask.getShowCreateTopicSQL(topicMeta), ZoneId.systemDefault(), null); + + Assert.assertEquals( + TopicConstant.REALTIME_MODE_BATCH_VALUE, + recreated.getTopicAttributes().get(TopicConstant.REALTIME_MODE_KEY)); + } + @Test public void testAlterTopicStatement() { final Statement statement = diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/topic/TopicMetaTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/topic/TopicMetaTest.java index 26af024d7a9..29a164389f2 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/topic/TopicMetaTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/topic/TopicMetaTest.java @@ -37,11 +37,19 @@ public class TopicMetaTest { new TopicMeta( "topic", 1L, - Map.of("Tag-Filter", "region = \"north\"", "source.CustomOption", "CaseSensitive")); + Map.of( + "Tag-Filter", + "region = \"north\"", + "RealTime.Mode", + " BATCH ", + "source.CustomOption", + "CaseSensitive")); final TopicMeta updated = original.deepCopyWithUpdatedAttributes(Map.of(" TAG-FILTER ", "region = \"North\"")); - Assert.assertEquals(2, updated.getConfig().getAttribute().size()); + Assert.assertEquals(3, updated.getConfig().getAttribute().size()); Assert.assertEquals("region = \"North\"", updated.getConfig().getTagFilter()); + Assert.assertEquals( + TopicConstant.REALTIME_MODE_BATCH_VALUE, updated.getConfig().getRealtimeMode()); Assert.assertEquals( "CaseSensitive", updated.getConfig().getAttribute().get("source.CustomOption")); Assert.assertEquals("region = \"north\"", original.getConfig().getTagFilter());
