VGalaxies commented on code in PR #19345:
URL: https://github.com/apache/iotdb/pull/19345#discussion_r4641498179


##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterValidator.java:
##########
@@ -141,10 +145,15 @@ public Void visitInPredicate(final InPredicate node, 
final Void context) {
               MAX_IN_VALUES));
     }
     accountNode(node.getValueList());
+    final Set<String> inValues = new HashSet<>(Math.max(16, (int) 
(values.size() / 0.75f) + 1));
     for (final Expression expression : values) {
-      accountNode(expression);
-      requireStringLiteral(expression, 
DataNodeMiscMessages.COLUMN_FILTER_IN_ELEMENT);
+      // IN list cardinality is already bounded by MAX_IN_VALUES above; do not 
also charge each
+      // literal against MAX_AST_NODES so the documented MAX_IN_VALUES limit 
remains reachable.
+      inValues.add(
+          requireStringLiteral(expression, 
DataNodeMiscMessages.COLUMN_FILTER_IN_ELEMENT)
+              .getValue());
     }
+    compiledInSets.put(node, Collections.unmodifiableSet(inValues));

Review Comment:
   Thanks for the note. Total IN literals remain bounded rather than unbounded: 
each `InPredicate` still charges its predicate, field identifier, and 
`InListExpression` against `MAX_AST_NODES` (256), and each individual `IN` list 
is capped at `MAX_IN_VALUES` (256). Keeping per-literal accounting out of 
`MAX_AST_NODES` is intentional so a single `IN` list of up to `MAX_IN_VALUES` 
(256) literals stays reachable as documented. We can introduce a separate 
aggregate cap later if needed.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueDeliveryTest.java:
##########
@@ -327,6 +328,167 @@ public void 
testLingerBatchRemainsReplayableAfterAddingActiveWriter() throws Exc
     assertTrue(queue.ack(CONSUMER, event.getCommitContext()));
   }
 
+  @Test
+  public void 
testFilteredEmptyEntriesMergeIntoLingerBatchOnOrderedReplayAndRedeliverCleanly()
+      throws Exception {
+    config.setSubscriptionConsensusBatchMaxTabletCount(8);
+    config.setSubscriptionConsensusBatchMaxDelayInMs(60_000);
+    memoryManager = new SubscriptionMemoryManager(tabletBytes * 8);
+    doAnswer(
+            invocation -> {
+              final long physicalTime = ((InsertNode) 
invocation.getArgument(0)).getPhysicalTime();
+              if (physicalTime >= 1002L && physicalTime <= 1004L) {
+                return Collections.emptyList();
+              }
+              return Collections.singletonList(tablet(physicalTime));
+            })
+        .when(converter)
+        .convert(any());
+    writeWal(5, true);
+    final ConsensusPrefetchingQueue queue = newQueue();
+    queue.setOrderMode(TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE);
+    queue.setActiveWriterNodeIds(Set.of(WRITER, 8));
+    writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 8, 0);
+    assertNull(queue.poll(CONSUMER));
+
+    // Process all 5 entries; filtered-empty entries 2..4 must merge into the 
active linger batch
+    // instead of flushing entry 1 prematurely.
+    for (int round = 0; round < 5; round++) {
+      queue.drivePrefetchOnce();
+    }
+    assertEquals(0, queue.getPrefetchedEventCount());
+    assertNull(queue.poll(CONSUMER));
+    assertEquals(tabletBytes * 2, queue.getRetainedTabletBytes());
+    assertTrue(committedProgress().getWriterPositions().isEmpty());
+
+    config.setSubscriptionConsensusBatchMaxDelayInMs(0);
+    writerSafeFrontierTracker.recordAppliedProgress(1005, WRITER, 5);
+    writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 
WRITER, 5);
+    for (int round = 0; round < 20 && queue.getPrefetchedEventCount() == 0; 
round++) {
+      queue.drivePrefetchOnce();
+    }
+
+    final SubscriptionEvent event = queue.poll(CONSUMER);
+    assertNotNull(event);
+    assertNull(queue.poll(CONSUMER));
+    assertEquals(new WriterProgress(1005, 5), 
event.getCommitContext().getWriterProgress());
+    final SubscriptionPollResponse first = deliver(event);
+    assertEquals(Arrays.asList(1001L, 1005L), timestamps(first));

Review Comment:
   `ConsensusPrefetchingQueueDeliveryTest` already saves `originalMaxDelay` in 
`@Before setUp()` (line 121) and restores 
`config.setSubscriptionConsensusBatchMaxDelayInMs(originalMaxDelay)` in `@After 
tearDown()` inside a `finally` block (line 167), so this mutation does not leak 
across tests.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueDeliveryTest.java:
##########
@@ -327,6 +328,167 @@ public void 
testLingerBatchRemainsReplayableAfterAddingActiveWriter() throws Exc
     assertTrue(queue.ack(CONSUMER, event.getCommitContext()));
   }
 
+  @Test
+  public void 
testFilteredEmptyEntriesMergeIntoLingerBatchOnOrderedReplayAndRedeliverCleanly()
+      throws Exception {
+    config.setSubscriptionConsensusBatchMaxTabletCount(8);
+    config.setSubscriptionConsensusBatchMaxDelayInMs(60_000);
+    memoryManager = new SubscriptionMemoryManager(tabletBytes * 8);
+    doAnswer(
+            invocation -> {
+              final long physicalTime = ((InsertNode) 
invocation.getArgument(0)).getPhysicalTime();
+              if (physicalTime >= 1002L && physicalTime <= 1004L) {
+                return Collections.emptyList();
+              }
+              return Collections.singletonList(tablet(physicalTime));
+            })
+        .when(converter)
+        .convert(any());
+    writeWal(5, true);
+    final ConsensusPrefetchingQueue queue = newQueue();
+    queue.setOrderMode(TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE);
+    queue.setActiveWriterNodeIds(Set.of(WRITER, 8));
+    writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 8, 0);
+    assertNull(queue.poll(CONSUMER));
+
+    // Process all 5 entries; filtered-empty entries 2..4 must merge into the 
active linger batch
+    // instead of flushing entry 1 prematurely.
+    for (int round = 0; round < 5; round++) {
+      queue.drivePrefetchOnce();
+    }
+    assertEquals(0, queue.getPrefetchedEventCount());
+    assertNull(queue.poll(CONSUMER));
+    assertEquals(tabletBytes * 2, queue.getRetainedTabletBytes());
+    assertTrue(committedProgress().getWriterPositions().isEmpty());
+
+    config.setSubscriptionConsensusBatchMaxDelayInMs(0);
+    writerSafeFrontierTracker.recordAppliedProgress(1005, WRITER, 5);
+    writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 
WRITER, 5);
+    for (int round = 0; round < 20 && queue.getPrefetchedEventCount() == 0; 
round++) {
+      queue.drivePrefetchOnce();
+    }
+
+    final SubscriptionEvent event = queue.poll(CONSUMER);
+    assertNotNull(event);
+    assertNull(queue.poll(CONSUMER));
+    assertEquals(new WriterProgress(1005, 5), 
event.getCommitContext().getWriterProgress());
+    final SubscriptionPollResponse first = deliver(event);
+    assertEquals(Arrays.asList(1001L, 1005L), timestamps(first));
+    assertTrue(event.isPayloadDetached());
+    assertEquals(0L, queue.getRetainedTabletBytes());
+
+    assertTrue(queue.nack(CONSUMER, event.getCommitContext()));
+    assertSame(event, queue.poll(CONSUMER));
+    assertEquals(first.getPayload(), deliver(event).getPayload());
+    assertTrue(queue.ack(CONSUMER, event.getCommitContext()));
+    assertEquals(
+        new WriterProgress(1005, 5), 
committedProgress().getWriterPositions().get(writerId()));

Review Comment:
   `ConsensusPrefetchingQueueDeliveryTest` already saves `originalMaxDelay` in 
`@Before setUp()` (line 121) and restores 
`config.setSubscriptionConsensusBatchMaxDelayInMs(originalMaxDelay)` in `@After 
tearDown()` inside a `finally` block (line 167), so this mutation does not leak 
across tests.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet() 
throws Exception {
     Assert.assertNull(TabletTagFilter.filter(tablet, 
TagFilterMatcher.matchNone()));
   }
 
+  @Test
+  public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+    final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+    final List<String> literals = new ArrayList<>(maxInLiterals);
+    for (int i = 0; i < maxInLiterals; i++) {
+      literals.add(String.format("\"d_%03d\"", i));
+    }

Review Comment:
   Thanks for the suggestion. `MAX_IN_VALUES` is currently fixed at 256 in 
`TagFilterValidator` and this PR is already approved; we will keep the test 
diff minimal here and derive the sentinel indices from `maxInLiterals` if 
`MAX_IN_VALUES` is changed in a future PR.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet() 
throws Exception {
     Assert.assertNull(TabletTagFilter.filter(tablet, 
TagFilterMatcher.matchNone()));
   }
 
+  @Test
+  public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+    final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+    final List<String> literals = new ArrayList<>(maxInLiterals);
+    for (int i = 0; i < maxInLiterals; i++) {
+      literals.add(String.format("\"d_%03d\"", i));
+    }
+    final TagFilterMatcher compiledMatcher =
+        matcher("device IN (" + String.join(", ", literals) + ")");
+
+    final List<String> columnNames = Arrays.asList("region", "device", 
"temperature");
+    final List<TSDataType> dataTypes =
+        Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE);
+    final List<ColumnCategory> categories =
+        Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG, 
ColumnCategory.FIELD);
+    final Tablet tablet = new Tablet("weather", columnNames, dataTypes, 
categories, 5);
+
+    tablet.addTimestamp(0, 1L);
+    tablet.addValue(0, 0, "north");
+    tablet.addValue(0, 1, "d_000");
+    tablet.addValue(0, 2, 10.0);
+
+    tablet.addTimestamp(1, 2L);
+    tablet.addValue(1, 0, "south");
+    tablet.addValue(1, 1, "d_255");

Review Comment:
   Thanks for the suggestion. `MAX_IN_VALUES` is currently fixed at 256 in 
`TagFilterValidator` and this PR is already approved; we will keep the test 
diff minimal here and derive the sentinel indices from `maxInLiterals` if 
`MAX_IN_VALUES` is changed in a future PR.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to