Copilot commented on code in PR #19345:
URL: https://github.com/apache/iotdb/pull/19345#discussion_r4640918996


##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueDeliveryTest.java:
##########
@@ -327,6 +328,167 @@ public void 
testLingerBatchRemainsReplayableAfterAddingActiveWriter() throws Exc
     assertTrue(queue.ack(CONSUMER, event.getCommitContext()));
   }
 
+  @Test
+  public void 
testFilteredEmptyEntriesMergeIntoLingerBatchOnOrderedReplayAndRedeliverCleanly()
+      throws Exception {
+    config.setSubscriptionConsensusBatchMaxTabletCount(8);
+    config.setSubscriptionConsensusBatchMaxDelayInMs(60_000);
+    memoryManager = new SubscriptionMemoryManager(tabletBytes * 8);
+    doAnswer(
+            invocation -> {
+              final long physicalTime = ((InsertNode) 
invocation.getArgument(0)).getPhysicalTime();
+              if (physicalTime >= 1002L && physicalTime <= 1004L) {
+                return Collections.emptyList();
+              }
+              return Collections.singletonList(tablet(physicalTime));
+            })
+        .when(converter)
+        .convert(any());
+    writeWal(5, true);
+    final ConsensusPrefetchingQueue queue = newQueue();
+    queue.setOrderMode(TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE);
+    queue.setActiveWriterNodeIds(Set.of(WRITER, 8));
+    writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 8, 0);
+    assertNull(queue.poll(CONSUMER));
+
+    // Process all 5 entries; filtered-empty entries 2..4 must merge into the 
active linger batch
+    // instead of flushing entry 1 prematurely.
+    for (int round = 0; round < 5; round++) {
+      queue.drivePrefetchOnce();
+    }
+    assertEquals(0, queue.getPrefetchedEventCount());
+    assertNull(queue.poll(CONSUMER));
+    assertEquals(tabletBytes * 2, queue.getRetainedTabletBytes());
+    assertTrue(committedProgress().getWriterPositions().isEmpty());
+
+    config.setSubscriptionConsensusBatchMaxDelayInMs(0);
+    writerSafeFrontierTracker.recordAppliedProgress(1005, WRITER, 5);
+    writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 
WRITER, 5);
+    for (int round = 0; round < 20 && queue.getPrefetchedEventCount() == 0; 
round++) {
+      queue.drivePrefetchOnce();
+    }
+
+    final SubscriptionEvent event = queue.poll(CONSUMER);
+    assertNotNull(event);
+    assertNull(queue.poll(CONSUMER));
+    assertEquals(new WriterProgress(1005, 5), 
event.getCommitContext().getWriterProgress());
+    final SubscriptionPollResponse first = deliver(event);
+    assertEquals(Arrays.asList(1001L, 1005L), timestamps(first));
+    assertTrue(event.isPayloadDetached());
+    assertEquals(0L, queue.getRetainedTabletBytes());
+
+    assertTrue(queue.nack(CONSUMER, event.getCommitContext()));
+    assertSame(event, queue.poll(CONSUMER));
+    assertEquals(first.getPayload(), deliver(event).getPayload());
+    assertTrue(queue.ack(CONSUMER, event.getCommitContext()));
+    assertEquals(
+        new WriterProgress(1005, 5), 
committedProgress().getWriterPositions().get(writerId()));

Review Comment:
   These tests mutate `subscriptionConsensusBatchMaxDelayInMs` but do not 
restore the original value. In this PR, `ConsensusPrefetchingQueueTest` was 
updated to save/restore the same config value, suggesting it’s a shared/global 
singleton and leaks across tests. To avoid order-dependent failures, save the 
original delay at the start of each test and restore it in a `finally` block 
(or via a common JUnit setup/teardown helper).



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet() 
throws Exception {
     Assert.assertNull(TabletTagFilter.filter(tablet, 
TagFilterMatcher.matchNone()));
   }
 
+  @Test
+  public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+    final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+    final List<String> literals = new ArrayList<>(maxInLiterals);
+    for (int i = 0; i < maxInLiterals; i++) {
+      literals.add(String.format("\"d_%03d\"", i));
+    }
+    final TagFilterMatcher compiledMatcher =
+        matcher("device IN (" + String.join(", ", literals) + ")");
+
+    final List<String> columnNames = Arrays.asList("region", "device", 
"temperature");
+    final List<TSDataType> dataTypes =
+        Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE);
+    final List<ColumnCategory> categories =
+        Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG, 
ColumnCategory.FIELD);
+    final Tablet tablet = new Tablet("weather", columnNames, dataTypes, 
categories, 5);
+
+    tablet.addTimestamp(0, 1L);
+    tablet.addValue(0, 0, "north");
+    tablet.addValue(0, 1, "d_000");
+    tablet.addValue(0, 2, 10.0);
+
+    tablet.addTimestamp(1, 2L);
+    tablet.addValue(1, 0, "south");
+    tablet.addValue(1, 1, "d_255");
+    tablet.addValue(1, 2, 20.0);
+
+    tablet.addTimestamp(2, 3L);
+    tablet.addValue(2, 0, "west");
+    tablet.addValue(2, 1, "d_999");
+    tablet.addValue(2, 2, 30.0);
+
+    tablet.addTimestamp(3, 4L);
+    tablet.addValue(3, 0, "east");
+    tablet.addValue(3, 2, 40.0);
+
+    tablet.addTimestamp(4, 5L);
+    tablet.addValue(4, 0, "north");
+    tablet.addValue(4, 1, "d_128");
+    tablet.addValue(4, 2, 50.0);
+    tablet.setRowSize(5);
+
+    final Tablet filtered = TabletTagFilter.filter(tablet, compiledMatcher);
+    Assert.assertNotNull(filtered);
+    Assert.assertEquals(3, filtered.getRowSize());
+    Assert.assertEquals(1L, filtered.getTimestamp(0));
+    Assert.assertEquals(2L, filtered.getTimestamp(1));
+    Assert.assertEquals(5L, filtered.getTimestamp(2));
+    Assert.assertEquals("d_000", filtered.getValue(0, 1).toString());
+    Assert.assertEquals("d_255", filtered.getValue(1, 1).toString());
+    Assert.assertEquals("d_128", filtered.getValue(2, 1).toString());

Review Comment:
   Similar to `TagFilterParserTest`, this test hard-codes `d_255` and `d_128`, 
which implicitly assumes `MAX_IN_VALUES >= 256`. If `MAX_IN_VALUES` is 
adjusted, the generated literal list and the expected retained rows can 
diverge. Prefer computing the “last” and “middle” expected device IDs from 
`maxInLiterals` (e.g., `maxInLiterals - 1` and `maxInLiterals / 2`) and using 
those in both the tablet setup and assertions.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParserTest.java:
##########
@@ -105,6 +108,77 @@ public void testRejectInvalidExpressions() {
     assertRejected("region REGEXP \"[\"", "illegal REGEXP pattern");
   }
 
+  @Test
+  public void testValidateAndCompileBuildsInPredicateHashSets() throws 
Exception {
+    final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+    final List<String> literals = new ArrayList<>(maxInLiterals);
+    for (int i = 0; i < maxInLiterals; i++) {
+      literals.add(String.format("\"d_%03d\"", i));
+    }

Review Comment:
   This test hard-codes sentinel values like `d_127` / `d_255`, which makes it 
brittle if `MAX_IN_VALUES` changes (even though the test is otherwise 
parameterized by `MAX_IN_VALUES`). Consider deriving these assertions from 
`maxInLiterals` (e.g., assert on index `0`, `maxInLiterals / 2`, and 
`maxInLiterals - 1`) so the test stays correct if the constant is tuned.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterValidator.java:
##########
@@ -141,10 +145,15 @@ public Void visitInPredicate(final InPredicate node, 
final Void context) {
               MAX_IN_VALUES));
     }
     accountNode(node.getValueList());
+    final Set<String> inValues = new HashSet<>(Math.max(16, (int) 
(values.size() / 0.75f) + 1));
     for (final Expression expression : values) {
-      accountNode(expression);
-      requireStringLiteral(expression, 
DataNodeMiscMessages.COLUMN_FILTER_IN_ELEMENT);
+      // IN list cardinality is already bounded by MAX_IN_VALUES above; do not 
also charge each
+      // literal against MAX_AST_NODES so the documented MAX_IN_VALUES limit 
remains reachable.
+      inValues.add(
+          requireStringLiteral(expression, 
DataNodeMiscMessages.COLUMN_FILTER_IN_ELEMENT)
+              .getValue());
     }
+    compiledInSets.put(node, Collections.unmodifiableSet(inValues));

Review Comment:
   Skipping `accountNode(expression)` for each `IN` literal makes 
`MAX_AST_NODES` effectively bypassable by composing many `IN (...)` predicates 
(each up to `MAX_IN_VALUES`) inside a single expression. This can allow 
significantly larger ASTs than intended and can become a memory/CPU DoS vector 
during parsing/validation/evaluation. A safer approach is to keep a global 
bound that includes total `IN` literal count (e.g., charge `values.size()` 
against `nodeCount` once per `InPredicate`, or increase `MAX_AST_NODES` to 
accommodate `MAX_IN_VALUES` while still enforcing an overall limit).



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueDeliveryTest.java:
##########
@@ -327,6 +328,167 @@ public void 
testLingerBatchRemainsReplayableAfterAddingActiveWriter() throws Exc
     assertTrue(queue.ack(CONSUMER, event.getCommitContext()));
   }
 
+  @Test
+  public void 
testFilteredEmptyEntriesMergeIntoLingerBatchOnOrderedReplayAndRedeliverCleanly()
+      throws Exception {
+    config.setSubscriptionConsensusBatchMaxTabletCount(8);
+    config.setSubscriptionConsensusBatchMaxDelayInMs(60_000);
+    memoryManager = new SubscriptionMemoryManager(tabletBytes * 8);
+    doAnswer(
+            invocation -> {
+              final long physicalTime = ((InsertNode) 
invocation.getArgument(0)).getPhysicalTime();
+              if (physicalTime >= 1002L && physicalTime <= 1004L) {
+                return Collections.emptyList();
+              }
+              return Collections.singletonList(tablet(physicalTime));
+            })
+        .when(converter)
+        .convert(any());
+    writeWal(5, true);
+    final ConsensusPrefetchingQueue queue = newQueue();
+    queue.setOrderMode(TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE);
+    queue.setActiveWriterNodeIds(Set.of(WRITER, 8));
+    writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 8, 0);
+    assertNull(queue.poll(CONSUMER));
+
+    // Process all 5 entries; filtered-empty entries 2..4 must merge into the 
active linger batch
+    // instead of flushing entry 1 prematurely.
+    for (int round = 0; round < 5; round++) {
+      queue.drivePrefetchOnce();
+    }
+    assertEquals(0, queue.getPrefetchedEventCount());
+    assertNull(queue.poll(CONSUMER));
+    assertEquals(tabletBytes * 2, queue.getRetainedTabletBytes());
+    assertTrue(committedProgress().getWriterPositions().isEmpty());
+
+    config.setSubscriptionConsensusBatchMaxDelayInMs(0);
+    writerSafeFrontierTracker.recordAppliedProgress(1005, WRITER, 5);
+    writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 
WRITER, 5);
+    for (int round = 0; round < 20 && queue.getPrefetchedEventCount() == 0; 
round++) {
+      queue.drivePrefetchOnce();
+    }
+
+    final SubscriptionEvent event = queue.poll(CONSUMER);
+    assertNotNull(event);
+    assertNull(queue.poll(CONSUMER));
+    assertEquals(new WriterProgress(1005, 5), 
event.getCommitContext().getWriterProgress());
+    final SubscriptionPollResponse first = deliver(event);
+    assertEquals(Arrays.asList(1001L, 1005L), timestamps(first));

Review Comment:
   These tests mutate `subscriptionConsensusBatchMaxDelayInMs` but do not 
restore the original value. In this PR, `ConsensusPrefetchingQueueTest` was 
updated to save/restore the same config value, suggesting it’s a shared/global 
singleton and leaks across tests. To avoid order-dependent failures, save the 
original delay at the start of each test and restore it in a `finally` block 
(or via a common JUnit setup/teardown helper).



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet() 
throws Exception {
     Assert.assertNull(TabletTagFilter.filter(tablet, 
TagFilterMatcher.matchNone()));
   }
 
+  @Test
+  public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+    final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+    final List<String> literals = new ArrayList<>(maxInLiterals);
+    for (int i = 0; i < maxInLiterals; i++) {
+      literals.add(String.format("\"d_%03d\"", i));
+    }

Review Comment:
   Similar to `TagFilterParserTest`, this test hard-codes `d_255` and `d_128`, 
which implicitly assumes `MAX_IN_VALUES >= 256`. If `MAX_IN_VALUES` is 
adjusted, the generated literal list and the expected retained rows can 
diverge. Prefer computing the “last” and “middle” expected device IDs from 
`maxInLiterals` (e.g., `maxInLiterals - 1` and `maxInLiterals / 2`) and using 
those in both the tablet setup and assertions.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet() 
throws Exception {
     Assert.assertNull(TabletTagFilter.filter(tablet, 
TagFilterMatcher.matchNone()));
   }
 
+  @Test
+  public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+    final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+    final List<String> literals = new ArrayList<>(maxInLiterals);
+    for (int i = 0; i < maxInLiterals; i++) {
+      literals.add(String.format("\"d_%03d\"", i));
+    }
+    final TagFilterMatcher compiledMatcher =
+        matcher("device IN (" + String.join(", ", literals) + ")");
+
+    final List<String> columnNames = Arrays.asList("region", "device", 
"temperature");
+    final List<TSDataType> dataTypes =
+        Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE);
+    final List<ColumnCategory> categories =
+        Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG, 
ColumnCategory.FIELD);
+    final Tablet tablet = new Tablet("weather", columnNames, dataTypes, 
categories, 5);
+
+    tablet.addTimestamp(0, 1L);
+    tablet.addValue(0, 0, "north");
+    tablet.addValue(0, 1, "d_000");
+    tablet.addValue(0, 2, 10.0);
+
+    tablet.addTimestamp(1, 2L);
+    tablet.addValue(1, 0, "south");
+    tablet.addValue(1, 1, "d_255");
+    tablet.addValue(1, 2, 20.0);
+
+    tablet.addTimestamp(2, 3L);
+    tablet.addValue(2, 0, "west");
+    tablet.addValue(2, 1, "d_999");
+    tablet.addValue(2, 2, 30.0);
+
+    tablet.addTimestamp(3, 4L);
+    tablet.addValue(3, 0, "east");
+    tablet.addValue(3, 2, 40.0);
+
+    tablet.addTimestamp(4, 5L);
+    tablet.addValue(4, 0, "north");
+    tablet.addValue(4, 1, "d_128");

Review Comment:
   Similar to `TagFilterParserTest`, this test hard-codes `d_255` and `d_128`, 
which implicitly assumes `MAX_IN_VALUES >= 256`. If `MAX_IN_VALUES` is 
adjusted, the generated literal list and the expected retained rows can 
diverge. Prefer computing the “last” and “middle” expected device IDs from 
`maxInLiterals` (e.g., `maxInLiterals - 1` and `maxInLiterals / 2`) and using 
those in both the tablet setup and assertions.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParserTest.java:
##########
@@ -105,6 +108,77 @@ public void testRejectInvalidExpressions() {
     assertRejected("region REGEXP \"[\"", "illegal REGEXP pattern");
   }
 
+  @Test
+  public void testValidateAndCompileBuildsInPredicateHashSets() throws 
Exception {
+    final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+    final List<String> literals = new ArrayList<>(maxInLiterals);
+    for (int i = 0; i < maxInLiterals; i++) {
+      literals.add(String.format("\"d_%03d\"", i));
+    }
+    final Expression maxInExpr =
+        PARSER.parseAndValidate("device IN (" + String.join(", ", literals) + 
")");
+    final TagFilterValidator.ValidationResult maxValidation =
+        TagFilterValidator.validateAndCompile(maxInExpr);
+
+    Assert.assertEquals(1, maxValidation.getCompiledInSets().size());
+    final Set<String> compiledMaxSet = 
maxValidation.getCompiledInSets().values().iterator().next();
+    Assert.assertEquals(maxInLiterals, compiledMaxSet.size());
+    Assert.assertTrue(compiledMaxSet.contains("d_000"));
+    Assert.assertTrue(compiledMaxSet.contains("d_127"));
+    Assert.assertTrue(compiledMaxSet.contains("d_255"));

Review Comment:
   This test hard-codes sentinel values like `d_127` / `d_255`, which makes it 
brittle if `MAX_IN_VALUES` changes (even though the test is otherwise 
parameterized by `MAX_IN_VALUES`). Consider deriving these assertions from 
`maxInLiterals` (e.g., assert on index `0`, `maxInLiterals / 2`, and 
`maxInLiterals - 1`) so the test stays correct if the constant is tuned.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet() 
throws Exception {
     Assert.assertNull(TabletTagFilter.filter(tablet, 
TagFilterMatcher.matchNone()));
   }
 
+  @Test
+  public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+    final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+    final List<String> literals = new ArrayList<>(maxInLiterals);
+    for (int i = 0; i < maxInLiterals; i++) {
+      literals.add(String.format("\"d_%03d\"", i));
+    }
+    final TagFilterMatcher compiledMatcher =
+        matcher("device IN (" + String.join(", ", literals) + ")");
+
+    final List<String> columnNames = Arrays.asList("region", "device", 
"temperature");
+    final List<TSDataType> dataTypes =
+        Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE);
+    final List<ColumnCategory> categories =
+        Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG, 
ColumnCategory.FIELD);
+    final Tablet tablet = new Tablet("weather", columnNames, dataTypes, 
categories, 5);
+
+    tablet.addTimestamp(0, 1L);
+    tablet.addValue(0, 0, "north");
+    tablet.addValue(0, 1, "d_000");
+    tablet.addValue(0, 2, 10.0);
+
+    tablet.addTimestamp(1, 2L);
+    tablet.addValue(1, 0, "south");
+    tablet.addValue(1, 1, "d_255");

Review Comment:
   Similar to `TagFilterParserTest`, this test hard-codes `d_255` and `d_128`, 
which implicitly assumes `MAX_IN_VALUES >= 256`. If `MAX_IN_VALUES` is 
adjusted, the generated literal list and the expected retained rows can 
diverge. Prefer computing the “last” and “middle” expected device IDs from 
`maxInLiterals` (e.g., `maxInLiterals - 1` and `maxInLiterals / 2`) and using 
those in both the tablet setup and assertions.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to