Copilot commented on code in PR #19345:
URL: https://github.com/apache/iotdb/pull/19345#discussion_r4640918996
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueDeliveryTest.java:
##########
@@ -327,6 +328,167 @@ public void
testLingerBatchRemainsReplayableAfterAddingActiveWriter() throws Exc
assertTrue(queue.ack(CONSUMER, event.getCommitContext()));
}
+ @Test
+ public void
testFilteredEmptyEntriesMergeIntoLingerBatchOnOrderedReplayAndRedeliverCleanly()
+ throws Exception {
+ config.setSubscriptionConsensusBatchMaxTabletCount(8);
+ config.setSubscriptionConsensusBatchMaxDelayInMs(60_000);
+ memoryManager = new SubscriptionMemoryManager(tabletBytes * 8);
+ doAnswer(
+ invocation -> {
+ final long physicalTime = ((InsertNode)
invocation.getArgument(0)).getPhysicalTime();
+ if (physicalTime >= 1002L && physicalTime <= 1004L) {
+ return Collections.emptyList();
+ }
+ return Collections.singletonList(tablet(physicalTime));
+ })
+ .when(converter)
+ .convert(any());
+ writeWal(5, true);
+ final ConsensusPrefetchingQueue queue = newQueue();
+ queue.setOrderMode(TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE);
+ queue.setActiveWriterNodeIds(Set.of(WRITER, 8));
+ writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 8, 0);
+ assertNull(queue.poll(CONSUMER));
+
+ // Process all 5 entries; filtered-empty entries 2..4 must merge into the
active linger batch
+ // instead of flushing entry 1 prematurely.
+ for (int round = 0; round < 5; round++) {
+ queue.drivePrefetchOnce();
+ }
+ assertEquals(0, queue.getPrefetchedEventCount());
+ assertNull(queue.poll(CONSUMER));
+ assertEquals(tabletBytes * 2, queue.getRetainedTabletBytes());
+ assertTrue(committedProgress().getWriterPositions().isEmpty());
+
+ config.setSubscriptionConsensusBatchMaxDelayInMs(0);
+ writerSafeFrontierTracker.recordAppliedProgress(1005, WRITER, 5);
+ writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000,
WRITER, 5);
+ for (int round = 0; round < 20 && queue.getPrefetchedEventCount() == 0;
round++) {
+ queue.drivePrefetchOnce();
+ }
+
+ final SubscriptionEvent event = queue.poll(CONSUMER);
+ assertNotNull(event);
+ assertNull(queue.poll(CONSUMER));
+ assertEquals(new WriterProgress(1005, 5),
event.getCommitContext().getWriterProgress());
+ final SubscriptionPollResponse first = deliver(event);
+ assertEquals(Arrays.asList(1001L, 1005L), timestamps(first));
+ assertTrue(event.isPayloadDetached());
+ assertEquals(0L, queue.getRetainedTabletBytes());
+
+ assertTrue(queue.nack(CONSUMER, event.getCommitContext()));
+ assertSame(event, queue.poll(CONSUMER));
+ assertEquals(first.getPayload(), deliver(event).getPayload());
+ assertTrue(queue.ack(CONSUMER, event.getCommitContext()));
+ assertEquals(
+ new WriterProgress(1005, 5),
committedProgress().getWriterPositions().get(writerId()));
Review Comment:
These tests mutate `subscriptionConsensusBatchMaxDelayInMs` but do not
restore the original value. In this PR, `ConsensusPrefetchingQueueTest` was
updated to save/restore the same config value, suggesting it’s a shared/global
singleton and leaks across tests. To avoid order-dependent failures, save the
original delay at the start of each test and restore it in a `finally` block
(or via a common JUnit setup/teardown helper).
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet()
throws Exception {
Assert.assertNull(TabletTagFilter.filter(tablet,
TagFilterMatcher.matchNone()));
}
+ @Test
+ public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+ final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+ final List<String> literals = new ArrayList<>(maxInLiterals);
+ for (int i = 0; i < maxInLiterals; i++) {
+ literals.add(String.format("\"d_%03d\"", i));
+ }
+ final TagFilterMatcher compiledMatcher =
+ matcher("device IN (" + String.join(", ", literals) + ")");
+
+ final List<String> columnNames = Arrays.asList("region", "device",
"temperature");
+ final List<TSDataType> dataTypes =
+ Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE);
+ final List<ColumnCategory> categories =
+ Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG,
ColumnCategory.FIELD);
+ final Tablet tablet = new Tablet("weather", columnNames, dataTypes,
categories, 5);
+
+ tablet.addTimestamp(0, 1L);
+ tablet.addValue(0, 0, "north");
+ tablet.addValue(0, 1, "d_000");
+ tablet.addValue(0, 2, 10.0);
+
+ tablet.addTimestamp(1, 2L);
+ tablet.addValue(1, 0, "south");
+ tablet.addValue(1, 1, "d_255");
+ tablet.addValue(1, 2, 20.0);
+
+ tablet.addTimestamp(2, 3L);
+ tablet.addValue(2, 0, "west");
+ tablet.addValue(2, 1, "d_999");
+ tablet.addValue(2, 2, 30.0);
+
+ tablet.addTimestamp(3, 4L);
+ tablet.addValue(3, 0, "east");
+ tablet.addValue(3, 2, 40.0);
+
+ tablet.addTimestamp(4, 5L);
+ tablet.addValue(4, 0, "north");
+ tablet.addValue(4, 1, "d_128");
+ tablet.addValue(4, 2, 50.0);
+ tablet.setRowSize(5);
+
+ final Tablet filtered = TabletTagFilter.filter(tablet, compiledMatcher);
+ Assert.assertNotNull(filtered);
+ Assert.assertEquals(3, filtered.getRowSize());
+ Assert.assertEquals(1L, filtered.getTimestamp(0));
+ Assert.assertEquals(2L, filtered.getTimestamp(1));
+ Assert.assertEquals(5L, filtered.getTimestamp(2));
+ Assert.assertEquals("d_000", filtered.getValue(0, 1).toString());
+ Assert.assertEquals("d_255", filtered.getValue(1, 1).toString());
+ Assert.assertEquals("d_128", filtered.getValue(2, 1).toString());
Review Comment:
Similar to `TagFilterParserTest`, this test hard-codes `d_255` and `d_128`,
which implicitly assumes `MAX_IN_VALUES >= 256`. If `MAX_IN_VALUES` is
adjusted, the generated literal list and the expected retained rows can
diverge. Prefer computing the “last” and “middle” expected device IDs from
`maxInLiterals` (e.g., `maxInLiterals - 1` and `maxInLiterals / 2`) and using
those in both the tablet setup and assertions.
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParserTest.java:
##########
@@ -105,6 +108,77 @@ public void testRejectInvalidExpressions() {
assertRejected("region REGEXP \"[\"", "illegal REGEXP pattern");
}
+ @Test
+ public void testValidateAndCompileBuildsInPredicateHashSets() throws
Exception {
+ final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+ final List<String> literals = new ArrayList<>(maxInLiterals);
+ for (int i = 0; i < maxInLiterals; i++) {
+ literals.add(String.format("\"d_%03d\"", i));
+ }
Review Comment:
This test hard-codes sentinel values like `d_127` / `d_255`, which makes it
brittle if `MAX_IN_VALUES` changes (even though the test is otherwise
parameterized by `MAX_IN_VALUES`). Consider deriving these assertions from
`maxInLiterals` (e.g., assert on index `0`, `maxInLiterals / 2`, and
`maxInLiterals - 1`) so the test stays correct if the constant is tuned.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterValidator.java:
##########
@@ -141,10 +145,15 @@ public Void visitInPredicate(final InPredicate node,
final Void context) {
MAX_IN_VALUES));
}
accountNode(node.getValueList());
+ final Set<String> inValues = new HashSet<>(Math.max(16, (int)
(values.size() / 0.75f) + 1));
for (final Expression expression : values) {
- accountNode(expression);
- requireStringLiteral(expression,
DataNodeMiscMessages.COLUMN_FILTER_IN_ELEMENT);
+ // IN list cardinality is already bounded by MAX_IN_VALUES above; do not
also charge each
+ // literal against MAX_AST_NODES so the documented MAX_IN_VALUES limit
remains reachable.
+ inValues.add(
+ requireStringLiteral(expression,
DataNodeMiscMessages.COLUMN_FILTER_IN_ELEMENT)
+ .getValue());
}
+ compiledInSets.put(node, Collections.unmodifiableSet(inValues));
Review Comment:
Skipping `accountNode(expression)` for each `IN` literal makes
`MAX_AST_NODES` effectively bypassable by composing many `IN (...)` predicates
(each up to `MAX_IN_VALUES`) inside a single expression. This can allow
significantly larger ASTs than intended and can become a memory/CPU DoS vector
during parsing/validation/evaluation. A safer approach is to keep a global
bound that includes total `IN` literal count (e.g., charge `values.size()`
against `nodeCount` once per `InPredicate`, or increase `MAX_AST_NODES` to
accommodate `MAX_IN_VALUES` while still enforcing an overall limit).
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueDeliveryTest.java:
##########
@@ -327,6 +328,167 @@ public void
testLingerBatchRemainsReplayableAfterAddingActiveWriter() throws Exc
assertTrue(queue.ack(CONSUMER, event.getCommitContext()));
}
+ @Test
+ public void
testFilteredEmptyEntriesMergeIntoLingerBatchOnOrderedReplayAndRedeliverCleanly()
+ throws Exception {
+ config.setSubscriptionConsensusBatchMaxTabletCount(8);
+ config.setSubscriptionConsensusBatchMaxDelayInMs(60_000);
+ memoryManager = new SubscriptionMemoryManager(tabletBytes * 8);
+ doAnswer(
+ invocation -> {
+ final long physicalTime = ((InsertNode)
invocation.getArgument(0)).getPhysicalTime();
+ if (physicalTime >= 1002L && physicalTime <= 1004L) {
+ return Collections.emptyList();
+ }
+ return Collections.singletonList(tablet(physicalTime));
+ })
+ .when(converter)
+ .convert(any());
+ writeWal(5, true);
+ final ConsensusPrefetchingQueue queue = newQueue();
+ queue.setOrderMode(TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE);
+ queue.setActiveWriterNodeIds(Set.of(WRITER, 8));
+ writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000, 8, 0);
+ assertNull(queue.poll(CONSUMER));
+
+ // Process all 5 entries; filtered-empty entries 2..4 must merge into the
active linger batch
+ // instead of flushing entry 1 prematurely.
+ for (int round = 0; round < 5; round++) {
+ queue.drivePrefetchOnce();
+ }
+ assertEquals(0, queue.getPrefetchedEventCount());
+ assertNull(queue.poll(CONSUMER));
+ assertEquals(tabletBytes * 2, queue.getRetainedTabletBytes());
+ assertTrue(committedProgress().getWriterPositions().isEmpty());
+
+ config.setSubscriptionConsensusBatchMaxDelayInMs(0);
+ writerSafeFrontierTracker.recordAppliedProgress(1005, WRITER, 5);
+ writerSafeFrontierTracker.observePendingWriterSafeTimeBarrier(2000,
WRITER, 5);
+ for (int round = 0; round < 20 && queue.getPrefetchedEventCount() == 0;
round++) {
+ queue.drivePrefetchOnce();
+ }
+
+ final SubscriptionEvent event = queue.poll(CONSUMER);
+ assertNotNull(event);
+ assertNull(queue.poll(CONSUMER));
+ assertEquals(new WriterProgress(1005, 5),
event.getCommitContext().getWriterProgress());
+ final SubscriptionPollResponse first = deliver(event);
+ assertEquals(Arrays.asList(1001L, 1005L), timestamps(first));
Review Comment:
These tests mutate `subscriptionConsensusBatchMaxDelayInMs` but do not
restore the original value. In this PR, `ConsensusPrefetchingQueueTest` was
updated to save/restore the same config value, suggesting it’s a shared/global
singleton and leaks across tests. To avoid order-dependent failures, save the
original delay at the start of each test and restore it in a `finally` block
(or via a common JUnit setup/teardown helper).
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet()
throws Exception {
Assert.assertNull(TabletTagFilter.filter(tablet,
TagFilterMatcher.matchNone()));
}
+ @Test
+ public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+ final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+ final List<String> literals = new ArrayList<>(maxInLiterals);
+ for (int i = 0; i < maxInLiterals; i++) {
+ literals.add(String.format("\"d_%03d\"", i));
+ }
Review Comment:
Similar to `TagFilterParserTest`, this test hard-codes `d_255` and `d_128`,
which implicitly assumes `MAX_IN_VALUES >= 256`. If `MAX_IN_VALUES` is
adjusted, the generated literal list and the expected retained rows can
diverge. Prefer computing the “last” and “middle” expected device IDs from
`maxInLiterals` (e.g., `maxInLiterals - 1` and `maxInLiterals / 2`) and using
those in both the tablet setup and assertions.
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet()
throws Exception {
Assert.assertNull(TabletTagFilter.filter(tablet,
TagFilterMatcher.matchNone()));
}
+ @Test
+ public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+ final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+ final List<String> literals = new ArrayList<>(maxInLiterals);
+ for (int i = 0; i < maxInLiterals; i++) {
+ literals.add(String.format("\"d_%03d\"", i));
+ }
+ final TagFilterMatcher compiledMatcher =
+ matcher("device IN (" + String.join(", ", literals) + ")");
+
+ final List<String> columnNames = Arrays.asList("region", "device",
"temperature");
+ final List<TSDataType> dataTypes =
+ Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE);
+ final List<ColumnCategory> categories =
+ Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG,
ColumnCategory.FIELD);
+ final Tablet tablet = new Tablet("weather", columnNames, dataTypes,
categories, 5);
+
+ tablet.addTimestamp(0, 1L);
+ tablet.addValue(0, 0, "north");
+ tablet.addValue(0, 1, "d_000");
+ tablet.addValue(0, 2, 10.0);
+
+ tablet.addTimestamp(1, 2L);
+ tablet.addValue(1, 0, "south");
+ tablet.addValue(1, 1, "d_255");
+ tablet.addValue(1, 2, 20.0);
+
+ tablet.addTimestamp(2, 3L);
+ tablet.addValue(2, 0, "west");
+ tablet.addValue(2, 1, "d_999");
+ tablet.addValue(2, 2, 30.0);
+
+ tablet.addTimestamp(3, 4L);
+ tablet.addValue(3, 0, "east");
+ tablet.addValue(3, 2, 40.0);
+
+ tablet.addTimestamp(4, 5L);
+ tablet.addValue(4, 0, "north");
+ tablet.addValue(4, 1, "d_128");
Review Comment:
Similar to `TagFilterParserTest`, this test hard-codes `d_255` and `d_128`,
which implicitly assumes `MAX_IN_VALUES >= 256`. If `MAX_IN_VALUES` is
adjusted, the generated literal list and the expected retained rows can
diverge. Prefer computing the “last” and “middle” expected device IDs from
`maxInLiterals` (e.g., `maxInLiterals - 1` and `maxInLiterals / 2`) and using
those in both the tablet setup and assertions.
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TagFilterParserTest.java:
##########
@@ -105,6 +108,77 @@ public void testRejectInvalidExpressions() {
assertRejected("region REGEXP \"[\"", "illegal REGEXP pattern");
}
+ @Test
+ public void testValidateAndCompileBuildsInPredicateHashSets() throws
Exception {
+ final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+ final List<String> literals = new ArrayList<>(maxInLiterals);
+ for (int i = 0; i < maxInLiterals; i++) {
+ literals.add(String.format("\"d_%03d\"", i));
+ }
+ final Expression maxInExpr =
+ PARSER.parseAndValidate("device IN (" + String.join(", ", literals) +
")");
+ final TagFilterValidator.ValidationResult maxValidation =
+ TagFilterValidator.validateAndCompile(maxInExpr);
+
+ Assert.assertEquals(1, maxValidation.getCompiledInSets().size());
+ final Set<String> compiledMaxSet =
maxValidation.getCompiledInSets().values().iterator().next();
+ Assert.assertEquals(maxInLiterals, compiledMaxSet.size());
+ Assert.assertTrue(compiledMaxSet.contains("d_000"));
+ Assert.assertTrue(compiledMaxSet.contains("d_127"));
+ Assert.assertTrue(compiledMaxSet.contains("d_255"));
Review Comment:
This test hard-codes sentinel values like `d_127` / `d_255`, which makes it
brittle if `MAX_IN_VALUES` changes (even though the test is otherwise
parameterized by `MAX_IN_VALUES`). Consider deriving these assertions from
`maxInLiterals` (e.g., assert on index `0`, `maxInLiterals / 2`, and
`maxInLiterals - 1`) so the test stays correct if the constant is tuned.
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/tagfilter/TabletTagFilterTest.java:
##########
@@ -134,6 +135,59 @@ public void testTrivialFilterReturnsOriginalTablet()
throws Exception {
Assert.assertNull(TabletTagFilter.filter(tablet,
TagFilterMatcher.matchNone()));
}
+ @Test
+ public void testInPredicateWithMaxValuesAndNullSemantics() throws Exception {
+ final int maxInLiterals = TagFilterValidator.MAX_IN_VALUES;
+ final List<String> literals = new ArrayList<>(maxInLiterals);
+ for (int i = 0; i < maxInLiterals; i++) {
+ literals.add(String.format("\"d_%03d\"", i));
+ }
+ final TagFilterMatcher compiledMatcher =
+ matcher("device IN (" + String.join(", ", literals) + ")");
+
+ final List<String> columnNames = Arrays.asList("region", "device",
"temperature");
+ final List<TSDataType> dataTypes =
+ Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE);
+ final List<ColumnCategory> categories =
+ Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG,
ColumnCategory.FIELD);
+ final Tablet tablet = new Tablet("weather", columnNames, dataTypes,
categories, 5);
+
+ tablet.addTimestamp(0, 1L);
+ tablet.addValue(0, 0, "north");
+ tablet.addValue(0, 1, "d_000");
+ tablet.addValue(0, 2, 10.0);
+
+ tablet.addTimestamp(1, 2L);
+ tablet.addValue(1, 0, "south");
+ tablet.addValue(1, 1, "d_255");
Review Comment:
Similar to `TagFilterParserTest`, this test hard-codes `d_255` and `d_128`,
which implicitly assumes `MAX_IN_VALUES >= 256`. If `MAX_IN_VALUES` is
adjusted, the generated literal list and the expected retained rows can
diverge. Prefer computing the “last” and “middle” expected device IDs from
`maxInLiterals` (e.g., `maxInLiterals - 1` and `maxInLiterals / 2`) and using
those in both the tablet setup and assertions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]