This is an automated email from the ASF dual-hosted git repository. Wei-hao-Li pushed a commit to branch fix-issue19207 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 02e5e2f8df17be179efb137108c0be0fe39da384 Author: Weihao Li <[email protected]> AuthorDate: Thu Nov 12 13:21:54 2026 +0800 fix Signed-off-by: Weihao Li <[email protected]> --- .../it/db/it/IoTDBWindowTVFRegionsIT.java | 127 ++++++++++++++++++++- .../planner/optimizations/ParallelizeGrouping.java | 103 +++++++++++++---- .../relational/analyzer/TableFunctionTest.java | 69 +++++++++++ .../planner/WindowFunctionOptimizationTest.java | 18 +++ 4 files changed, 290 insertions(+), 27 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFRegionsIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFRegionsIT.java index f98dabd5e96..72ea4c32541 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFRegionsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFRegionsIT.java @@ -38,9 +38,9 @@ import static org.junit.Assert.fail; /** * Multi-region coverage for nested set-semantic TVFs and window functions whose PARTITION BY - * reorders a subset of the child's partition keys. Every device is a single row of {@code t}, so - * one outer partition spans all DataRegions and any accidental per-region split changes the window - * count. + * reorders or prefixes the child's partition key, and for window functions over a hash aggregation. + * Every device is a single row of {@code t}, so one outer partition spans all DataRegions and any + * accidental per-region split changes the window count. */ @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -48,6 +48,7 @@ public class IoTDBWindowTVFRegionsIT { private static final String DATABASE_NAME = "test"; private static final int DEVICE_NUM = 40; + private static final int HASH_AGGREGATION_ROW_NUM = 6000; @BeforeClass public static void setUp() throws Exception { @@ -60,6 +61,7 @@ public class IoTDBWindowTVFRegionsIT { .setDefaultDataRegionGroupNumPerDatabase(4); EnvFactory.getEnv().initClusterEnvironment(); insertData(); + insertHashAggregationData(); } @AfterClass @@ -86,6 +88,125 @@ public class IoTDBWindowTVFRegionsIT { } } + private static void insertHashAggregationData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE t2(d STRING TAG, s1 INT32 FIELD, s3 INT32 FIELD)"); + for (int start = 0; start < HASH_AGGREGATION_ROW_NUM; start += 1000) { + StringBuilder sql = new StringBuilder("INSERT INTO t2(time, d, s1, s3) VALUES "); + for (int row = start; row < start + 1000; row++) { + // Pseudo-random s1 in {0, 1, 2}; s3 is unique. + int s1 = (int) ((((long) row * 0x9E3779B1L) & 0xFFFFFFFFL) >> 20) % 3; + if (row > start) { + sql.append(", "); + } + sql.append('(') + .append(row + 1) + .append(", 'd1', ") + .append(s1) + .append(", ") + .append(row) + .append(')'); + } + statement.execute(sql.toString()); + } + statement.execute("FLUSH"); + } catch (Exception e) { + e.printStackTrace(); + fail(String.valueOf(e.getMessage())); + } + } + + @Test + public void testSameOrderPrefixPartitionStaysSingleInstance() { + // The outer PARTITION BY (a, b) is a strict prefix of the inner one. One outer partition still + // spans every DataRegion, so the outer SESSION must stay single-instance. + tableResultSetEqualTest( + "SELECT window_start, window_end, count(*) AS cnt FROM SESSION(DATA => (" + + " SELECT window_start AS inner_start, window_end AS inner_end, time, a, b, c, s1 " + + " FROM SESSION(DATA => t PARTITION BY (a, b, c) ORDER BY time, GAP => 1ms)" + + ") PARTITION BY (a, b) ORDER BY time, GAP => 100ms) " + + "GROUP BY window_start, window_end ORDER BY window_start", + new String[] {"window_start", "window_end", "cnt"}, + new String[] {"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.040Z,40,"}, + DATABASE_NAME); + } + + @Test + public void testSameOrderPrefixWindowPartitionStaysSingleInstance() { + tableResultSetEqualTest( + "SELECT max(cnt) AS max_cnt, min(cnt) AS min_cnt FROM (" + + " SELECT count(s1) OVER (PARTITION BY a, b) AS cnt " + + " FROM SESSION(DATA => t PARTITION BY (a, b, c) ORDER BY time, GAP => 1ms))", + new String[] {"max_cnt", "min_cnt"}, + new String[] {"40,40,"}, + DATABASE_NAME); + } + + @Test + public void testHashAggregationWindowKeepsPartitionsWhole() { + // A hash aggregation's output is not clustered by its grouping keys. The window on top must + // see every (s1, s3) row of a partition together, not blocks with mixed s1 values. + tableResultSetEqualTest( + "SELECT s1, cnt, count(*) AS n FROM (" + + " SELECT s1, count(*) OVER (PARTITION BY s1) AS cnt " + + " FROM (SELECT s1, s3 FROM t2 GROUP BY s1, s3)) " + + "GROUP BY s1, cnt ORDER BY s1, cnt", + new String[] {"s1", "cnt", "n"}, + new String[] {"0,1980,1980,", "1,2013,2013,", "2,2007,2007,"}, + DATABASE_NAME); + } + + @Test + public void testHashAggregationSessionKeepsPartitionsWhole() { + tableResultSetEqualTest( + "SELECT s1, window_start, window_end, count(*) AS cnt FROM SESSION(DATA => (" + + " SELECT s1, s3, max(time) AS time FROM t2 GROUP BY s1, s3" + + ") PARTITION BY (s1) ORDER BY time, GAP => 100000ms) " + + "GROUP BY s1, window_start, window_end ORDER BY s1, window_start", + new String[] {"s1", "window_start", "window_end", "cnt"}, + new String[] { + "0,1970-01-01T00:00:00.001Z,1970-01-01T00:00:05.997Z,1980,", + "1,1970-01-01T00:00:00.005Z,1970-01-01T00:00:05.999Z,2013,", + "2,1970-01-01T00:00:00.002Z,1970-01-01T00:00:06.000Z,2007," + }, + DATABASE_NAME); + } + + @Test + public void testSameOrderPrefixTopKRankingStaysSingleInstance() { + // row_number() over (a, b) on top of row_number() over (a, b, c): the outer partition is the + // whole table, so the global top 2 is two rows. A per-region split returns 2 rows per region. + tableResultSetEqualTest( + "SELECT time, c, rn2 FROM (" + + " SELECT *, row_number() OVER (PARTITION BY a, b ORDER BY time) AS rn2 " + + " FROM (SELECT time, a, b, c, s1, " + + " row_number() OVER (PARTITION BY a, b, c ORDER BY time) AS rn1 " + + " FROM t)) " + + "WHERE rn2 <= 2 ORDER BY time", + new String[] {"time", "c", "rn2"}, + new String[] {"1970-01-01T00:00:00.001Z,c00,1,", "1970-01-01T00:00:00.002Z,c01,2,"}, + DATABASE_NAME); + } + + @Test + public void testSameOrderPrefixMatchRecognizeStaysSingleInstance() { + // MATCH_RECOGNIZE partitioned by (a, b) over a SESSION partitioned by (a, b, c): the outer + // partition is the whole table, so the pattern matches one run of 40 rows. + tableResultSetEqualTest( + "SELECT * FROM (SELECT time, a, b, c, s1 FROM SESSION(DATA => t " + + " PARTITION BY (a, b, c) ORDER BY time, GAP => 1ms)) " + + "MATCH_RECOGNIZE (PARTITION BY a, b ORDER BY time " + + " MEASURES RPR_FIRST(A.time) AS first_t, RPR_LAST(A.time) AS last_t, " + + " COUNT(A.s1) AS cnt " + + " ONE ROW PER MATCH PATTERN (A+) DEFINE A AS true) " + + "ORDER BY first_t", + new String[] {"a", "b", "first_t", "last_t", "cnt"}, + new String[] {"A,B,1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.040Z,40,"}, + DATABASE_NAME); + } + @Test public void testReorderedSubsetPartitionStaysSingleInstance() { // The outer PARTITION BY (b, a) is a strict subset of the inner one in a different order. One diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java index 406780fd820..095542f5675 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java @@ -73,10 +73,25 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizati * <li>GroupNode[PK={tag1,tag2}, OK={s1}] -> TopKNode[sort={tag1,tag2,tag3}] * <li>GroupNode[PK={tag1,tag2}, OK={}] -> AggregationNode[group={tag1,tag2}] * <li>GroupNode[PK={tag1,tag2}, OK={s1}] -> AggregationNode[group={tag1,tag2,s1}] - * <li>GroupNode[PK={tag1}, OK={}] -> TableFunctionNode[partition={tag1,tag2}] + * <li>GroupNode[PK={tag1,tag2}] -> TableFunctionNode[partition={tag2,tag1}] * </ul> * </ul> * + * <p>The child states its guarantee in one of two forms, and they are not interchangeable: + * + * <ul> + * <li>A single globally ordered stream ({@link SortNode}, {@link StreamSortNode}, {@link + * TopKNode}). Such a stream is clustered by every prefix of its sort keys, so a positional + * prefix match is enough ({@link Rewriter#checkPrefixMatch}). + * <li>Per-instance partitioning or grouping (a {@link GroupNode}'s own partition keys, a table + * function's or pattern recognition's PARTITION BY, an aggregation's grouping keys). This + * only says that each instance's stream is clustered by those keys: a required prefix may + * span several instances (several DataRegions, an unordered fan-in) and be interleaved, and a + * hash aggregation does not cluster its output at all. Only an exact key set makes the + * consumer's partitions the child's partitions ({@link + * Rewriter#checkPartitionKeysMatch}). + * </ul> + * * <p>Otherwise, the GroupNode should be transformed into a SortNode. */ public class ParallelizeGrouping implements PlanOptimizer { @@ -120,17 +135,13 @@ public class ParallelizeGrouping implements PlanOptimizer { } /** - * We need to make sure: context#partitionKey can be satisfied by the prefix of childOrder. + * Whether the child's single ordered stream satisfies the required partition prefix. * - * <p>When the child exposes exactly the partition keys, the prefix is compared as a column set: - * partition columns are identified by their values, so a different writing order still produces - * the same partitions. This lets nested set-semantic table functions reuse their input - * clustering when the same PARTITION BY columns are listed in a different order. The set - * comparison is deliberately limited to this case: for a strict subset the child's partitions - * are only a refinement of the required ones, and after a fan-in over unordered sources - * (multiple DataRegions, hash aggregation) the subset need not be contiguous in the input. - * - * <p>Otherwise the prefix is compared positionally, as before. + * <p>Used when the child reports a globally sorted stream ({@link SortNode}, {@link + * StreamSortNode}, {@link TopKNode}). Such a stream is clustered by every prefix of its sort + * keys, so the prefix matches positionally. A permutation of the same key set also matches: + * partition columns are identified by their values, and the group node re-establishes the + * requested order inside every partition run. * * <p>The ORDER BY part of the requirement needs no check here. A group node sorts within every * partition run by its full ordering scheme, so the input only has to be clustered; whatever @@ -154,6 +165,32 @@ public class ParallelizeGrouping implements PlanOptimizer { sameKeySet || requiredPrefix.equals(actualPrefix) ? ENABLE_PARALLEL : TO_SORT; } + /** + * Whether the child's per-instance partitioning satisfies the required partition keys. + * + * <p>Used for partition and grouping claims: a {@link GroupNode}'s own partition keys, a table + * function's or pattern recognition's PARTITION BY, and an aggregation's grouping keys. + * + * <p>Only an exact key set, in any order, is accepted. Each instance only guarantees that its + * own stream is clustered by the full key set; a required prefix of those keys can be + * interleaved across instances (several DataRegions, an unordered fan-in) or not clustered at + * all (hash aggregation). Only when the consumer's partitions are the child's partitions can + * the consumer be split per stream. + */ + private void checkPartitionKeysMatch(Context context, List<Symbol> childPartitionKeys) { + if (context.canSkip()) { + return; + } + int partitionKeyCount = context.partitionKeyCount; + List<Symbol> requiredKeys = context.sortKey.getOrderBy().subList(0, partitionKeyCount); + context.canOptimized = + partitionKeyCount == childPartitionKeys.size() + && ImmutableSet.copyOf(requiredKeys) + .equals(ImmutableSet.copyOf(childPartitionKeys)) + ? ENABLE_PARALLEL + : TO_SORT; + } + @Override public PlanNode visitGroup(GroupNode node, Context context) { // A GroupNode without partition keys is a pure global ordering requirement. @@ -166,19 +203,24 @@ public class ParallelizeGrouping implements PlanOptimizer { false); } - checkPrefixMatch( - context, node.getOrderingScheme().getOrderBy().subList(0, node.getPartitionKeyCount())); + // Decide the group's own fate first: the claim it makes to its ancestors depends on + // whether it survives as a partitioned group or degrades to one globally ordered sort. Context newContext = new Context(node.getOrderingScheme(), node.getPartitionKeyCount()); PlanNode child = node.getChild().accept(this, newContext); - switch (newContext.canOptimized) { - case ENABLE_PARALLEL: - GroupNode newNode = (GroupNode) node.clone(); - newNode.addChild(child); - return newNode; - case TO_SORT: - default: + if (newContext.canOptimized == ENABLE_PARALLEL) { + // The group survives and its output is only per-instance partitioned. + checkPartitionKeysMatch( + context, node.getOrderingScheme().getOrderBy().subList(0, node.getPartitionKeyCount())); + if (!context.canSkip() && context.canOptimized != ENABLE_PARALLEL) { return new SortNode(node.getPlanNodeId(), child, node.getOrderingScheme(), false, false); + } + GroupNode newNode = (GroupNode) node.clone(); + newNode.addChild(child); + return newNode; } + // The group degrades to a sort, so its output is one stream ordered by the full scheme. + checkPrefixMatch(context, node.getOrderingScheme().getOrderBy()); + return new SortNode(node.getPlanNodeId(), child, node.getOrderingScheme(), false, false); } @Override @@ -230,7 +272,7 @@ public class ParallelizeGrouping implements PlanOptimizer { if (!dataOrganizationSpecification.isPresent()) { context.canOptimized = TO_SORT; } else { - checkPrefixMatch(context, dataOrganizationSpecification.get().getPartitionBy()); + checkPartitionKeysMatch(context, dataOrganizationSpecification.get().getPartitionBy()); } } return visitPlan(node, context); @@ -251,7 +293,7 @@ public class ParallelizeGrouping implements PlanOptimizer { if (!orderingScheme.isPresent()) { context.canOptimized = TO_SORT; } else { - checkPrefixMatch(context, partitionBy); + checkPartitionKeysMatch(context, partitionBy); } } return visitPlan(node, context); @@ -310,13 +352,26 @@ public class ParallelizeGrouping implements PlanOptimizer { @Override public PlanNode visitAggregation(AggregationNode node, Context context) { - checkPrefixMatch(context, node.getGroupingKeys()); + if (!context.canSkip()) { + if (node.getPreGroupedSymbols().isEmpty()) { + // A hash aggregation's output is not clustered by its grouping keys. + context.canOptimized = TO_SORT; + } else { + checkPartitionKeysMatch(context, node.getGroupingKeys()); + } + } return visitPlan(node, context); } @Override public PlanNode visitAggregationTableScan(AggregationTableScanNode node, Context context) { - checkPrefixMatch(context, node.getGroupingKeys()); + if (!context.canSkip()) { + if (node.getPreGroupedSymbols().isEmpty()) { + context.canOptimized = TO_SORT; + } else { + checkPartitionKeysMatch(context, node.getGroupingKeys()); + } + } return node; } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java index 725d4646f07..feda108180f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java @@ -21,9 +21,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.plan.relational.function.tvf.ForecastTableFunction; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GroupNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TableFunctionProcessorNode; import org.apache.iotdb.commons.udf.builtin.relational.tvf.FFTTableFunction; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; +import org.apache.iotdb.db.queryengine.plan.relational.planner.PlanNodeSearcher; import org.apache.iotdb.db.queryengine.plan.relational.planner.PlanTester; import org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern; import org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.TableFunctionProcessorMatcher; @@ -65,6 +69,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.topK; import static org.apache.iotdb.udf.api.type.Type.DOUBLE; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class TableFunctionTest { @@ -323,6 +328,70 @@ public class TableFunctionTest { sort(orderBy("tag2", "tag1", "time"), tableFunctionProcessor(inner, scanAllColumns()))); } + @Test + public void testNestedSetSemanticSameOrderPrefixFallsBackToSort() { + PlanTester planTester = new PlanTester(); + String sql = + "SELECT * FROM SESSION(DATA => (SELECT * FROM SESSION(DATA => table1 " + + "PARTITION BY (tag1, tag2, tag3) ORDER BY time, GAP => 1m)) " + + "PARTITION BY (tag1, tag2) ORDER BY time, GAP => 2m)"; + planTester.createPlan(sql); + Consumer<TableFunctionProcessorMatcher.Builder> outer = + sessionMatcher("window_start", "window_end", 120_000L); + Consumer<TableFunctionProcessorMatcher.Builder> inner = + sessionMatcher("window_start_0", "window_end_1", 60_000L); + // A strict prefix in the same order has the same problem as a reordered one: one outer + // partition can span several inner ones, which a fan-in does not keep contiguous. + assertPlan( + planTester.getFragmentPlan(0), + output( + tableFunctionProcessor( + outer, + mergeSort(orderBy("tag1", "tag2", "time"), exchange(), exchange(), exchange())))); + assertPlan( + planTester.getFragmentPlan(1), + sort(orderBy("tag1", "tag2", "time"), tableFunctionProcessor(inner, scanAllColumns()))); + } + + @Test + public void testSessionOverGroupByPrefixFallsBackToSort() { + PlanTester planTester = new PlanTester(); + String sql = + "SELECT * FROM SESSION(DATA => (SELECT tag1, tag2, tag3, max(time) AS time FROM table1 " + + "GROUP BY tag1, tag2, tag3) PARTITION BY (tag1, tag2) ORDER BY time, GAP => 1m)"; + LogicalQueryPlan logicalPlan = planTester.createPlan(sql); + // A grouping is only clustered per instance; a strict prefix of the grouping keys must be + // re-sorted before one instance can consume it. + assertOuterFunctionDegradedToSort(logicalPlan); + } + + @Test + public void testSessionOverHashAggregationFallsBackToSort() { + PlanTester planTester = new PlanTester(); + // s1 and s2 are fields, so the aggregation cannot stream and hashes its input instead. + String sql = + "SELECT * FROM SESSION(DATA => (SELECT s1, s2, max(time) AS time FROM table1 " + + "GROUP BY s1, s2) PARTITION BY (s1, s2) ORDER BY time, GAP => 1m)"; + LogicalQueryPlan logicalPlan = planTester.createPlan(sql); + // The grouping keys are exactly the partition keys, but a hash aggregation's output is not + // clustered by them, so the group must still degrade to a sort. + assertOuterFunctionDegradedToSort(logicalPlan); + } + + private static void assertOuterFunctionDegradedToSort(LogicalQueryPlan logicalPlan) { + TableFunctionProcessorNode outerFunction = + PlanNodeSearcher.searchFrom(logicalPlan.getRootNode()) + .where(TableFunctionProcessorNode.class::isInstance) + .findFirst() + .map(TableFunctionProcessorNode.class::cast) + .orElseThrow(() -> new AssertionError("no table function in plan")); + // GroupNode extends SortNode, so both checks are needed. + assertTrue( + "outer function must sort its input instead of reusing the grouping", + outerFunction.getChild() instanceof SortNode + && !(outerFunction.getChild() instanceof GroupNode)); + } + @Test public void testNestedSetSemanticDifferentOrderByKeepsOuterGroup() { PlanTester planTester = new PlanTester(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionOptimizationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionOptimizationTest.java index 4db03ccd3a2..394ea5d8171 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionOptimizationTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionOptimizationTest.java @@ -142,6 +142,24 @@ public class WindowFunctionOptimizationTest { output(project(window(mergeSort(exchange(), exchange(), exchange()))))); } + @Test + public void testSameOrderPrefixWindowPartitionKeepsSingleInstance() { + PlanTester planTester = new PlanTester(); + + String sql = + "SELECT count(s1) OVER (PARTITION BY tag1, tag2) FROM " + + "(SELECT *, max(s1) OVER (PARTITION BY tag1, tag2, tag3) AS m FROM table1)"; + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + PlanMatchPattern tableScan = tableScan("testdb.table1"); + + // The outer PARTITION BY is a strict prefix of the inner one. A grouping is only clustered + // per instance, so the outer window must not reuse it: it stays single-instance above a sort. + assertPlan(logicalQueryPlan, output(project(window(sort(project(group(tableScan))))))); + assertPlan( + planTester.getFragmentPlan(0), + output(project(window(mergeSort(exchange(), exchange(), exchange()))))); + } + @Test public void testTopKRankingPushDown() { PlanTester planTester = new PlanTester();
