This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 050fcb5aa98 Parallelize process of set semantic TVF when the conetent
of partition keys were same (#19155)
050fcb5aa98 is described below
commit 050fcb5aa98bd5e25019d60b77073998e4df3fc9
Author: Weihao Li <[email protected]>
AuthorDate: Tue Nov 10 16:10:15 2026 +0800
Parallelize process of set semantic TVF when the conetent of partition keys
were same (#19155)
---
.../relational/it/db/it/IoTDBWindowTVFIT.java | 94 ++++++++
.../it/db/it/IoTDBWindowTVFRegionsIT.java | 135 ++++++++++++
.../planner/optimizations/ParallelizeGrouping.java | 40 +++-
.../relational/analyzer/TableFunctionTest.java | 241 +++++++++++++++++++++
.../planner/WindowFunctionOptimizationTest.java | 19 ++
.../planner/assertions/MergeSortMatcher.java | 67 ++++++
.../planner/assertions/PlanMatchPattern.java | 4 +
7 files changed, 588 insertions(+), 12 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java
index 0adddc0cf69..2f82952b8ea 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java
@@ -251,6 +251,100 @@ public class IoTDBWindowTVFIT {
DATABASE_NAME);
}
+ @Test
+ public void testNestedSessionWithPermutedPartitionColumns() {
+ String[] expectedHeader =
+ new String[] {
+ "window_start",
+ "window_end",
+ "inner_start",
+ "inner_end",
+ "time",
+ "factory_id",
+ "device_id",
+ "s1"
+ };
+ String[] retArray =
+ new String[] {
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.001Z,F1,device_1,10.0,",
+
"1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.005Z,F1,device_1,15.0,",
+
"1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.009Z,F1,device_1,5.0,",
+
"1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,F1,device_2,20.0,",
+
"1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,F1,device_2,25.0,",
+
"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,F2,device_1,30.0,"
+ };
+ // The outer PARTITION BY lists the same columns as the inner one in a
different order; the
+ // result must be identical to the same-order variant (next test).
+ String sql =
+ "SELECT window_start, window_end, inner_start, inner_end, time,
factory_id, device_id, s1 "
+ + "FROM SESSION(DATA => ("
+ + " SELECT window_start AS inner_start, window_end AS
inner_end, time, factory_id, device_id, s1 "
+ + " FROM SESSION(DATA => table5 PARTITION BY (factory_id,
device_id) ORDER BY time, GAP => 5ms)"
+ + ") PARTITION BY (device_id, factory_id) ORDER BY time, GAP =>
3ms) "
+ + "ORDER BY factory_id, device_id, time";
+ tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
+ }
+
+ @Test
+ public void testNestedSessionWithSamePartitionColumnOrder() {
+ String[] expectedHeader =
+ new String[] {
+ "window_start",
+ "window_end",
+ "inner_start",
+ "inner_end",
+ "time",
+ "factory_id",
+ "device_id",
+ "s1"
+ };
+ String[] retArray =
+ new String[] {
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.001Z,F1,device_1,10.0,",
+
"1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.005Z,F1,device_1,15.0,",
+
"1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.009Z,F1,device_1,5.0,",
+
"1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,F1,device_2,20.0,",
+
"1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,F1,device_2,25.0,",
+
"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,F2,device_1,30.0,"
+ };
+ // Reference case: both levels list the PARTITION BY columns in the same
order.
+ String sql =
+ "SELECT window_start, window_end, inner_start, inner_end, time,
factory_id, device_id, s1 "
+ + "FROM SESSION(DATA => ("
+ + " SELECT window_start AS inner_start, window_end AS
inner_end, time, factory_id, device_id, s1 "
+ + " FROM SESSION(DATA => table5 PARTITION BY (factory_id,
device_id) ORDER BY time, GAP => 5ms)"
+ + ") PARTITION BY (factory_id, device_id) ORDER BY time, GAP =>
3ms) "
+ + "ORDER BY factory_id, device_id, time";
+ tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
+ }
+
+ @Test
+ public void testThreeLevelNestedSessionWithPermutedPartitionColumns() {
+ String[] expectedHeader =
+ new String[] {"window_start", "window_end", "time", "factory_id",
"device_id", "s1"};
+ String[] retArray =
+ new String[] {
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.001Z,F1,device_1,10.0,",
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.005Z,F1,device_1,15.0,",
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.009Z,1970-01-01T00:00:00.009Z,F1,device_1,5.0,",
+
"1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.002Z,F1,device_2,20.0,",
+
"1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,1970-01-01T00:00:00.011Z,F1,device_2,25.0,",
+
"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.003Z,F2,device_1,30.0,"
+ };
+ // Adjacent levels list the same PARTITION BY columns in different orders.
+ String sql =
+ "SELECT window_start, window_end, time, factory_id, device_id, s1 "
+ + "FROM SESSION(DATA => ("
+ + " SELECT window_start AS mid_start, window_end AS mid_end,
inner_start, inner_end, time, factory_id, device_id, s1 "
+ + " FROM SESSION(DATA => ("
+ + " SELECT window_start AS inner_start, window_end AS
inner_end, time, factory_id, device_id, s1 "
+ + " FROM SESSION(DATA => table5 PARTITION BY (factory_id,
device_id) ORDER BY time, GAP => 5ms)"
+ + " ) PARTITION BY (device_id, factory_id) ORDER BY time, GAP
=> 3ms)"
+ + ") PARTITION BY (factory_id, device_id) ORDER BY time, GAP =>
5ms) "
+ + "ORDER BY factory_id, device_id, time";
+ tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
+ }
+
@Test
public void testVariationFunction() {
String[] expectedHeader = new String[] {"window_index", "time",
"stock_id", "price", "s1"};
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFRegionsIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFRegionsIT.java
new file mode 100644
index 00000000000..f98dabd5e96
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFRegionsIT.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.db.it;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
+import static org.junit.Assert.fail;
+
+/**
+ * Multi-region coverage for nested set-semantic TVFs and window functions
whose PARTITION BY
+ * reorders a subset of the child's partition keys. Every device is a single
row of {@code t}, so
+ * one outer partition spans all DataRegions and any accidental per-region
split changes the window
+ * count.
+ */
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBWindowTVFRegionsIT {
+
+ private static final String DATABASE_NAME = "test";
+ private static final int DEVICE_NUM = 40;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSchemaRegionGroupExtensionPolicy("CUSTOM")
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultSchemaRegionGroupNumPerDatabase(1)
+ .setDefaultDataRegionGroupNumPerDatabase(4);
+ EnvFactory.getEnv().initClusterEnvironment();
+ insertData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ private static void insertData() {
+ try (Connection connection = EnvFactory.getEnv().getTableConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + DATABASE_NAME);
+ statement.execute("USE " + DATABASE_NAME);
+ statement.execute(
+ "CREATE TABLE t(a STRING TAG, b STRING TAG, c STRING TAG, s1 DOUBLE
FIELD)");
+ for (int i = 0; i < DEVICE_NUM; i++) {
+ statement.execute(
+ String.format(
+ "INSERT INTO t(time, a, b, c, s1) VALUES (%d, 'A', 'B',
'c%02d', %d.0)",
+ i + 1, i, i + 1));
+ }
+ statement.execute("FLUSH");
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testReorderedSubsetPartitionStaysSingleInstance() {
+ // The outer PARTITION BY (b, a) is a strict subset of the inner one in a
different order. One
+ // outer partition spans every DataRegion, so the outer SESSION must stay
single-instance:
+ // exactly one window covering all 40 devices. A per-region split would
return one window per
+ // region, each counting only that region's devices.
+ tableResultSetEqualTest(
+ "SELECT window_start, window_end, count(*) AS cnt FROM SESSION(DATA =>
("
+ + " SELECT window_start AS inner_start, window_end AS
inner_end, time, a, b, c, s1 "
+ + " FROM SESSION(DATA => t PARTITION BY (a, b, c) ORDER BY
time, GAP => 1ms)"
+ + ") PARTITION BY (b, a) ORDER BY time, GAP => 100ms) "
+ + "GROUP BY window_start, window_end ORDER BY window_start",
+ new String[] {"window_start", "window_end", "cnt"},
+ new String[] {"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.040Z,40,"},
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testReorderedSubsetWindowPartitionStaysSingleInstance() {
+ // Same shape for a plain window function: count(s1) over (b, a) must see
rows from every
+ // region, so every row gets the full-device count.
+ tableResultSetEqualTest(
+ "SELECT max(cnt) AS max_cnt, min(cnt) AS min_cnt FROM ("
+ + " SELECT count(s1) OVER (PARTITION BY b, a) AS cnt "
+ + " FROM SESSION(DATA => t PARTITION BY (a, b, c) ORDER BY
time, GAP => 1ms))",
+ new String[] {"max_cnt", "min_cnt"},
+ new String[] {"40,40,"},
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testPermutedExactPartitionColumnsStayParallel() {
+ // The intended case: the outer PARTITION BY lists exactly the inner
columns in a different
+ // order, so each device is its own outer partition and both levels run
per region. 40 devices
+ // yield 40 windows of one row each.
+ tableResultSetEqualTest(
+ "SELECT count(*) AS windows, max(cnt) AS max_cnt, min(cnt) AS min_cnt
FROM ("
+ + " SELECT window_start, window_end, count(*) AS cnt FROM
SESSION(DATA => ("
+ + " SELECT window_start AS inner_start, window_end AS
inner_end, time, a, b, c, s1 "
+ + " FROM SESSION(DATA => t PARTITION BY (a, b, c) ORDER BY
time, GAP => 1ms)"
+ + " ) PARTITION BY (c, b, a) ORDER BY time, GAP => 100ms) "
+ + " GROUP BY window_start, window_end)",
+ new String[] {"windows", "max_cnt", "min_cnt"},
+ new String[] {"40,1,1,"},
+ DATABASE_NAME);
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java
index ac3fc65fe66..406780fd820 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java
@@ -42,6 +42,8 @@ import
org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableS
import
org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExternalTsFileAggregationScanNode;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExternalTsFileScanNode;
+import com.google.common.collect.ImmutableSet;
+
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -117,25 +119,39 @@ public class ParallelizeGrouping implements PlanOptimizer
{
.collect(Collectors.toList()));
}
- /** We need to make sure: context#partitionKey can match the prefix of
childOrderSchema */
+ /**
+ * We need to make sure: context#partitionKey can be satisfied by the
prefix of childOrder.
+ *
+ * <p>When the child exposes exactly the partition keys, the prefix is
compared as a column set:
+ * partition columns are identified by their values, so a different
writing order still produces
+ * the same partitions. This lets nested set-semantic table functions
reuse their input
+ * clustering when the same PARTITION BY columns are listed in a different
order. The set
+ * comparison is deliberately limited to this case: for a strict subset
the child's partitions
+ * are only a refinement of the required ones, and after a fan-in over
unordered sources
+ * (multiple DataRegions, hash aggregation) the subset need not be
contiguous in the input.
+ *
+ * <p>Otherwise the prefix is compared positionally, as before.
+ *
+ * <p>The ORDER BY part of the requirement needs no check here. A group
node sorts within every
+ * partition run by its full ordering scheme, so the input only has to be
clustered; whatever
+ * order rows have inside a partition is re-established locally by the
group's own sort.
+ */
private void checkPrefixMatch(Context context, List<Symbol> childOrder) {
if (context.canSkip()) {
return;
}
- if (context.partitionKeyCount > childOrder.size()) {
+ int partitionKeyCount = context.partitionKeyCount;
+ if (partitionKeyCount > childOrder.size()) {
context.canOptimized = TO_SORT;
return;
}
- OrderingScheme prefix = context.sortKey;
- for (int i = 0; i < context.partitionKeyCount; i++) {
- Symbol lhs = prefix.getOrderBy().get(i);
- Symbol rhs = childOrder.get(i);
- if (!lhs.equals(rhs)) {
- context.canOptimized = TO_SORT;
- return;
- }
- }
- context.canOptimized = ENABLE_PARALLEL;
+ List<Symbol> requiredPrefix = context.sortKey.getOrderBy().subList(0,
partitionKeyCount);
+ List<Symbol> actualPrefix = childOrder.subList(0, partitionKeyCount);
+ boolean sameKeySet =
+ partitionKeyCount == childOrder.size()
+ &&
ImmutableSet.copyOf(requiredPrefix).equals(ImmutableSet.copyOf(actualPrefix));
+ context.canOptimized =
+ sameKeySet || requiredPrefix.equals(actualPrefix) ? ENABLE_PARALLEL
: TO_SORT;
}
@Override
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java
index 637113026f4..725d4646f07 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java
@@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableSet;
import org.junit.Test;
import java.util.Collections;
+import java.util.List;
import java.util.function.Consumer;
import static
org.apache.iotdb.commons.queryengine.plan.relational.function.tvf.ForecastTableFunction.DEFAULT_OUTPUT_INTERVAL;
@@ -226,6 +227,246 @@ public class TableFunctionTest {
assertPlan(planTester.getFragmentPlan(5), tableScan);
}
+ private static final String NESTED_SESSION_SQL =
+ "SELECT * FROM SESSION(DATA => (SELECT * FROM SESSION(DATA => table1 "
+ + "PARTITION BY (tag1, tag2, tag3) ORDER BY time, GAP => 1m)) "
+ + "PARTITION BY (tag3, tag2, tag1) ORDER BY time, GAP => 2m)";
+
+ private static Consumer<TableFunctionProcessorMatcher.Builder>
sessionMatcher(
+ String windowStart, String windowEnd, long gap) {
+ return builder ->
+ builder
+ .name("session")
+ .properOutputs(windowStart, windowEnd)
+ .requiredSymbols("time")
+ .handle(new MapTableFunctionHandle.Builder().addProperty("GAP",
gap).build());
+ }
+
+ private static PlanMatchPattern scanAllColumns() {
+ return tableScan(
+ "testdb.table1",
+ ImmutableList.of("time", "tag1", "tag2", "tag3", "attr1", "attr2",
"s1", "s2", "s3"),
+ ImmutableSet.of("time", "tag1", "tag2", "tag3", "attr1", "attr2",
"s1", "s2", "s3"));
+ }
+
+ private static List<PlanMatchPattern.Ordering> orderBy(String... fields) {
+ ImmutableList.Builder<PlanMatchPattern.Ordering> result =
ImmutableList.builder();
+ for (String field : fields) {
+ result.add(sort(field, ASCENDING, LAST));
+ }
+ return result.build();
+ }
+
+ @Test
+ public void testNestedSetSemanticPermutedPartitionsKeepBothGroups() {
+ PlanTester planTester = new PlanTester();
+ LogicalQueryPlan logicalQueryPlan =
planTester.createPlan(NESTED_SESSION_SQL);
+ Consumer<TableFunctionProcessorMatcher.Builder> outer =
+ sessionMatcher("window_start", "window_end", 120_000L);
+ Consumer<TableFunctionProcessorMatcher.Builder> inner =
+ sessionMatcher("window_start_0", "window_end_1", 60_000L);
+ // PARTITION BY columns are the same set in a different order and ORDER BY
matches, so both
+ // grouping nodes survive: the outer keeps its declared order, the inner
keeps its own.
+ assertPlan(
+ logicalQueryPlan,
+ anyTree(
+ tableFunctionProcessor(
+ outer,
+ group(
+ orderBy("tag3", "tag2", "tag1", "time"),
+ 3,
+ tableFunctionProcessor(
+ inner,
+ group(orderBy("tag1", "tag2", "tag3", "time"), 3,
scanAllColumns()))))));
+ // Both levels are split per region; only cross-region devices need a
merged stream.
+ assertPlan(planTester.getFragmentPlan(0), output(collect(exchange(),
exchange(), exchange())));
+ assertPlan(
+ planTester.getFragmentPlan(1),
+ tableFunctionProcessor(outer, tableFunctionProcessor(inner,
scanAllColumns())));
+ assertPlan(
+ planTester.getFragmentPlan(2),
+ tableFunctionProcessor(outer, tableFunctionProcessor(inner,
scanAllColumns())));
+ assertPlan(
+ planTester.getFragmentPlan(3),
+ tableFunctionProcessor(
+ outer,
+ tableFunctionProcessor(
+ inner,
+ mergeSort(orderBy("tag3", "tag2", "tag1", "time"), exchange(),
exchange()))));
+ assertPlan(planTester.getFragmentPlan(4), scanAllColumns());
+ assertPlan(planTester.getFragmentPlan(5), scanAllColumns());
+ }
+
+ @Test
+ public void testNestedSetSemanticReorderedSubsetFallsBackToSort() {
+ PlanTester planTester = new PlanTester();
+ String sql =
+ "SELECT * FROM SESSION(DATA => (SELECT * FROM SESSION(DATA => table1 "
+ + "PARTITION BY (tag1, tag2, tag3) ORDER BY time, GAP => 1m)) "
+ + "PARTITION BY (tag2, tag1) ORDER BY time, GAP => 2m)";
+ planTester.createPlan(sql);
+ Consumer<TableFunctionProcessorMatcher.Builder> outer =
+ sessionMatcher("window_start", "window_end", 120_000L);
+ Consumer<TableFunctionProcessorMatcher.Builder> inner =
+ sessionMatcher("window_start_0", "window_end_1", 60_000L);
+ // A reordered strict subset must not reuse the inner partitioning: one
outer partition can
+ // span several inner ones, which a fan-in does not keep contiguous. The
outer function stays
+ // single-instance above a full merge sort, as before this optimization.
+ assertPlan(
+ planTester.getFragmentPlan(0),
+ output(
+ tableFunctionProcessor(
+ outer,
+ mergeSort(orderBy("tag2", "tag1", "time"), exchange(),
exchange(), exchange()))));
+ assertPlan(
+ planTester.getFragmentPlan(1),
+ sort(orderBy("tag2", "tag1", "time"), tableFunctionProcessor(inner,
scanAllColumns())));
+ }
+
+ @Test
+ public void testNestedSetSemanticDifferentOrderByKeepsOuterGroup() {
+ PlanTester planTester = new PlanTester();
+ String sql =
+ "SELECT * FROM SESSION(DATA => (SELECT * FROM SESSION(DATA => table1 "
+ + "PARTITION BY (tag1, tag2, tag3) ORDER BY time, GAP => 1m)) "
+ + "PARTITION BY (tag3, tag2, tag1) ORDER BY s1, GAP => 2m)";
+ LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql);
+ Consumer<TableFunctionProcessorMatcher.Builder> outer =
+ sessionMatcher("window_start", "window_end", 120_000L);
+ Consumer<TableFunctionProcessorMatcher.Builder> inner =
+ sessionMatcher("window_start_0", "window_end_1", 60_000L);
+ // Only the partition prefix has to be clustered: the outer group
re-establishes its s1 order
+ // inside every partition, so a different ORDER BY does not degrade it to
a full sort.
+ assertPlan(
+ logicalQueryPlan,
+ anyTree(
+ tableFunctionProcessor(
+ outer,
+ group(
+ orderBy("tag3", "tag2", "tag1", "s1"),
+ 3,
+ tableFunctionProcessor(
+ inner,
+ group(orderBy("tag1", "tag2", "tag3", "time"), 3,
scanAllColumns()))))));
+ // Both levels are still split per region.
+ assertPlan(planTester.getFragmentPlan(0), output(collect(exchange(),
exchange(), exchange())));
+ }
+
+ @Test
+ public void testNestedSetSemanticGlobalOrderByDeclaredOrder() {
+ PlanTester planTester = new PlanTester();
+ planTester.createPlan(NESTED_SESSION_SQL + " ORDER BY tag3, tag2, tag1,
time");
+ Consumer<TableFunctionProcessorMatcher.Builder> outer =
+ sessionMatcher("window_start", "window_end", 120_000L);
+ Consumer<TableFunctionProcessorMatcher.Builder> inner =
+ sessionMatcher("window_start_0", "window_end_1", 60_000L);
+ // The requested global order equals the outer declared order, so no
per-region sort is needed
+ // and the cross-region merge uses exactly that order.
+ assertPlan(
+ planTester.getFragmentPlan(0),
+ output(
+ mergeSort(
+ orderBy("tag3", "tag2", "tag1", "time"), exchange(),
exchange(), exchange())));
+ assertPlan(
+ planTester.getFragmentPlan(1),
+ tableFunctionProcessor(outer, tableFunctionProcessor(inner,
scanAllColumns())));
+ assertPlan(
+ planTester.getFragmentPlan(3),
+ tableFunctionProcessor(
+ outer,
+ tableFunctionProcessor(
+ inner,
+ mergeSort(orderBy("tag3", "tag2", "tag1", "time"), exchange(),
exchange()))));
+ }
+
+ @Test
+ public void testNestedSetSemanticGlobalOrderByPermutesPartitionColumns() {
+ PlanTester planTester = new PlanTester();
+ // The global order permutes the partition columns again; the distributed
planner must prove it
+ // by rewriting grouping/device comparators instead of reusing the TVF
declared order.
+ planTester.createPlan(NESTED_SESSION_SQL + " ORDER BY tag2, tag3, tag1,
time");
+ Consumer<TableFunctionProcessorMatcher.Builder> outer =
+ sessionMatcher("window_start", "window_end", 120_000L);
+ Consumer<TableFunctionProcessorMatcher.Builder> inner =
+ sessionMatcher("window_start_0", "window_end_1", 60_000L);
+ assertPlan(
+ planTester.getFragmentPlan(0),
+ output(
+ mergeSort(
+ orderBy("tag2", "tag3", "tag1", "time"), exchange(),
exchange(), exchange())));
+ assertPlan(
+ planTester.getFragmentPlan(1),
+ tableFunctionProcessor(outer, tableFunctionProcessor(inner,
scanAllColumns())));
+ assertPlan(
+ planTester.getFragmentPlan(3),
+ tableFunctionProcessor(
+ outer,
+ tableFunctionProcessor(
+ inner,
+ mergeSort(orderBy("tag2", "tag3", "tag1", "time"), exchange(),
exchange()))));
+ }
+
+ @Test
+ public void testThreeLevelNestedSetSemanticPermutedPartitions() {
+ PlanTester planTester = new PlanTester();
+ String sql =
+ "SELECT * FROM SESSION(DATA => (SELECT * FROM SESSION(DATA => (SELECT
* FROM SESSION(DATA => table1 "
+ + "PARTITION BY (tag1, tag2, tag3) ORDER BY time, GAP => 1m)) "
+ + "PARTITION BY (tag3, tag2, tag1) ORDER BY time, GAP => 2m)) "
+ + "PARTITION BY (tag2, tag1, tag3) ORDER BY time, GAP => 3m)";
+ LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql);
+ Consumer<TableFunctionProcessorMatcher.Builder> outerMost =
+ sessionMatcher("window_start", "window_end", 180_000L);
+ Consumer<TableFunctionProcessorMatcher.Builder> middle =
+ sessionMatcher("window_start_0", "window_end_1", 120_000L);
+ Consumer<TableFunctionProcessorMatcher.Builder> inner =
+ sessionMatcher("window_start_2", "window_end_3", 60_000L);
+ // Every level lists the same PARTITION BY columns in a different order,
so all three groups
+ // survive and are executed in parallel per region.
+ assertPlan(
+ logicalQueryPlan,
+ anyTree(
+ tableFunctionProcessor(
+ outerMost,
+ group(
+ orderBy("tag2", "tag1", "tag3", "time"),
+ 3,
+ tableFunctionProcessor(
+ middle,
+ group(
+ orderBy("tag3", "tag2", "tag1", "time"),
+ 3,
+ tableFunctionProcessor(
+ inner,
+ group(
+ orderBy("tag1", "tag2", "tag3", "time"),
+ 3,
+ scanAllColumns()))))))));
+ // Every level is split per region; only cross-region devices need a
merged stream.
+ assertPlan(planTester.getFragmentPlan(0), output(collect(exchange(),
exchange(), exchange())));
+ assertPlan(
+ planTester.getFragmentPlan(1),
+ tableFunctionProcessor(
+ outerMost,
+ tableFunctionProcessor(middle, tableFunctionProcessor(inner,
scanAllColumns()))));
+ assertPlan(
+ planTester.getFragmentPlan(2),
+ tableFunctionProcessor(
+ outerMost,
+ tableFunctionProcessor(middle, tableFunctionProcessor(inner,
scanAllColumns()))));
+ assertPlan(
+ planTester.getFragmentPlan(3),
+ tableFunctionProcessor(
+ outerMost,
+ tableFunctionProcessor(
+ middle,
+ tableFunctionProcessor(
+ inner,
+ mergeSort(orderBy("tag2", "tag1", "tag3", "time"),
exchange(), exchange())))));
+ assertPlan(planTester.getFragmentPlan(4), scanAllColumns());
+ assertPlan(planTester.getFragmentPlan(5), scanAllColumns());
+ }
+
@Test
public void testLeafFunction() {
PlanTester planTester = new PlanTester();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionOptimizationTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionOptimizationTest.java
index 9a1c96ba7b5..4db03ccd3a2 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionOptimizationTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionOptimizationTest.java
@@ -123,6 +123,25 @@ public class WindowFunctionOptimizationTest {
sort(tableScan)))))));
}
+ @Test
+ public void testReorderedSubsetWindowPartitionKeepsSingleInstance() {
+ PlanTester planTester = new PlanTester();
+
+ String sql =
+ "SELECT count(s1) OVER (PARTITION BY tag2, tag1) FROM "
+ + "(SELECT *, max(s1) OVER (PARTITION BY tag1, tag2, tag3) AS m
FROM table1)";
+ LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql);
+ PlanMatchPattern tableScan = tableScan("testdb.table1");
+
+ // The outer PARTITION BY is a reordered strict subset of the inner one.
It must not reuse the
+ // inner grouping, whose partitions are finer than the outer window's, so
the outer window
+ // stays single-instance above a sort.
+ assertPlan(logicalQueryPlan,
output(project(window(sort(project(group(tableScan)))))));
+ assertPlan(
+ planTester.getFragmentPlan(0),
+ output(project(window(mergeSort(exchange(), exchange(),
exchange())))));
+ }
+
@Test
public void testTopKRankingPushDown() {
PlanTester planTester = new PlanTester();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/MergeSortMatcher.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/MergeSortMatcher.java
new file mode 100644
index 00000000000..c739b265db5
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/MergeSortMatcher.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.planner.assertions;
+
+import org.apache.iotdb.commons.queryengine.common.SessionInfo;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode;
+import
org.apache.iotdb.commons.queryengine.plan.relational.planner.node.MergeSortNode;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
+import
org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.Ordering;
+
+import java.util.List;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+import static com.google.common.base.Preconditions.checkState;
+import static java.util.Objects.requireNonNull;
+import static
org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.MatchResult.NO_MATCH;
+import static
org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.MatchResult.match;
+import static
org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.Util.orderingSchemeMatches;
+
+class MergeSortMatcher implements Matcher {
+ private final List<Ordering> orderBy;
+
+ public MergeSortMatcher(List<Ordering> orderBy) {
+ this.orderBy = requireNonNull(orderBy, "orderBy is null");
+ }
+
+ @Override
+ public boolean shapeMatches(PlanNode node) {
+ return node instanceof MergeSortNode;
+ }
+
+ @Override
+ public MatchResult detailMatches(
+ PlanNode node, SessionInfo sessionInfo, Metadata metadata, SymbolAliases
symbolAliases) {
+ checkState(
+ shapeMatches(node),
+ "Plan testing framework error: shapeMatches returned false in
detailMatches in %s",
+ this.getClass().getName());
+ MergeSortNode mergeSortNode = (MergeSortNode) node;
+ if (!orderingSchemeMatches(orderBy, mergeSortNode.getOrderingScheme(),
symbolAliases)) {
+ return NO_MATCH;
+ }
+ return match();
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this).add("orderBy", orderBy).toString();
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java
index 6d5102d408a..9ffd9b9bdec 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java
@@ -868,6 +868,10 @@ public final class PlanMatchPattern {
return node(MergeSortNode.class, sources);
}
+ public static PlanMatchPattern mergeSort(List<Ordering> orderBy,
PlanMatchPattern... sources) {
+ return node(MergeSortNode.class, sources).with(new
MergeSortMatcher(orderBy));
+ }
+
public static PlanMatchPattern collect(PlanMatchPattern... sources) {
return node(CollectNode.class, sources);
}