JackieTien97 commented on code in PR #18725:
URL: https://github.com/apache/iotdb/pull/18725#discussion_r4358871544
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java:
##########
@@ -842,4 +846,187 @@ public void testM4RejectsDescendingOrderBy() {
e.getMessage());
}
}
+
+ // Aliases use the planner's actual symbol names so the same matcher also
resolves them through
+ // the ExchangeNodes of the distributed plan (which bind symbols to their
own names).
+ private static final ImmutableMap<String, String> TABLE1_COLUMNS =
+ ImmutableMap.<String, String>builder()
+ .put("time", "time")
+ .put("tag1_0", "tag1")
+ .put("tag2_1", "tag2")
+ .put("tag3_2", "tag3")
+ .put("attr1_3", "attr1")
+ .put("attr2_4", "attr2")
+ .put("s1_5", "s1")
+ .put("s2_6", "s2")
+ .put("s3_7", "s3")
+ .buildOrThrow();
+
+ private static final String LTTB_DATA_ARGUMENT =
+ "DATA => table1 PARTITION BY (tag1, tag2, tag3, attr1, attr2) ORDER BY
time";
+
+ private static MapTableFunctionHandle.Builder lttbHandle(String mode) {
+ return new MapTableFunctionHandle.Builder()
+ .addProperty(LTTBTableFunction.MODE_PROPERTY, mode)
+ .addProperty(
+ LTTBTableFunction.PARTITION_TYPES_PROPERTY,
"STRING,STRING,STRING,STRING,STRING")
+ .addProperty(LTTBTableFunction.PARTICIPANT_TYPES_PROPERTY,
"INT64,INT64,DOUBLE");
+ }
+
+ private static TableFunctionProcessorMatcher.Builder lttbMatcher(
+ TableFunctionProcessorMatcher.Builder builder, String... windowColumns) {
+ ImmutableList.Builder<String> properOutputs = ImmutableList.builder();
+ properOutputs.add(windowColumns);
+ properOutputs.add(
+ "lttb_tag1",
+ "lttb_tag2",
+ "lttb_tag3",
+ "lttb_attr1",
+ "lttb_attr2",
+ "lttb_s1_time",
+ "lttb_s1",
+ "lttb_s2_time",
+ "lttb_s2",
+ "lttb_s3_time",
+ "lttb_s3");
+ return builder
+ .name("lttb")
+ .properOutputs(properOutputs.build().toArray(new String[0]))
+ .requiredSymbols(
+ "time", "tag1_0", "tag2_1", "tag3_2", "attr1_3", "attr2_4",
"s1_5", "s2_6", "s3_7");
+ }
+
+ @Test
+ public void testLTTBTargetCountMode() {
+ PlanTester planTester = new PlanTester();
+ String sql = "SELECT * FROM LTTB(" + LTTB_DATA_ARGUMENT + ", TIMECOL =>
'time', N => 100)";
+ LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql);
+ PlanMatchPattern tableScan = tableScan("testdb.table1", TABLE1_COLUMNS);
+
+ Consumer<TableFunctionProcessorMatcher.Builder> tableFunctionMatcher =
+ builder ->
+ lttbMatcher(builder, "window_index")
+ .handle(
+ lttbHandle("TARGET_COUNT")
+ .addProperty(LTTBTableFunction.N_PARAMETER_NAME, 100L)
+ .build());
+
+ // PARTITION BY several columns plans a GroupNode (a SortNode subclass)
below the function
+ assertPlan(
+ logicalQueryPlan, anyTree(tableFunctionProcessor(tableFunctionMatcher,
group(tableScan))));
+ // LTTB is not mergeable: the whole partition is gathered and ordered
before the function runs
+ assertPlan(
+ planTester.getFragmentPlan(0),
+ output(
+ tableFunctionProcessor(
+ tableFunctionMatcher, mergeSort(exchange(), exchange(),
exchange()))));
+ }
+
+ @Test
+ public void testLTTBTimeWindowMode() {
+ PlanTester planTester = new PlanTester();
+ String sql =
+ "SELECT * FROM LTTB("
+ + LTTB_DATA_ARGUMENT
+ + ", TIMECOL => 'time', SIZE => 1h, SLIDE => 30m, "
+ + "ORIGIN => 1970-01-01T00:00:00.000+00:00)";
+ Consumer<TableFunctionProcessorMatcher.Builder> tableFunctionMatcher =
+ builder ->
+ lttbMatcher(builder, "window_start", "window_end")
+ .handle(
+ lttbHandle("TIME_WINDOW")
+ .addProperty(LTTBTableFunction.SIZE_PARAMETER_NAME,
3600000L)
+ .addProperty(LTTBTableFunction.SLIDE_PARAMETER_NAME,
1800000L)
+ .addProperty(LTTBTableFunction.ORIGIN_PARAMETER_NAME,
0L)
+ .build());
+ assertPlan(
+ planTester.createPlan(sql),
+ anyTree(
+ tableFunctionProcessor(
+ tableFunctionMatcher, group(tableScan("testdb.table1",
TABLE1_COLUMNS)))));
+ }
+
+ @Test
+ public void testLTTBCountWindowMode() {
+ PlanTester planTester = new PlanTester();
+ String sql = "SELECT * FROM LTTB(" + LTTB_DATA_ARGUMENT + ", TIMECOL =>
'time', SIZE => 5)";
+ Consumer<TableFunctionProcessorMatcher.Builder> tableFunctionMatcher =
+ builder ->
+ lttbMatcher(builder, "window_index")
+ .handle(
+ lttbHandle("COUNT_WINDOW")
+ .addProperty(LTTBTableFunction.SIZE_PARAMETER_NAME, 5L)
+ .addProperty(LTTBTableFunction.SLIDE_PARAMETER_NAME,
5L)
+ .build());
+ assertPlan(
+ planTester.createPlan(sql),
+ anyTree(
+ tableFunctionProcessor(
+ tableFunctionMatcher, group(tableScan("testdb.table1",
TABLE1_COLUMNS)))));
+ }
+
+ /**
+ * A window function input plans a per-branch GroupNode that is not
registered in the ordering
+ * map, so the non-split branch has to take the ordering from the GroupNode
itself. Without that
+ * the children are merged by a CollectNode and partitions are mixed.
+ */
+ @Test
+ public void testLTTBOverWindowFunctionInputIsMergedInOrder() {
+ PlanTester planTester = new PlanTester();
+ String sql =
+ "SELECT * FROM LTTB(DATA => (SELECT time, tag3, s1, "
+ + "row_number() OVER (PARTITION BY tag3, s1 ORDER BY time) AS rn
FROM table1) "
+ + "PARTITION BY tag3 ORDER BY time, TIMECOL => 'time', N => 4)";
Review Comment:
[planner test] This test does not cover the plan shape that failed
`PARTITION BY tag3, s1` does not partition by device, so the window is not
planned per region. It runs once, over a `MergeSort`, as the assertion shows.
The failure from the previous round needs a window that is planned per region,
with a `GroupNode` on top of each branch, under an LTTB that is not split. A
window partitioned by all tags, for example `PARTITION BY tag3, tag1, tag2`,
gives that plan. At 9a10836 such a query (with `WHERE time < 100`, which reads
two regions) is planned as:
```
fragment 0: Output -> TableFunctionProcessor (lttb) -> MergeSort [tag3,
time] -> Exchange, Exchange
fragments 1-2: Group [tag3, time] -> Project -> Window [tag3, tag1, tag2] ->
DeviceTableScan (one region each)
```
Please assert this shape. A `Collect` instead of the `MergeSort`, or an LTTB
in the region fragments, would mean that the bug is back.
The Javadoc and the comment (lines 969–971 and 982–985) still describe the
change from b7c5a83, where the per-branch `GroupNode` "is not registered in the
ordering map". Since the merge with master, `visitGroup` registers the
per-branch `GroupNode`s and the non-split branch merges by `commonOrdering`.
Please update both.
##########
integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java:
##########
@@ -0,0 +1,513 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.db.it;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail;
+import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBLTTBTableFunctionIT {
+
+ private static final String DATABASE_NAME = "lttb";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ // Four data region groups per database, so a partition can be split
across regions and a
+ // per-fragment LTTB or an unordered merge would produce visibly wrong
results.
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(4);
+ EnvFactory.getEnv().initClusterEnvironment();
+ try (Connection connection = EnvFactory.getEnv().getTableConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + DATABASE_NAME);
+ statement.execute("USE " + DATABASE_NAME);
+ statement.execute(
+ "CREATE TABLE metrics(device STRING TAG, s1 DOUBLE FIELD, s2 DOUBLE
FIELD)");
+ statement.execute(
+ "INSERT INTO metrics(time, device, s1, s2) VALUES "
+ + "(0, 'd1', 0.0, 0.0),"
+ + "(1, 'd1', 1.0, 0.0),"
+ + "(2, 'd1', 8.0, null),"
+ + "(3, 'd1', 2.0, 9.0),"
+ + "(4, 'd1', 0.0, 1.0),"
+ + "(5, 'd1', 0.0, 0.0),"
+ + "(0, 'd2', 0.0, 0.0),"
+ + "(1, 'd2', null, null)");
+ statement.execute(
+ "CREATE TABLE sparse(device STRING TAG, value DOUBLE FIELD, marker
INT32 FIELD)");
+ statement.execute(
+ "INSERT INTO sparse(time, device, value, marker) VALUES "
+ + "(0, 'd1', 0.0, 1), (1, 'd1', 1.0, 1), (2, 'd1', 2.0, 1),"
+ + "(3, 'd1', null, 1), (4, 'd1', null, 1), (5, 'd1', null, 1),"
+ + "(6, 'd1', 0.0, 1), (7, 'd1', 0.0, 1), (8, 'd1', 0.0, 1)");
+ // Two devices share one site tag; both write t = 0 so the partition has
a duplicate
+ // timestamp.
+ statement.execute("CREATE TABLE shared(site STRING TAG, device STRING
TAG, s1 DOUBLE FIELD)");
+ statement.execute(
+ "INSERT INTO shared(time, site, device, s1) VALUES "
+ + "(0, 'a', 'x1', 1.0), (1, 'a', 'x1', 2.0), (0, 'a', 'x2',
3.0), (1, 'a', 'x2', 4.0)");
+ // Eight devices of one query partition, each writing t = 0..3, so every
timestamp occurs
+ // eight times in the partition. The eight devices own eight distinct
series slots, which the
+ // partition allocator spreads over the four DataRegionGroups, so the
rows sharing t = 0 are
+ // stored in different regions and the duplicate only becomes visible
once the fragments are
+ // merged.
+ statement.execute(
+ "CREATE TABLE frag_dup(part STRING TAG, device STRING TAG, s1 DOUBLE
FIELD)");
+ for (int device = 0; device < 8; device++) {
+ statement.execute(
+ "INSERT INTO frag_dup(time, part, device, s1) VALUES "
+ + "(0, 'p', 'f"
+ + device
+ + "', "
+ + (device + 1)
+ + ".0),"
+ + "(1, 'p', 'f"
+ + device
+ + "', "
+ + (device + 2)
+ + ".0),"
+ + "(2, 'p', 'f"
+ + device
+ + "', "
+ + (device + 3)
+ + ".0),"
+ + "(3, 'p', 'f"
+ + device
+ + "', "
+ + (device + 4)
+ + ".0)");
+ }
+ statement.execute("CREATE TABLE integral_values(device STRING TAG, value
INT64 FIELD)");
+ long baseline = 1L << 60;
+ long[] deltas = {0, 1, 8, 2, 0, 0};
+ for (int i = 0; i < deltas.length; i++) {
+ statement.execute(
+ "INSERT INTO integral_values(time, device, value) VALUES ("
+ + i
+ + ", 'd1', "
+ + (baseline + deltas[i])
+ + ")");
+ }
+ // Two partitions ('p' and 'q'), eight devices each, interleaved
timestamps: the devices of
+ // one
+ // partition carry disjoint time ranges, so the partition is ordered
once the rows are merged
+ // and a fragment-local sample would return one sample per region.
+ statement.execute(
+ "CREATE TABLE regions(part STRING TAG, device STRING TAG, s1 DOUBLE
FIELD)");
+ for (int device = 0; device < 8; device++) {
+ for (int row = 0; row < 4; row++) {
+ long time = device * 4L + row;
+ statement.execute(
+ "INSERT INTO regions(time, part, device, s1) VALUES ("
+ + time
+ + ", 'p', 'p"
+ + device
+ + "', "
+ + (time % 7)
+ + ".0)");
+ statement.execute(
+ "INSERT INTO regions(time, part, device, s1) VALUES ("
+ + time
+ + ", 'q', 'q"
+ + device
+ + "', "
+ + (time % 5)
+ + ".0)");
+ }
+ }
+ statement.execute("FLUSH");
+ } catch (Exception e) {
+ fail("insertData failed: " + e.getMessage());
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testTargetCountPreservesEndpointsAndAlignsColumns() {
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 'time', N => 3) ORDER BY device, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1", "s2_time",
"s2"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.002Z,8.0,1970-01-01T00:00:00.003Z,9.0,",
+ "0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0,"
+ },
Review Comment:
[IT] This test fails at 9a10836
It fails with both `-PTableSimpleIT` and `-PTableClusterIT`:
```
java.lang.ArrayIndexOutOfBoundsException: Index 3 out of bounds for length 3
at
org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqual(TestUtils.java:372)
...
at
org.apache.iotdb.relational.it.db.it.IoTDBLTTBTableFunctionIT.testTargetCountPreservesEndpointsAndAlignsColumns(IoTDBLTTBTableFunctionIT.java:172)
```
Since `d2` now has values at t = 0 (line 75), its partition returns a row,
so the query returns 4 rows. The fourth is
`0|d2|1970-01-01T00:00:00.000Z|0.0|1970-01-01T00:00:00.000Z|0.0|`. With that
row added, the test passes on both profiles:
```suggestion
new String[] {
"0,d1,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0,",
"0,d1,1970-01-01T00:00:00.002Z,8.0,1970-01-01T00:00:00.003Z,9.0,",
"0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0,",
"0,d2,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0,"
},
```
##########
integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java:
##########
@@ -0,0 +1,513 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.db.it;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail;
+import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBLTTBTableFunctionIT {
+
+ private static final String DATABASE_NAME = "lttb";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ // Four data region groups per database, so a partition can be split
across regions and a
+ // per-fragment LTTB or an unordered merge would produce visibly wrong
results.
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(4);
+ EnvFactory.getEnv().initClusterEnvironment();
+ try (Connection connection = EnvFactory.getEnv().getTableConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + DATABASE_NAME);
+ statement.execute("USE " + DATABASE_NAME);
+ statement.execute(
+ "CREATE TABLE metrics(device STRING TAG, s1 DOUBLE FIELD, s2 DOUBLE
FIELD)");
+ statement.execute(
+ "INSERT INTO metrics(time, device, s1, s2) VALUES "
+ + "(0, 'd1', 0.0, 0.0),"
+ + "(1, 'd1', 1.0, 0.0),"
+ + "(2, 'd1', 8.0, null),"
+ + "(3, 'd1', 2.0, 9.0),"
+ + "(4, 'd1', 0.0, 1.0),"
+ + "(5, 'd1', 0.0, 0.0),"
+ + "(0, 'd2', 0.0, 0.0),"
+ + "(1, 'd2', null, null)");
+ statement.execute(
+ "CREATE TABLE sparse(device STRING TAG, value DOUBLE FIELD, marker
INT32 FIELD)");
+ statement.execute(
+ "INSERT INTO sparse(time, device, value, marker) VALUES "
+ + "(0, 'd1', 0.0, 1), (1, 'd1', 1.0, 1), (2, 'd1', 2.0, 1),"
+ + "(3, 'd1', null, 1), (4, 'd1', null, 1), (5, 'd1', null, 1),"
+ + "(6, 'd1', 0.0, 1), (7, 'd1', 0.0, 1), (8, 'd1', 0.0, 1)");
+ // Two devices share one site tag; both write t = 0 so the partition has
a duplicate
+ // timestamp.
+ statement.execute("CREATE TABLE shared(site STRING TAG, device STRING
TAG, s1 DOUBLE FIELD)");
+ statement.execute(
+ "INSERT INTO shared(time, site, device, s1) VALUES "
+ + "(0, 'a', 'x1', 1.0), (1, 'a', 'x1', 2.0), (0, 'a', 'x2',
3.0), (1, 'a', 'x2', 4.0)");
+ // Eight devices of one query partition, each writing t = 0..3, so every
timestamp occurs
+ // eight times in the partition. The eight devices own eight distinct
series slots, which the
+ // partition allocator spreads over the four DataRegionGroups, so the
rows sharing t = 0 are
+ // stored in different regions and the duplicate only becomes visible
once the fragments are
+ // merged.
+ statement.execute(
+ "CREATE TABLE frag_dup(part STRING TAG, device STRING TAG, s1 DOUBLE
FIELD)");
+ for (int device = 0; device < 8; device++) {
+ statement.execute(
+ "INSERT INTO frag_dup(time, part, device, s1) VALUES "
+ + "(0, 'p', 'f"
+ + device
+ + "', "
+ + (device + 1)
+ + ".0),"
+ + "(1, 'p', 'f"
+ + device
+ + "', "
+ + (device + 2)
+ + ".0),"
+ + "(2, 'p', 'f"
+ + device
+ + "', "
+ + (device + 3)
+ + ".0),"
+ + "(3, 'p', 'f"
+ + device
+ + "', "
+ + (device + 4)
+ + ".0)");
+ }
+ statement.execute("CREATE TABLE integral_values(device STRING TAG, value
INT64 FIELD)");
+ long baseline = 1L << 60;
+ long[] deltas = {0, 1, 8, 2, 0, 0};
+ for (int i = 0; i < deltas.length; i++) {
+ statement.execute(
+ "INSERT INTO integral_values(time, device, value) VALUES ("
+ + i
+ + ", 'd1', "
+ + (baseline + deltas[i])
+ + ")");
+ }
+ // Two partitions ('p' and 'q'), eight devices each, interleaved
timestamps: the devices of
+ // one
+ // partition carry disjoint time ranges, so the partition is ordered
once the rows are merged
+ // and a fragment-local sample would return one sample per region.
+ statement.execute(
+ "CREATE TABLE regions(part STRING TAG, device STRING TAG, s1 DOUBLE
FIELD)");
+ for (int device = 0; device < 8; device++) {
+ for (int row = 0; row < 4; row++) {
+ long time = device * 4L + row;
+ statement.execute(
+ "INSERT INTO regions(time, part, device, s1) VALUES ("
+ + time
+ + ", 'p', 'p"
+ + device
+ + "', "
+ + (time % 7)
+ + ".0)");
+ statement.execute(
+ "INSERT INTO regions(time, part, device, s1) VALUES ("
+ + time
+ + ", 'q', 'q"
+ + device
+ + "', "
+ + (time % 5)
+ + ".0)");
+ }
+ }
+ statement.execute("FLUSH");
+ } catch (Exception e) {
+ fail("insertData failed: " + e.getMessage());
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testTargetCountPreservesEndpointsAndAlignsColumns() {
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 'time', N => 3) ORDER BY device, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1", "s2_time",
"s2"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.002Z,8.0,1970-01-01T00:00:00.003Z,9.0,",
+ "0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testCountWindowAndTimeWindowOutput() {
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics
WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', SIZE =>
2) "
+ + "ORDER BY window_index",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.001Z,1.0,",
+ "1,d1,1970-01-01T00:00:00.002Z,8.0,",
+ "2,d1,1970-01-01T00:00:00.004Z,0.0,",
+ "2,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics
WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', SIZE =>
2ms, "
+ + "ORIGIN => 1970-01-01T00:00:00.000+00:00) ORDER BY window_start",
+ new String[] {"window_start", "window_end", "device", "s1_time", "s1"},
+ new String[] {
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.001Z,1.0,",
+
"1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.004Z,d1,1970-01-01T00:00:00.002Z,8.0,",
+
"1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0,",
+
"1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsAreRejectedWith701() {
+ // the same six d1 rows twice: (0, 1, 2, 3, 4, 5) each occurs twice in one
partition
+ String duplicated =
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics
WHERE device = 'd1' "
+ + "UNION ALL SELECT time, device, s1 FROM metrics WHERE device =
'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ String expected =
+ "701: LTTB does not support duplicate timestamps, but timestamp 0
appears more than once"
+ + " in the same partition.";
+
+ tableAssertTestFail(duplicated + "N => 3)", expected, DATABASE_NAME);
+ tableAssertTestFail(duplicated + "SIZE => 2)", expected, DATABASE_NAME);
+ tableAssertTestFail(
+ duplicated + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)",
+ expected,
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsInDifferentPartitionsRemainValid() {
+ // each partition is unique on its own, so the query must succeed and keep
both rows
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics "
+ + "WHERE time = 0 AND (device = 'd1' OR device = 'd2')) "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', N => 3) "
+ + "ORDER BY device",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
"0,d2,1970-01-01T00:00:00.000Z,0.0,",
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testWindowSkipsNullBucketForLookahead() {
+ String data =
+ "DATA => (SELECT time, device, value FROM sparse) PARTITION BY device
ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 3) ORDER BY window_index",
+ new String[] {"window_index", "device", "value_time", "value"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.002Z,2.0,",
+ "2,d1,1970-01-01T00:00:00.006Z,0.0,",
+ "2,d1,1970-01-01T00:00:00.008Z,0.0,"
+ },
+ DATABASE_NAME);
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 3ms) ORDER BY window_start",
+ new String[] {"window_start", "window_end", "device", "value_time",
"value"},
+ new String[] {
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,2.0,",
+
"1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.006Z,0.0,",
+
"1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.008Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testNonFiniteValuesAreKeptInShortSeries() {
+ // three non-NULL input points whose values are NaN/Infinity for N => 3
+ String nonFinite =
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 / 0.0 AS s1 FROM
metrics "
+ + "WHERE device = 'd1' AND time <= 2) "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+
+ tableResultSetEqualTest(
+ nonFinite + "N => 3) ORDER BY s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,NaN,",
+ "0,d1,1970-01-01T00:00:00.001Z,Infinity,",
+ "0,d1,1970-01-01T00:00:00.002Z,Infinity,",
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testInt64SamplingPreservesSmallPeaksAtLargeBaselines() {
+ String prefix =
+ "SELECT * FROM LTTB(DATA => integral_values PARTITION BY device ORDER
BY time, "
+ + "TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ prefix + "N => 3) ORDER BY value_time",
+ new String[] {"window_index", "device", "value_time", "value"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,1152921504606846976,",
+ "0,d1,1970-01-01T00:00:00.002Z,1152921504606846984,",
+ "0,d1,1970-01-01T00:00:00.005Z,1152921504606846976,"
+ },
+ DATABASE_NAME);
+ tableResultSetEqualTest(
+ prefix + "SIZE => 2) ORDER BY window_index",
+ new String[] {"window_index", "device", "value_time", "value"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,1152921504606846976,",
+ "0,d1,1970-01-01T00:00:00.001Z,1152921504606846977,",
+ "1,d1,1970-01-01T00:00:00.002Z,1152921504606846984,",
+ "2,d1,1970-01-01T00:00:00.004Z,1152921504606846976,",
+ "2,d1,1970-01-01T00:00:00.005Z,1152921504606846976,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testSlideSmallerThanSizeKeepsOneAnchorChain() {
+ // SIZE => 2, SLIDE => 1: overlapping count windows. F(t0) is reported by
the first window and
+ // L(t5) by the last one, and the windows in between continue the same
anchor chain.
+ String data =
+ "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 2, SLIDE => 1) ORDER BY
window_index, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.001Z,1.0,",
+ "1,d1,1970-01-01T00:00:00.002Z,8.0,",
+ "2,d1,1970-01-01T00:00:00.003Z,2.0,",
+ "3,d1,1970-01-01T00:00:00.003Z,2.0,",
+ "4,d1,1970-01-01T00:00:00.004Z,0.0,",
+ "5,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ // The time-window form with SLIDE < SIZE starts a window one slide before
the first point.
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 2ms, SLIDE => 1ms) ORDER BY
window_start, s1_time",
+ new String[] {"window_start", "window_end", "device", "s1_time", "s1"},
+ new String[] {
+
"1969-12-31T23:59:59.999Z,1970-01-01T00:00:00.001Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.001Z,1.0,",
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,8.0,",
+
"1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.004Z,d1,1970-01-01T00:00:00.003Z,2.0,",
+
"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.005Z,d1,1970-01-01T00:00:00.003Z,2.0,",
+
"1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0,",
+
"1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.007Z,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testSlideLargerThanSizeSkipsTheGap() {
+ // SIZE => 1, SLIDE => 2 keeps only the windows that receive a row: [0,1),
[2,3), [4,5).
+ String data =
+ "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 1, SLIDE => 2) ORDER BY
window_index, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "1,d1,1970-01-01T00:00:00.002Z,8.0,",
+ "2,d1,1970-01-01T00:00:00.004Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testTimeWindowWithUnalignedOriginShiftsBoundaries() {
+ String data =
+ "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB("
+ + data
+ + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.001+00:00) "
+ + "ORDER BY window_start, s1_time",
+ new String[] {"window_start", "window_end", "device", "s1_time", "s1"},
+ new String[] {
+
"1969-12-31T23:59:59.999Z,1970-01-01T00:00:00.001Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,8.0,",
+
"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.005Z,d1,1970-01-01T00:00:00.003Z,2.0,",
+
"1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.007Z,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testInvalidTimeColumnIsRejected() {
+ tableAssertTestFail(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 's1', N => 3)",
+ "701: ",
+ DATABASE_NAME);
+ tableAssertTestFail(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 'missing', N => 3)",
+ "701: ",
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsAcrossDevicesInOnePartitionAreRejected() {
+ // Both devices are tagged 'shared', so PARTITION BY site puts t = 0 of
each device into one
+ // partition and the duplicate check has to reject it.
+ tableAssertTestFail(
+ "SELECT * FROM LTTB(DATA => (SELECT time, site, s1 FROM shared ORDER
BY time) "
+ + "PARTITION BY site ORDER BY time, TIMECOL => 'time', N => 3)",
+ "701: LTTB does not support duplicate timestamps, but timestamp 0
appears more than once"
+ + " in the same partition.",
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsAcrossDistributedFragmentsAreRejected() {
+ // Every timestamp of the partition is written by all eight devices, whose
series slots are
+ // spread over the four DataRegionGroups of the database, so the duplicate
rows are read from
+ // different regions. The function runs once over the merged partition, so
the check sees the
+ // duplicates regardless of how many fragments supplied them.
+ String expected =
+ "701: LTTB does not support duplicate timestamps, but timestamp 0
appears more than once"
+ + " in the same partition.";
+ String prefix =
+ "SELECT * FROM LTTB(DATA => (SELECT time, part, s1 FROM frag_dup) "
+ + "PARTITION BY part ORDER BY time, TIMECOL => 'time', ";
+
+ tableAssertTestFail(prefix + "N => 3)", expected, DATABASE_NAME);
+ tableAssertTestFail(prefix + "SIZE => 3)", expected, DATABASE_NAME);
+ tableAssertTestFail(
+ prefix + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)",
expected, DATABASE_NAME);
+ }
+
+ /**
+ * A partitioned GroupNode does not make a partition complete within one
branch, so LTTB has to
+ * run once over the partitions merged in order. With four data region
groups and a partition
+ * whose devices carry disjoint time ranges, a per-region sample or an
unordered merge changes the
+ * result; {@code N => 1000} returns every eligible point, so all three
inputs must agree.
+ */
+ @Test
+ public void testNonDirectScanInputsMatchTheDirectScan() throws SQLException {
+ String direct =
+ "SELECT window_index, part, s1_time, s1 FROM LTTB("
+ + "DATA => regions PARTITION BY part ORDER BY time, TIMECOL =>
'time', N => 1000) "
+ + "ORDER BY part, s1_time";
+ String windowFunctionInput =
+ "SELECT window_index, part, s1_time, s1 FROM LTTB("
+ + "DATA => (SELECT time, part, s1, "
+ + "row_number() OVER (PARTITION BY part, device ORDER BY time) AS
rn FROM regions) "
+ + "PARTITION BY part ORDER BY time, TIMECOL => 'time', N => 1000) "
+ + "ORDER BY part, s1_time";
+ String sessionInput =
+ "SELECT window_index, part, s1_time, s1 FROM LTTB("
+ + "DATA => (SELECT time, part, s1 FROM SESSION("
+ + "DATA => regions PARTITION BY (part, device) ORDER BY time,
TIMECOL => 'time', "
+ + "GAP => 1m)) PARTITION BY part ORDER BY time, TIMECOL => 'time',
N => 1000) "
+ + "ORDER BY part, s1_time";
+
+ List<String> expected = queryRows(direct);
+ // 8 devices x 4 rows per partition, 2 partitions.
+ assertEquals(64, expected.size());
+ assertEquals(expected, queryRows(windowFunctionInput));
+ assertEquals(expected, queryRows(sessionInput));
+ }
Review Comment:
[IT] This test fails at 9a10836, and with `N => 1000` it cannot detect a
per-region LTTB
**1. The direct query fails** with both profiles. `DATA => regions` passes
the STRING tag `device` to LTTB as a participant column:
```
java.sql.SQLException: 701: Only column with double, float, int32, int64 can
be calculated by the function, device is the STRING.
```
Projecting it away, as the other two inputs do, fixes this: `DATA => (SELECT
time, part, s1 FROM regions)`.
**2. With `N => 1000` the test cannot detect a split.** Every LTTB then
returns all the points it receives, so the per-region results add up to the
direct result. The Javadoc says that "a per-region sample ... changes the
result", but that does not hold for `N => 1000`. I checked this on 1C1D with
the projection fix and with the LTTB exclusion removed from
`canSplitTableFunctionProcessor`. The window-function and `SESSION` inputs then
ran four LTTB instances each, one per region:
| `N` | rows: direct / window / `SESSION` | inputs agree |
|---|---|---|
| 1000 | 64 / 64 / 64 | yes |
| 8 | 16 / 32 / 32 | no |
Only an `N` that actually samples catches the split. The version below runs
both values of `N`. It passes with both profiles at 9a10836 and fails once the
exclusion is removed:
```suggestion
/**
* A partitioned GroupNode does not make a partition complete within one
branch, so LTTB has to
* run once over the partitions merged in order. With four data region
groups and a partition
* whose devices carry disjoint time ranges, all three inputs must agree.
{@code N => 1000}
* returns every point, so it checks that each partition receives exactly
its rows; {@code N => 8}
* samples, so it also catches an LTTB that runs once per region.
*/
@Test
public void testNonDirectScanInputsMatchTheDirectScan() throws
SQLException {
for (int n : new int[] {1000, 8}) {
String rest =
" PARTITION BY part ORDER BY time, TIMECOL => 'time', N => "
+ n
+ ") ORDER BY part, s1_time";
String direct =
"SELECT window_index, part, s1_time, s1 FROM LTTB("
+ "DATA => (SELECT time, part, s1 FROM regions)"
+ rest;
String windowFunctionInput =
"SELECT window_index, part, s1_time, s1 FROM LTTB("
+ "DATA => (SELECT time, part, s1, "
+ "row_number() OVER (PARTITION BY part, device ORDER BY time)
AS rn FROM regions)"
+ rest;
String sessionInput =
"SELECT window_index, part, s1_time, s1 FROM LTTB("
+ "DATA => (SELECT time, part, s1 FROM SESSION("
+ "DATA => regions PARTITION BY (part, device) ORDER BY time,
TIMECOL => 'time', "
+ "GAP => 1m))"
+ rest;
List<String> expected = queryRows(direct);
// 2 partitions of 32 points each: all of them for N => 1000, 8 each
for N => 8.
assertEquals(n == 1000 ? 64 : 16, expected.size());
assertEquals(expected, queryRows(windowFunctionInput));
assertEquals(expected, queryRows(sessionInput));
}
}
```
##########
integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java:
##########
@@ -0,0 +1,513 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.db.it;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail;
+import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBLTTBTableFunctionIT {
+
+ private static final String DATABASE_NAME = "lttb";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ // Four data region groups per database, so a partition can be split
across regions and a
+ // per-fragment LTTB or an unordered merge would produce visibly wrong
results.
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(4);
+ EnvFactory.getEnv().initClusterEnvironment();
+ try (Connection connection = EnvFactory.getEnv().getTableConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + DATABASE_NAME);
+ statement.execute("USE " + DATABASE_NAME);
+ statement.execute(
+ "CREATE TABLE metrics(device STRING TAG, s1 DOUBLE FIELD, s2 DOUBLE
FIELD)");
+ statement.execute(
+ "INSERT INTO metrics(time, device, s1, s2) VALUES "
+ + "(0, 'd1', 0.0, 0.0),"
+ + "(1, 'd1', 1.0, 0.0),"
+ + "(2, 'd1', 8.0, null),"
+ + "(3, 'd1', 2.0, 9.0),"
+ + "(4, 'd1', 0.0, 1.0),"
+ + "(5, 'd1', 0.0, 0.0),"
+ + "(0, 'd2', 0.0, 0.0),"
+ + "(1, 'd2', null, null)");
+ statement.execute(
+ "CREATE TABLE sparse(device STRING TAG, value DOUBLE FIELD, marker
INT32 FIELD)");
+ statement.execute(
+ "INSERT INTO sparse(time, device, value, marker) VALUES "
+ + "(0, 'd1', 0.0, 1), (1, 'd1', 1.0, 1), (2, 'd1', 2.0, 1),"
+ + "(3, 'd1', null, 1), (4, 'd1', null, 1), (5, 'd1', null, 1),"
+ + "(6, 'd1', 0.0, 1), (7, 'd1', 0.0, 1), (8, 'd1', 0.0, 1)");
+ // Two devices share one site tag; both write t = 0 so the partition has
a duplicate
+ // timestamp.
+ statement.execute("CREATE TABLE shared(site STRING TAG, device STRING
TAG, s1 DOUBLE FIELD)");
+ statement.execute(
+ "INSERT INTO shared(time, site, device, s1) VALUES "
+ + "(0, 'a', 'x1', 1.0), (1, 'a', 'x1', 2.0), (0, 'a', 'x2',
3.0), (1, 'a', 'x2', 4.0)");
+ // Eight devices of one query partition, each writing t = 0..3, so every
timestamp occurs
+ // eight times in the partition. The eight devices own eight distinct
series slots, which the
+ // partition allocator spreads over the four DataRegionGroups, so the
rows sharing t = 0 are
+ // stored in different regions and the duplicate only becomes visible
once the fragments are
+ // merged.
+ statement.execute(
+ "CREATE TABLE frag_dup(part STRING TAG, device STRING TAG, s1 DOUBLE
FIELD)");
+ for (int device = 0; device < 8; device++) {
+ statement.execute(
+ "INSERT INTO frag_dup(time, part, device, s1) VALUES "
+ + "(0, 'p', 'f"
+ + device
+ + "', "
+ + (device + 1)
+ + ".0),"
+ + "(1, 'p', 'f"
+ + device
+ + "', "
+ + (device + 2)
+ + ".0),"
+ + "(2, 'p', 'f"
+ + device
+ + "', "
+ + (device + 3)
+ + ".0),"
+ + "(3, 'p', 'f"
+ + device
+ + "', "
+ + (device + 4)
+ + ".0)");
+ }
+ statement.execute("CREATE TABLE integral_values(device STRING TAG, value
INT64 FIELD)");
+ long baseline = 1L << 60;
+ long[] deltas = {0, 1, 8, 2, 0, 0};
+ for (int i = 0; i < deltas.length; i++) {
+ statement.execute(
+ "INSERT INTO integral_values(time, device, value) VALUES ("
+ + i
+ + ", 'd1', "
+ + (baseline + deltas[i])
+ + ")");
+ }
+ // Two partitions ('p' and 'q'), eight devices each, interleaved
timestamps: the devices of
+ // one
+ // partition carry disjoint time ranges, so the partition is ordered
once the rows are merged
+ // and a fragment-local sample would return one sample per region.
+ statement.execute(
+ "CREATE TABLE regions(part STRING TAG, device STRING TAG, s1 DOUBLE
FIELD)");
+ for (int device = 0; device < 8; device++) {
+ for (int row = 0; row < 4; row++) {
+ long time = device * 4L + row;
+ statement.execute(
+ "INSERT INTO regions(time, part, device, s1) VALUES ("
+ + time
+ + ", 'p', 'p"
+ + device
+ + "', "
+ + (time % 7)
+ + ".0)");
+ statement.execute(
+ "INSERT INTO regions(time, part, device, s1) VALUES ("
+ + time
+ + ", 'q', 'q"
+ + device
+ + "', "
+ + (time % 5)
+ + ".0)");
+ }
+ }
+ statement.execute("FLUSH");
+ } catch (Exception e) {
+ fail("insertData failed: " + e.getMessage());
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testTargetCountPreservesEndpointsAndAlignsColumns() {
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 'time', N => 3) ORDER BY device, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1", "s2_time",
"s2"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.002Z,8.0,1970-01-01T00:00:00.003Z,9.0,",
+ "0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testCountWindowAndTimeWindowOutput() {
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics
WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', SIZE =>
2) "
+ + "ORDER BY window_index",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.001Z,1.0,",
+ "1,d1,1970-01-01T00:00:00.002Z,8.0,",
+ "2,d1,1970-01-01T00:00:00.004Z,0.0,",
+ "2,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics
WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', SIZE =>
2ms, "
+ + "ORIGIN => 1970-01-01T00:00:00.000+00:00) ORDER BY window_start",
+ new String[] {"window_start", "window_end", "device", "s1_time", "s1"},
+ new String[] {
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.001Z,1.0,",
+
"1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.004Z,d1,1970-01-01T00:00:00.002Z,8.0,",
+
"1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0,",
+
"1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsAreRejectedWith701() {
+ // the same six d1 rows twice: (0, 1, 2, 3, 4, 5) each occurs twice in one
partition
+ String duplicated =
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics
WHERE device = 'd1' "
+ + "UNION ALL SELECT time, device, s1 FROM metrics WHERE device =
'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ String expected =
+ "701: LTTB does not support duplicate timestamps, but timestamp 0
appears more than once"
+ + " in the same partition.";
+
+ tableAssertTestFail(duplicated + "N => 3)", expected, DATABASE_NAME);
+ tableAssertTestFail(duplicated + "SIZE => 2)", expected, DATABASE_NAME);
+ tableAssertTestFail(
+ duplicated + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)",
+ expected,
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsInDifferentPartitionsRemainValid() {
+ // each partition is unique on its own, so the query must succeed and keep
both rows
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics "
+ + "WHERE time = 0 AND (device = 'd1' OR device = 'd2')) "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', N => 3) "
+ + "ORDER BY device",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
"0,d2,1970-01-01T00:00:00.000Z,0.0,",
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testWindowSkipsNullBucketForLookahead() {
+ String data =
+ "DATA => (SELECT time, device, value FROM sparse) PARTITION BY device
ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 3) ORDER BY window_index",
+ new String[] {"window_index", "device", "value_time", "value"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.002Z,2.0,",
+ "2,d1,1970-01-01T00:00:00.006Z,0.0,",
+ "2,d1,1970-01-01T00:00:00.008Z,0.0,"
+ },
+ DATABASE_NAME);
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 3ms) ORDER BY window_start",
+ new String[] {"window_start", "window_end", "device", "value_time",
"value"},
+ new String[] {
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,2.0,",
+
"1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.006Z,0.0,",
+
"1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.008Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testNonFiniteValuesAreKeptInShortSeries() {
+ // three non-NULL input points whose values are NaN/Infinity for N => 3
+ String nonFinite =
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 / 0.0 AS s1 FROM
metrics "
+ + "WHERE device = 'd1' AND time <= 2) "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+
+ tableResultSetEqualTest(
+ nonFinite + "N => 3) ORDER BY s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,NaN,",
+ "0,d1,1970-01-01T00:00:00.001Z,Infinity,",
+ "0,d1,1970-01-01T00:00:00.002Z,Infinity,",
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testInt64SamplingPreservesSmallPeaksAtLargeBaselines() {
+ String prefix =
+ "SELECT * FROM LTTB(DATA => integral_values PARTITION BY device ORDER
BY time, "
+ + "TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ prefix + "N => 3) ORDER BY value_time",
+ new String[] {"window_index", "device", "value_time", "value"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,1152921504606846976,",
+ "0,d1,1970-01-01T00:00:00.002Z,1152921504606846984,",
+ "0,d1,1970-01-01T00:00:00.005Z,1152921504606846976,"
+ },
+ DATABASE_NAME);
+ tableResultSetEqualTest(
+ prefix + "SIZE => 2) ORDER BY window_index",
+ new String[] {"window_index", "device", "value_time", "value"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,1152921504606846976,",
+ "0,d1,1970-01-01T00:00:00.001Z,1152921504606846977,",
+ "1,d1,1970-01-01T00:00:00.002Z,1152921504606846984,",
+ "2,d1,1970-01-01T00:00:00.004Z,1152921504606846976,",
+ "2,d1,1970-01-01T00:00:00.005Z,1152921504606846976,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testSlideSmallerThanSizeKeepsOneAnchorChain() {
+ // SIZE => 2, SLIDE => 1: overlapping count windows. F(t0) is reported by
the first window and
+ // L(t5) by the last one, and the windows in between continue the same
anchor chain.
+ String data =
+ "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 2, SLIDE => 1) ORDER BY
window_index, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.001Z,1.0,",
+ "1,d1,1970-01-01T00:00:00.002Z,8.0,",
+ "2,d1,1970-01-01T00:00:00.003Z,2.0,",
+ "3,d1,1970-01-01T00:00:00.003Z,2.0,",
+ "4,d1,1970-01-01T00:00:00.004Z,0.0,",
+ "5,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ // The time-window form with SLIDE < SIZE starts a window one slide before
the first point.
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 2ms, SLIDE => 1ms) ORDER BY
window_start, s1_time",
+ new String[] {"window_start", "window_end", "device", "s1_time", "s1"},
+ new String[] {
+
"1969-12-31T23:59:59.999Z,1970-01-01T00:00:00.001Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.001Z,1.0,",
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,8.0,",
+
"1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.004Z,d1,1970-01-01T00:00:00.003Z,2.0,",
+
"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.005Z,d1,1970-01-01T00:00:00.003Z,2.0,",
+
"1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0,",
+
"1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.007Z,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testSlideLargerThanSizeSkipsTheGap() {
+ // SIZE => 1, SLIDE => 2 keeps only the windows that receive a row: [0,1),
[2,3), [4,5).
+ String data =
+ "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 1, SLIDE => 2) ORDER BY
window_index, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "1,d1,1970-01-01T00:00:00.002Z,8.0,",
+ "2,d1,1970-01-01T00:00:00.004Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testTimeWindowWithUnalignedOriginShiftsBoundaries() {
+ String data =
+ "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB("
+ + data
+ + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.001+00:00) "
+ + "ORDER BY window_start, s1_time",
+ new String[] {"window_start", "window_end", "device", "s1_time", "s1"},
+ new String[] {
+
"1969-12-31T23:59:59.999Z,1970-01-01T00:00:00.001Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,8.0,",
+
"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.005Z,d1,1970-01-01T00:00:00.003Z,2.0,",
+
"1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.007Z,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testInvalidTimeColumnIsRejected() {
+ tableAssertTestFail(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 's1', N => 3)",
+ "701: ",
+ DATABASE_NAME);
+ tableAssertTestFail(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 'missing', N => 3)",
+ "701: ",
+ DATABASE_NAME);
Review Comment:
[IT] Please assert the full error messages
`701: ` matches any semantic error, so these assertions would also pass if
the query failed for an unrelated reason. With the messages that the queries
actually return, the test passes on both profiles:
```suggestion
tableAssertTestFail(
"SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY
time, "
+ "TIMECOL => 's1', N => 3)",
"701: The type of the column [s1] is not as expected.",
DATABASE_NAME);
tableAssertTestFail(
"SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY
time, "
+ "TIMECOL => 'missing', N => 3)",
"701: Required column [missing] not found in the source table
argument.",
DATABASE_NAME);
```
##########
integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java:
##########
@@ -0,0 +1,513 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.db.it;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail;
+import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBLTTBTableFunctionIT {
+
+ private static final String DATABASE_NAME = "lttb";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ // Four data region groups per database, so a partition can be split
across regions and a
+ // per-fragment LTTB or an unordered merge would produce visibly wrong
results.
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(4);
+ EnvFactory.getEnv().initClusterEnvironment();
+ try (Connection connection = EnvFactory.getEnv().getTableConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + DATABASE_NAME);
+ statement.execute("USE " + DATABASE_NAME);
+ statement.execute(
+ "CREATE TABLE metrics(device STRING TAG, s1 DOUBLE FIELD, s2 DOUBLE
FIELD)");
+ statement.execute(
+ "INSERT INTO metrics(time, device, s1, s2) VALUES "
+ + "(0, 'd1', 0.0, 0.0),"
+ + "(1, 'd1', 1.0, 0.0),"
+ + "(2, 'd1', 8.0, null),"
+ + "(3, 'd1', 2.0, 9.0),"
+ + "(4, 'd1', 0.0, 1.0),"
+ + "(5, 'd1', 0.0, 0.0),"
+ + "(0, 'd2', 0.0, 0.0),"
+ + "(1, 'd2', null, null)");
+ statement.execute(
+ "CREATE TABLE sparse(device STRING TAG, value DOUBLE FIELD, marker
INT32 FIELD)");
+ statement.execute(
+ "INSERT INTO sparse(time, device, value, marker) VALUES "
+ + "(0, 'd1', 0.0, 1), (1, 'd1', 1.0, 1), (2, 'd1', 2.0, 1),"
+ + "(3, 'd1', null, 1), (4, 'd1', null, 1), (5, 'd1', null, 1),"
+ + "(6, 'd1', 0.0, 1), (7, 'd1', 0.0, 1), (8, 'd1', 0.0, 1)");
+ // Two devices share one site tag; both write t = 0 so the partition has
a duplicate
+ // timestamp.
+ statement.execute("CREATE TABLE shared(site STRING TAG, device STRING
TAG, s1 DOUBLE FIELD)");
+ statement.execute(
+ "INSERT INTO shared(time, site, device, s1) VALUES "
+ + "(0, 'a', 'x1', 1.0), (1, 'a', 'x1', 2.0), (0, 'a', 'x2',
3.0), (1, 'a', 'x2', 4.0)");
+ // Eight devices of one query partition, each writing t = 0..3, so every
timestamp occurs
+ // eight times in the partition. The eight devices own eight distinct
series slots, which the
+ // partition allocator spreads over the four DataRegionGroups, so the
rows sharing t = 0 are
+ // stored in different regions and the duplicate only becomes visible
once the fragments are
+ // merged.
+ statement.execute(
+ "CREATE TABLE frag_dup(part STRING TAG, device STRING TAG, s1 DOUBLE
FIELD)");
+ for (int device = 0; device < 8; device++) {
+ statement.execute(
+ "INSERT INTO frag_dup(time, part, device, s1) VALUES "
+ + "(0, 'p', 'f"
+ + device
+ + "', "
+ + (device + 1)
+ + ".0),"
+ + "(1, 'p', 'f"
+ + device
+ + "', "
+ + (device + 2)
+ + ".0),"
+ + "(2, 'p', 'f"
+ + device
+ + "', "
+ + (device + 3)
+ + ".0),"
+ + "(3, 'p', 'f"
+ + device
+ + "', "
+ + (device + 4)
+ + ".0)");
+ }
+ statement.execute("CREATE TABLE integral_values(device STRING TAG, value
INT64 FIELD)");
+ long baseline = 1L << 60;
+ long[] deltas = {0, 1, 8, 2, 0, 0};
+ for (int i = 0; i < deltas.length; i++) {
+ statement.execute(
+ "INSERT INTO integral_values(time, device, value) VALUES ("
+ + i
+ + ", 'd1', "
+ + (baseline + deltas[i])
+ + ")");
+ }
+ // Two partitions ('p' and 'q'), eight devices each, interleaved
timestamps: the devices of
+ // one
+ // partition carry disjoint time ranges, so the partition is ordered
once the rows are merged
+ // and a fragment-local sample would return one sample per region.
+ statement.execute(
+ "CREATE TABLE regions(part STRING TAG, device STRING TAG, s1 DOUBLE
FIELD)");
+ for (int device = 0; device < 8; device++) {
+ for (int row = 0; row < 4; row++) {
+ long time = device * 4L + row;
+ statement.execute(
+ "INSERT INTO regions(time, part, device, s1) VALUES ("
+ + time
+ + ", 'p', 'p"
+ + device
+ + "', "
+ + (time % 7)
+ + ".0)");
+ statement.execute(
+ "INSERT INTO regions(time, part, device, s1) VALUES ("
+ + time
+ + ", 'q', 'q"
+ + device
+ + "', "
+ + (time % 5)
+ + ".0)");
+ }
+ }
+ statement.execute("FLUSH");
+ } catch (Exception e) {
+ fail("insertData failed: " + e.getMessage());
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testTargetCountPreservesEndpointsAndAlignsColumns() {
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 'time', N => 3) ORDER BY device, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1", "s2_time",
"s2"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.002Z,8.0,1970-01-01T00:00:00.003Z,9.0,",
+ "0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testCountWindowAndTimeWindowOutput() {
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics
WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', SIZE =>
2) "
+ + "ORDER BY window_index",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.001Z,1.0,",
+ "1,d1,1970-01-01T00:00:00.002Z,8.0,",
+ "2,d1,1970-01-01T00:00:00.004Z,0.0,",
+ "2,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics
WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', SIZE =>
2ms, "
+ + "ORIGIN => 1970-01-01T00:00:00.000+00:00) ORDER BY window_start",
+ new String[] {"window_start", "window_end", "device", "s1_time", "s1"},
+ new String[] {
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.001Z,1.0,",
+
"1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.004Z,d1,1970-01-01T00:00:00.002Z,8.0,",
+
"1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0,",
+
"1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsAreRejectedWith701() {
+ // the same six d1 rows twice: (0, 1, 2, 3, 4, 5) each occurs twice in one
partition
+ String duplicated =
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics
WHERE device = 'd1' "
+ + "UNION ALL SELECT time, device, s1 FROM metrics WHERE device =
'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ String expected =
+ "701: LTTB does not support duplicate timestamps, but timestamp 0
appears more than once"
+ + " in the same partition.";
+
+ tableAssertTestFail(duplicated + "N => 3)", expected, DATABASE_NAME);
+ tableAssertTestFail(duplicated + "SIZE => 2)", expected, DATABASE_NAME);
+ tableAssertTestFail(
+ duplicated + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)",
+ expected,
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsInDifferentPartitionsRemainValid() {
+ // each partition is unique on its own, so the query must succeed and keep
both rows
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics "
+ + "WHERE time = 0 AND (device = 'd1' OR device = 'd2')) "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', N => 3) "
+ + "ORDER BY device",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
"0,d2,1970-01-01T00:00:00.000Z,0.0,",
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testWindowSkipsNullBucketForLookahead() {
+ String data =
+ "DATA => (SELECT time, device, value FROM sparse) PARTITION BY device
ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 3) ORDER BY window_index",
+ new String[] {"window_index", "device", "value_time", "value"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.002Z,2.0,",
+ "2,d1,1970-01-01T00:00:00.006Z,0.0,",
+ "2,d1,1970-01-01T00:00:00.008Z,0.0,"
+ },
+ DATABASE_NAME);
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 3ms) ORDER BY window_start",
+ new String[] {"window_start", "window_end", "device", "value_time",
"value"},
+ new String[] {
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,2.0,",
+
"1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.006Z,0.0,",
+
"1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.008Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testNonFiniteValuesAreKeptInShortSeries() {
+ // three non-NULL input points whose values are NaN/Infinity for N => 3
+ String nonFinite =
+ "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 / 0.0 AS s1 FROM
metrics "
+ + "WHERE device = 'd1' AND time <= 2) "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+
+ tableResultSetEqualTest(
+ nonFinite + "N => 3) ORDER BY s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,NaN,",
+ "0,d1,1970-01-01T00:00:00.001Z,Infinity,",
+ "0,d1,1970-01-01T00:00:00.002Z,Infinity,",
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testInt64SamplingPreservesSmallPeaksAtLargeBaselines() {
+ String prefix =
+ "SELECT * FROM LTTB(DATA => integral_values PARTITION BY device ORDER
BY time, "
+ + "TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ prefix + "N => 3) ORDER BY value_time",
+ new String[] {"window_index", "device", "value_time", "value"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,1152921504606846976,",
+ "0,d1,1970-01-01T00:00:00.002Z,1152921504606846984,",
+ "0,d1,1970-01-01T00:00:00.005Z,1152921504606846976,"
+ },
+ DATABASE_NAME);
+ tableResultSetEqualTest(
+ prefix + "SIZE => 2) ORDER BY window_index",
+ new String[] {"window_index", "device", "value_time", "value"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,1152921504606846976,",
+ "0,d1,1970-01-01T00:00:00.001Z,1152921504606846977,",
+ "1,d1,1970-01-01T00:00:00.002Z,1152921504606846984,",
+ "2,d1,1970-01-01T00:00:00.004Z,1152921504606846976,",
+ "2,d1,1970-01-01T00:00:00.005Z,1152921504606846976,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testSlideSmallerThanSizeKeepsOneAnchorChain() {
+ // SIZE => 2, SLIDE => 1: overlapping count windows. F(t0) is reported by
the first window and
+ // L(t5) by the last one, and the windows in between continue the same
anchor chain.
+ String data =
+ "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 2, SLIDE => 1) ORDER BY
window_index, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "0,d1,1970-01-01T00:00:00.001Z,1.0,",
+ "1,d1,1970-01-01T00:00:00.002Z,8.0,",
+ "2,d1,1970-01-01T00:00:00.003Z,2.0,",
+ "3,d1,1970-01-01T00:00:00.003Z,2.0,",
+ "4,d1,1970-01-01T00:00:00.004Z,0.0,",
+ "5,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ // The time-window form with SLIDE < SIZE starts a window one slide before
the first point.
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 2ms, SLIDE => 1ms) ORDER BY
window_start, s1_time",
+ new String[] {"window_start", "window_end", "device", "s1_time", "s1"},
+ new String[] {
+
"1969-12-31T23:59:59.999Z,1970-01-01T00:00:00.001Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.001Z,1.0,",
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,8.0,",
+
"1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.004Z,d1,1970-01-01T00:00:00.003Z,2.0,",
+
"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.005Z,d1,1970-01-01T00:00:00.003Z,2.0,",
+
"1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0,",
+
"1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.007Z,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testSlideLargerThanSizeSkipsTheGap() {
+ // SIZE => 1, SLIDE => 2 keeps only the windows that receive a row: [0,1),
[2,3), [4,5).
+ String data =
+ "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB(" + data + "SIZE => 1, SLIDE => 2) ORDER BY
window_index, s1_time",
+ new String[] {"window_index", "device", "s1_time", "s1"},
+ new String[] {
+ "0,d1,1970-01-01T00:00:00.000Z,0.0,",
+ "1,d1,1970-01-01T00:00:00.002Z,8.0,",
+ "2,d1,1970-01-01T00:00:00.004Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testTimeWindowWithUnalignedOriginShiftsBoundaries() {
+ String data =
+ "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') "
+ + "PARTITION BY device ORDER BY time, TIMECOL => 'time', ";
+ tableResultSetEqualTest(
+ "SELECT * FROM LTTB("
+ + data
+ + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.001+00:00) "
+ + "ORDER BY window_start, s1_time",
+ new String[] {"window_start", "window_end", "device", "s1_time", "s1"},
+ new String[] {
+
"1969-12-31T23:59:59.999Z,1970-01-01T00:00:00.001Z,d1,1970-01-01T00:00:00.000Z,0.0,",
+
"1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,8.0,",
+
"1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.005Z,d1,1970-01-01T00:00:00.003Z,2.0,",
+
"1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.007Z,d1,1970-01-01T00:00:00.005Z,0.0,"
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testInvalidTimeColumnIsRejected() {
+ tableAssertTestFail(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 's1', N => 3)",
+ "701: ",
+ DATABASE_NAME);
+ tableAssertTestFail(
+ "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time,
"
+ + "TIMECOL => 'missing', N => 3)",
+ "701: ",
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsAcrossDevicesInOnePartitionAreRejected() {
+ // Both devices are tagged 'shared', so PARTITION BY site puts t = 0 of
each device into one
+ // partition and the duplicate check has to reject it.
+ tableAssertTestFail(
+ "SELECT * FROM LTTB(DATA => (SELECT time, site, s1 FROM shared ORDER
BY time) "
+ + "PARTITION BY site ORDER BY time, TIMECOL => 'time', N => 3)",
+ "701: LTTB does not support duplicate timestamps, but timestamp 0
appears more than once"
+ + " in the same partition.",
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDuplicateTimestampsAcrossDistributedFragmentsAreRejected() {
+ // Every timestamp of the partition is written by all eight devices, whose
series slots are
+ // spread over the four DataRegionGroups of the database, so the duplicate
rows are read from
+ // different regions. The function runs once over the merged partition, so
the check sees the
+ // duplicates regardless of how many fragments supplied them.
+ String expected =
+ "701: LTTB does not support duplicate timestamps, but timestamp 0
appears more than once"
+ + " in the same partition.";
+ String prefix =
+ "SELECT * FROM LTTB(DATA => (SELECT time, part, s1 FROM frag_dup) "
+ + "PARTITION BY part ORDER BY time, TIMECOL => 'time', ";
+
+ tableAssertTestFail(prefix + "N => 3)", expected, DATABASE_NAME);
+ tableAssertTestFail(prefix + "SIZE => 3)", expected, DATABASE_NAME);
+ tableAssertTestFail(
+ prefix + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)",
expected, DATABASE_NAME);
Review Comment:
[IT] Please also cover the window-function and `SESSION` inputs
The previous round asked for duplicates across the devices of one partition
with three inputs: the direct scan, the window-function input and the `SESSION`
input. This test covers only the direct scan, and the other two inputs are the
ones that broke before. Both raise 701 at 9a10836, and this version passes with
both profiles:
```suggestion
String[] inputs = {
"(SELECT time, part, s1 FROM frag_dup)",
"(SELECT time, part, s1, "
+ "row_number() OVER (PARTITION BY part, device ORDER BY time) AS
rn FROM frag_dup)",
"(SELECT time, part, s1 FROM SESSION(DATA => frag_dup PARTITION BY
(part, device) "
+ "ORDER BY time, TIMECOL => 'time', GAP => 1m))"
};
for (String input : inputs) {
String prefix =
"SELECT * FROM LTTB(DATA => "
+ input
+ " PARTITION BY part ORDER BY time, TIMECOL => 'time', ";
tableAssertTestFail(prefix + "N => 3)", expected, DATABASE_NAME);
tableAssertTestFail(prefix + "SIZE => 3)", expected, DATABASE_NAME);
tableAssertTestFail(
prefix + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)",
expected,
DATABASE_NAME);
}
```
These queries also fail with 701 when LTTB runs once per region; I checked
that with the exclusion removed. With eight devices in four regions, at least
one region holds two of them, so the duplicate is visible within one fragment.
The split itself is therefore caught only by
`testNonDirectScanInputsMatchTheDirectScan` with a sampling `N`.
##########
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java:
##########
@@ -0,0 +1,1847 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.udf.builtin.relational.tvf;
+
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.exception.SemanticException;
+import org.apache.iotdb.commons.i18n.CommonMessages;
+import
org.apache.iotdb.commons.queryengine.plan.relational.function.TableFunctionProcessorMemoryBudget;
+import org.apache.iotdb.udf.api.exception.UDFException;
+import org.apache.iotdb.udf.api.relational.TableFunction;
+import org.apache.iotdb.udf.api.relational.access.Record;
+import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle;
+import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis;
+import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle;
+import
org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider;
+import org.apache.iotdb.udf.api.relational.table.argument.Argument;
+import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema;
+import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument;
+import org.apache.iotdb.udf.api.relational.table.argument.TableArgument;
+import
org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor;
+import
org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification;
+import
org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification;
+import
org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification;
+import org.apache.iotdb.udf.api.type.Type;
+
+import org.apache.tsfile.block.column.ColumnBuilder;
+import org.apache.tsfile.utils.RamUsageEstimator;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static
org.apache.iotdb.udf.api.relational.table.argument.ScalarArgumentChecker.POSITIVE_LONG_CHECKER;
+
+/**
+ * Largest-Triangle-Three-Buckets (LTTB) downsampling as a built-in Table
Model table function.
+ *
+ * <p>The function shares its table-argument conventions with {@link
M4TableFunction}: {@code DATA}
+ * has set semantics, {@code TIMECOL} names the {@code TIMESTAMP} ordering
column, partition columns
+ * are echoed into the proper output, and every other numeric column is a
participant column that is
+ * downsampled independently. Two mutually exclusive execution modes exist:
+ *
+ * <ul>
+ * <li><b>Target-count mode</b> ({@code N}): the whole partition is buffered
and the standard LTTB
+ * algorithm reduces each participant column to exactly {@code N} points
(or all points when
+ * the column has no more than {@code N} eligible points). The output
uses a fixed {@code
+ * window_index = 0}.
+ * <li><b>Window/bucket mode</b> ({@code SIZE}, optional {@code
SLIDE}/{@code ORIGIN}): windows
+ * are built exactly like M4 (time windows for duration literals, count
windows for integers),
+ * but {@code SIZE} defines a bucket rather than the input range of a
separate standard LTTB
+ * invocation. The result is standard LTTB whose buckets are the windows:
+ * <ul>
+ * <li>Per partition and participant column, the first eligible point
{@code F} and the last
+ * eligible point {@code L} that fall in a window are buckets of
their own and are
+ * always returned, as in the original algorithm. {@code F} is
reported by the first
+ * window that contains it and {@code L} by the last one.
+ * <li>Every other window contributes one bucket made of its remaining
eligible points. The
+ * anchor chain starts at {@code F}: for a window bucket {@code A}
is the previously
+ * selected point, {@code C} is the average point of the next
window bucket that has
+ * points ({@code L} when there is none), ties go to the earliest
point, and buckets
+ * without points are skipped. Overlapping windows ({@code SLIDE <
SIZE}) keep one
+ * chain, and the state is independent between partitions and
columns.
+ * </ul>
+ * </ul>
+ *
+ * <p>Bucket mode emits, per window, up to three rows per participant column:
{@code F}, the point
+ * selected from the window bucket, and {@code L}. Rows are padded by position
as in M4, and a
+ * window in which no participant column has an eligible point emits no row.
Target-count mode emits
+ * as many rows as the column with the most selected points and no row when
that number is zero.
+ *
+ * <p>{@code NULL} participant values are ignored per column; every non-null
value, including {@code
+ * NaN} and the infinities, is eligible and is reproduced in the output. A
non-finite value cannot
+ * serve as a triangle vertex, though: the anchor is the most recent finite
selected point (or the
+ * first finite eligible point), the lookahead average uses finite points only
and skips buckets
+ * without one, and a bucket without a finite candidate returns its earliest
point without moving
+ * the anchor. Ties in the triangle area are broken deterministically by the
earliest timestamp
+ * (stable input order).
+ *
+ * <p>The timestamps of the input rows of one partition must be unique. As
soon as two rows of the
+ * same partition carry the same timestamp the query fails with a semantic
error ({@code 701}); the
+ * check runs at the partition input boundary, before per-column {@code NULL}
filtering and before
+ * window expansion, so a row that several overlapping windows share is still
a single input row.
+ * Equal timestamps in different partitions remain valid. Since the input is
ordered by {@code
+ * TIMECOL}, equal timestamps are adjacent, and the check compares each row
with the previous one
+ * and carries the last seen timestamp across input blocks.
+ */
+public class LTTBTableFunction implements TableFunction {
+
+ public static final String DATA_PARAMETER_NAME = "DATA";
+ public static final String TIMECOL_PARAMETER_NAME = "TIMECOL";
+ public static final String N_PARAMETER_NAME = "N";
+ public static final String SIZE_PARAMETER_NAME = "SIZE";
+ public static final String SLIDE_PARAMETER_NAME = "SLIDE";
+ public static final String ORIGIN_PARAMETER_NAME = "ORIGIN";
+
+ /** Internal boolean argument injected by the analyzer: {@code true} if SIZE
is a duration. */
+ public static final String WINDOW_MODE_PARAMETER_NAME = "__LTTB_WINDOW_MODE";
+
+ public static final long UNSPECIFIED_N = Long.MIN_VALUE;
+ public static final long UNSPECIFIED_SIZE = Long.MIN_VALUE;
+ public static final long UNSPECIFIED_SLIDE = Long.MIN_VALUE;
+ public static final long MIN_TARGET_COUNT = 3L;
+
+ /** Hard cap on eligible points buffered per participant column in one
partition or window. */
+ public static final int MAX_BUFFERED_POINTS = 1 << 22;
+
+ /**
+ * Fallback for {@code lttb_buffer_size_in_bytes} when it is not positive.
The configured default
+ * in {@code iotdb-system.properties.template} is {@code 0}, which {@code
IoTDBDescriptor}
+ * resolves to {@code min(32 MiB, operator memory / query_thread_count /
2)}; this constant
+ * therefore only matches {@link
org.apache.iotdb.commons.conf.CommonConfig}'s initial value and
+ * keeps a misconfigured node bounded.
+ */
+ public static final long DEFAULT_MAX_BUFFER_BYTES = 32L * 1024 * 1024;
+
+ public static final String MODE_PROPERTY = "__LTTB_MODE";
+ public static final String PARTITION_TYPES_PROPERTY =
"__LTTB_PARTITION_TYPES";
+ public static final String PARTICIPANT_TYPES_PROPERTY =
"__LTTB_PARTICIPANT_TYPES";
+
+ private static final String OUTPUT_WINDOW_START_COLUMN = "window_start";
+ private static final String OUTPUT_WINDOW_END_COLUMN = "window_end";
+ private static final String OUTPUT_WINDOW_INDEX_COLUMN = "window_index";
+ private static final String TIME_SUFFIX = "_time";
+ private static final int INITIAL_CAPACITY = 64;
+ private static final int NOT_SELECTED = -1;
+
+ /** Execution mode persisted in the {@link MapTableFunctionHandle}. */
+ public enum Mode {
+ TARGET_COUNT,
+ TIME_WINDOW,
+ COUNT_WINDOW
+ }
+
+ /**
+ * The live value of the hot-reloadable {@code lttb_buffer_size_in_bytes}
configuration parameter.
+ * Values that are not positive fall back to {@link
#DEFAULT_MAX_BUFFER_BYTES} so a misconfigured
+ * node still bounds memory instead of buffering without limit.
+ *
+ * <p>This is read once per {@linkplain TableFunctionProcessorProvider}, in
{@link
+ * #getProcessorProvider(TableFunctionHandle)}. Every processor of that
provider enforces the same
+ * value, and the operator reserves the same value, so a hot reload cannot
make the two differ for
+ * a running query; it applies to providers created afterwards.
+ */
+ public static long maxBufferBytes() {
+ long configured =
CommonDescriptor.getInstance().getConfig().getLttbBufferSize();
+ return configured > 0 ? configured : DEFAULT_MAX_BUFFER_BYTES;
+ }
+
+ @Override
+ public List<ParameterSpecification> getArgumentsSpecifications() {
+ return Arrays.asList(
+
TableParameterSpecification.builder().name(DATA_PARAMETER_NAME).setSemantics().build(),
+ ScalarParameterSpecification.builder()
+ .name(TIMECOL_PARAMETER_NAME)
+ .type(Type.STRING)
+ .build(),
+ ScalarParameterSpecification.builder()
+ .name(N_PARAMETER_NAME)
+ .type(Type.INT64)
+ .defaultValue(UNSPECIFIED_N)
+ .build(),
+ ScalarParameterSpecification.builder()
+ .name(SIZE_PARAMETER_NAME)
+ .type(Type.INT64)
+ .defaultValue(UNSPECIFIED_SIZE)
+ .addChecker(POSITIVE_LONG_CHECKER)
+ .build(),
+ ScalarParameterSpecification.builder()
+ .name(SLIDE_PARAMETER_NAME)
+ .type(Type.INT64)
+ .defaultValue(UNSPECIFIED_SLIDE)
+ .addChecker(POSITIVE_LONG_CHECKER)
+ .build(),
+ ScalarParameterSpecification.builder()
+ .name(ORIGIN_PARAMETER_NAME)
+ .type(Type.TIMESTAMP)
+ .defaultValue(0L)
+ .build());
+ }
+
+ @Override
+ public TableFunctionAnalysis analyze(Map<String, Argument> arguments) throws
UDFException {
+ int timeColumnIndex =
+ WindowTVFUtils.checkOrderByColumn(arguments, DATA_PARAMETER_NAME,
TIMECOL_PARAMETER_NAME);
+ TableArgument tableArgument = (TableArgument)
arguments.get(DATA_PARAMETER_NAME);
+ List<Integer> partitionIndexes =
WindowTVFUtils.getPartitionIndexes(tableArgument);
+ Set<Integer> excludedIndexes = new HashSet<>(partitionIndexes);
+ excludedIndexes.add(timeColumnIndex);
+
+ long n = getLong(arguments, N_PARAMETER_NAME);
+ long size = getLong(arguments, SIZE_PARAMETER_NAME);
+ long slide = getLong(arguments, SLIDE_PARAMETER_NAME);
+ boolean nSpecified = n != UNSPECIFIED_N;
+ boolean sizeSpecified = size != UNSPECIFIED_SIZE;
+ boolean slideSpecified = slide != UNSPECIFIED_SLIDE;
+ if (nSpecified == sizeSpecified) {
+ throw new SemanticException(
+ CommonMessages
+
.EXCEPTION_EXACTLY_ONE_OF_THE_N_AND_SIZE_ARGUMENTS_MUST_BE_SPECIFIED_FOR_LTTB_54AF0733);
+ }
+
+ Mode mode;
+ if (nSpecified) {
+ if (slideSpecified) {
+ throw new SemanticException(
+ CommonMessages
+
.EXCEPTION_THE_N_ARGUMENT_OF_LTTB_CANNOT_BE_COMBINED_WITH_THE_SLIDE_OR_ORIGIN_ARGUMENTS_C2FF1FE1);
+ }
+ if (n < MIN_TARGET_COUNT) {
+ throw new SemanticException(
+
CommonMessages.EXCEPTION_THE_N_ARGUMENT_OF_LTTB_MUST_BE_AT_LEAST_3_29CE2B87);
+ }
+ mode = Mode.TARGET_COUNT;
+ } else {
+ if (size <= 0) {
+ throw new UDFException(
+ CommonMessages
+
.EXCEPTION_INVALID_SCALAR_ARGUMENT_SIZE_SHOULD_BE_A_POSITIVE_VALUE_3ECF76E0);
+ }
+ if (!slideSpecified) {
+ slide = size;
+ } else if (slide <= 0) {
+ throw new UDFException(
+ CommonMessages
+
.EXCEPTION_INVALID_SCALAR_ARGUMENT_SLIDE_SHOULD_BE_A_POSITIVE_VALUE_F019E091);
+ }
+ boolean isTimeWindow =
+ arguments.containsKey(WINDOW_MODE_PARAMETER_NAME)
+ && (boolean) ((ScalarArgument)
arguments.get(WINDOW_MODE_PARAMETER_NAME)).getValue();
+ mode = isTimeWindow ? Mode.TIME_WINDOW : Mode.COUNT_WINDOW;
+ }
+
+ DescribedSchema.Builder schemaBuilder = new DescribedSchema.Builder();
+ if (mode == Mode.TIME_WINDOW) {
+ schemaBuilder
+ .addField(OUTPUT_WINDOW_START_COLUMN, Type.TIMESTAMP)
+ .addField(OUTPUT_WINDOW_END_COLUMN, Type.TIMESTAMP);
+ } else {
+ schemaBuilder.addField(OUTPUT_WINDOW_INDEX_COLUMN, Type.INT64);
+ }
+
+ List<Type> partitionTypes = new ArrayList<>();
+ for (int partitionIndex : partitionIndexes) {
+ Type type = tableArgument.getFieldTypes().get(partitionIndex);
+ partitionTypes.add(type);
+
schemaBuilder.addField(tableArgument.getFieldNames().get(partitionIndex).get(),
type);
+ }
+
+ // Rejects every non-numeric participant column with the shared "not
allowed columns" message.
+ List<Integer> participantIndexes =
+ WindowTVFUtils.getCalculationIndexes(tableArgument, excludedIndexes,
null);
+ if (participantIndexes.isEmpty()) {
+ throw new
SemanticException(CommonMessages.EXCEPTION_NO_CALCULATE_COLUMNS);
+ }
+ List<Type> participantTypes = new ArrayList<>();
+ for (int participantIndex : participantIndexes) {
+ Type type = tableArgument.getFieldTypes().get(participantIndex);
+ String columnName =
tableArgument.getFieldNames().get(participantIndex).get();
+ participantTypes.add(type);
+ schemaBuilder.addField(columnName + TIME_SUFFIX, Type.TIMESTAMP);
+ schemaBuilder.addField(columnName, type);
+ }
+
+ MapTableFunctionHandle.Builder handleBuilder =
+ new MapTableFunctionHandle.Builder()
+ .addProperty(MODE_PROPERTY, mode.name())
+ .addProperty(PARTITION_TYPES_PROPERTY,
WindowTVFUtils.joinTypes(partitionTypes))
+ .addProperty(PARTICIPANT_TYPES_PROPERTY,
WindowTVFUtils.joinTypes(participantTypes));
+ if (mode == Mode.TARGET_COUNT) {
+ handleBuilder.addProperty(N_PARAMETER_NAME, n);
+ } else {
+ handleBuilder.addProperty(SIZE_PARAMETER_NAME,
size).addProperty(SLIDE_PARAMETER_NAME, slide);
+ if (mode == Mode.TIME_WINDOW) {
+ handleBuilder.addProperty(ORIGIN_PARAMETER_NAME, getLong(arguments,
ORIGIN_PARAMETER_NAME));
+ }
+ }
+
+ List<Integer> requiredColumns = new ArrayList<>();
+ requiredColumns.add(timeColumnIndex);
+ requiredColumns.addAll(partitionIndexes);
+ requiredColumns.addAll(participantIndexes);
+
+ return TableFunctionAnalysis.builder()
+ .properColumnSchema(schemaBuilder.build())
+ .requireRecordSnapshot(false)
+ .requiredColumns(DATA_PARAMETER_NAME, requiredColumns)
+ .handle(handleBuilder.build())
+ .build();
+ }
+
+ private static long getLong(Map<String, Argument> arguments, String name) {
+ return (long) ((ScalarArgument) arguments.get(name)).getValue();
+ }
+
+ @Override
+ public TableFunctionHandle createTableFunctionHandle() {
+ return new MapTableFunctionHandle();
+ }
+
+ @Override
+ public TableFunctionProcessorProvider getProcessorProvider(
+ TableFunctionHandle tableFunctionHandle) {
+ MapTableFunctionHandle handle = (MapTableFunctionHandle)
tableFunctionHandle;
+ Mode mode = Mode.valueOf((String) handle.getProperty(MODE_PROPERTY));
+ Type[] partitionTypes =
+ WindowTVFUtils.parseTypes((String)
handle.getProperty(PARTITION_TYPES_PROPERTY));
+ Type[] participantTypes =
+ WindowTVFUtils.parseTypes((String)
handle.getProperty(PARTICIPANT_TYPES_PROPERTY));
+ // Resolved once so that every processor of this provider enforces the
same limit the operator
+ // reserves, even if lttb_buffer_size_in_bytes is hot-reloaded while the
query runs.
+ long maxBufferedBytes = maxBufferBytes();
+
+ return new LTTBProcessorProvider(
+ mode, partitionTypes, participantTypes, handle, maxBufferedBytes);
+ }
+
+ /**
+ * The provider of one LTTB invocation. It reports the budget it resolved at
construction, so the
+ * operator reserves exactly the limit its processors enforce.
+ */
+ private static final class LTTBProcessorProvider
+ implements TableFunctionProcessorProvider,
TableFunctionProcessorMemoryBudget {
+ private final Mode mode;
+ private final Type[] partitionTypes;
+ private final Type[] participantTypes;
+ private final MapTableFunctionHandle handle;
+ private final long maxBufferedBytes;
+
+ private LTTBProcessorProvider(
+ Mode mode,
+ Type[] partitionTypes,
+ Type[] participantTypes,
+ MapTableFunctionHandle handle,
+ long maxBufferedBytes) {
+ this.mode = mode;
+ this.partitionTypes = partitionTypes;
+ this.participantTypes = participantTypes;
+ this.handle = handle;
+ this.maxBufferedBytes = maxBufferedBytes;
+ }
+
+ @Override
+ public long getEstimatedMaxBufferedBytes() {
+ return maxBufferedBytes;
+ }
+
+ @Override
+ public TableFunctionDataProcessor getDataProcessor() {
+ ParticipantColumn[] participantColumns =
+ createParticipantColumns(participantTypes, partitionTypes.length +
1);
+ switch (mode) {
+ case TARGET_COUNT:
+ return new TargetCountLTTBDataProcessor(
+ (long) handle.getProperty(N_PARAMETER_NAME),
+ partitionTypes,
+ participantColumns,
+ maxBufferedBytes);
+ case TIME_WINDOW:
+ return new TimeWindowLTTBDataProcessor(
+ (long) handle.getProperty(SIZE_PARAMETER_NAME),
+ (long) handle.getProperty(SLIDE_PARAMETER_NAME),
+ (long) handle.getProperty(ORIGIN_PARAMETER_NAME),
+ partitionTypes,
+ participantColumns,
+ maxBufferedBytes);
+ case COUNT_WINDOW:
+ return new CountWindowLTTBDataProcessor(
+ (long) handle.getProperty(SIZE_PARAMETER_NAME),
+ (long) handle.getProperty(SLIDE_PARAMETER_NAME),
+ partitionTypes,
+ participantColumns,
+ maxBufferedBytes);
+ default:
+ throw new IllegalStateException(String.valueOf(mode));
+ }
+ }
+ }
+
+ private static ParticipantColumn[] createParticipantColumns(Type[] types,
int firstInputIndex) {
+ ParticipantColumn[] columns = new ParticipantColumn[types.length];
+ for (int i = 0; i < types.length; i++) {
+ columns[i] = new ParticipantColumn(firstInputIndex + i, types[i]);
+ }
+ return columns;
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // LTTB core
+ //
---------------------------------------------------------------------------------------------
+
+ /**
+ * Standard LTTB selection over an ordered point sequence.
+ *
+ * <p>When {@code count <= n} every index is returned. Otherwise the first
and last points are
+ * retained, bucket {@code i} covers {@code [floor(i*(count-2)/(n-2))+1,
+ * floor((i+1)*(count-2)/(n-2))+1)}, and for every bucket the candidate
forming the largest
+ * triangle with the anchor and the lookahead average wins. The last bucket
uses the final point
+ * as the lookahead, and ties are broken by the earliest timestamp.
+ *
+ * <p>{@code NaN} and infinite values are eligible points, but they cannot
serve as a triangle
+ * vertex: an undefined area would make every candidate tie and the bucket
would return its
+ * earliest point regardless of the data. The anchor is therefore the most
recent finite selected
+ * point (or the first finite point), the lookahead average uses only finite
points and skips
+ * buckets without one, and a bucket with no finite candidate returns its
earliest point without
+ * moving the anchor. Non-finite points are still selected and returned when
their bucket has no
+ * finite alternative.
+ *
+ * @return ascending indexes of the selected points
+ */
+ static int[] selectTargetCount(long[] times, double[] values, int count,
long n) {
+ return selectTargetCount(times, values, null, count, n);
+ }
+
+ private static int[] selectTargetCount(
+ long[] times, double[] values, long[] integralValues, int count, long n)
{
+ if (count <= n) {
+ int[] all = new int[count];
+ for (int i = 0; i < count; i++) {
+ all[i] = i;
+ }
+ return all;
+ }
+
+ int[] selected = new int[(int) n];
+ int selectedCount = 0;
+ selected[selectedCount++] = 0;
+ long bucketCount = n - 2;
+ long intermediate = count - 2L;
+ // The anchor starts at the earliest finite point so that every area it
participates in has a
+ // representable origin. It only moves to a selected point; a bucket with
no finite candidate
+ // leaves it in place.
+ int anchor = 0;
+ while (anchor < count && !isFiniteValue(values, integralValues, anchor)) {
+ anchor++;
+ }
+ boolean hasAnchor = anchor < count;
+ for (long bucket = 0; bucket < bucketCount; bucket++) {
+ int currentStart = (int) (bucket * intermediate / bucketCount) + 1;
+ int currentEnd = (int) ((bucket + 1) * intermediate / bucketCount) + 1;
+ int nextEnd =
+ bucket == bucketCount - 1 ? count : (int) ((bucket + 2) *
intermediate / bucketCount) + 1;
+
+ long anchorTime = times[hasAnchor ? anchor : 0];
+ double anchorValue = hasAnchor ? values[anchor] : 0;
+ long integralAnchor = hasAnchor && integralValues != null ?
integralValues[anchor] : 0;
+ double avgTime;
+ double avgValue;
+ int lookaheadEnd;
+ if (bucket == bucketCount - 1) {
+ lookaheadEnd = count;
+ } else {
+ lookaheadEnd = nextEnd;
+ }
+ if (bucket == bucketCount - 1) {
+ avgTime = differenceAsDouble(times[count - 1], anchorTime);
+ avgValue = relativeValue(values, integralValues, count - 1,
anchorValue, integralAnchor);
Review Comment:
[non-finite] In `N` mode, `C` does not follow the documented rule in two
cases
The class Javadoc (lines 96–99) and the Javadoc of `selectTargetCount`
(lines 424–430) say that `C` uses only finite points and skips buckets without
one. `SIZE` mode does this: `pickInBucket` uses the next window bucket that has
a finite point, and otherwise the last finite eligible point. `N` mode differs
in two cases.
**1. The last bucket (these lines).** `C` is the final point even when that
point is non-finite. Every area in the bucket is then undefined, so
`argMaxArea` hands the bucket to `argMaxAreaExactly`. That method finds no
finite point in `[count - 1, count)`, scores every candidate 0 and returns the
first finite candidate:
| input (t = 0, 1, ...) | args | 9a10836 | documented rule |
|---|---|---|---|
| `0, 1, 9, 2, 0, NaN` | `N => 3` | t0:0, **t1:1**, t5:NaN | t0:0, **t2:9**,
t5:NaN |
| `0, 1, 9, 2, 0, Infinity` | `N => 3` | t0:0, **t1:1**, t5:Infinity | t0:0,
**t2:9**, t5:Infinity |
| `0, 1, 9, 2, NaN, NaN` | `N => 3` | t0:0, **t1:1**, t5:NaN | t0:0,
**t2:9**, t5:NaN |
| `0, 5, 1, 2, 9, 3, NaN` | `N => 4` | t0:0, t1:5, **t3:2**, t6:NaN | t0:0,
t1:5, **t4:9**, t6:NaN |
With a finite final point, `0, 1, 9, 2, 0, 0` returns t0:0, t2:9, t5:0.
`SIZE` mode handles a non-finite `L` correctly.
**2. A next bucket without a finite point (lines 493–503).** `C` becomes the
last finite point of the partition instead of the average of the next bucket
that has one:
| input (t = 0, 1, ...) | args | 9a10836 | documented rule |
|---|---|---|---|
| `0, 5, 4, NaN, NaN, 10, 10, -20` | `N => 5` | t0:0, **t2:4**, t3:NaN,
t6:10, t7:-20 | t0:0, **t1:5**, t3:NaN, t6:10, t7:-20 |
For the first bucket, `C` is t7:-20 rather than the average of t5 and t6.
**Suggested fix.** Use one lookahead rule for every bucket: `C` averages the
finite points of the next bucket that has one, with the final point as the last
bucket, and is the last finite point when there is no such bucket.
`argMaxAreaExactly`, which today always receives `[currentEnd, lookaheadEnd)`,
should receive the same points. I tried this locally. All the cases above then
match the documented rule, and the 41 LTTB unit tests, the expected results
from the previous round and a differential test of 30,000 random cases still
pass.
**Tests.** The previous round asked for expected-result tests of the
non-finite rule. None were added.
`testTargetCountSkipsNonFiniteCandidatesWhenSampling` only checks the endpoints
and the length, and the other tests use only non-finite values or no more than
`N` points. Please add the cases from the previous round and the ones above, in
both `N` and `SIZE` mode.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]