This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 74a0d306f81 perf: avoid reading tsfile on distinct aggregation on
tag/attribute column
74a0d306f81 is described below
commit 74a0d306f8138c14406d4fb2ccfc120b9adf5183
Author: shizy <[email protected]>
AuthorDate: Sat Apr 12 08:26:20 2025 +0800
perf: avoid reading tsfile on distinct aggregation on tag/attribute column
---
.../it/query/recent/IoTDBDistinctTagIT.java | 257 +++++++++++++++++++++
.../relational/AbstractAggTableScanOperator.java | 9 +-
2 files changed, 259 insertions(+), 7 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java
new file mode 100644
index 00000000000..f0d0ca89ff6
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.query.recent;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBDistinctTagIT {
+ private static final String DATABASE_NAME = "test";
+
+ protected static final String[] SQLs =
+ new String[] {
+ "CREATE DATABASE IF NOT EXISTS test",
+ "USE test",
+ // test flush
+ "CREATE TABLE IF NOT EXISTS t1(deviceId STRING TAG, attr1 STRING
ATTRIBUTE, s1 INT64 FIELD)",
+ "insert into t1(time, deviceId, attr1, s1) values(1000, 'd1', 'a1',
10)",
+ "insert into t1(time, deviceId, attr1, s1) values(1000, 'd2', 'a2',
11)",
+ "insert into t1(time, deviceId, attr1, s1) values(2000, 'd2', 'xx',
12)",
+ "insert into t1(time, deviceId, attr1, s1) values(4000, 'd1', 'a1',
13)",
+ "flush",
+ "insert into t1(time, deviceId, attr1, s1) values(5000, 'd3', 'a3',
10)",
+ "insert into t1(time, deviceId, attr1, s1) values(6000, 'd4', 'a4',
11)",
+ "insert into t1(time, deviceId, attr1, s1) values(3000, 'd2', 'a2',
12)",
+ "insert into t1(time, deviceId, attr1, s1) values(2000, 'd1', 'a1',
13)",
+ "flush",
+
+ // test memory
+ "CREATE TABLE IF NOT EXISTS t2(deviceId STRING TAG, attr1 STRING
ATTRIBUTE, s1 INT64 FIELD)",
+ "insert into t2(time, deviceId, attr1, s1) values(1000, 'd1', 'a1',
10)",
+ "insert into t2(time, deviceId, attr1, s1) values(1000, 'd2', 'a2',
11)",
+ "insert into t2(time, deviceId, attr1, s1) values(2000, 'd2', 'xx',
12)",
+ "insert into t2(time, deviceId, attr1, s1) values(4000, 'd1', 'a1',
13)",
+ "insert into t2(time, deviceId, attr1, s1) values(5000, 'd3', 'a3',
10)",
+ "insert into t2(time, deviceId, attr1, s1) values(6000, 'd4', 'a4',
11)",
+ "insert into t2(time, deviceId, attr1, s1) values(3000, 'd2', 'a2',
12)",
+ "insert into t2(time, deviceId, attr1, s1) values(2000, 'd1', 'a1',
13)",
+ };
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+
EnvFactory.getEnv().getConfig().getCommonConfig().setPartitionInterval(1000);
+ EnvFactory.getEnv().initClusterEnvironment();
+ prepareData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testDistinct() {
+ // distinct(deviceId)
+ String[] expectedHeader = new String[] {"deviceId"};
+ String[] retArray = new String[] {"d1,", "d2,", "d3,", "d4,"};
+ tableResultSetEqualTest(
+ "select distinct(deviceId) from test.t1 order by deviceId",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "select distinct(deviceId) from test.t2 order by deviceId",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ // distinct(attr1)
+ expectedHeader = new String[] {"attr1"};
+ retArray = new String[] {"a1,", "a2,", "a3,", "a4,"};
+ tableResultSetEqualTest(
+ "select distinct(attr1) from test.t1 order by attr1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "select distinct(attr1) from test.t2 order by attr1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDistinctWithTimeFilter() {
+ // distinct(deviceId) ... where time > 3000;
+ String[] expectedHeader = new String[] {"deviceId"};
+ String[] retArray = new String[] {"d1,", "d3,", "d4,"};
+ tableResultSetEqualTest(
+ "select distinct(deviceId) from test.t1 where time > 3000 order by
deviceId",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "select distinct(deviceId) from test.t2 where time > 3000 order by
deviceId",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ // distinct(attr1) ... where time = 5000;
+ expectedHeader = new String[] {"attr1"};
+ retArray = new String[] {"a3,"};
+ tableResultSetEqualTest(
+ "select distinct(attr1) from test.t1 where time = 5000 order by attr1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "select distinct(attr1) from test.t2 where time = 5000 order by attr1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDistinctWithPushDownFilter() {
+ // distinct(deviceId) ... where s1 = 11;
+ String[] expectedHeader = new String[] {"deviceId"};
+ String[] retArray = new String[] {"d2,", "d4,"};
+ tableResultSetEqualTest(
+ "select distinct(deviceId) from test.t1 where s1 = 11 order by
deviceId",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "select distinct(deviceId) from test.t2 where s1 = 11 order by
deviceId",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ // distinct(attr1) ... where s1 > 11;
+ expectedHeader = new String[] {"attr1"};
+ retArray = new String[] {"a1,", "a2,"};
+ tableResultSetEqualTest(
+ "select distinct(attr1) from test.t1 where s1 > 11 order by attr1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "select distinct(attr1) from test.t2 where s1 > 11 order by attr1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void testDistinctWithDelete() throws SQLException {
+ String[] sqls =
+ new String[] {
+ "USE test",
+ // test flush
+ "CREATE TABLE IF NOT EXISTS t3(deviceId STRING TAG, attr1 STRING
ATTRIBUTE, s1 INT64 FIELD)",
+ "insert into t3(time, deviceId, attr1, s1) values(1000, 'd1', 'a1',
10)",
+ "insert into t3(time, deviceId, attr1, s1) values(1000, 'd2', 'a2',
11)",
+ "insert into t3(time, deviceId, attr1, s1) values(2000, 'd2', 'xx',
12)",
+ "insert into t3(time, deviceId, attr1, s1) values(4000, 'd1', 'a1',
13)",
+ "flush",
+ "delete from test.t3",
+
+ // test memory
+ "CREATE TABLE IF NOT EXISTS t4(deviceId STRING TAG, attr1 STRING
ATTRIBUTE, s1 INT64 FIELD)",
+ "insert into t4(time, deviceId, attr1, s1) values(1000, 'd1', 'a1',
10)",
+ "insert into t4(time, deviceId, attr1, s1) values(1000, 'd2', 'a2',
11)",
+ "insert into t4(time, deviceId, attr1, s1) values(2000, 'd2', 'xx',
12)",
+ "insert into t4(time, deviceId, attr1, s1) values(4000, 'd1', 'a1',
13)",
+ "delete from test.t4",
+ };
+
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ for (String sql : sqls) {
+ statement.execute(sql);
+ }
+ }
+
+ // distinct(deviceId)
+ String[] expectedHeader = new String[] {"deviceId"};
+ String[] retArray = new String[] {};
+ tableResultSetEqualTest(
+ "select distinct(deviceId) from test.t3 order by deviceId",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+ tableResultSetEqualTest(
+ "select distinct(deviceId) from test.t4 order by deviceId",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ // distinct(attr1)
+ expectedHeader = new String[] {"attr1"};
+ retArray = new String[] {};
+ tableResultSetEqualTest(
+ "select distinct(attr1) from test.t3 order by attr1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "select distinct(attr1) from test.t4 order by attr1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+ }
+
+ private static void prepareData() {
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : SQLs) {
+ statement.execute(sql);
+ }
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractAggTableScanOperator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractAggTableScanOperator.java
index bd0f9c16ef9..52d829f7cf6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractAggTableScanOperator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractAggTableScanOperator.java
@@ -667,11 +667,6 @@ public abstract class AbstractAggTableScanOperator extends
AbstractDataSourceOpe
return false;
}
- // no aggregation function, just output ids or attributes
- if (aggregators.isEmpty()) {
- return false;
- }
-
for (TableAggregator aggregator : aggregators) {
if (!aggregator.hasFinalResult()) {
return false;
@@ -684,8 +679,8 @@ public abstract class AbstractAggTableScanOperator extends
AbstractDataSourceOpe
private void checkIfAllAggregatorHasFinalResult() {
if (allAggregatorsHasFinalResult
- && timeIterator.getType()
- == ITableTimeRangeIterator.TimeIteratorType.SINGLE_TIME_ITERATOR) {
+ && (timeIterator.getType() ==
ITableTimeRangeIterator.TimeIteratorType.SINGLE_TIME_ITERATOR
+ || tableAggregators.isEmpty())) {
nextDevice();
inputTsBlock = null;
