This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch fix_groupby_agg in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 307914ce5bac9835b91f77ccbcfe0c3c9987ee9e Author: Beyyes <[email protected]> AuthorDate: Wed Oct 16 11:06:35 2024 +0800 fix it --- .../db/it/IoTDBMultiIDsWithAttributesTableIT.java | 124 ++++++++++++++++++++- .../TableSingleTimeWindowIterator.java | 13 ++- .../TableAggregationTableScanOperator.java | 7 ++ .../relational/aggregation/AvgAccumulator.java | 3 +- .../plan/planner/TableOperatorGenerator.java | 10 +- 5 files changed, 150 insertions(+), 7 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java index 3d997449470..08ac67178b0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java @@ -633,12 +633,35 @@ public class IoTDBMultiIDsWithAttributesTableIT { "d2,l4,3,3,3,0,0,0,3,27.0,", "d2,l5,3,3,3,1,0,0,3,30.0,", }; - String sql = + sql = "select device, level, " + "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num " + "from table0 group by device,level order by device, level"; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "device", + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "sum_num" + }; + retArray = + new String[] { + "d1,3,3,3,0,3,3,3,20.0,", + }; + sql = + "select device, " + + "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num " + + "from table0 where device='d1' and level='l1' group by device order by device"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test @@ -776,7 +799,106 @@ public class IoTDBMultiIDsWithAttributesTableIT { // ")\n"; } + @Test + public void aggregationNoDataTest() { + expectedHeader = + new String[] { + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "sum_num", + "avg_num" + }; + + retArray = + new String[] { + "0,0,0,0,0,0,0,null,null,", + }; + sql = + "select count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + retArray = + new String[] { + "2,2,2,0,2,1,2,24.0,12.0,", + }; + sql = + "select count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 or time=1971-04-27T01:46:40.000+08:00"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "device", + "level", + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "sum_num", + "avg_num" + }; + retArray = new String[] {}; + sql = + "select device, level, count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 group by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + retArray = new String[] {"d1,l2,1,1,1,0,1,1,1,12.0,12.0,", "d2,l2,1,1,1,0,1,0,1,12.0,12.0,"}; + sql = + "select device, level, count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 or time=1971-04-27T01:46:40.000+08:00 group by device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "device", + "level", + "bin", + "count_num", + "count_star", + "count_device", + "count_date", + "count_attr1", + "count_attr2", + "count_time", + "sum_num", + "avg_num" + }; + retArray = new String[] {}; + sql = + "select device, level, date_bin(1d, time) as bin, count(num) as count_num, count(*) as count_star, " + + "count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 group by 3, device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + retArray = + new String[] { + "d1,l2,1971-04-26T00:00:00.000Z,1,1,1,0,1,1,1,12.0,12.0,", + "d2,l2,1971-04-26T00:00:00.000Z,1,1,1,0,1,0,1,12.0,12.0," + }; + sql = + "select device, level, date_bin(1d, time) as bin, count(num) as count_num, count(*) as count_star, " + + "count(device) as count_device, count(date) as count_date, " + + "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num," + + "avg(num) as avg_num from table0 where time=32 or time=1971-04-27T01:46:40.000+08:00 group by 3, device, level"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + // ================================================================== // ============================ Join Test =========================== + // ================================================================== // no filter @Test public void innerJoinTest1() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/TableSingleTimeWindowIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/TableSingleTimeWindowIterator.java index b3b61dbabbc..d23decf12ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/TableSingleTimeWindowIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/TableSingleTimeWindowIterator.java @@ -28,10 +28,12 @@ public class TableSingleTimeWindowIterator implements ITableTimeRangeIterator { private TimeRange curTimeRange; - public TableSingleTimeWindowIterator(long startTime, long endTime) { - curTimeRange = new TimeRange(startTime, endTime); + public TableSingleTimeWindowIterator(TimeRange curTimeRange) { + this.curTimeRange = curTimeRange; } + public TableSingleTimeWindowIterator() {} + @Override public TimeIteratorType getType() { return TimeIteratorType.SINGLE_TIME_ITERATOR; @@ -59,12 +61,15 @@ public class TableSingleTimeWindowIterator implements ITableTimeRangeIterator { @Override public void resetCurTimeRange() { - // do nothing + this.curTimeRange = null; } @Override public void updateCurTimeRange(long startTime) { - // not used + // only meets real data, init the curTimeRange + if (curTimeRange == null) { + curTimeRange = new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableAggregationTableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableAggregationTableScanOperator.java index e0f71e62c06..d81b6add2ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableAggregationTableScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableAggregationTableScanOperator.java @@ -621,6 +621,7 @@ public class TableAggregationTableScanOperator extends AbstractSeriesAggregation private void updateCurTimeRange(long startTime) { if (timeIterator.getType() == ITableTimeRangeIterator.TimeIteratorType.SINGLE_TIME_ITERATOR) { + timeIterator.updateCurTimeRange(startTime); return; } @@ -637,6 +638,12 @@ public class TableAggregationTableScanOperator extends AbstractSeriesAggregation /** Append a row of aggregation results to the result tsBlock. */ public void appendAggregationResult( TsBlockBuilder tsBlockBuilder, List<? extends TableAggregator> aggregators) { + + // no date in current time range, just output empty + if (!timeIterator.hasCachedTimeRange()) { + return; + } + ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); int groupKeySize = groupingKeySchemas == null ? 0 : groupingKeySchemas.size(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java index 90dd7cb957a..11b1d07030d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java @@ -25,6 +25,7 @@ import org.apache.tsfile.file.metadata.statistics.IntegerStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -89,7 +90,7 @@ public class AvgAccumulator implements TableAccumulator { @Override public void addIntermediate(Column argument) { checkArgument( - argument instanceof BinaryColumn, + argument instanceof BinaryColumn || argument instanceof RunLengthEncodedColumn, "intermediate input and output of Avg should be BinaryColumn"); for (int i = 0; i < argument.getPositionCount(); i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index a1dcefb0bd4..f3c5fac2138 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -137,6 +137,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.Type; @@ -1513,7 +1514,14 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution } } if (timeRangeIterator == null) { - timeRangeIterator = new TableSingleTimeWindowIterator(Long.MIN_VALUE, Long.MAX_VALUE); + if (node.getGroupingKeys().isEmpty()) { + // global aggregation, has no group by, output init value if no data + timeRangeIterator = + new TableSingleTimeWindowIterator(new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE)); + } else { + // aggregation with group by, only has data the result will not be empty + timeRangeIterator = new TableSingleTimeWindowIterator(); + } } final OperatorContext operatorContext =
