This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2691d4dac1e Fix NPE of approx_most_frequent and approx_count_distinct
on empty input (#19168)
2691d4dac1e is described below
commit 2691d4dac1e0da454e2d9c4998b0f85c6ccf153a
Author: alpass163 <[email protected]>
AuthorDate: Sat Nov 7 20:20:11 2026 +0800
Fix NPE of approx_most_frequent and approx_count_distinct on empty input
(#19168)
---
.../it/query/recent/IoTDBTableAggregationIT.java | 90 +++++++++++++++++++---
.../AbstractApproxMostFrequentAccumulator.java | 19 ++++-
.../ApproxCountDistinctAccumulator.java | 12 ++-
.../BlobApproxMostFrequentAccumulator.java | 4 +
.../approximate/HyperLogLogStateFactory.java | 2 +-
.../aggregation/approximate/SpaceSaving.java | 4 +
...stractGroupedApproxMostFrequentAccumulator.java | 18 +++--
.../BlobGroupedApproxMostFrequentAccumulator.java | 9 ++-
.../GroupedApproxCountDistinctAccumulator.java | 12 ++-
9 files changed, 145 insertions(+), 25 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java
index 99188903b99..6bf49726a0d 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java
@@ -57,6 +57,7 @@ public class IoTDBTableAggregationIT {
"CREATE DATABASE " + DATABASE_NAME,
"USE " + DATABASE_NAME,
"CREATE TABLE table1(province STRING TAG, city STRING TAG, region
STRING TAG, device_id STRING TAG, color STRING ATTRIBUTE, type STRING
ATTRIBUTE, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5
BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP
FIELD, s10 DATE FIELD)",
+ "CREATE TABLE empty_table(device_id STRING TAG, s1 INT32 FIELD, s2
INT64 FIELD, s6 TEXT FIELD, s8 BLOB FIELD)",
"INSERT INTO
table1(time,province,city,region,device_id,color,type,s1,s3,s6,s8,s9) values
(2024-09-24T06:15:30.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',30,30.0,'shanghai_huangpu_red_A_d01_30',
X'cafebabe30',2024-09-24T06:15:30.000+00:00)",
"INSERT INTO
table1(time,province,city,region,device_id,color,type,s2,s3,s4,s6,s7,s9,s10)
values
(2024-09-24T06:15:35.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',35000,35.0,35.0,'shanghai_huangpu_red_A_d01_35','shanghai_huangpu_red_A_d01_35',2024-09-24T06:15:35.000+00:00,'2024-09-24')",
"INSERT INTO
table1(time,province,city,region,device_id,color,type,s1,s3,s5,s7,s9) values
(2024-09-24T06:15:40.000+00:00,'shanghai','shanghai','huangpu','d01','red','A',40,40.0,true,'shanghai_huangpu_red_A_d01_40',2024-09-24T06:15:40.000+00:00)",
@@ -4377,26 +4378,26 @@ public class IoTDBTableAggregationIT {
expectedHeader = new String[] {"time", "province", "_col2"};
retArray =
new String[] {
- "2024-09-24T06:15:30.000Z,beijing,{},",
+ "2024-09-24T06:15:30.000Z,beijing,null,",
"2024-09-24T06:15:31.000Z,beijing,{\"31000\":2},",
"2024-09-24T06:15:35.000Z,beijing,{\"35000\":2},",
- "2024-09-24T06:15:36.000Z,beijing,{},",
+ "2024-09-24T06:15:36.000Z,beijing,null,",
"2024-09-24T06:15:40.000Z,beijing,{\"40000\":2},",
- "2024-09-24T06:15:41.000Z,beijing,{},",
+ "2024-09-24T06:15:41.000Z,beijing,null,",
"2024-09-24T06:15:46.000Z,beijing,{\"46000\":2},",
"2024-09-24T06:15:50.000Z,beijing,{\"50000\":4},",
- "2024-09-24T06:15:51.000Z,beijing,{},",
- "2024-09-24T06:15:55.000Z,beijing,{},",
- "2024-09-24T06:15:30.000Z,shanghai,{},",
+ "2024-09-24T06:15:51.000Z,beijing,null,",
+ "2024-09-24T06:15:55.000Z,beijing,null,",
+ "2024-09-24T06:15:30.000Z,shanghai,null,",
"2024-09-24T06:15:31.000Z,shanghai,{\"31000\":2},",
"2024-09-24T06:15:35.000Z,shanghai,{\"35000\":2},",
- "2024-09-24T06:15:36.000Z,shanghai,{},",
+ "2024-09-24T06:15:36.000Z,shanghai,null,",
"2024-09-24T06:15:40.000Z,shanghai,{\"40000\":2},",
- "2024-09-24T06:15:41.000Z,shanghai,{},",
+ "2024-09-24T06:15:41.000Z,shanghai,null,",
"2024-09-24T06:15:46.000Z,shanghai,{\"46000\":2},",
"2024-09-24T06:15:50.000Z,shanghai,{\"50000\":4},",
- "2024-09-24T06:15:51.000Z,shanghai,{},",
- "2024-09-24T06:15:55.000Z,shanghai,{},",
+ "2024-09-24T06:15:51.000Z,shanghai,null,",
+ "2024-09-24T06:15:55.000Z,shanghai,null,",
};
tableResultSetEqualTest(
@@ -4406,6 +4407,75 @@ public class IoTDBTableAggregationIT {
DATABASE_NAME);
}
+ @Test
+ public void blobGroupedApproxMostFrequentTest() {
+ // Groups whose BLOB column is entirely NULL must yield NULL instead of an
empty map.
+ // Under a PARTIAL -> FINAL aggregation the partial side emits NULL for
these groups and
+ // the final side never initializes their state, so evaluateFinal must
handle that too.
+ tableResultSetEqualTest(
+ "select time,province,approx_most_frequent(s8, 1, 10) from table1
group by 1,2 order by 2,1",
+ new String[] {"time", "province", "_col2"},
+ new String[] {
+ "2024-09-24T06:15:30.000Z,beijing,{\"0xcafebabe30\":2},",
+ "2024-09-24T06:15:31.000Z,beijing,{\"0xcafebabe31\":2},",
+ "2024-09-24T06:15:35.000Z,beijing,null,",
+ "2024-09-24T06:15:36.000Z,beijing,null,",
+ "2024-09-24T06:15:40.000Z,beijing,null,",
+ "2024-09-24T06:15:41.000Z,beijing,{\"0xcafebabe41\":2},",
+ "2024-09-24T06:15:46.000Z,beijing,null,",
+ "2024-09-24T06:15:50.000Z,beijing,{\"0xcafebabe50\":2},",
+ "2024-09-24T06:15:51.000Z,beijing,null,",
+ "2024-09-24T06:15:55.000Z,beijing,{\"0xcafebabe55\":4},",
+ "2024-09-24T06:15:30.000Z,shanghai,{\"0xcafebabe30\":2},",
+ "2024-09-24T06:15:31.000Z,shanghai,{\"0xcafebabe31\":2},",
+ "2024-09-24T06:15:35.000Z,shanghai,null,",
+ "2024-09-24T06:15:36.000Z,shanghai,null,",
+ "2024-09-24T06:15:40.000Z,shanghai,null,",
+ "2024-09-24T06:15:41.000Z,shanghai,{\"0xcafebabe41\":2},",
+ "2024-09-24T06:15:46.000Z,shanghai,null,",
+ "2024-09-24T06:15:50.000Z,shanghai,{\"0xcafebabe50\":2},",
+ "2024-09-24T06:15:51.000Z,shanghai,null,",
+ "2024-09-24T06:15:55.000Z,shanghai,{\"0xcafebabe55\":4},",
+ },
+ DATABASE_NAME);
+ }
+
+ @Test
+ public void approxAggregationEmptyInputTest() {
+ // No block reaches the aggregation operator, so the accumulator state is
never created
+ tableResultSetEqualTest(
+ "select approx_most_frequent(s1, 2, 10), approx_most_frequent(s2, 2,
10), approx_most_frequent(s6, 2, 10), approx_most_frequent(s8, 2, 10) from
empty_table",
+ buildHeaders(4),
+ new String[] {"null,null,null,null,"},
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "select approx_count_distinct(s1), approx_count_distinct(s6) from
empty_table",
+ buildHeaders(2),
+ new String[] {"0,0,"},
+ DATABASE_NAME);
+
+ tableResultSetEqualTest(
+ "select device_id, approx_most_frequent(s1, 2, 10) from empty_table
group by device_id",
+ new String[] {"device_id", "_col1"},
+ new String[] {},
+ DATABASE_NAME);
+
+ // All rows are filtered out by the predicate
+ tableResultSetEqualTest(
+ "select approx_most_frequent(s1, 2, 10), approx_count_distinct(s1)
from table1 where s1 < 0",
+ buildHeaders(2),
+ new String[] {"null,0,"},
+ DATABASE_NAME);
+
+ // Rows exist but every value is null, so there is nothing to count
+ tableResultSetEqualTest(
+ "select approx_most_frequent(s2, 2, 10), approx_count_distinct(s2)
from table1 where s2 is null",
+ buildHeaders(2),
+ new String[] {"null,0,"},
+ DATABASE_NAME);
+ }
+
@Test
public void approxPercentileTest() {
tableResultSetEqualTest(
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java
index 0b2d9ab5f0a..b929b6c27a2 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AbstractApproxMostFrequentAccumulator.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.calc.execution.operator.source.relational.aggregation;
+import
org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSaving;
import
org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSavingStateFactory;
import org.apache.iotdb.calc.i18n.CalcMessages;
@@ -36,13 +37,23 @@ public abstract class
AbstractApproxMostFrequentAccumulator<T> implements TableA
@Override
public void evaluateIntermediate(ColumnBuilder columnBuilder) {
- columnBuilder.writeBinary(new Binary(state.getSpaceSaving().serialize()));
+ SpaceSaving<T> spaceSaving = state.getSpaceSaving();
+ if (spaceSaving == null || spaceSaving.isEmpty()) {
+ columnBuilder.appendNull();
+ return;
+ }
+ columnBuilder.writeBinary(new Binary(spaceSaving.serialize()));
}
@Override
public void evaluateFinal(ColumnBuilder columnBuilder) {
+ SpaceSaving<T> spaceSaving = state.getSpaceSaving();
+ if (spaceSaving == null || spaceSaving.isEmpty()) {
+ columnBuilder.appendNull();
+ return;
+ }
columnBuilder.writeBinary(
- new Binary(new Gson().toJson(state.getSpaceSaving().getBuckets()),
StandardCharsets.UTF_8));
+ new Binary(new Gson().toJson(spaceSaving.getBuckets()),
StandardCharsets.UTF_8));
}
@Override
@@ -58,6 +69,8 @@ public abstract class
AbstractApproxMostFrequentAccumulator<T> implements TableA
@Override
public void reset() {
- state.getSpaceSaving().reset();
+ if (state.getSpaceSaving() != null) {
+ state.getSpaceSaving().reset();
+ }
}
}
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java
index 65824142a07..97cf60e8d8b 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java
@@ -106,11 +106,19 @@ public class ApproxCountDistinctAccumulator implements
TableAccumulator {
columnBuilder instanceof BinaryColumnBuilder,
CalcMessages
.EXCEPTION_INTERMEDIATE_INPUT_AND_OUTPUT_OF_APPROX_COUNT_DISTINCT_SHOULD_BE_BINARYCOLUMN_F444D3BE);
+ if (state.getHyperLogLog() == null) {
+ columnBuilder.appendNull();
+ return;
+ }
columnBuilder.writeBinary(new Binary(state.getHyperLogLog().serialize()));
}
@Override
public void evaluateFinal(ColumnBuilder columnBuilder) {
+ if (state.getHyperLogLog() == null) {
+ columnBuilder.writeLong(0);
+ return;
+ }
columnBuilder.writeLong(state.getHyperLogLog().cardinality());
}
@@ -127,7 +135,9 @@ public class ApproxCountDistinctAccumulator implements
TableAccumulator {
@Override
public void reset() {
- state.getHyperLogLog().reset();
+ if (state.getHyperLogLog() != null) {
+ state.getHyperLogLog().reset();
+ }
}
public void addBooleanInput(Column valueColumn, AggregationMask mask,
HyperLogLog hll) {
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/BlobApproxMostFrequentAccumulator.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/BlobApproxMostFrequentAccumulator.java
index 9f2df5d9455..c70e0855c83 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/BlobApproxMostFrequentAccumulator.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/BlobApproxMostFrequentAccumulator.java
@@ -31,6 +31,10 @@ import java.util.stream.Collectors;
public class BlobApproxMostFrequentAccumulator extends
BinaryApproxMostFrequentAccumulator {
@Override
public void evaluateFinal(ColumnBuilder columnBuilder) {
+ if (state.getSpaceSaving() == null || state.getSpaceSaving().isEmpty()) {
+ columnBuilder.appendNull();
+ return;
+ }
Map<Binary, Long> buckets = state.getSpaceSaving().getBuckets();
Map<String, Long> formatedBuckets =
buckets.entrySet().stream()
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/HyperLogLogStateFactory.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/HyperLogLogStateFactory.java
index cc0780efa1a..5746019d324 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/HyperLogLogStateFactory.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/HyperLogLogStateFactory.java
@@ -50,7 +50,7 @@ public class HyperLogLogStateFactory {
public long getEstimatedSize() {
// not used
- return INSTANCE_SIZE + hll.getEstimatedSize();
+ return hll == null ? INSTANCE_SIZE : INSTANCE_SIZE +
hll.getEstimatedSize();
}
public void merge(HyperLogLog other) {
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/SpaceSaving.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/SpaceSaving.java
index 0ac134cfa0a..610620a92e3 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/SpaceSaving.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/approximate/SpaceSaving.java
@@ -124,6 +124,10 @@ public class SpaceSaving<K> {
return buckets.buildOrThrow();
}
+ public boolean isEmpty() {
+ return streamSummary.size() == 0;
+ }
+
public byte[] serialize() {
List<Counter<K>> counters = streamSummary.topK(capacity);
// Calculate the size of the keys
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxMostFrequentAccumulator.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxMostFrequentAccumulator.java
index 96826bf0dde..925c74ce261 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxMostFrequentAccumulator.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/AbstractGroupedApproxMostFrequentAccumulator.java
@@ -19,6 +19,7 @@
package
org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped;
+import
org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSaving;
import
org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSavingStateFactory;
import
org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.SpaceSavingBigArray;
@@ -42,15 +43,22 @@ public abstract class
AbstractGroupedApproxMostFrequentAccumulator<T>
@Override
public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) {
- columnBuilder.writeBinary(new
Binary(state.getSpaceSavings().get(groupId).serialize()));
+ SpaceSaving<T> spaceSaving = state.getSpaceSavings().get(groupId);
+ if (spaceSaving == null || spaceSaving.isEmpty()) {
+ columnBuilder.appendNull();
+ return;
+ }
+ columnBuilder.writeBinary(new Binary(spaceSaving.serialize()));
}
@Override
public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) {
- Binary result =
- new Binary(
- new
Gson().toJson(state.getSpaceSavings().get(groupId).getBuckets()),
- StandardCharsets.UTF_8);
+ SpaceSaving<T> spaceSaving = state.getSpaceSavings().get(groupId);
+ if (spaceSaving == null || spaceSaving.isEmpty()) {
+ columnBuilder.appendNull();
+ return;
+ }
+ Binary result = new Binary(new Gson().toJson(spaceSaving.getBuckets()),
StandardCharsets.UTF_8);
columnBuilder.writeBinary(result);
}
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/BlobGroupedApproxMostFrequentAccumulator.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/BlobGroupedApproxMostFrequentAccumulator.java
index b83583ee4e8..36f7278224f 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/BlobGroupedApproxMostFrequentAccumulator.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/BlobGroupedApproxMostFrequentAccumulator.java
@@ -19,6 +19,8 @@
package
org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped;
+import
org.apache.iotdb.calc.execution.operator.source.relational.aggregation.approximate.SpaceSaving;
+
import com.google.gson.Gson;
import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.utils.Binary;
@@ -32,7 +34,12 @@ public class BlobGroupedApproxMostFrequentAccumulator
extends BinaryGroupedApproxMostFrequentAccumulator {
@Override
public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) {
- Map<Binary, Long> buckets =
state.getSpaceSavings().get(groupId).getBuckets();
+ SpaceSaving<Binary> spaceSaving = state.getSpaceSavings().get(groupId);
+ if (spaceSaving == null || spaceSaving.isEmpty()) {
+ columnBuilder.appendNull();
+ return;
+ }
+ Map<Binary, Long> buckets = spaceSaving.getBuckets();
Map<String, Long> formatedBuckets =
buckets.entrySet().stream()
.collect(
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java
index ae6e2ec828b..40811e24f69 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java
@@ -101,14 +101,18 @@ public class GroupedApproxCountDistinctAccumulator
implements GroupedAccumulator
@Override
public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) {
- HyperLogLogBigArray hlls = state.getHyperLogLogs();
- columnBuilder.writeBinary(new Binary(hlls.get(groupId).serialize()));
+ HyperLogLog hll = state.getHyperLogLogs().get(groupId);
+ if (hll == null) {
+ columnBuilder.appendNull();
+ return;
+ }
+ columnBuilder.writeBinary(new Binary(hll.serialize()));
}
@Override
public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) {
- HyperLogLogBigArray hlls = state.getHyperLogLogs();
- columnBuilder.writeLong(hlls.get(groupId).cardinality());
+ HyperLogLog hll = state.getHyperLogLogs().get(groupId);
+ columnBuilder.writeLong(hll == null ? 0 : hll.cardinality());
}
@Override