This is an automated email from the ASF dual-hosted git repository.
CritasWang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb-extras.git
The following commit(s) were added to refs/heads/master by this push:
new e46e63b Keep MAX buckets whose values are all non-positive (#116)
e46e63b is described below
commit e46e63b4226d59b778b37191097176eeaace4376
Author: ZIHAN DAI <[email protected]>
AuthorDate: Mon Aug 3 12:00:19 2026 +1000
Keep MAX buckets whose values are all non-positive (#116)
The milliseconds aggregation path projected MAX over the numeric COALESCE
expression. IoTDB's grouped max accumulator seeds FLOAT/DOUBLE state with
Float/Double.MIN_VALUE -- the smallest positive value, not the most
negative one -- and only marks a group initialized when value >= state, so
a bucket whose numeric maximum is zero or negative comes back NULL. This
DAO reads a NULL aggregate as an empty bucket and skips it, so a MAX
downsampling query over, for example, a sub-zero sensor silently lost
whole buckets instead of failing.
Project -MIN(-x) instead: the grouped min accumulator seeds with MAX_VALUE
and is not affected, and IEEE-754 negation is exact, so the result is
identical over finite values on servers with and without the upstream fix.
The projection is shared with the calendar path, whose non-grouped
accumulators track an explicit initialized flag rather than a sentinel, so
the substitution is exact there too and an empty bucket still yields NULL.
The long and string channels are already correct and keep using MAX.
Verified against apache/iotdb 2.0.8 and 2.0.10 containers. See
apache/iotdb#18300, fixed on master but not yet in a released version.
Signed-off-by: Zihan Dai <[email protected]>
---
.../thingsboard/table/IoTDBTableTimeseriesDao.java | 18 +++-
.../table/IoTDBTableTimeseriesAggregationIT.java | 98 ++++++++++++++++++++++
.../table/IoTDBTableTimeseriesDaoTest.java | 10 ++-
3 files changed, 123 insertions(+), 3 deletions(-)
diff --git
a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java
b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java
index 216ffcd..37c244c 100644
---
a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java
+++
b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java
@@ -652,10 +652,24 @@ public class IoTDBTableTimeseriesDao extends
IoTDBTableBaseDao
+ AGG_STR_COLUMN
+ ", "
+ numericCountProjection();
+ // The numeric MAX is projected as -MIN(-x) rather than MAX(x).
IoTDB's GROUPED max
+ // accumulator seeds FLOAT/DOUBLE state with Float/Double.MIN_VALUE --
the smallest
+ // POSITIVE value, not the most negative one -- and only marks a group
initialized when
+ // `value >= state`, so a bucket whose numeric maximum is zero or
negative comes back
+ // NULL. This DAO reads a NULL aggregate as "empty bucket" and skips
it, so a MAX
+ // downsampling query over e.g. a sub-zero sensor would silently lose
whole buckets
+ // instead of failing. Every release up to and including 2.0.10 is
affected; fixed on
+ // master by apache/iotdb#18300, which is not in a released version
yet. The grouped MIN
+ // accumulator seeds with MAX_VALUE and is not affected, and IEEE-754
negation is exact,
+ // so -MIN(-x) is an exact substitute for MAX(x) over finite values,
on affected and
+ // fixed servers alike. This projection is shared with the calendar
path, whose
+ // non-grouped accumulators track an explicit initialized flag rather
than a sentinel, so
+ // the substitution is exact there too and an empty bucket still
yields NULL. MAX(long_v)
+ // and MAX(str_v) are already correct (true Long.MIN_VALUE seed /
flag-based) and stay.
case MAX ->
- "MAX("
+ "-1 * MIN(-1 * ("
+ NUMERIC_VALUE
- + ") AS "
+ + ")) AS "
+ AGG_NUM_COLUMN
+ ", MAX(long_v) AS "
+ MAX_LONG_COLUMN
diff --git
a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java
b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java
index 3e65e50..302f4fa 100644
---
a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java
+++
b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java
@@ -694,6 +694,104 @@ class IoTDBTableTimeseriesAggregationIT {
}
}
+ @Test
+ void maxKeepsBucketsWhoseValuesAreAllNonPositiveAgainstRealIoTDB() throws
Exception {
+ TestScope scope =
+ scope(
+ "agg_nonpositive",
+ "55555555-5555-5555-5555-555555555511",
+ "66666666-6666-6666-6666-666666666611");
+ bootstrapSchema(scope.database());
+ try (ITableSessionPool pool = newPool(scope.database())) {
+ IoTDBTableConfig config = config(8);
+ IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool,
config);
+ IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer,
config);
+ try {
+ // Regression for apache/iotdb#18300 against REAL IoTDB. The server's
GROUPED max
+ // accumulator seeds FLOAT/DOUBLE state with Double.MIN_VALUE -- the
smallest POSITIVE
+ // value -- so on every release up to and including 2.0.10 a bucket
whose maximum is zero
+ // or negative makes MAX(<expression>) return NULL. This DAO reads a
NULL aggregate as an
+ // empty bucket and skips it, so such buckets used to VANISH from the
series: a MAX
+ // downsampling query over a sub-zero sensor silently returned fewer
points, with no
+ // error anywhere. The projection computes -MIN(-x) instead, which the
unaffected grouped
+ // MIN accumulator evaluates exactly. Every bucket here has a
non-positive maximum, so on
+ // the old projection this test fails with "bucket count expected 2
but was 0".
+ // Bucket [1000,2000): doubles -5.0, -3.0 -> midpoint 1500, MAX =
-3.0, MIN = -5.0
+ // Bucket [2000,3000): doubles 0.0, 0.0 -> midpoint 2500, MAX =
0.0, MIN = 0.0
+ saveAll(
+ dao,
+ scope,
+ List.of(
+ entry(1000L, "np", DataType.DOUBLE, -5.0D),
+ entry(1500L, "np", DataType.DOUBLE, -3.0D),
+ entry(2100L, "np", DataType.DOUBLE, 0.0D),
+ entry(2400L, "np", DataType.DOUBLE, 0.0D)));
+
+ ReadTsKvQueryResult max = aggregate(dao, scope, "np", Aggregation.MAX);
+ assertNumericBuckets(
+ max,
+ new long[] {1500L, 2500L},
+ new DataType[] {DataType.DOUBLE, DataType.DOUBLE},
+ new double[] {-3.0D, 0.0D},
+ 2400L);
+
+ // MIN is not affected by the upstream bug; it must be byte-for-byte
unchanged.
+ ReadTsKvQueryResult min = aggregate(dao, scope, "np", Aggregation.MIN);
+ assertNumericBuckets(
+ min,
+ new long[] {1500L, 2500L},
+ new DataType[] {DataType.DOUBLE, DataType.DOUBLE},
+ new double[] {-5.0D, 0.0D},
+ 2400L);
+ } finally {
+ dao.destroy();
+ writer.destroy();
+ }
+ }
+ }
+
+ @Test
+ void
maxOverNonPositiveLongOnlyAndMixedBucketsKeepsResultTypeAgainstRealIoTDB()
throws Exception {
+ TestScope scope =
+ scope(
+ "agg_nonpositive_types",
+ "55555555-5555-5555-5555-555555555512",
+ "66666666-6666-6666-6666-666666666612");
+ bootstrapSchema(scope.database());
+ try (ITableSessionPool pool = newPool(scope.database())) {
+ IoTDBTableConfig config = config(8);
+ IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool,
config);
+ IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer,
config);
+ try {
+ // Same regression across the two typed channels, since they take
different code paths:
+ // Bucket [1000,2000): LONG-ONLY -5, -3 -> midpoint 1500, MAX = -3
kept LONG-typed via
+ // the direct MAX(long_v) channel (LongBigArray seeds with the
true Long.MIN_VALUE,
+ // so that channel was always correct).
+ // Bucket [2000,3000): MIXED long -5 + double -3.0 -> midpoint 2500,
MAX = -3.0 promoted
+ // to DOUBLE, which reads the numeric channel and therefore
exercises the fix.
+ saveAll(
+ dao,
+ scope,
+ List.of(
+ entry(1000L, "nt", DataType.LONG, -5L),
+ entry(1500L, "nt", DataType.LONG, -3L),
+ entry(2100L, "nt", DataType.LONG, -5L),
+ entry(2400L, "nt", DataType.DOUBLE, -3.0D)));
+
+ ReadTsKvQueryResult max = aggregate(dao, scope, "nt", Aggregation.MAX);
+ assertNumericBuckets(
+ max,
+ new long[] {1500L, 2500L},
+ new DataType[] {DataType.LONG, DataType.DOUBLE},
+ new double[] {-3.0D, -3.0D},
+ 2400L);
+ } finally {
+ dao.destroy();
+ writer.destroy();
+ }
+ }
+ }
+
private ReadTsKvQueryResult calendarAggregate(
IoTDBTableTimeseriesDao dao,
TestScope scope,
diff --git
a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java
b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java
index 9672f50..66fa857 100644
---
a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java
+++
b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java
@@ -1737,8 +1737,16 @@ class IoTDBTableTimeseriesDaoTest {
"CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0
END) AS INT64) "
+ "AS count_double"),
statements.get(2));
+ // The numeric MAX is projected as -MIN(-x): IoTDB's grouped max
accumulator seeds
+ // FLOAT/DOUBLE state with Double.MIN_VALUE (the smallest POSITIVE value),
so MAX over a
+ // bucket whose maximum is zero or negative returns NULL and the bucket
would be dropped
+ // (apache/iotdb#18300). The grouped MIN accumulator is unaffected and
IEEE-754 negation is
+ // exact, so this is an exact substitute over finite values. The long and
string channels are
+ // already correct and keep using MAX.
assertTrue(
- statements.get(3).contains("MAX(COALESCE(double_v, CAST(long_v AS
DOUBLE))) AS agg_num")
+ statements
+ .get(3)
+ .contains("-1 * MIN(-1 * (COALESCE(double_v, CAST(long_v AS
DOUBLE)))) AS agg_num")
&& statements.get(3).contains("MAX(long_v) AS max_long")
&& statements.get(3).contains("MAX(str_v) AS agg_str")
&& statements
