This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9a386abdc0e Fix MAD UDF exact calculation and silent approximate
result (#19162)
9a386abdc0e is described below
commit 9a386abdc0ef905219a43839f7ce58e5d748f28b
Author: alpass163 <[email protected]>
AuthorDate: Fri Nov 6 17:02:22 2026 +0800
Fix MAD UDF exact calculation and silent approximate result (#19162)
---
.../iotdb/library/i18n/LibraryUdfMessages.java | 6 ++
.../iotdb/library/i18n/LibraryUdfMessages.java | 5 +
.../org/apache/iotdb/library/dprofile/UDAFMad.java | 10 +-
.../dprofile/util/ExactOrderStatistics.java | 16 +--
.../apache/iotdb/library/dprofile/UDAFMadTest.java | 114 +++++++++++++++++++++
.../dprofile/util/ExactOrderStatisticsTest.java | 73 +++++++++++++
6 files changed, 215 insertions(+), 9 deletions(-)
diff --git
a/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
b/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
index 2e79a474747..62bba86ae2b 100644
---
a/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
+++
b/library-udf/src/main/i18n/en/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
@@ -123,6 +123,12 @@ public final class LibraryUdfMessages {
// MADSketch
public static final String NO_VALUES_IN_TIME_SERIES = "No values in the time
series";
+ // UDAFMad
+ public static final String
+
EXCEPTION_MAD_SKETCH_CANNOT_PRODUCE_A_RESULT_WITH_THE_GIVEN_ERROR_PLEASE_RETRY_WITH_A_SMALLER_ERROR_OR_USE_ERROR_0_FOR_THE_EXACT_RESULT_E168B5FE
=
+ "MAD sketch cannot produce a result with the given error. Please
retry with a smaller"
+ + " 'error', or use 'error'=0 for the exact result.";
+
// SlidingCollector
public static final String COMBINING_NOT_POSSIBLE = "Combining not possible";
diff --git
a/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
b/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
index 42e145ffd65..b3a3d552b22 100644
---
a/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
+++
b/library-udf/src/main/i18n/zh/org/apache/iotdb/library/i18n/LibraryUdfMessages.java
@@ -111,6 +111,11 @@ public final class LibraryUdfMessages {
// MADSketch
public static final String NO_VALUES_IN_TIME_SERIES = "时间序列中没有值";
+ // UDAFMad
+ public static final String
+
EXCEPTION_MAD_SKETCH_CANNOT_PRODUCE_A_RESULT_WITH_THE_GIVEN_ERROR_PLEASE_RETRY_WITH_A_SMALLER_ERROR_OR_USE_ERROR_0_FOR_THE_EXACT_RESULT_E168B5FE
=
+ "在给定的 error 下,MAD 近似计算无法得出结果。请减小 error 后重试,或使用 error=0 进行精确计算。";
+
// SlidingCollector
public static final String COMBINING_NOT_POSSIBLE = "无法合并";
diff --git
a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java
b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java
index 8a5f99abdb6..58987cc5866 100644
--- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java
+++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.library.dprofile;
import org.apache.iotdb.library.dprofile.util.ExactOrderStatistics;
import org.apache.iotdb.library.dprofile.util.MADSketch;
+import org.apache.iotdb.library.dprofile.util.Mad;
import org.apache.iotdb.library.i18n.LibraryUdfMessages;
import org.apache.iotdb.library.util.Util;
import org.apache.iotdb.udf.api.UDTF;
@@ -30,6 +31,7 @@ import
org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations;
import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator;
import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters;
import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy;
+import org.apache.iotdb.udf.api.exception.UDFException;
import org.apache.iotdb.udf.api.type.Type;
/** calculate the exact or approximate median absolute deviation (mad). */
@@ -78,7 +80,13 @@ public class UDAFMad implements UDTF {
if (exact) {
collector.putDouble(0, statistics.getMad());
} else {
- collector.putDouble(0, sketch.getMad().result);
+ Mad mad = sketch.getMad();
+ if (mad.errorBound == Double.MAX_VALUE) {
+ throw new UDFException(
+ LibraryUdfMessages
+
.EXCEPTION_MAD_SKETCH_CANNOT_PRODUCE_A_RESULT_WITH_THE_GIVEN_ERROR_PLEASE_RETRY_WITH_A_SMALLER_ERROR_OR_USE_ERROR_0_FOR_THE_EXACT_RESULT_E168B5FE);
+ }
+ collector.putDouble(0, mad.result);
}
}
}
diff --git
a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java
b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java
index c4c49f0fdb9..c225a40ed6a 100644
---
a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java
+++
b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java
@@ -127,9 +127,9 @@ public class ExactOrderStatistics {
throw new NoSuchElementException();
} else {
double median = getMedian(nums);
- DoubleArrayList dal = new DoubleArrayList();
+ DoubleArrayList dal = new DoubleArrayList(nums.size());
for (int i = 0; i < nums.size(); ++i) {
- dal.set(i, Math.abs(nums.get(i) - median));
+ dal.add(Math.abs(nums.get(i) - median));
}
return getMedian(dal);
}
@@ -144,9 +144,9 @@ public class ExactOrderStatistics {
throw new NoSuchElementException();
} else {
double median = getMedian(nums);
- DoubleArrayList dal = new DoubleArrayList();
+ DoubleArrayList dal = new DoubleArrayList(nums.size());
for (int i = 0; i < nums.size(); ++i) {
- dal.set(i, Math.abs(nums.get(i) - median));
+ dal.add(Math.abs(nums.get(i) - median));
}
return getMedian(dal);
}
@@ -157,9 +157,9 @@ public class ExactOrderStatistics {
throw new NoSuchElementException();
} else {
double median = getMedian(nums);
- DoubleArrayList dal = new DoubleArrayList();
+ DoubleArrayList dal = new DoubleArrayList(nums.size());
for (int i = 0; i < nums.size(); ++i) {
- dal.set(i, Math.abs(nums.get(i) - median));
+ dal.add(Math.abs(nums.get(i) - median));
}
return getMedian(dal);
}
@@ -170,9 +170,9 @@ public class ExactOrderStatistics {
throw new NoSuchElementException();
} else {
double median = getMedian(nums);
- DoubleArrayList dal = new DoubleArrayList();
+ DoubleArrayList dal = new DoubleArrayList(nums.size());
for (int i = 0; i < nums.size(); ++i) {
- dal.set(i, Math.abs(nums.get(i) - median));
+ dal.add(Math.abs(nums.get(i) - median));
}
return getMedian(dal);
}
diff --git
a/library-udf/src/test/java/org/apache/iotdb/library/dprofile/UDAFMadTest.java
b/library-udf/src/test/java/org/apache/iotdb/library/dprofile/UDAFMadTest.java
new file mode 100644
index 00000000000..c4d64cf3075
--- /dev/null
+++
b/library-udf/src/test/java/org/apache/iotdb/library/dprofile/UDAFMadTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.library.dprofile;
+
+import org.apache.iotdb.udf.api.access.Row;
+import org.apache.iotdb.udf.api.collector.PointCollector;
+import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations;
+import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters;
+import org.apache.iotdb.udf.api.exception.UDFException;
+import org.apache.iotdb.udf.api.type.Type;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Proxy;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class UDAFMadTest {
+
+ private static final double[] VALUES = {
+ 2403.71, 2404.36, 2402.88, 2403.95, 2404.12, 2401.63, 2403.52, 2404.77,
2402.34, 2403.18,
+ 2405.01, 2397.77, 2404.63, 2402.07, 2403.85, 2404.44, 2402.61, 2403.29,
2404.90, 2402.15,
+ 2403.63, 2404.05, 2402.72, 2403.97
+ };
+ private static final double EXPECTED_MAD = 0.78;
+
+ @Test
+ public void testExactMad() throws Exception {
+ UDAFMad mad = new UDAFMad();
+ mad.beforeStart(parameters(0), new
UDTFConfigurations(ZoneId.systemDefault()));
+ RecordingCollector collector = new RecordingCollector();
+ for (double value : VALUES) {
+ mad.transform(doubleRow(value), collector.collector);
+ }
+ mad.terminate(collector.collector);
+ Assert.assertEquals(EXPECTED_MAD, collector.value, 1e-9);
+ }
+
+ @Test
+ public void testSketchRejectsDegenerateResult() throws Exception {
+ UDAFMad mad = new UDAFMad();
+ mad.beforeStart(parameters(0.01), new
UDTFConfigurations(ZoneId.systemDefault()));
+ RecordingCollector collector = new RecordingCollector();
+ for (double value : VALUES) {
+ mad.transform(doubleRow(value), collector.collector);
+ }
+ Assert.assertThrows(UDFException.class, () ->
mad.terminate(collector.collector));
+ }
+
+ private static UDFParameters parameters(double error) {
+ Map<String, String> attributes = new HashMap<>();
+ if (error != 0) {
+ attributes.put("error", Double.toString(error));
+ }
+ return new UDFParameters(Arrays.asList("s0"), Arrays.asList(Type.DOUBLE),
attributes);
+ }
+
+ private static Row doubleRow(double value) {
+ return (Row)
+ Proxy.newProxyInstance(
+ Row.class.getClassLoader(),
+ new Class<?>[] {Row.class},
+ (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "getTime":
+ return 0L;
+ case "getDouble":
+ return value;
+ case "getDataType":
+ return Type.DOUBLE;
+ case "size":
+ return 1;
+ default:
+ return null;
+ }
+ });
+ }
+
+ private static class RecordingCollector {
+
+ private double value = Double.NaN;
+ private final PointCollector collector =
+ (PointCollector)
+ Proxy.newProxyInstance(
+ PointCollector.class.getClassLoader(),
+ new Class<?>[] {PointCollector.class},
+ (proxy, method, args) -> {
+ if ("putDouble".equals(method.getName())) {
+ value = (Double) args[1];
+ }
+ return null;
+ });
+ }
+}
diff --git
a/library-udf/src/test/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatisticsTest.java
b/library-udf/src/test/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatisticsTest.java
new file mode 100644
index 00000000000..7d11e594e3c
--- /dev/null
+++
b/library-udf/src/test/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatisticsTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.library.dprofile.util;
+
+import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList;
+import org.eclipse.collections.impl.list.mutable.primitive.FloatArrayList;
+import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList;
+import org.eclipse.collections.impl.list.mutable.primitive.LongArrayList;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ExactOrderStatisticsTest {
+
+ private static final double[] VALUES = {
+ 2403.71, 2404.36, 2402.88, 2403.95, 2404.12, 2401.63, 2403.52, 2404.77,
2402.34, 2403.18,
+ 2405.01, 2397.77, 2404.63, 2402.07, 2403.85, 2404.44, 2402.61, 2403.29,
2404.90, 2402.15,
+ 2403.63, 2404.05, 2402.72, 2403.97
+ };
+ private static final double EXPECTED_MAD = 0.78;
+
+ @Test
+ public void testGetMadDouble() {
+ DoubleArrayList nums = new DoubleArrayList(VALUES.length);
+ for (double value : VALUES) {
+ nums.add(value);
+ }
+ Assert.assertEquals(EXPECTED_MAD, ExactOrderStatistics.getMad(nums), 1e-9);
+ }
+
+ @Test
+ public void testGetMadFloat() {
+ FloatArrayList nums = new FloatArrayList(VALUES.length);
+ for (double value : VALUES) {
+ nums.add((float) value);
+ }
+ Assert.assertEquals(EXPECTED_MAD, ExactOrderStatistics.getMad(nums), 1e-3);
+ }
+
+ @Test
+ public void testGetMadInt() {
+ IntArrayList nums = new IntArrayList(VALUES.length);
+ for (int i = 1; i <= VALUES.length; i++) {
+ nums.add(i);
+ }
+ Assert.assertEquals(6.0, ExactOrderStatistics.getMad(nums), 0);
+ }
+
+ @Test
+ public void testGetMadLong() {
+ LongArrayList nums = new LongArrayList(VALUES.length);
+ for (long i = 1; i <= VALUES.length; i++) {
+ nums.add(i);
+ }
+ Assert.assertEquals(6.0, ExactOrderStatistics.getMad(nums), 0);
+ }
+}