This is an automated email from the ASF dual-hosted git repository.
yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 88902bcf571 Compare numbers of different types by value in
isDistinctFrom (#19791)
88902bcf571 is described below
commit 88902bcf57172a018662a94f141336adfee3644c
Author: Yash Mayya <[email protected]>
AuthorDate: Thu Oct 15 13:50:36 2026 -0700
Compare numbers of different types by value in isDistinctFrom (#19791)
---
.../common/function/scalar/ObjectFunctions.java | 68 +++++++++++-
.../function/scalar/ObjectFunctionsTest.java | 117 +++++++++++++++++++++
.../PostAggregationFunctionTest.java | 6 ++
.../src/test/resources/queries/Comparisons.json | 68 ++++++++++++
4 files changed, 258 insertions(+), 1 deletion(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ObjectFunctions.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ObjectFunctions.java
index b1eb047fe43..458378fcf62 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ObjectFunctions.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ObjectFunctions.java
@@ -18,6 +18,8 @@
*/
package org.apache.pinot.common.function.scalar;
+import java.math.BigDecimal;
+import java.util.Objects;
import javax.annotation.Nullable;
import org.apache.pinot.spi.annotations.ScalarFunction;
@@ -36,6 +38,8 @@ public class ObjectFunctions {
return !isNull(obj);
}
+ /// Numbers are compared by value, so `5` and `5L` are not distinct. Arrays
(e.g. `BYTES` values, which are `byte[]`)
+ /// are compared by content. Other values are compared with [Object#equals].
@ScalarFunction(nullableParameters = true)
public static boolean isDistinctFrom(@Nullable Object obj1, @Nullable Object
obj2) {
if (obj1 == null && obj2 == null) {
@@ -44,7 +48,10 @@ public class ObjectFunctions {
if (obj1 == null || obj2 == null) {
return true;
}
- return !obj1.equals(obj2);
+ if (isNumber(obj1) && isNumber(obj2)) {
+ return !numberEquals((Number) obj1, (Number) obj2);
+ }
+ return !Objects.deepEquals(obj1, obj2);
}
@ScalarFunction(nullableParameters = true)
@@ -52,6 +59,65 @@ public class ObjectFunctions {
return !isDistinctFrom(obj1, obj2);
}
+ /// Covers the number types that scalar functions get for INT, LONG, FLOAT,
DOUBLE and BIG_DECIMAL values. Other
+ /// numbers (e.g. `Short` or `BigInteger`) are not compared by value, but
with [Objects#deepEquals].
+ private static boolean isNumber(Object obj) {
+ return obj instanceof Integer || obj instanceof Long || obj instanceof
Float || obj instanceof Double
+ || obj instanceof BigDecimal;
+ }
+
+ private static boolean isFloatingPoint(Number number) {
+ return number instanceof Float || number instanceof Double;
+ }
+
+ /// Compares two numbers by value, so that numbers of different types (e.g.
`Integer` and `Long`) can be equal. This
+ /// is close to `BinaryOperatorTransformFunction`, which runs
`isDistinctFrom` in the single-stage engine and in the
+ /// multi-stage leaf stage. It differs only in two edge cases: a `long`
above 2^53 is compared exactly with a
+ /// floating point number, and NaN or infinity compared with a `BigDecimal`
does not throw.
+ /// - Two integers are compared as `long`.
+ /// - Two floating point numbers are compared with [Double#compare], like
[Double#equals]: a `float` is widened
+ /// (`1.1f` does not equal `1.1`), NaN equals NaN, and `-0.0` does not
equal `0.0`.
+ /// - An integer and a floating point number are compared exactly.
+ /// - When either number is a `BigDecimal`, both are compared as
`BigDecimal`, which ignores the scale (`1.5`
+ /// equals `1.50`). A floating point number is converted with
[BigDecimal#valueOf(double)], so `0.1` equals the
+ /// `BigDecimal` `0.1`. NaN and infinity do not equal any `BigDecimal`.
`BigDecimal` has no negative zero, so
+ /// `-0.0` equals the `BigDecimal` `0`, the same as in the leaf stage,
but does not equal `0.0` or `0`.
+ private static boolean numberEquals(Number number1, Number number2) {
+ if (number1 instanceof BigDecimal || number2 instanceof BigDecimal) {
+ BigDecimal value1 = toBigDecimal(number1);
+ BigDecimal value2 = toBigDecimal(number2);
+ return value1 != null && value2 != null && value1.compareTo(value2) == 0;
+ }
+ boolean isFloatingPoint1 = isFloatingPoint(number1);
+ boolean isFloatingPoint2 = isFloatingPoint(number2);
+ if (!isFloatingPoint1 && !isFloatingPoint2) {
+ return number1.longValue() == number2.longValue();
+ }
+ if (isFloatingPoint1 && isFloatingPoint2) {
+ return Double.compare(number1.doubleValue(), number2.doubleValue()) == 0;
+ }
+ long longValue = isFloatingPoint1 ? number2.longValue() :
number1.longValue();
+ double doubleValue = isFloatingPoint1 ? number1.doubleValue() :
number2.doubleValue();
+ if (Math.abs(longValue) <= 1L << 53) {
+ // The long converts to double exactly
+ return Double.compare(longValue, doubleValue) == 0;
+ }
+ return Double.isFinite(doubleValue) && new
BigDecimal(doubleValue).compareTo(BigDecimal.valueOf(longValue)) == 0;
+ }
+
+ /// Returns `null` for NaN and infinity.
+ @Nullable
+ private static BigDecimal toBigDecimal(Number number) {
+ if (number instanceof BigDecimal) {
+ return (BigDecimal) number;
+ }
+ if (isFloatingPoint(number)) {
+ double value = number.doubleValue();
+ return Double.isFinite(value) ? BigDecimal.valueOf(value) : null;
+ }
+ return BigDecimal.valueOf(number.longValue());
+ }
+
@Nullable
@ScalarFunction(nullableParameters = true, isVarArg = true)
public static Object coalesce(Object... objects) {
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/function/scalar/ObjectFunctionsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/function/scalar/ObjectFunctionsTest.java
new file mode 100644
index 00000000000..d3cef0cfe04
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/function/scalar/ObjectFunctionsTest.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.function.scalar;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+
+public class ObjectFunctionsTest {
+
+ @DataProvider
+ public static Object[][] distinctFromCases() {
+ return new Object[][]{
+ // Nulls
+ {null, null, false},
+ {null, 1, true},
+ // Same types
+ {5, 5, false},
+ {5, 6, true},
+ {5L, 5L, false},
+ {1.5, 1.5, false},
+ {1.5f, 1.5f, false},
+ {"a", "a", false},
+ {"a", "b", true},
+ {true, true, false},
+ {new Timestamp(1000L), new Timestamp(1000L), false},
+ // Integers of different types
+ {5, 5L, false},
+ {5, 6L, true},
+ {Integer.MAX_VALUE, (long) Integer.MAX_VALUE, false},
+ {-1, 0xFFFFFFFFL, true},
+ // Integers and floating point numbers
+ {5, 5.0, false},
+ {5, 5.0f, false},
+ {5L, 5.5, true},
+ {16777217, 16777216f, true},
+ {9007199254740993L, 9007199254740992.0, true},
+ {1L << 60, 0x1p60, false},
+ {Long.MAX_VALUE, (double) Long.MAX_VALUE, true},
+ {Long.MIN_VALUE, -0x1p63, false},
+ // A float is widened to double, so 1.1f (1.100000023841858) is
distinct from 1.1
+ {1.5f, 1.5, false},
+ {1.1f, 1.1, true},
+ {1.1f, (double) 1.1f, false},
+ // BigDecimal ignores the scale, and compares a floating point number
by its decimal string
+ {new BigDecimal("1.5"), new BigDecimal("1.50"), false},
+ {new BigDecimal("1.5"), new BigDecimal("1.51"), true},
+ {new BigDecimal("5.00"), 5, false},
+ {new BigDecimal("5.00"), 5L, false},
+ {new BigDecimal("5.01"), 5L, true},
+ {new BigDecimal("1.50"), 1.5, false},
+ {new BigDecimal("0.1"), 0.1, false},
+ {new BigDecimal("0.10000000000000000001"), 0.1, true},
+ {new BigDecimal("1.50"), 1.5f, false},
+ {new BigDecimal("1.1"), 1.1f, true},
+ // Like Double.equals(), -0.0 is distinct from 0.0. BigDecimal has no
-0.0.
+ {-0.0, 0.0, true},
+ {-0.0f, 0.0f, true},
+ {-0.0f, 0.0, true},
+ {-0.0, -0.0f, false},
+ {-0.0, 0, true},
+ {-0.0, 0L, true},
+ {-0.0, BigDecimal.ZERO, false},
+ // NaN is not distinct from NaN, and is distinct from any other number
+ {Double.NaN, Double.NaN, false},
+ {Float.NaN, Double.NaN, false},
+ {Double.NaN, 0.0, true},
+ {Double.NaN, 0, true},
+ {Double.NaN, Long.MAX_VALUE, true},
+ {Double.NaN, BigDecimal.ZERO, true},
+ {Float.NaN, BigDecimal.ONE, true},
+ // Infinity
+ {Double.POSITIVE_INFINITY, Float.POSITIVE_INFINITY, false},
+ {Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, true},
+ {Double.POSITIVE_INFINITY, Long.MAX_VALUE, true},
+ {Double.POSITIVE_INFINITY, new BigDecimal("1E+400"), true},
+ // Arrays are compared by content
+ {new byte[]{1, 2}, new byte[]{1, 2}, false},
+ {new byte[]{1, 2}, new byte[]{1, 3}, true},
+ {new int[]{1, 2}, new int[]{1, 2}, false},
+ {new String[]{"a"}, new String[]{"a"}, false},
+ {new String[]{"a"}, new String[]{"b"}, true},
+ // Other values are compared with equals()
+ {"5", 5, true},
+ {true, 1, true},
+ {new Timestamp(1000L), 1000L, true}
+ };
+ }
+
+ @Test(dataProvider = "distinctFromCases")
+ public void testIsDistinctFrom(Object value1, Object value2, boolean
expected) {
+ assertEquals(ObjectFunctions.isDistinctFrom(value1, value2), expected);
+ assertEquals(ObjectFunctions.isDistinctFrom(value2, value1), expected);
+ assertEquals(ObjectFunctions.isNotDistinctFrom(value1, value2), !expected);
+ assertEquals(ObjectFunctions.isNotDistinctFrom(value2, value1), !expected);
+ }
+}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/query/postaggregation/PostAggregationFunctionTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/query/postaggregation/PostAggregationFunctionTest.java
index d7f002d62bd..76ec42b5cbd 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/query/postaggregation/PostAggregationFunctionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/query/postaggregation/PostAggregationFunctionTest.java
@@ -83,6 +83,12 @@ public class PostAggregationFunctionTest {
assertTrue((Boolean) function.invoke(new Object[]{"a", null}));
assertTrue((Boolean) function.invoke(new Object[]{"a", "b"}));
+ // isDistinctFrom compares numbers of different types by value, e.g.
isDistinctFrom(MAX(intCol), 5)
+ function = new PostAggregationFunction("isDistinctFrom",
+ new ColumnDataType[]{ColumnDataType.DOUBLE, ColumnDataType.LONG});
+ assertFalse((Boolean) function.invoke(new Object[]{5.0, 5L}));
+ assertTrue((Boolean) function.invoke(new Object[]{5.5, 5L}));
+
// isNotDistinctFrom
function = new PostAggregationFunction("isNotDistinctFrom",
new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.STRING});
diff --git a/pinot-query-runtime/src/test/resources/queries/Comparisons.json
b/pinot-query-runtime/src/test/resources/queries/Comparisons.json
index 38cbe0a435f..97200680cd6 100644
--- a/pinot-query-runtime/src/test/resources/queries/Comparisons.json
+++ b/pinot-query-runtime/src/test/resources/queries/Comparisons.json
@@ -541,5 +541,73 @@
{ "sql": "SELECT a BETWEEN 'a' AND 'c' FROM {tbl}" },
{ "sql" : "SELECT 'c' BETWEEN 'a' AND 'b' FROM {tbl}"}
]
+ },
+ "distinct_from_function_mixed_numeric_types": {
+ "tables": {
+ "tbl": {
+ "schema": [
+ {"name": "intCol", "type": "INT"},
+ {"name": "longCol", "type": "LONG"},
+ {"name": "floatCol", "type": "FLOAT"},
+ {"name": "doubleCol", "type": "DOUBLE"},
+ {"name": "bigDecimalCol", "type": "BIG_DECIMAL"}
+ ],
+ "inputs": [
+ [5, 5, 5.0, 5.0, "5"],
+ [5, 6, 5.5, 5.5, "5.5"],
+ [0, 0, 0.0, 0.0, "0"],
+ [-2, 2, 1.1, 1.1, "1.1"],
+ [2147483647, 2147483647, -1.5, -1.5, "-1.5"]
+ ]
+ }
+ },
+ "queries": [
+ {
+ "description": "The GROUP BY subquery moves the call above the leaf
stage, where it runs the scalar function",
+ "sql": "SELECT t.intCol, isDistinctFrom(t.intCol, CAST(5 AS BIGINT)),
isNotDistinctFrom(t.intCol, CAST(5 AS BIGINT)) FROM (SELECT intCol FROM {tbl}
GROUP BY intCol) t",
+ "h2Sql": "SELECT t.intCol, t.intCol IS DISTINCT FROM CAST(5 AS
BIGINT), t.intCol IS NOT DISTINCT FROM CAST(5 AS BIGINT) FROM (SELECT intCol
FROM {tbl} GROUP BY intCol) t"
+ },
+ {
+ "description": "Numbers of different types are compared by value",
+ "sql": "SELECT t.intCol, isDistinctFrom(t.intCol, t.longCol),
isDistinctFrom(t.intCol, t.floatCol), isDistinctFrom(t.intCol, t.doubleCol),
isDistinctFrom(t.intCol, t.bigDecimalCol), isDistinctFrom(t.longCol,
t.doubleCol), isDistinctFrom(t.floatCol, t.doubleCol),
isDistinctFrom(t.doubleCol, t.bigDecimalCol), isNotDistinctFrom(t.longCol,
t.floatCol) FROM (SELECT intCol, longCol, floatCol, doubleCol, bigDecimalCol
FROM {tbl} GROUP BY intCol, longCol, floatCol, doubleCol, bigDecimalCol) t",
+ "h2Sql": "SELECT t.intCol, t.intCol IS DISTINCT FROM t.longCol,
t.intCol IS DISTINCT FROM t.floatCol, t.intCol IS DISTINCT FROM t.doubleCol,
t.intCol IS DISTINCT FROM t.bigDecimalCol, t.longCol IS DISTINCT FROM
t.doubleCol, t.floatCol IS DISTINCT FROM t.doubleCol, t.doubleCol IS DISTINCT
FROM t.bigDecimalCol, t.longCol IS NOT DISTINCT FROM t.floatCol FROM (SELECT
intCol, longCol, floatCol, doubleCol, bigDecimalCol FROM {tbl} GROUP BY intCol,
longCol, floatCol, doubleCol, bigDecim [...]
+ },
+ {
+ "description": "NaN is not distinct from NaN, and is distinct from any
BIG_DECIMAL",
+ "sql": "SELECT t.doubleCol, isDistinctFrom(SQRT(t.doubleCol),
CAST(SQRT(t.doubleCol) AS REAL)), isDistinctFrom(SQRT(t.doubleCol),
t.bigDecimalCol) FROM (SELECT intCol, longCol, floatCol, doubleCol,
bigDecimalCol FROM {tbl} GROUP BY intCol, longCol, floatCol, doubleCol,
bigDecimalCol) t",
+ "h2Sql": "SELECT t.doubleCol, SQRT(t.doubleCol) IS DISTINCT FROM
CAST(SQRT(t.doubleCol) AS REAL), SQRT(t.doubleCol) IS DISTINCT FROM
t.bigDecimalCol FROM (SELECT intCol, longCol, floatCol, doubleCol,
bigDecimalCol FROM {tbl} GROUP BY intCol, longCol, floatCol, doubleCol,
bigDecimalCol) t"
+ },
+ {
+ "description": "BIG_DECIMAL values with different scales are not
distinct",
+ "sql": "SELECT t.bigDecimalCol, isDistinctFrom(t.bigDecimalCol,
CAST(1.1 AS DECIMAL(38, 2))), isNotDistinctFrom(t.bigDecimalCol, CAST(-1.5 AS
DECIMAL(38, 3))) FROM (SELECT intCol, longCol, floatCol, doubleCol,
bigDecimalCol FROM {tbl} GROUP BY intCol, longCol, floatCol, doubleCol,
bigDecimalCol) t",
+ "h2Sql": "SELECT t.bigDecimalCol, t.bigDecimalCol IS DISTINCT FROM
CAST(1.1 AS DECIMAL(38, 2)), t.bigDecimalCol IS NOT DISTINCT FROM CAST(-1.5 AS
DECIMAL(38, 3)) FROM (SELECT intCol, longCol, floatCol, doubleCol,
bigDecimalCol FROM {tbl} GROUP BY intCol, longCol, floatCol, doubleCol,
bigDecimalCol) t"
+ },
+ {
+ "description": "All-literal calls are folded during planning",
+ "sql": "SELECT isDistinctFrom(5, CAST(5 AS BIGINT)), isDistinctFrom(5,
5.0), isDistinctFrom(1.5, 1.50), isNotDistinctFrom(CAST(5 AS BIGINT), 5.0) FROM
{tbl}",
+ "h2Sql": "SELECT 5 IS DISTINCT FROM CAST(5 AS BIGINT), 5 IS DISTINCT
FROM 5.0, 1.5 IS DISTINCT FROM 1.50, CAST(5 AS BIGINT) IS NOT DISTINCT FROM 5.0
FROM {tbl}"
+ }
+ ]
+ },
+ "distinct_from_function_bytes": {
+ "tables": {
+ "tbl": {
+ "schema": [
+ {"name": "bytesCol1", "type": "BYTES"},
+ {"name": "bytesCol2", "type": "BYTES"}
+ ],
+ "inputs": [
+ ["deadbeef", "deadbeef"],
+ ["00", "01"]
+ ]
+ }
+ },
+ "queries": [
+ {
+ "description": "BYTES values are compared by content above the leaf
stage",
+ "sql": "SELECT isDistinctFrom(t.bytesCol1, t.bytesCol2),
isNotDistinctFrom(t.bytesCol1, t.bytesCol2) FROM (SELECT bytesCol1, bytesCol2
FROM {tbl} GROUP BY bytesCol1, bytesCol2) t",
+ "h2Sql": "SELECT t.bytesCol1 IS DISTINCT FROM t.bytesCol2, t.bytesCol1
IS NOT DISTINCT FROM t.bytesCol2 FROM (SELECT bytesCol1, bytesCol2 FROM {tbl}
GROUP BY bytesCol1, bytesCol2) t"
+ }
+ ]
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
