This is an automated email from the ASF dual-hosted git repository.
yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new ad850480ef2 Revert "Map DECIMAL(p, 0) to a type that holds all its
values in the multi-stage engine (#19790)" (#19949)
ad850480ef2 is described below
commit ad850480ef2ea656a8c921b625578017b7d5e9ab
Author: Yash Mayya <[email protected]>
AuthorDate: Mon Oct 19 15:11:52 2026 -0700
Revert "Map DECIMAL(p, 0) to a type that holds all its values in the
multi-stage engine (#19790)" (#19949)
This reverts commit dc7837cb617645ff48aa8146c71c0f23b6ef83e4.
---
.../rel/rules/PinotEvaluateLiteralRule.java | 7 +-
.../planner/logical/RelToPlanNodeConverter.java | 40 +------
.../query/planner/logical/RexExpressionUtils.java | 44 ++-----
.../physical/v2/PRelToPlanNodeConverter.java | 8 +-
.../logical/RelToPlanNodeConverterTest.java | 53 +--------
.../planner/logical/RexExpressionUtilsTest.java | 36 +-----
.../physical/v2/PRelToPlanNodeConverterTest.java | 28 -----
.../src/test/resources/queries/AggregatePlans.json | 26 -----
.../runtime/operator/operands/FilterOperand.java | 40 +------
.../query/runtime/operator/FilterOperatorTest.java | 36 ------
.../src/test/resources/queries/NumericTypes.json | 127 ---------------------
.../pinot/segment/spi/AggregationFunctionType.java | 111 +-----------------
12 files changed, 20 insertions(+), 536 deletions(-)
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java
b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java
index cee1ec9d50f..856b776c966 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java
@@ -189,12 +189,7 @@ public class PinotEvaluateLiteralRule {
// to is determined by the operator's return type. Pinot's CAST function
implementation requires two arguments:
// the value to be cast and the target type.
argumentTypes = new ColumnDataType[]{argumentTypes[0],
ColumnDataType.STRING};
- // A CAST to some DECIMAL types converts through LONG to drop the
fraction, as at runtime (see
- // RexExpressionUtils.handleCast()). The LONG result converts to the
type of the CAST after the call.
- ColumnDataType castType =
- RelToPlanNodeConverter.isCastThroughLong(rexCall.getType(),
operands.get(0).getType()) ? ColumnDataType.LONG
- :
RelToPlanNodeConverter.convertToColumnDataType(rexCall.getType());
- arguments = new Object[]{arguments[0], castType.name()};
+ arguments = new Object[]{arguments[0],
RelToPlanNodeConverter.convertToColumnDataType(rexCall.getType()).name()};
}
String canonicalName =
FunctionRegistry.canonicalize(PinotRuleUtils.extractFunctionName(rexCall));
FunctionInfo functionInfo =
FunctionRegistry.lookupFunctionInfo(canonicalName, argumentTypes);
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
index 48165051996..6198f1cbcb0 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
@@ -1103,6 +1103,7 @@ public final class RelToPlanNodeConverter {
/// the proper primitive type for Pinot.
/// TODO: Revisit this method:
/// - Currently we are converting exact value to approximate value
+ /// - Integer can only cover all values with precision 9; Long can only
cover all values with precision 18
///
/// [org.apache.pinot.common.utils.request.RequestUtils#getLiteralExpression(
/// org.apache.calcite.sql.SqlLiteral)]
@@ -1114,12 +1115,9 @@ public final class RelToPlanNodeConverter {
int precision = relDataType.getPrecision();
int scale = relDataType.getScale();
if (scale == 0) {
- // Pick the type by the precision, so that it holds every value of the
DECIMAL type: INT holds every number with
- // up to 9 digits, and LONG every number with up to 18 digits. For
example, Calcite gives an integer literal
- // beyond the BIGINT range the type DECIMAL(p, 0) with p >= 19, which
LONG would wrap around.
- if (precision <= 9) {
+ if (precision <= 10) {
return isArray ? ColumnDataType.INT_ARRAY : ColumnDataType.INT;
- } else if (precision <= 18) {
+ } else if (precision <= 38) {
return isArray ? ColumnDataType.LONG_ARRAY : ColumnDataType.LONG;
} else {
// TODO: Modify it to return ColumnDataType.BIG_DECIMAL_ARRAY after
`1.6.0` release
@@ -1139,38 +1137,6 @@ public final class RelToPlanNodeConverter {
}
}
- /// Returns whether a CAST of a value of the operand type to the target type
converts the value to LONG first.
- ///
- /// [#resolveDecimal] gives DECIMAL(p, 0) with 19 <= p <= 38 the type
BIG_DECIMAL, which keeps the fraction of a
- /// value. A CAST to these types from a type that is not an integer type
(e.g. DOUBLE) converts the value through
- /// LONG, as it did when the planner gave these types the type LONG, so that
the result is an integer. As before, such
- /// a CAST does not hold values beyond the LONG range: a DOUBLE saturates, a
BIG_DECIMAL wraps around, and a STRING
- /// fails to parse. A CAST to DECIMAL(p, 0) with p > 38 keeps the fraction,
as before.
- public static boolean isCastThroughLong(RelDataType targetType, RelDataType
operandType) {
- if (targetType.getSqlTypeName() != SqlTypeName.DECIMAL ||
targetType.getScale() != 0) {
- return false;
- }
- int precision = targetType.getPrecision();
- if (precision <= 18 || precision > 38) {
- return false;
- }
- switch (operandType.getSqlTypeName()) {
- case TINYINT:
- case SMALLINT:
- case INTEGER:
- case BIGINT:
- case UTINYINT:
- case USMALLINT:
- case UINTEGER:
- case UBIGINT:
- return false;
- case DECIMAL:
- return operandType.getScale() != 0;
- default:
- return true;
- }
- }
-
public static String getTableNameFromTableScan(TableScan tableScan) {
return getTableNameFromRelTable(tableScan.getTable());
}
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java
index 9777cbc4ad0..8bcf2bf6b63 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java
@@ -367,18 +367,9 @@ public class RexExpressionUtils {
// CAST is being rewritten into "rexCall.CAST<targetType>(inputValue)",
// - e.g. result type has already been converted into the CAST RexCall,
so we assert single operand.
assert rexCall.operands.size() == 1;
- RexNode operand = rexCall.operands.get(0);
- RexExpression value = fromRexNode(operand);
- if (RelToPlanNodeConverter.isCastThroughLong(rexCall.type,
operand.getType())) {
- // Drop the fraction, which a CAST to BIG_DECIMAL keeps
- value = cast(value, ColumnDataType.LONG);
- }
- return cast(value,
RelToPlanNodeConverter.convertToColumnDataType(rexCall.type));
- }
-
- private static RexExpression.FunctionCall cast(RexExpression value,
ColumnDataType castType) {
List<RexExpression> operands = new ArrayList<>(2);
- operands.add(value);
+ operands.add(fromRexNode(rexCall.operands.get(0)));
+ ColumnDataType castType =
RelToPlanNodeConverter.convertToColumnDataType(rexCall.type);
operands.add(new RexExpression.Literal(ColumnDataType.STRING,
castType.name()));
return new RexExpression.FunctionCall(castType, SqlKind.CAST.name(),
operands);
}
@@ -397,20 +388,15 @@ public class RexExpressionUtils {
if (leftOperand instanceof RexLiteral) {
return evaluateLiteralIn((RexLiteral) leftOperand,
sarg.rangeSet.asRanges(), sarg.nullAs);
}
- RexExpression inExpr = dataType == ColumnDataType.BIG_DECIMAL
- ? convertPoints(SqlKind.OR, SqlKind.EQUALS, leftOperand,
sarg.rangeSet.asRanges(), dataType)
- : new RexExpression.FunctionCall(ColumnDataType.BOOLEAN,
SqlKind.IN.name(),
- toSearchFunctionOperands(leftOperand, sarg.rangeSet.asRanges(),
dataType));
+ RexExpression inExpr = new
RexExpression.FunctionCall(ColumnDataType.BOOLEAN, SqlKind.IN.name(),
+ toSearchFunctionOperands(leftOperand, sarg.rangeSet.asRanges(),
dataType));
return addNullCheckIfRequired(leftOperand, sarg.nullAs, inExpr);
} else if (sarg.isComplementedPoints()) {
if (leftOperand instanceof RexLiteral) {
return evaluateLiteralNotIn((RexLiteral) leftOperand,
sarg.rangeSet.complement().asRanges(), sarg.nullAs);
}
- Set<Range> points = sarg.rangeSet.complement().asRanges();
- RexExpression notInExpr = dataType == ColumnDataType.BIG_DECIMAL
- ? convertPoints(SqlKind.AND, SqlKind.NOT_EQUALS, leftOperand,
points, dataType)
- : new RexExpression.FunctionCall(ColumnDataType.BOOLEAN,
SqlKind.NOT_IN.name(),
- toSearchFunctionOperands(leftOperand, points, dataType));
+ RexExpression notInExpr = new
RexExpression.FunctionCall(ColumnDataType.BOOLEAN, SqlKind.NOT_IN.name(),
+ toSearchFunctionOperands(leftOperand,
sarg.rangeSet.complement().asRanges(), dataType));
return addNullCheckIfRequired(leftOperand, sarg.nullAs, notInExpr);
} else {
if (leftOperand instanceof RexLiteral) {
@@ -435,7 +421,8 @@ public class RexExpressionUtils {
/// result is `AND(NOT_IN(x, <list>), x > 0)`.
/// - When both forms apply, the one with fewer values is used.
///
- /// `BIG_DECIMAL` ranges always become comparisons, like single
`BIG_DECIMAL` values (see [#convertPoints]).
+ /// `BIG_DECIMAL` ranges always become comparisons: an intermediate stage
matches `IN` values with `equals`, which
+ /// depends on the scale of the value.
private static RexExpression convertRanges(ColumnDataType dataType, RexNode
leftOperand, RangeSet rangeSet) {
Set<Range> ranges = rangeSet.asRanges();
if (dataType == ColumnDataType.BIG_DECIMAL) {
@@ -476,21 +463,6 @@ public class RexExpressionUtils {
return convertRangesToOr(dataType, leftOperand, ranges);
}
- /// Converts single values to one comparison each, e.g. `OR(x = 1, x = 2)`
for `x IN (1, 2)` and
- /// `AND(x <> 1, x <> 2)` for `x NOT IN (1, 2)`. Used for `BIG_DECIMAL`
values: an intermediate stage of an older
- /// server matches `IN` values with `equals`, which depends on the scale of
the value (1.0 does not equal 1), and a
- /// comparison does not.
- private static RexExpression convertPoints(SqlKind combineKind, SqlKind
comparisonKind, RexNode leftOperand,
- Set<Range> points, ColumnDataType dataType) {
- RexExpression convertedLeftOperand = fromRexNode(leftOperand);
- List<RexExpression> terms = new ArrayList<>(points.size());
- for (Range point : points) {
- terms.add(new RexExpression.FunctionCall(ColumnDataType.BOOLEAN,
comparisonKind.name(),
- List.of(convertedLeftOperand, fromRexLiteralValue(dataType,
point.lowerEndpoint()))));
- }
- return combine(combineKind, terms);
- }
-
private static RexExpression combine(SqlKind kind, List<RexExpression>
terms) {
if (terms.size() == 1) {
return terms.get(0);
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverter.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverter.java
index a37f2f2a0f3..cc9d4416593 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverter.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverter.java
@@ -363,6 +363,7 @@ public class PRelToPlanNodeConverter {
/// the proper primitive type for Pinot.
/// TODO: Revisit this method:
/// - Currently we are converting exact value to approximate value
+ /// - Integer can only cover all values with precision 9; Long can only
cover all values with precision 18
///
/// [org.apache.pinot.common.utils.request.RequestUtils#getLiteralExpression(
/// org.apache.calcite.sql.SqlLiteral)]
@@ -374,12 +375,9 @@ public class PRelToPlanNodeConverter {
int precision = relDataType.getPrecision();
int scale = relDataType.getScale();
if (scale == 0) {
- // Pick the type by the precision, so that it holds every value of the
DECIMAL type: INT holds every number with
- // up to 9 digits, and LONG every number with up to 18 digits. For
example, Calcite gives an integer literal
- // beyond the BIGINT range the type DECIMAL(p, 0) with p >= 19, which
LONG would wrap around.
- if (precision <= 9) {
+ if (precision <= 10) {
return isArray ? ColumnDataType.INT_ARRAY : ColumnDataType.INT;
- } else if (precision <= 18) {
+ } else if (precision <= 38) {
return isArray ? ColumnDataType.LONG_ARRAY : ColumnDataType.LONG;
} else {
// TODO: Modify it to return ColumnDataType.BIG_DECIMAL_ARRAY after
`1.6.0` release
diff --git
a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverterTest.java
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverterTest.java
index e6d9cff5a48..69d6acca0cd 100644
---
a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverterTest.java
+++
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverterTest.java
@@ -115,34 +115,15 @@ public class RelToPlanNodeConverterTest {
@Test
public void testBigDecimal() {
- // DECIMAL(p, 0) maps to the narrowest type that holds every value: INT up
to 9 digits, LONG up to 18 digits
- Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
9)),
- DataSchema.ColumnDataType.INT);
Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
10)),
- DataSchema.ColumnDataType.LONG);
- Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
18)),
- DataSchema.ColumnDataType.LONG);
- Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
19)),
- DataSchema.ColumnDataType.BIG_DECIMAL);
+ DataSchema.ColumnDataType.INT);
Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
38)),
- DataSchema.ColumnDataType.BIG_DECIMAL);
+ DataSchema.ColumnDataType.LONG);
Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
39)),
DataSchema.ColumnDataType.BIG_DECIMAL);
- Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
- new ArraySqlType(new BasicSqlType(RelDataTypeSystem.DEFAULT,
SqlTypeName.DECIMAL, 9), true)),
- DataSchema.ColumnDataType.INT_ARRAY);
- Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
- new ArraySqlType(new BasicSqlType(RelDataTypeSystem.DEFAULT,
SqlTypeName.DECIMAL, 18), true)),
- DataSchema.ColumnDataType.LONG_ARRAY);
- Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
- new ArraySqlType(new BasicSqlType(RelDataTypeSystem.DEFAULT,
SqlTypeName.DECIMAL, 19), true)),
- DataSchema.ColumnDataType.DOUBLE_ARRAY);
Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
14, 10)),
@@ -155,36 +136,6 @@ public class RelToPlanNodeConverterTest {
DataSchema.ColumnDataType.BIG_DECIMAL);
}
- @Test
- public void testIsCastThroughLong() {
- RelDataType doubleType = new BasicSqlType(RelDataTypeSystem.DEFAULT,
SqlTypeName.DOUBLE);
- RelDataType decimal19 = new BasicSqlType(RelDataTypeSystem.DEFAULT,
SqlTypeName.DECIMAL, 19);
- RelDataType decimal38 = new BasicSqlType(RelDataTypeSystem.DEFAULT,
SqlTypeName.DECIMAL, 38);
- // Types that can have a fraction convert through LONG to DECIMAL(p, 0)
with 19 <= p <= 38
- Assert.assertTrue(RelToPlanNodeConverter.isCastThroughLong(decimal19,
doubleType));
- Assert.assertTrue(RelToPlanNodeConverter.isCastThroughLong(decimal38,
doubleType));
- Assert.assertTrue(RelToPlanNodeConverter.isCastThroughLong(decimal19,
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL, 10,
2)));
- Assert.assertTrue(RelToPlanNodeConverter.isCastThroughLong(decimal19,
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.VARCHAR)));
- // Integer types convert directly
- Assert.assertFalse(RelToPlanNodeConverter.isCastThroughLong(decimal19,
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER)));
- Assert.assertFalse(RelToPlanNodeConverter.isCastThroughLong(decimal19,
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT)));
- Assert.assertFalse(RelToPlanNodeConverter.isCastThroughLong(decimal19,
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL, 20)));
- // Other target types: LONG drops the fraction itself, and BIG_DECIMAL
beyond 38 digits keeps it as before
- Assert.assertFalse(RelToPlanNodeConverter.isCastThroughLong(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL, 18),
doubleType));
- Assert.assertFalse(RelToPlanNodeConverter.isCastThroughLong(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL, 39),
doubleType));
- Assert.assertFalse(RelToPlanNodeConverter.isCastThroughLong(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL, 20,
2), doubleType));
- Assert.assertFalse(RelToPlanNodeConverter.isCastThroughLong(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
doubleType));
- }
-
@Test
public void testConvertToColumnDataTypeForArray() {
Assert.assertEquals(RelToPlanNodeConverter.convertToColumnDataType(
diff --git
a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RexExpressionUtilsTest.java
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RexExpressionUtilsTest.java
index 313940e7b2b..536b2ecb948 100644
---
a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RexExpressionUtilsTest.java
+++
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RexExpressionUtilsTest.java
@@ -697,7 +697,7 @@ public class RexExpressionUtilsTest {
call(SqlKind.LESS_THAN_OR_EQUAL, ref(), intLiteral(10)))));
}
- /// BIG_DECIMAL points next to a range stay comparisons, because older
servers match IN values with `equals`.
+ /// BIG_DECIMAL points next to a range stay comparisons, because
intermediate stages match IN values with `equals`.
@Test
public void testHandleSearchBigDecimalPointsAndRange() {
RexInputRef inputRef =
_rexBuilder.makeInputRef(_typeFactory.createSqlType(SqlTypeName.DECIMAL, 38,
2), 0);
@@ -716,26 +716,6 @@ public class RexExpressionUtilsTest {
}
}
- /// `x IN (1, 2)` on BIG_DECIMAL values becomes `x = 1 OR x = 2`, because
older servers match IN values with
- /// `equals`, which depends on the scale of the value (e.g. a SUM of 1.0
does not equal 1).
- @Test
- public void testHandleSearchBigDecimalPoints() {
- RexExpression points = call(SqlKind.OR, call(SqlKind.EQUALS, ref(),
bigDecimalLiteral(1)),
- call(SqlKind.EQUALS, ref(), bigDecimalLiteral(2)));
- Assert.assertEquals(convertBigDecimalSearch(RexUnknownAs.UNKNOWN,
points(1, 2)), points);
- Assert.assertEquals(convertBigDecimalSearch(RexUnknownAs.FALSE, points(1,
2)),
- call(SqlKind.AND, points, call(SqlKind.IS_NOT_NULL, ref())));
- }
-
- /// `x NOT IN (1, 2)` on BIG_DECIMAL values becomes `x <> 1 AND x <> 2`.
- @Test
- public void testHandleSearchBigDecimalComplementedPoints() {
- ImmutableRangeSet<BigDecimal> rangeSet =
ImmutableRangeSet.copyOf(points(1, 2).complement());
- Assert.assertEquals(convertBigDecimalSearch(RexUnknownAs.UNKNOWN,
rangeSet),
- call(SqlKind.AND, call(SqlKind.NOT_EQUALS, ref(),
bigDecimalLiteral(1)),
- call(SqlKind.NOT_EQUALS, ref(), bigDecimalLiteral(2))));
- }
-
/// Point ranges for the values `from` to `to`, both included.
private static ImmutableRangeSet<BigDecimal> points(int from, int to) {
ImmutableRangeSet.Builder<BigDecimal> builder =
ImmutableRangeSet.builder();
@@ -769,20 +749,6 @@ public class RexExpressionUtilsTest {
return call(kind, operands);
}
- private RexExpression convertBigDecimalSearch(RexUnknownAs nullAs,
ImmutableRangeSet<BigDecimal> rangeSet) {
- // DECIMAL(19, 0) is a BIG_DECIMAL type
- RexInputRef inputRef =
_rexBuilder.makeInputRef(_typeFactory.createSqlType(SqlTypeName.DECIMAL, 19,
0), 0);
- Sarg<BigDecimal> sarg = Sarg.of(nullAs, rangeSet);
- RexLiteral searchLiteral =
- _rexBuilder.makeSearchArgumentLiteral(sarg,
_typeFactory.createSqlType(SqlTypeName.DECIMAL, 19, 0));
- return RexExpressionUtils.fromRexCall(
- (RexCall) _rexBuilder.makeCall(SqlStdOperatorTable.SEARCH, inputRef,
searchLiteral));
- }
-
- private static RexExpression bigDecimalLiteral(int value) {
- return new RexExpression.Literal(ColumnDataType.BIG_DECIMAL,
BigDecimal.valueOf(value));
- }
-
private RexExpression convertIntSearch(RexUnknownAs nullAs,
ImmutableRangeSet<BigDecimal> rangeSet) {
RexInputRef inputRef =
_rexBuilder.makeInputRef(_typeFactory.createSqlType(SqlTypeName.INTEGER), 0);
Sarg<BigDecimal> sarg = Sarg.of(nullAs, rangeSet);
diff --git
a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverterTest.java
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverterTest.java
index ade062753d7..e2ea5affb79 100644
---
a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverterTest.java
+++
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverterTest.java
@@ -18,10 +18,8 @@
*/
package org.apache.pinot.query.planner.physical.v2;
-import org.apache.calcite.rel.type.RelDataTypeSystem;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.type.ArraySqlType;
-import org.apache.calcite.sql.type.BasicSqlType;
import org.apache.calcite.sql.type.ObjectSqlType;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.pinot.common.utils.DataSchema;
@@ -62,32 +60,6 @@ public class PRelToPlanNodeConverterTest {
new ArraySqlType(new ObjectSqlType(SqlTypeName.UBIGINT,
SqlIdentifier.STAR, true, null, null), true)));
}
- @Test
- public void testConvertDecimalTypes() {
- // DECIMAL(p, 0) maps to the narrowest type that holds every value: INT up
to 9 digits, LONG up to 18 digits
- Assert.assertEquals(PRelToPlanNodeConverter.convertToColumnDataType(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
9)),
- DataSchema.ColumnDataType.INT);
- Assert.assertEquals(PRelToPlanNodeConverter.convertToColumnDataType(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
10)),
- DataSchema.ColumnDataType.LONG);
- Assert.assertEquals(PRelToPlanNodeConverter.convertToColumnDataType(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
18)),
- DataSchema.ColumnDataType.LONG);
- Assert.assertEquals(PRelToPlanNodeConverter.convertToColumnDataType(
- new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
19)),
- DataSchema.ColumnDataType.BIG_DECIMAL);
- Assert.assertEquals(PRelToPlanNodeConverter.convertToColumnDataType(
- new ArraySqlType(new BasicSqlType(RelDataTypeSystem.DEFAULT,
SqlTypeName.DECIMAL, 9), true)),
- DataSchema.ColumnDataType.INT_ARRAY);
- Assert.assertEquals(PRelToPlanNodeConverter.convertToColumnDataType(
- new ArraySqlType(new BasicSqlType(RelDataTypeSystem.DEFAULT,
SqlTypeName.DECIMAL, 18), true)),
- DataSchema.ColumnDataType.LONG_ARRAY);
- Assert.assertEquals(PRelToPlanNodeConverter.convertToColumnDataType(
- new ArraySqlType(new BasicSqlType(RelDataTypeSystem.DEFAULT,
SqlTypeName.DECIMAL, 19), true)),
- DataSchema.ColumnDataType.DOUBLE_ARRAY);
- }
-
@Test
public void testConvertUuidTypes() {
Assert.assertEquals(PRelToPlanNodeConverter.convertToColumnDataType(
diff --git a/pinot-query-planner/src/test/resources/queries/AggregatePlans.json
b/pinot-query-planner/src/test/resources/queries/AggregatePlans.json
index a3fe22031c8..364a54fda9d 100644
--- a/pinot-query-planner/src/test/resources/queries/AggregatePlans.json
+++ b/pinot-query-planner/src/test/resources/queries/AggregatePlans.json
@@ -1,32 +1,6 @@
{
"aggregates_planning_tests": {
"queries": [
- {
- "description": "ARRAY_AGG as BIG_DECIMAL casts only DECIMAL values
that the planner gives INT or LONG",
- "sql": "EXPLAIN PLAN FOR SELECT ARRAY_AGG(col4, 'BIG_DECIMAL'),
ARRAY_AGG(CAST(col7 AS DECIMAL(20, 0)), 'BIG_DECIMAL'), ARRAY_AGG(CAST(col3 AS
DECIMAL(5, 0)), 'BIG_DECIMAL') FROM a",
- "output": [
- "Execution Plan",
- "\nPinotLogicalAggregate(group=[{}], agg#0=[ARRAYAGG($0,
_UTF-8'BIG_DECIMAL')], agg#1=[ARRAYAGG($1, _UTF-8'BIG_DECIMAL')],
agg#2=[ARRAYAGG($2, _UTF-8'BIG_DECIMAL')], aggType=[FINAL])",
- "\n PinotLogicalExchange(distribution=[hash])",
- "\n PinotLogicalAggregate(group=[{}], agg#0=[ARRAYAGG($0,
_UTF-8'BIG_DECIMAL')], agg#1=[ARRAYAGG($2, _UTF-8'BIG_DECIMAL')],
agg#2=[ARRAYAGG($3, _UTF-8'BIG_DECIMAL')], aggType=[LEAF])",
- "\n LogicalProject(col4=[$3], $f1=[_UTF-8'BIG_DECIMAL'],
$f2=[CAST($6):DECIMAL(20, 0) NOT NULL], $f3=[CAST(CAST($2):DECIMAL(5, 0) NOT
NULL):DECIMAL(2000, 1000) NOT NULL])",
- "\n PinotLogicalTableScan(table=[[default, a]])",
- "\n"
- ]
- },
- {
- "description": "ARRAY_AGG casts its value to its data type, so that
the result type is the type of the array that it returns",
- "sql": "EXPLAIN PLAN FOR SELECT ARRAY_AGG(col3, 'LONG'),
ARRAY_AGG(col7, 'LONG'), ARRAY_AGG(col3, 'INT') FROM a",
- "output": [
- "Execution Plan",
- "\nPinotLogicalAggregate(group=[{}], agg#0=[ARRAYAGG($0,
_UTF-8'LONG')], agg#1=[ARRAYAGG($1, _UTF-8'LONG')], agg#2=[ARRAYAGG($2,
_UTF-8'INT')], aggType=[FINAL])",
- "\n PinotLogicalExchange(distribution=[hash])",
- "\n PinotLogicalAggregate(group=[{}], agg#0=[ARRAYAGG($0,
_UTF-8'LONG')], agg#1=[ARRAYAGG($2, _UTF-8'LONG')], agg#2=[ARRAYAGG($3,
_UTF-8'INT')], aggType=[LEAF])",
- "\n LogicalProject($f0=[CAST($2):BIGINT NOT NULL],
$f1=[_UTF-8'LONG'], col7=[$6], col3=[$2], $f4=[_UTF-8'INT'])",
- "\n PinotLogicalTableScan(table=[[default, a]])",
- "\n"
- ]
- },
{
"description": "Select AVG aggregation for a BIG_DECIMAL column,
TODO:(No need to cast in LogicalProject)",
"sql": "EXPLAIN PLAN FOR SELECT AVG(a.col4) as avg FROM a WHERE a.col3
>= (SELECT AVG(a.col4) FROM a)",
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/operands/FilterOperand.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/operands/FilterOperand.java
index 7dbcebb79dc..ea4f7903203 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/operands/FilterOperand.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/operands/FilterOperand.java
@@ -20,7 +20,6 @@ package org.apache.pinot.query.runtime.operator.operands;
import com.google.common.base.Preconditions;
import com.google.common.collect.Ordering;
-import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.function.IntPredicate;
@@ -133,49 +132,12 @@ public abstract class FilterOperand implements
TransformOperand {
if (result == null) {
return null;
}
- if (valuesEqual(firstResult, result)) {
+ if (firstResult.equals(result)) {
return _isNotIn ? 0 : 1;
}
}
return _isNotIn ? 1 : 0;
}
-
- /// Compares numbers by value, like a comparison does and like the
single-stage engine does for IN. `equals()` also
- /// compares the scale of a BigDecimal, so 1.0 (e.g. a SUM, which is a
DOUBLE) does not equal 1, and the class of a
- /// number, so a LONG does not equal the same INT. The planner gives the
values of an IN the same type, so the
- /// classes differ only if that changes.
- private static boolean valuesEqual(Object value1, Object value2) {
- if (value1 instanceof Number && value2 instanceof Number
- && (value1 instanceof BigDecimal || value1.getClass() !=
value2.getClass())) {
- return numbersEqual((Number) value1, (Number) value2);
- }
- return value1.equals(value2);
- }
-
- private static boolean numbersEqual(Number value1, Number value2) {
- if (value1 instanceof BigDecimal || value2 instanceof BigDecimal) {
- BigDecimal bigDecimal1 = toBigDecimal(value1);
- BigDecimal bigDecimal2 = toBigDecimal(value2);
- return bigDecimal1 != null && bigDecimal2 != null &&
bigDecimal1.compareTo(bigDecimal2) == 0;
- }
- if (value1 instanceof Double || value1 instanceof Float || value2
instanceof Double || value2 instanceof Float) {
- return Double.compare(value1.doubleValue(), value2.doubleValue()) == 0;
- }
- return value1.longValue() == value2.longValue();
- }
-
- /// Returns the BigDecimal of a number, or `null` for NaN and the
infinities, which have none.
- @Nullable
- private static BigDecimal toBigDecimal(Number value) {
- if (value instanceof BigDecimal) {
- return (BigDecimal) value;
- }
- if (value instanceof Double || value instanceof Float) {
- double doubleValue = value.doubleValue();
- return Double.isFinite(doubleValue) ? BigDecimal.valueOf(doubleValue)
: null;
- }
- return BigDecimal.valueOf(value.longValue());
- }
}
public static class IsTrue extends FilterOperand {
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/FilterOperatorTest.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/FilterOperatorTest.java
index 03f9d97c464..b39a2d7fabb 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/FilterOperatorTest.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/FilterOperatorTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.query.runtime.operator;
-import java.math.BigDecimal;
import java.util.List;
import org.apache.calcite.sql.SqlKind;
import org.apache.pinot.common.utils.DataSchema;
@@ -203,41 +202,6 @@ public class FilterOperatorTest {
assertEquals(resultRows.get(0), new Object[]{3, 2});
}
- @Test
- public void shouldCompareBigDecimalsByValueInInFilter() {
- DataSchema inputSchema = new DataSchema(new String[]{"bigDecimal0"}, new
ColumnDataType[]{
- ColumnDataType.BIG_DECIMAL
- });
- when(_input.nextBlock()).thenReturn(OperatorTestUtil.block(inputSchema,
new Object[]{new BigDecimal("1.0")},
- new Object[]{new BigDecimal("2.5")}, new Object[]{new
BigDecimal("3")}));
- // 1.0 has another scale than 1, and 3 is a LONG
- RexExpression.FunctionCall in = new
RexExpression.FunctionCall(ColumnDataType.BOOLEAN, SqlKind.IN.name(),
- List.of(new RexExpression.InputRef(0),
- new RexExpression.Literal(ColumnDataType.BIG_DECIMAL, new
BigDecimal("1")),
- new RexExpression.Literal(ColumnDataType.LONG, 3L)));
- FilterOperator operator = getOperator(inputSchema, in);
- List<Object[]> resultRows = ((MseBlock.Data)
operator.nextBlock()).asRowHeap().getRows();
- assertEquals(resultRows.size(), 2);
- assertEquals(resultRows.get(0), new Object[]{new BigDecimal("1.0")});
- assertEquals(resultRows.get(1), new Object[]{new BigDecimal("3")});
- }
-
- @Test
- public void shouldCompareNumbersOfOtherClassesInNotInFilter() {
- DataSchema inputSchema = new DataSchema(new String[]{"long0"}, new
ColumnDataType[]{
- ColumnDataType.LONG
- });
- when(_input.nextBlock()).thenReturn(
- OperatorTestUtil.block(inputSchema, new Object[]{1L}, new
Object[]{2L}, new Object[]{3L}));
- RexExpression.FunctionCall notIn = new
RexExpression.FunctionCall(ColumnDataType.BOOLEAN, SqlKind.NOT_IN.name(),
- List.of(new RexExpression.InputRef(0), new
RexExpression.Literal(ColumnDataType.INT, 1),
- new RexExpression.Literal(ColumnDataType.DOUBLE, 3.0)));
- FilterOperator operator = getOperator(inputSchema, notIn);
- List<Object[]> resultRows = ((MseBlock.Data)
operator.nextBlock()).asRowHeap().getRows();
- assertEquals(resultRows.size(), 1);
- assertEquals(resultRows.get(0), new Object[]{2L});
- }
-
@Test
public void shouldHandleBooleanFunction() {
DataSchema inputSchema = new DataSchema(new String[]{"string1"}, new
ColumnDataType[]{
diff --git a/pinot-query-runtime/src/test/resources/queries/NumericTypes.json
b/pinot-query-runtime/src/test/resources/queries/NumericTypes.json
index 75b728a7e62..a5cc4e4e782 100644
--- a/pinot-query-runtime/src/test/resources/queries/NumericTypes.json
+++ b/pinot-query-runtime/src/test/resources/queries/NumericTypes.json
@@ -113,132 +113,5 @@
"sql": "SELECT floatv + 1, doublev + 1 FROM {floats}"
}
]
- },
- "exact_literals_beyond_int_and_long": {
- "comment": "Calcite gives an integer literal beyond the BIGINT range, and
CAST(... AS DECIMAL(p, 0)), the type DECIMAL(p, 0). 7766279631452241919 is
99999999999999999999 wrapped around to a LONG, and -1294967296 is 3000000000
wrapped around to an INT. The fractions of doubleCol and bigDecimalCol are
below 0.5, because Pinot drops the fraction in a CAST to an integer type, and
H2 rounds it.",
- "tables": {
- "tbl": {
- "schema": [
- {"name": "intCol", "type": "INT"},
- {"name": "longCol", "type": "LONG"},
- {"name": "doubleCol", "type": "DOUBLE"},
- {"name": "bigDecimalCol", "type": "BIG_DECIMAL"}
- ],
- "inputs": [
- [1, 1, 1.2, "2.25"],
- [100, 7766279631452241919, -1.3, "-7.1"],
- [-1294967296, -2, 0.4, "0"],
- [2147483647, 5, 2.25, "3.3"]
- ]
- }
- },
- "queries": [
- {
- "description": "equality with a literal beyond the BIGINT range in the
leaf stage",
- "sql": "SELECT intCol FROM {tbl} WHERE longCol = 99999999999999999999"
- },
- {
- "description": "range filter with a literal beyond the BIGINT range in
the leaf stage",
- "sql": "SELECT intCol FROM {tbl} WHERE longCol < 9223372036854775808"
- },
- {
- "description": "IN filter with a literal beyond the BIGINT range in
the leaf stage",
- "sql": "SELECT intCol FROM {tbl} WHERE longCol IN (1,
99999999999999999999)"
- },
- {
- "description": "range filter with a negative literal beyond the BIGINT
range in the leaf stage",
- "sql": "SELECT intCol FROM {tbl} WHERE longCol > -9223372036854775809"
- },
- {
- "description": "arithmetic with a literal beyond the BIGINT range in
the leaf stage",
- "sql": "SELECT intCol + 99999999999999999999 FROM {tbl}"
- },
- {
- "description": "comparison with a literal beyond the BIGINT range in
an intermediate stage",
- "sql": "SELECT longCol, longCol < 9223372036854775808, longCol =
99999999999999999999 FROM (SELECT longCol FROM {tbl} GROUP BY longCol)"
- },
- {
- "description": "arithmetic with a literal beyond the BIGINT range in
an intermediate stage",
- "sql": "SELECT intCol + 99999999999999999999 FROM (SELECT intCol FROM
{tbl} GROUP BY intCol)"
- },
- {
- "description": "arithmetic with a literal beyond the BIGINT range that
is evaluated during planning",
- "sql": "SELECT intCol, 99999999999999999999 + 1, 99999999999999999999
* 2 FROM {tbl}"
- },
- {
- "description": "BIGINT value promoted to DECIMAL(19, 0) keeps its
exact value",
- "sql": "SELECT intCol, COALESCE(longCol, CAST(1 AS DECIMAL(5, 0)))
FROM {tbl}"
- },
- {
- "description": "BIGINT value promoted to DECIMAL(19, 0) in an
intermediate stage keeps its exact value",
- "sql": "SELECT longCol, COALESCE(longCol, CAST(1 AS DECIMAL(5, 0)))
FROM (SELECT longCol FROM {tbl} GROUP BY longCol)"
- },
- {
- "description": "group by a value that can be a literal beyond the
BIGINT range",
- "sql": "SELECT CASE WHEN intCol > 0 THEN longCol ELSE
99999999999999999999 END AS v, COUNT(*) FROM {tbl} GROUP BY CASE WHEN intCol >
0 THEN longCol ELSE 99999999999999999999 END"
- },
- {
- "description": "equality with a DECIMAL(10, 0) literal beyond the INT
range in the leaf stage",
- "sql": "SELECT longCol FROM {tbl} WHERE intCol = CAST(3000000000 AS
DECIMAL(10, 0))"
- },
- {
- "description": "range filter with a DECIMAL(10, 0) literal beyond the
INT range in the leaf stage",
- "sql": "SELECT longCol FROM {tbl} WHERE intCol < CAST(3000000000 AS
DECIMAL(10, 0))"
- },
- {
- "description": "comparison with a DECIMAL(10, 0) literal beyond the
INT range in an intermediate stage",
- "sql": "SELECT intCol, intCol < CAST(3000000000 AS DECIMAL(10, 0)),
intCol = CAST(3000000000 AS DECIMAL(10, 0)) FROM (SELECT intCol FROM {tbl}
GROUP BY intCol)"
- },
- {
- "description": "aggregations of DECIMAL(p, 0) values beyond the INT
range",
- "sql": "SELECT SUM(CAST(intCol AS DECIMAL(10, 0))), SUM(CAST(intCol AS
DECIMAL(20, 0))), MAX(CAST(intCol AS DECIMAL(20, 0))) FROM {tbl} WHERE intCol >
0"
- },
- {
- "description": "group by aggregation of DECIMAL(10, 0) values beyond
the INT range",
- "sql": "SELECT intCol > 0, SUM(CAST(intCol AS DECIMAL(10, 0))) FROM
{tbl} GROUP BY intCol > 0"
- },
- {
- "description": "IN on the SUM of DECIMAL(19, 0) values, which is a
BIG_DECIMAL with a scale that the IN values do not have",
- "sql": "SELECT SUM(CAST(intCol AS DECIMAL(19, 0))) IN (CAST(1 AS
DECIMAL(19, 0)), CAST(2 AS DECIMAL(19, 0))) FROM {tbl} WHERE intCol = 1",
- "h2Sql": "SELECT SUM(intCol) IN (1, 2) FROM {tbl} WHERE intCol = 1"
- },
- {
- "description": "IN and NOT IN on the SUM of DECIMAL(p, 0) values in
HAVING",
- "sql": "SELECT intCol, SUM(CAST(intCol AS DECIMAL(19, 0))) IN (1,
100), SUM(CAST(intCol AS DECIMAL(40, 0))) NOT IN (1, 100) FROM {tbl} GROUP BY
intCol HAVING SUM(CAST(intCol AS DECIMAL(19, 0))) IN (1, 100, 2147483647)"
- },
- {
- "description": "CAST of values with a fraction to DECIMAL(19, 0) and
DECIMAL(38, 0) in the leaf stage drops the fraction",
- "sql": "SELECT intCol, CAST(doubleCol AS DECIMAL(19, 0)),
CAST(bigDecimalCol AS DECIMAL(38, 0)) FROM {tbl} WHERE doubleCol > 0"
- },
- {
- "description": "CAST of values with a fraction to DECIMAL(19, 0) in an
intermediate stage drops the fraction",
- "sql": "SELECT doubleCol, CAST(doubleCol AS DECIMAL(19, 0)),
CAST(bigDecimalCol AS DECIMAL(19, 0)) FROM (SELECT doubleCol, bigDecimalCol
FROM {tbl} GROUP BY doubleCol, bigDecimalCol)"
- },
- {
- "description": "CAST of a value with a fraction to DECIMAL(19, 0) that
is evaluated during planning drops the fraction",
- "sql": "SELECT intCol, CAST(roundDecimal(1.55, 1) AS DECIMAL(19, 0))
FROM {tbl}",
- "h2Sql": "SELECT intCol, CAST(1 AS DECIMAL(19, 0)) FROM {tbl}"
- },
- {
- "description": "ARRAY_AGG of DECIMAL(p, 0) values returns arrays of
its data type",
- "sql": "SELECT arraySumLong(ARRAY_AGG(CAST(longCol AS DECIMAL(19, 0)),
'LONG')), arraySumInt(ARRAY_AGG(CAST(intCol AS DECIMAL(10, 0)), 'INT')) FROM
{tbl} WHERE intCol < 1000",
- "h2Sql": "SELECT SUM(longCol), SUM(intCol) FROM {tbl} WHERE intCol <
1000"
- },
- {
- "description": "ARRAY_AGG of DOUBLE values as FLOAT and BIG_DECIMAL
keeps the fraction",
- "sql": "SELECT arrayElementAtFloat(ARRAY_AGG(doubleCol, 'FLOAT'), 1),
arrayElementAtDouble(ARRAY_AGG(doubleCol, 'BIG_DECIMAL'), 1) FROM {tbl} WHERE
intCol = 1",
- "h2Sql": "SELECT CAST(doubleCol AS REAL), doubleCol FROM {tbl} WHERE
intCol = 1"
- },
- {
- "description": "ARRAY_AGG of DECIMAL values as BIG_DECIMAL",
- "sql": "SELECT arrayElementAtDouble(ARRAY_AGG(CAST(longCol AS
DECIMAL(20, 0)), 'BIG_DECIMAL'), 1),
arrayElementAtDouble(ARRAY_AGG(CAST(doubleCol AS DECIMAL(10, 2)),
'BIG_DECIMAL'), 1), arrayElementAtDouble(ARRAY_AGG(CAST(intCol AS DECIMAL(5,
0)), 'BIG_DECIMAL'), 1) FROM {tbl} WHERE intCol = 1",
- "h2Sql": "SELECT CAST(longCol AS DOUBLE), doubleCol, CAST(intCol AS
DOUBLE) FROM {tbl} WHERE intCol = 1"
- },
- {
- "description": "ARRAY_AGG of values of another type returns arrays of
its data type",
- "sql": "SELECT arraySumLong(ARRAY_AGG(intCol, 'LONG')),
ARRAYLENGTH(ARRAY_AGG(doubleCol, 'FLOAT')), ARRAYLENGTH(ARRAY_AGG(longCol,
'BIG_DECIMAL', true)) FROM {tbl}",
- "h2Sql": "SELECT SUM(intCol), COUNT(*), COUNT(DISTINCT longCol) FROM
{tbl}"
- }
- ]
}
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/AggregationFunctionType.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/AggregationFunctionType.java
index 9430d81c5d1..06d04efbb00 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/AggregationFunctionType.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/AggregationFunctionType.java
@@ -26,23 +26,14 @@ import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.sql.SqlCall;
-import org.apache.calcite.sql.SqlCallBinding;
import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlOperandCountRange;
-import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlOperatorBinding;
-import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
-import org.apache.calcite.sql.type.SqlOperandCountRanges;
import org.apache.calcite.sql.type.SqlOperandTypeChecker;
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Strings;
import org.apache.pinot.spi.utils.CommonConstants;
@@ -198,7 +189,7 @@ public enum AggregationFunctionType {
ReturnTypes.ARG1, OperandTypes.VARIADIC, SqlTypeName.OTHER,
SqlTypeName.BIGINT),
// Array aggregate functions
- ARRAYAGG("arrayAgg", new ArrayOfComponentReturnTypeInference(), new
ArrayAggOperandTypeChecker(), SqlTypeName.OTHER),
+ ARRAYAGG("arrayAgg", new ArrayOfComponentReturnTypeInference(),
OperandTypes.VARIADIC, SqlTypeName.OTHER),
LISTAGG("listAgg", SqlTypeName.OTHER, SqlTypeName.VARCHAR),
SUMARRAYLONG("sumArrayLong", new
ArrayReturnTypeInference(SqlTypeName.BIGINT), OperandTypes.ARRAY,
SqlTypeName.OTHER),
@@ -465,106 +456,6 @@ public enum AggregationFunctionType {
}
}
- /// Casts the value of `ARRAY_AGG(value, 'dataType'[, isDistinct])` to the
data type, so that the return type that
- /// [ArrayOfComponentReturnTypeInference] infers from the value is the type
of the returned array. The function picks
- /// its accumulator by the data type, not by the type of the value: for
example, `ARRAY_AGG(intCol, 'LONG')` returns
- /// longs. The cast changes the operand instead of the return type
inference, because Calcite infers the type of an
- /// aggregate call again after the validation (e.g.
`Aggregate.typeMatchesInferred()`), and it cannot read the value
- /// of a literal operand then.
- private static class ArrayAggOperandTypeChecker implements
SqlOperandTypeChecker {
- @Override
- public boolean checkOperandTypes(SqlCallBinding callBinding, boolean
throwOnFailure) {
- if (callBinding.getOperandCount() < 2 ||
!callBinding.isOperandLiteral(1, false)
- || !SqlTypeUtil.inCharFamily(callBinding.getOperandType(1))) {
- // The function fails at runtime without a data type
- return true;
- }
- String dataTypeName = callBinding.getOperandLiteralValue(1,
String.class);
- if (dataTypeName == null) {
- return true;
- }
- RelDataTypeFactory typeFactory = callBinding.getTypeFactory();
- RelDataType dataType = toRelDataType(dataTypeName, typeFactory);
- RelDataType valueType = callBinding.getOperandType(0);
- // Keep multi-value columns, for which the return type inference uses
the type of the elements. If the validation
- // checks the call again, the value already has the data type, so
needsCast() prevents a second cast.
- if (dataType == null || valueType.getComponentType() != null ||
!needsCast(valueType, dataType)) {
- return true;
- }
- RelDataType castType = typeFactory.createTypeWithNullability(dataType,
valueType.isNullable());
- SqlCall call = callBinding.getCall();
- SqlNode cast = SqlStdOperatorTable.CAST.createCall(SqlParserPos.ZERO,
call.operand(0),
-
SqlTypeUtil.convertTypeToSpec(castType).withNullable(castType.isNullable()));
- call.setOperand(0, cast);
- callBinding.getValidator().setValidatedNodeType(cast, castType);
- return true;
- }
-
- /// Returns the SQL type of a Pinot data type name, or `null` when it is
not a data type that ARRAY_AGG supports.
- @Nullable
- private static RelDataType toRelDataType(String dataTypeName,
RelDataTypeFactory typeFactory) {
- switch (dataTypeName.toUpperCase()) {
- case "INT":
- return typeFactory.createSqlType(SqlTypeName.INTEGER);
- case "LONG":
- return typeFactory.createSqlType(SqlTypeName.BIGINT);
- case "FLOAT":
- return typeFactory.createSqlType(SqlTypeName.REAL);
- case "DOUBLE":
- return typeFactory.createSqlType(SqlTypeName.DOUBLE);
- case "BIG_DECIMAL":
- return typeFactory.createSqlType(SqlTypeName.DECIMAL);
- case "BOOLEAN":
- return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
- case "TIMESTAMP":
- return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
- case "STRING":
- case "JSON":
- return typeFactory.createSqlType(SqlTypeName.VARCHAR);
- case "BYTES":
- return typeFactory.createSqlType(SqlTypeName.VARBINARY);
- case "UUID":
- return typeFactory.createSqlType(SqlTypeName.UUID);
- default:
- return null;
- }
- }
-
- private static boolean needsCast(RelDataType valueType, RelDataType
dataType) {
- SqlTypeName valueTypeName = valueType.getSqlTypeName();
- switch (valueTypeName) {
- case ANY:
- case OTHER:
- // The type is not known, so a cast cannot help
- return false;
- case CHAR:
- case BINARY:
- // Arrays of CHAR and BINARY values are STRING and BYTES arrays,
like arrays of VARCHAR and VARBINARY values
- return valueTypeName.getFamily() !=
dataType.getSqlTypeName().getFamily();
- case DECIMAL:
- if (dataType.getSqlTypeName() != SqlTypeName.DECIMAL) {
- return true;
- }
- // The multi-stage engine converts the BIG_DECIMAL values of
ARRAY_AGG to the array type of the plan. It can
- // do this for the array type of every DECIMAL value, except
DECIMAL(p, 0) with p <= 18, whose array type is
- // an INT or LONG array (see
RelToPlanNodeConverter.resolveDecimal()).
- return valueType.getScale() == 0 && valueType.getPrecision() <= 18;
- default:
- return valueTypeName != dataType.getSqlTypeName();
- }
- }
-
- @Override
- public SqlOperandCountRange getOperandCountRange() {
- return SqlOperandCountRanges.any();
- }
-
- @Override
- public String getAllowedSignatures(SqlOperator op, String opName) {
- return opName + "(<VALUE>, <DATA_TYPE>[, <IS_DISTINCT>])";
- }
- }
-
/// Used for aggregation functions that always return BIGINT. The "IfEmpty"
logic ensures that the return type is
/// nullable for pure aggregation queries (no group-by) and filtered
aggregation queries. Return values can be null
/// if there are no matching rows (even if the operand type is not nullable).
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
