This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 973d94ebccbe5bb36d6de87aac9b39a2bc28dd15 Author: morrySnow <[email protected]> AuthorDate: Mon Apr 15 15:00:36 2024 +0800 [opt](Nereids) prefer slot type to support delete task better (#33559) --- .../apache/doris/nereids/types/DecimalV2Type.java | 4 + .../doris/nereids/util/TypeCoercionUtils.java | 96 ++++++++++++++++++++++ .../doris/nereids/util/TypeCoercionUtilsTest.java | 67 +++++++++++++++ 3 files changed, 167 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java index 8a5a199a8d4..c0f023ed739 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java @@ -173,6 +173,10 @@ public class DecimalV2Type extends FractionalType { return scale; } + public int getRange() { + return precision - scale; + } + @Override public DataType conversion() { if (Config.enable_decimal_conversion && shouldConversion) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index afcdb30f2dc..214fbc18049 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -78,6 +78,7 @@ import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; @@ -120,6 +121,7 @@ import java.math.BigInteger; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -933,6 +935,18 @@ public class TypeCoercionUtils { Optional<DataType> commonType = findWiderTypeForTwoForComparison( left.getDataType(), right.getDataType(), false); + + if (commonType.isPresent()) { + commonType = Optional.of(downgradeDecimalAndDateLikeType( + commonType.get(), + left, + right)); + commonType = Optional.of(downgradeDecimalAndDateLikeType( + commonType.get(), + right, + left)); + } + if (commonType.isPresent()) { if (!supportCompare(commonType.get())) { throw new AnalysisException("data type " + commonType.get() @@ -982,6 +996,13 @@ public class TypeCoercionUtils { + " could not used in InPredicate " + inPredicate.toSql()); } + if (optionalCommonType.isPresent()) { + optionalCommonType = Optional.of(downgradeDecimalAndDateLikeType( + optionalCommonType.get(), + inPredicate.getCompareExpr(), + inPredicate.getOptions().toArray(new Expression[0]))); + } + return optionalCommonType .map(commonType -> { List<Expression> newChildren = inPredicate.children().stream() @@ -992,6 +1013,81 @@ public class TypeCoercionUtils { .orElse(inPredicate); } + /** + * if the expression like slot vs literal, then we prefer cast literal to slot type to ensure delete task could run. + * currently process decimalv2 vs decimalv3, datev1 vs datev2, datetimev1 vs datetimev2. + */ + private static DataType downgradeDecimalAndDateLikeType( + DataType commonType, + Expression target, + Expression... compareExpressions) { + if (shouldDowngrade(DecimalV3Type.class, DecimalV2Type.class, + commonType, target, + d -> ((DecimalV3Type) d).getRange() <= DecimalV2Type.MAX_PRECISION - DecimalV2Type.MAX_SCALE + && ((DecimalV3Type) d).getScale() <= DecimalV2Type.MAX_SCALE, + o -> o.isLiteral() + && (o.getDataType().isDecimalV2Type() || o.getDataType().isDecimalV3Type()), + compareExpressions)) { + DecimalV3Type decimalV3Type = (DecimalV3Type) commonType; + return DecimalV2Type.createDecimalV2Type(decimalV3Type.getPrecision(), decimalV3Type.getScale()); + } + // cast to datev1 for datev1 slot in (datev1 or datev2 literal) + if (shouldDowngrade(DateV2Type.class, DateType.class, + commonType, target, + d -> true, + o -> o.isLiteral() + && (o.getDataType().isDateType() || o.getDataType().isDateV2Type()), + compareExpressions)) { + return DateType.INSTANCE; + } + // cast to datetimev1 for datetimev1 slot in (date like literal) if scale is 0 + if (shouldDowngrade(DateTimeV2Type.class, DateTimeType.class, + commonType, target, + d -> ((DateTimeV2Type) d).getScale() == 0, + o -> o.isLiteral() && o.getDataType().isDateLikeType(), + compareExpressions)) { + return DateTimeType.INSTANCE; + } + return commonType; + } + + /** + * check should downgrade from commonTypeClazz to targetTypeClazz. + * @param commonTypeClazz before downgrade type + * @param targetTypeClazz try to downgrade to type + * @param commonType original common type + * @param target target expression aka slot + * @param commonTypePredicate constraint for original type + * @param otherPredicate constraint for other expressions aka literals + * @param others literals + * + * @return true for should downgrade + */ + private static boolean shouldDowngrade( + Class<? extends DataType> commonTypeClazz, + Class<? extends DataType> targetTypeClazz, + DataType commonType, + Expression target, + Function<DataType, Boolean> commonTypePredicate, + Function<Expression, Boolean> otherPredicate, + Expression... others) { + if (!commonTypeClazz.isInstance(commonType)) { + return false; + } + if (!targetTypeClazz.isInstance(target.getDataType())) { + return false; + } + if (!commonTypePredicate.apply(commonType)) { + return false; + } + for (Expression other : others) { + if (!otherPredicate.apply(other)) { + return false; + } + } + return true; + } + /** * process case when type coercion. */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java index d9a2946ba88..be465d9c371 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java @@ -20,10 +20,17 @@ package org.apache.doris.nereids.util; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.Subtract; import org.apache.doris.nereids.trees.expressions.literal.CharLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral; import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; @@ -56,6 +63,7 @@ import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.IntegralType; +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -735,4 +743,63 @@ public class TypeCoercionUtilsTest { Assertions.assertEquals(expression.child(0), new Cast(multiply.child(0), DecimalV3Type.createDecimalV3Type(10, 3))); } + + @Test + public void testProcessInDowngrade() { + // DecimalV2 slot vs DecimalV3 literal + InPredicate decimalDowngrade = new InPredicate( + new SlotReference("c1", DecimalV2Type.createDecimalV2Type(15, 6)), + ImmutableList.of( + new DecimalV3Literal(BigDecimal.valueOf(12345.1234567)), + new DecimalLiteral(BigDecimal.valueOf(12345.1234)))); + decimalDowngrade = (InPredicate) TypeCoercionUtils.processInPredicate(decimalDowngrade); + Assertions.assertEquals(DecimalV2Type.createDecimalV2Type(16, 7), decimalDowngrade.getCompareExpr().getDataType()); + + // DateV1 slot vs DateV2 literal + InPredicate dateDowngrade = new InPredicate( + new SlotReference("c1", DateType.INSTANCE), + ImmutableList.of( + new DateLiteral(2024, 4, 12), + new DateV2Literal(2024, 4, 12))); + dateDowngrade = (InPredicate) TypeCoercionUtils.processInPredicate(dateDowngrade); + Assertions.assertEquals(DateType.INSTANCE, dateDowngrade.getCompareExpr().getDataType()); + + // DatetimeV1 slot vs DateLike literal + InPredicate datetimeDowngrade = new InPredicate( + new SlotReference("c1", DateTimeType.INSTANCE), + ImmutableList.of( + new DateLiteral(2024, 4, 12), + new DateV2Literal(2024, 4, 12), + new DateTimeLiteral(2024, 4, 12, 18, 25, 30), + new DateTimeV2Literal(2024, 4, 12, 18, 25, 30, 0))); + datetimeDowngrade = (InPredicate) TypeCoercionUtils.processInPredicate(datetimeDowngrade); + Assertions.assertEquals(DateTimeType.INSTANCE, datetimeDowngrade.getCompareExpr().getDataType()); + } + + @Test + public void testProcessComparisonPredicateDowngrade() { + // DecimalV2 slot vs DecimalV3 literal + EqualTo decimalDowngrade = new EqualTo( + new SlotReference("c1", DecimalV2Type.createDecimalV2Type(15, 6)), + new DecimalV3Literal(BigDecimal.valueOf(12345.1234567)) + ); + decimalDowngrade = (EqualTo) TypeCoercionUtils.processComparisonPredicate(decimalDowngrade); + Assertions.assertEquals(DecimalV2Type.createDecimalV2Type(16, 7), decimalDowngrade.left().getDataType()); + + // DateV1 slot vs DateV2 literal (this case cover right slot vs left literal) + EqualTo dateDowngrade = new EqualTo( + new DateV2Literal(2024, 4, 12), + new SlotReference("c1", DateType.INSTANCE) + ); + dateDowngrade = (EqualTo) TypeCoercionUtils.processComparisonPredicate(dateDowngrade); + Assertions.assertEquals(DateType.INSTANCE, dateDowngrade.left().getDataType()); + + // DatetimeV1 slot vs DateLike literal + EqualTo datetimeDowngrade = new EqualTo( + new SlotReference("c1", DateTimeType.INSTANCE), + new DateTimeV2Literal(2024, 4, 12, 18, 25, 30, 0) + ); + datetimeDowngrade = (EqualTo) TypeCoercionUtils.processComparisonPredicate(datetimeDowngrade); + Assertions.assertEquals(DateTimeType.INSTANCE, datetimeDowngrade.left().getDataType()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
