This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new c25f3524ae0 [fix](iceberg) fix iceberg predicate conversion bug #33283
(#33290)
c25f3524ae0 is described below
commit c25f3524ae0ae9ed0418b8421958914d68101088
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Sun Apr 7 08:13:03 2024 +0800
[fix](iceberg) fix iceberg predicate conversion bug #33283 (#33290)
---
.../doris/external/iceberg/util/IcebergUtils.java | 74 ++-
.../planner/external/iceberg/IcebergScanNode.java | 19 +-
.../iceberg/util/IcebergPredicateTest.java | 256 +++++++++
.../external/iceberg/util/TestIcebergPredict.java | 135 -----
.../iceberg/test_iceberg_predicate_conversion.out | 611 +++++++++++++++++++++
.../test_iceberg_predicate_conversion.groovy | 75 +++
6 files changed, 1026 insertions(+), 144 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java
index 0dd5927d373..872096a7f32 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java
@@ -56,8 +56,11 @@ import org.apache.iceberg.Snapshot;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableOperations;
import org.apache.iceberg.TableScan;
+import org.apache.iceberg.expressions.And;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.expressions.Not;
+import org.apache.iceberg.expressions.Or;
import org.apache.iceberg.expressions.Unbound;
import org.apache.iceberg.transforms.PartitionSpecVisitor;
import org.apache.iceberg.types.Type.TypeID;
@@ -254,6 +257,10 @@ public class IcebergUtils {
Expression right =
convertToIcebergExpr(compoundPredicate.getChild(1), schema);
if (left != null && right != null) {
expression = Expressions.and(left, right);
+ } else if (left != null) {
+ return left;
+ } else if (right != null) {
+ return right;
}
break;
}
@@ -356,6 +363,9 @@ public class IcebergUtils {
}
LiteralExpr literalExpr = (LiteralExpr) inExpr.getChild(i);
Object value = extractDorisLiteral(nestedField.type(),
literalExpr);
+ if (value == null) {
+ return null;
+ }
valueList.add(value);
}
if (inExpr.isNotIn()) {
@@ -366,16 +376,63 @@ public class IcebergUtils {
expression = Expressions.in(colName, valueList);
}
}
- if (expression != null && expression instanceof Unbound) {
- try {
- ((Unbound<?, ?>) expression).bind(schema.asStruct(), true);
- return expression;
- } catch (Exception e) {
- LOG.warn("Failed to check expression: " + e.getMessage());
- return null;
+
+ return checkConversion(expression, schema);
+ }
+
+ private static Expression checkConversion(Expression expression, Schema
schema) {
+ if (expression == null) {
+ return null;
+ }
+ switch (expression.op()) {
+ case AND: {
+ And andExpr = (And) expression;
+ Expression left = checkConversion(andExpr.left(), schema);
+ Expression right = checkConversion(andExpr.right(), schema);
+ if (left != null && right != null) {
+ return andExpr;
+ } else if (left != null) {
+ return left;
+ } else if (right != null) {
+ return right;
+ } else {
+ return null;
+ }
}
+ case OR: {
+ Or orExpr = (Or) expression;
+ Expression left = checkConversion(orExpr.left(), schema);
+ Expression right = checkConversion(orExpr.right(), schema);
+ if (left == null || right == null) {
+ return null;
+ } else {
+ return orExpr;
+ }
+ }
+ case NOT: {
+ Not notExpr = (Not) expression;
+ Expression child = checkConversion(notExpr.child(), schema);
+ if (child == null) {
+ return null;
+ } else {
+ return notExpr;
+ }
+ }
+ case TRUE:
+ case FALSE:
+ return expression;
+ default:
+ if (!(expression instanceof Unbound)) {
+ return null;
+ }
+ try {
+ ((Unbound<?, ?>) expression).bind(schema.asStruct(), true);
+ return expression;
+ } catch (Exception e) {
+ LOG.debug("Failed to check expression: {}",
e.getMessage());
+ return null;
+ }
}
- return null;
}
private static Object extractDorisLiteral(org.apache.iceberg.types.Type
icebergType, Expr expr) {
@@ -394,6 +451,7 @@ public class IcebergUtils {
DateLiteral dateLiteral = (DateLiteral) expr;
switch (icebergTypeID) {
case STRING:
+ case DATE:
return dateLiteral.getStringValue();
case TIMESTAMP:
return dateLiteral.unixTimestamp(TimeUtils.getTimeZone())
* MILLIS_TO_NANO_TIME;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
index e7ca3484ae0..943d0817def 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
@@ -41,6 +41,7 @@ import org.apache.doris.planner.external.TableFormatType;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.spi.Split;
import org.apache.doris.statistics.StatisticalType;
+import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TFileAttributes;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TFileRangeDesc;
@@ -51,7 +52,8 @@ import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPushAggOp;
import org.apache.doris.thrift.TTableFormatFileDesc;
-import avro.shaded.com.google.common.base.Preconditions;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import org.apache.hadoop.fs.Path;
import org.apache.iceberg.BaseTable;
import org.apache.iceberg.CombinedScanTask;
@@ -92,6 +94,7 @@ public class IcebergScanNode extends FileQueryScanNode {
private IcebergSource source;
private Table icebergTable;
+ private List<String> pushdownIcebergPredicates = Lists.newArrayList();
/**
* External file scan node for Query iceberg table
@@ -201,6 +204,7 @@ public class IcebergScanNode extends FileQueryScanNode {
}
for (Expression predicate : expressions) {
scan = scan.filter(predicate);
+ this.pushdownIcebergPredicates.add(predicate.toString());
}
// get splits
@@ -446,4 +450,17 @@ public class IcebergScanNode extends FileQueryScanNode {
}
}
}
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel
detailLevel) {
+ if (pushdownIcebergPredicates.isEmpty()) {
+ return super.getNodeExplainString(prefix, detailLevel);
+ }
+ StringBuilder sb = new StringBuilder();
+ for (String predicate : pushdownIcebergPredicates) {
+ sb.append(prefix).append(prefix).append(predicate).append("\n");
+ }
+ return super.getNodeExplainString(prefix, detailLevel)
+ + String.format("%sicebergPredicatePushdown=\n%s\n", prefix,
sb);
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/util/IcebergPredicateTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/util/IcebergPredicateTest.java
new file mode 100644
index 00000000000..ae42e588bc5
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/util/IcebergPredicateTest.java
@@ -0,0 +1,256 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.external.iceberg.util;
+
+import org.apache.doris.analysis.BinaryPredicate;
+import org.apache.doris.analysis.BoolLiteral;
+import org.apache.doris.analysis.CompoundPredicate;
+import org.apache.doris.analysis.CompoundPredicate.Operator;
+import org.apache.doris.analysis.DateLiteral;
+import org.apache.doris.analysis.DecimalLiteral;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.FloatLiteral;
+import org.apache.doris.analysis.InPredicate;
+import org.apache.doris.analysis.IntLiteral;
+import org.apache.doris.analysis.LiteralExpr;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.analysis.TableName;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Lists;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.types.Types;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class IcebergPredicateTest {
+
+ public static Schema schema;
+
+ @BeforeClass
+ public static void before() throws AnalysisException {
+ schema = new Schema(
+ Types.NestedField.required(1, "c_int",
Types.IntegerType.get()),
+ Types.NestedField.required(2, "c_long", Types.LongType.get()),
+ Types.NestedField.required(3, "c_bool",
Types.BooleanType.get()),
+ Types.NestedField.required(4, "c_float",
Types.FloatType.get()),
+ Types.NestedField.required(5, "c_double",
Types.DoubleType.get()),
+ Types.NestedField.required(6, "c_dec",
Types.DecimalType.of(20, 10)),
+ Types.NestedField.required(7, "c_date", Types.DateType.get()),
+ Types.NestedField.required(8, "c_ts",
Types.TimestampType.withoutZone()),
+ Types.NestedField.required(10, "c_str", Types.StringType.get())
+ );
+ }
+
+ @Test
+ public void testBinaryPredicate() throws AnalysisException {
+ List<LiteralExpr> literalList = new ArrayList<LiteralExpr>() {{
+ add(new BoolLiteral(true));
+ add(new DateLiteral("2023-01-02", Type.DATEV2));
+ add(new DateLiteral("2024-01-02 12:34:56.123456",
Type.DATETIMEV2));
+ add(new DecimalLiteral(new BigDecimal("1.23")));
+ add(new FloatLiteral(1.23, Type.FLOAT));
+ add(new FloatLiteral(3.456, Type.DOUBLE));
+ add(new IntLiteral(1, Type.TINYINT));
+ add(new IntLiteral(1, Type.SMALLINT));
+ add(new IntLiteral(1, Type.INT));
+ add(new IntLiteral(1, Type.BIGINT));
+ add(new StringLiteral("abc"));
+ add(new StringLiteral("2023-01-02"));
+ add(new StringLiteral("2023-01-02 01:02:03.456789"));
+ }};
+
+ List<SlotRef> slotRefs = new ArrayList<SlotRef>() {{
+ add(new SlotRef(new TableName(), "c_int"));
+ add(new SlotRef(new TableName(), "c_long"));
+ add(new SlotRef(new TableName(), "c_bool"));
+ add(new SlotRef(new TableName(), "c_float"));
+ add(new SlotRef(new TableName(), "c_double"));
+ add(new SlotRef(new TableName(), "c_dec"));
+ add(new SlotRef(new TableName(), "c_date"));
+ add(new SlotRef(new TableName(), "c_ts"));
+ add(new SlotRef(new TableName(), "c_str"));
+ }};
+
+ // true indicates support for pushdown
+ Boolean[][] expects = new Boolean[][] {
+ { // int
+ false, false, false, false, false, false, true, true,
true, true, false, false, false
+ },
+ { // long
+ false, false, false, false, false, false, true, true,
true, true, false, false, false
+ },
+ { // boolean
+ true, false, false, false, false, false, false, false,
false, false, false, false, false
+ },
+ { // float
+ false, false, false, false, true, false, true, true,
true, true, false, false, false
+ },
+ { // double
+ false, false, false, true, true, true, true, true,
true, true, false, false, false
+ },
+ { // decimal
+ false, false, false, true, true, true, true, true,
true, true, false, false, false
+ },
+ { // date
+ false, true, false, false, false, false, true, true,
true, true, false, true, false
+ },
+ { // timestamp
+ false, true, true, false, false, false, false, false,
false, true, false, false, false
+ },
+ { // string
+ true, true, true, true, false, false, false, false,
false, false, true, true, true
+ }
+ };
+
+ ArrayListMultimap<Boolean, Expr> validPredicateMap =
ArrayListMultimap.create();
+
+ // binary predicate
+ for (int i = 0; i < slotRefs.size(); i++) {
+ final int loc = i;
+ List<Boolean> ret = literalList.stream().map(literal -> {
+ BinaryPredicate expr = new
BinaryPredicate(BinaryPredicate.Operator.EQ, slotRefs.get(loc), literal);
+ Expression expression =
IcebergUtils.convertToIcebergExpr(expr, schema);
+ validPredicateMap.put(expression != null, expr);
+ return expression != null;
+ }).collect(Collectors.toList());
+ Assert.assertArrayEquals(expects[i], ret.toArray());
+ }
+
+ // in predicate
+ for (int i = 0; i < slotRefs.size(); i++) {
+ final int loc = i;
+ List<Boolean> ret = literalList.stream().map(literal -> {
+ InPredicate expr = new InPredicate(slotRefs.get(loc),
Lists.newArrayList(literal), false);
+ Expression expression =
IcebergUtils.convertToIcebergExpr(expr, schema);
+ validPredicateMap.put(expression != null, expr);
+ return expression != null;
+ }).collect(Collectors.toList());
+ Assert.assertArrayEquals(expects[i], ret.toArray());
+ }
+
+ // not in predicate
+ for (int i = 0; i < slotRefs.size(); i++) {
+ final int loc = i;
+ List<Boolean> ret = literalList.stream().map(literal -> {
+ InPredicate expr = new InPredicate(slotRefs.get(loc),
Lists.newArrayList(literal), true);
+ Expression expression =
IcebergUtils.convertToIcebergExpr(expr, schema);
+ validPredicateMap.put(expression != null, expr);
+ return expression != null;
+ }).collect(Collectors.toList());
+ Assert.assertArrayEquals(expects[i], ret.toArray());
+ }
+
+ // bool literal
+ Expression trueExpr = IcebergUtils.convertToIcebergExpr(new
BoolLiteral(true), schema);
+ Expression falseExpr = IcebergUtils.convertToIcebergExpr(new
BoolLiteral(false), schema);
+ Assert.assertEquals(Expressions.alwaysTrue(), trueExpr);
+ Assert.assertEquals(Expressions.alwaysFalse(), falseExpr);
+ validPredicateMap.put(true, new BoolLiteral(true));
+ validPredicateMap.put(true, new BoolLiteral(false));
+
+ List<Expr> validExprs = validPredicateMap.get(true);
+ List<Expr> invalidExprs = validPredicateMap.get(false);
+ // OR predicate
+ // both valid
+ for (int i = 0; i < validExprs.size(); i++) {
+ for (int j = 0; j < validExprs.size(); j++) {
+ CompoundPredicate orPredicate = new
CompoundPredicate(Operator.OR,
+ validExprs.get(i), validExprs.get(j));
+ Expression expression =
IcebergUtils.convertToIcebergExpr(orPredicate, schema);
+ Assert.assertNotNull("pred: " + orPredicate.toSql(),
expression);
+ }
+ }
+ // both invalid
+ for (int i = 0; i < invalidExprs.size(); i++) {
+ for (int j = 0; j < invalidExprs.size(); j++) {
+ CompoundPredicate orPredicate = new
CompoundPredicate(Operator.OR,
+ invalidExprs.get(i), invalidExprs.get(j));
+ Expression expression =
IcebergUtils.convertToIcebergExpr(orPredicate, schema);
+ Assert.assertNull("pred: " + orPredicate.toSql(), expression);
+ }
+ }
+ // valid or invalid
+ for (int i = 0; i < validExprs.size(); i++) {
+ for (int j = 0; j < invalidExprs.size(); j++) {
+ CompoundPredicate orPredicate = new
CompoundPredicate(Operator.OR,
+ validExprs.get(i), invalidExprs.get(j));
+ Expression expression =
IcebergUtils.convertToIcebergExpr(orPredicate, schema);
+ Assert.assertNull("pred: " + orPredicate.toSql(), expression);
+ }
+ }
+
+ // AND predicate
+ // both valid
+ for (int i = 0; i < validExprs.size(); i++) {
+ for (int j = 0; j < validExprs.size(); j++) {
+ CompoundPredicate andPredicate = new
CompoundPredicate(Operator.AND,
+ validExprs.get(i), validExprs.get(j));
+ Expression expression =
IcebergUtils.convertToIcebergExpr(andPredicate, schema);
+ Assert.assertNotNull("pred: " + andPredicate.toSql(),
expression);
+ }
+ }
+ // both invalid
+ for (int i = 0; i < invalidExprs.size(); i++) {
+ for (int j = 0; j < invalidExprs.size(); j++) {
+ CompoundPredicate andPredicate = new
CompoundPredicate(Operator.AND,
+ invalidExprs.get(i), invalidExprs.get(j));
+ Expression expression =
IcebergUtils.convertToIcebergExpr(andPredicate, schema);
+ Assert.assertNull("pred: " + andPredicate.toSql(), expression);
+ }
+ }
+ // valid and invalid
+ for (int i = 0; i < validExprs.size(); i++) {
+ for (int j = 0; j < invalidExprs.size(); j++) {
+ CompoundPredicate andPredicate = new
CompoundPredicate(Operator.AND,
+ validExprs.get(i), invalidExprs.get(j));
+ Expression expression =
IcebergUtils.convertToIcebergExpr(andPredicate, schema);
+ Assert.assertNotNull("pred: " + andPredicate.toSql(),
expression);
+
Assert.assertEquals(IcebergUtils.convertToIcebergExpr(validExprs.get(i),
schema).toString(),
+ expression.toString());
+ }
+ }
+
+ // NOT predicate
+ // valid
+ for (int i = 0; i < validExprs.size(); i++) {
+ CompoundPredicate notPredicate = new
CompoundPredicate(Operator.NOT,
+ validExprs.get(i), null);
+ Expression expression =
IcebergUtils.convertToIcebergExpr(notPredicate, schema);
+ Assert.assertNotNull("pred: " + notPredicate.toSql(), expression);
+ }
+ // invalid
+ for (int i = 0; i < invalidExprs.size(); i++) {
+ CompoundPredicate notPredicate = new
CompoundPredicate(Operator.NOT,
+ invalidExprs.get(i), null);
+ Expression expression =
IcebergUtils.convertToIcebergExpr(notPredicate, schema);
+ Assert.assertNull("pred: " + notPredicate.toSql(), expression);
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/util/TestIcebergPredict.java
b/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/util/TestIcebergPredict.java
deleted file mode 100644
index fa589926992..00000000000
---
a/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/util/TestIcebergPredict.java
+++ /dev/null
@@ -1,135 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.external.iceberg.util;
-
-import org.apache.doris.analysis.BinaryPredicate;
-import org.apache.doris.analysis.BoolLiteral;
-import org.apache.doris.analysis.DateLiteral;
-import org.apache.doris.analysis.DecimalLiteral;
-import org.apache.doris.analysis.FloatLiteral;
-import org.apache.doris.analysis.IntLiteral;
-import org.apache.doris.analysis.LiteralExpr;
-import org.apache.doris.analysis.SlotRef;
-import org.apache.doris.analysis.StringLiteral;
-import org.apache.doris.analysis.TableName;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.common.AnalysisException;
-
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.expressions.Expression;
-import org.apache.iceberg.types.Types;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class TestIcebergPredict {
-
- public static Schema schema;
-
- @BeforeClass
- public static void before() throws AnalysisException {
- schema = new Schema(
- Types.NestedField.required(1, "c_int", Types.IntegerType.get()),
- Types.NestedField.required(2, "c_long", Types.LongType.get()),
- Types.NestedField.required(3, "c_bool", Types.BooleanType.get()),
- Types.NestedField.required(4, "c_float", Types.FloatType.get()),
- Types.NestedField.required(5, "c_double", Types.DoubleType.get()),
- Types.NestedField.required(6, "c_dec", Types.DecimalType.of(20,
10)),
- Types.NestedField.required(7, "c_date", Types.DateType.get()),
- Types.NestedField.required(8, "c_ts",
Types.TimestampType.withoutZone()),
- Types.NestedField.required(10, "c_str", Types.StringType.get())
- );
- }
-
- @Test
- public void testBinaryPredicate() throws AnalysisException {
- List<LiteralExpr> literalList = new ArrayList<LiteralExpr>() {{
- add(new BoolLiteral(true));
- add(new DateLiteral("2023-01-02", Type.DATEV2));
- add(new DateLiteral("2024-01-02 12:34:56.123456",
Type.DATETIMEV2));
- add(new DecimalLiteral(new BigDecimal("1.23")));
- add(new FloatLiteral(1.23, Type.FLOAT));
- add(new FloatLiteral(3.456, Type.DOUBLE));
- add(new IntLiteral(1, Type.TINYINT));
- add(new IntLiteral(1, Type.SMALLINT));
- add(new IntLiteral(1, Type.INT));
- add(new IntLiteral(1, Type.BIGINT));
- add(new StringLiteral("abc"));
- add(new StringLiteral("2023-01-02"));
- add(new StringLiteral("2023-01-02 01:02:03.456789"));
- }};
-
- List<SlotRef> slotRefs = new ArrayList<SlotRef>() {{
- add(new SlotRef(new TableName(), "c_int"));
- add(new SlotRef(new TableName(), "c_long"));
- add(new SlotRef(new TableName(), "c_bool"));
- add(new SlotRef(new TableName(), "c_float"));
- add(new SlotRef(new TableName(), "c_double"));
- add(new SlotRef(new TableName(), "c_dec"));
- add(new SlotRef(new TableName(), "c_date"));
- add(new SlotRef(new TableName(), "c_ts"));
- add(new SlotRef(new TableName(), "c_str"));
- }};
-
- // true indicates support for pushdown
- Boolean[][] expects = new Boolean[][] {
- { // int
- false, false, false, false, false, false, true, true, true,
true, false, false, false
- },
- { // long
- false, false, false, false, false, false, true, true, true,
true, false, false, false
- },
- { // boolean
- true, false, false, false, false, false, false, false, false,
false, false, false, false
- },
- { // float
- false, false, false, false, true, false, true, true, true,
true, false, false, false
- },
- { // double
- false, false, false, true, true, true, true, true, true, true,
false, false, false
- },
- { // decimal
- false, false, false, true, true, true, true, true, true, true,
false, false, false
- },
- { // date
- false, false, false, false, false, false, true, true, true,
true, false, true, false
- },
- { // timestamp
- false, true, true, false, false, false, false, false, false,
true, false, false, false
- },
- { // string
- true, true, true, true, false, false, false, false, false,
false, true, true, true
- }
- };
-
- for (int i = 0; i < slotRefs.size(); i++) {
- final int loc = i;
- List<Boolean> ret = literalList.stream().map(literal -> {
- BinaryPredicate expr = new
BinaryPredicate(BinaryPredicate.Operator.EQ, slotRefs.get(loc), literal);
- Expression expression =
IcebergUtils.convertToIcebergExpr(expr, schema);
- return expression != null;
- }).collect(Collectors.toList());
- Assert.assertArrayEquals(expects[i], ret.toArray());
- }
- }
-}
diff --git
a/regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out
b/regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out
new file mode 100644
index 00000000000..0569e0d01d8
--- /dev/null
+++
b/regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out
@@ -0,0 +1,611 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !q01 --
+11801003 35210325
+
+-- !q02 --
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1996-05-06
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+1997-05-18
+
+-- !q03 --
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1996-05-06 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+1997-05-18 MAIL
+
+-- !q04 --
+1992-01-02 REG AIR
+1992-01-02 SHIP
+1992-01-03 REG AIR
+1992-01-03 TRUCK
+1992-01-04 AIR
+1992-01-04 FOB
+1992-01-04 RAIL
+1992-01-04 REG AIR
+1992-01-04 TRUCK
+1992-01-05 AIR
+
+-- !q04 --
+2023-03-07T20:35:59.064
+2023-03-07T20:35:59.087
+2023-03-07T20:35:59.110
+2023-03-07T20:35:59.129
+2023-03-07T20:35:59.224
+
diff --git
a/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy
b/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy
new file mode 100644
index 00000000000..f6dc03639ec
--- /dev/null
+++
b/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_predicate_conversion",
"p2,external,hive,external_remote,external_remote_hive") {
+ String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
+ String extHiveHmsPort =
context.config.otherConfigs.get("extHiveHmsPort")
+
+ sql """drop catalog if exists test_iceberg_predicate_conversion;"""
+ sql """
+ create catalog if not exists test_iceberg_predicate_conversion
properties (
+ 'type'='hms',
+ 'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+ );
+ """
+
+ sql """switch test_iceberg_predicate_conversion;"""
+ sql """ use `iceberg_catalog`; """
+
+ def sqlstr = """select glue_int, glue_varchar from iceberg_glue_types
where glue_varchar > date '2023-03-07' """
+ def plan = """ref(name="glue_varchar") > "2023-03-07 00:00:00""""
+ order_qt_q01 """${sqlstr}"""
+ explain {
+ sql("""${sqlstr}""")
+ contains """${plan}"""
+ }
+
+ sqlstr = """select l_shipdate from lineitem where l_shipdate in
("1997-05-18", "1996-05-06"); """
+ plan = """ref(name="l_shipdate") in ("1996-05-06", "1997-05-18")"""
+ order_qt_q02 """${sqlstr}"""
+ explain {
+ sql("""${sqlstr}""")
+ contains """${plan}"""
+ }
+
+ sqlstr = """select l_shipdate, l_shipmode from lineitem where
l_shipdate in ("1997-05-18", "1996-05-06") and l_shipmode = "MAIL";"""
+ order_qt_q03 """${sqlstr}"""
+ explain {
+ sql("""${sqlstr}""")
+ contains """ref(name="l_shipdate") in ("1996-05-06",
"1997-05-18")"""
+ contains """ref(name="l_shipmode") == "MAIL""""
+ }
+
+ sqlstr = """select l_shipdate, l_shipmode from lineitem where
l_shipdate in ("1997-05-18", "1996-05-06") or NOT(l_shipmode = "MAIL") order by
l_shipdate, l_shipmode limit 10"""
+ plan = """(ref(name="l_shipdate") in ("1996-05-06", "1997-05-18") or
not(ref(name="l_shipmode") == "MAIL"))"""
+ order_qt_q04 """${sqlstr}"""
+ explain {
+ sql("""${sqlstr}""")
+ contains """${plan}"""
+ }
+
+ sqlstr = """select glue_timstamp from iceberg_glue_types where
glue_timstamp > '2023-03-07 20:35:59' order by glue_timstamp limit 5"""
+ plan = """ref(name="glue_timstamp") > 1678192559000000"""
+ order_qt_q04 """${sqlstr}"""
+ explain {
+ sql("""${sqlstr}""")
+ contains """${plan}"""
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
