This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 00826185c1 [fix](tvf view)Support Table valued function view for
nereids (#23317)
00826185c1 is described below
commit 00826185c178b169bac858eb3df00ebb5f2423d8
Author: Jibing-Li <[email protected]>
AuthorDate: Fri Aug 25 21:23:16 2023 +0800
[fix](tvf view)Support Table valued function view for nereids (#23317)
Nereids doesn't support view based table value function, because tvf view
doesn't contain the proper qualifier (catalog, db and table name). This pr is
to support this function.
Also, fix nereids table value function explain output exprs incorrect bug.
---
.../doris/analysis/TableValuedFunctionRef.java | 4 +-
.../glue/translator/PhysicalPlanTranslator.java | 4 +-
.../trees/plans/logical/LogicalTVFRelation.java | 6 +-
.../doris/tablefunction/TableValuedFunctionIf.java | 1 +
.../external_table_p2/tvf/test_tvf_view_p2.out | 28 +++++++++
.../external_table_p2/tvf/test_tvf_view_p2.groovy | 69 ++++++++++++++++++++++
6 files changed, 107 insertions(+), 5 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
index 166b3297ea..b1e7c7c89e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
@@ -41,7 +41,7 @@ public class TableValuedFunctionRef extends TableRef {
private Map<String, String> params;
public TableValuedFunctionRef(String funcName, String alias, Map<String,
String> params) throws AnalysisException {
- super(new TableName(null, null, "_table_valued_function_" + funcName),
alias);
+ super(new TableName(null, null, TableValuedFunctionIf.TVF_TABLE_PREFIX
+ funcName), alias);
this.funcName = funcName;
this.params = params;
this.tableFunction = TableValuedFunctionIf.getTableFunction(funcName,
params);
@@ -49,7 +49,7 @@ public class TableValuedFunctionRef extends TableRef {
if (hasExplicitAlias()) {
return;
}
- aliases = new String[] { "_table_valued_function_" + funcName };
+ aliases = new String[] { TableValuedFunctionIf.TVF_TABLE_PREFIX +
funcName };
}
public TableValuedFunctionRef(TableValuedFunctionRef other) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 2551cb6441..ced08fe6d5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -674,8 +674,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
// TODO: it is weird update label in this way
// set label for explain
for (Slot slot : slots) {
- String tableColumnName = "_table_valued_function_" +
tvfRelation.getFunction().getName()
- + "." + slots.get(0).getName();
+ String tableColumnName = TableValuedFunctionIf.TVF_TABLE_PREFIX +
tvfRelation.getFunction().getName()
+ + "." + slot.getName();
context.findSlotRef(slot.getExprId()).setLabel(tableColumnName);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
index 6c5f554b2f..4527ffa316 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
@@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.algebra.TVFRelation;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.tablefunction.TableValuedFunctionIf;
import com.google.common.collect.ImmutableList;
@@ -39,16 +40,19 @@ import java.util.Optional;
public class LogicalTVFRelation extends LogicalRelation implements TVFRelation
{
private final TableValuedFunction function;
+ private final ImmutableList<String> qualifier;
public LogicalTVFRelation(RelationId id, TableValuedFunction function) {
super(id, PlanType.LOGICAL_TVF_RELATION);
this.function = function;
+ qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX +
function.getName());
}
public LogicalTVFRelation(RelationId id, TableValuedFunction function,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties) {
super(id, PlanType.LOGICAL_TVF_RELATION, groupExpression,
logicalProperties);
this.function = function;
+ qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX +
function.getName());
}
@Override
@@ -94,7 +98,7 @@ public class LogicalTVFRelation extends LogicalRelation
implements TVFRelation {
public List<Slot> computeOutput() {
return function.getTable().getBaseSchema()
.stream()
- .map(col -> SlotReference.fromColumn(col, ImmutableList.of()))
+ .map(col -> SlotReference.fromColumn(col, qualifier))
.collect(ImmutableList.toImmutableList());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
index fe3a6a0a94..d1b5deeb28 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
@@ -30,6 +30,7 @@ import java.util.Map;
public abstract class TableValuedFunctionIf {
private FunctionGenTable table = null;
+ public static final String TVF_TABLE_PREFIX = "_table_valued_function_";
public FunctionGenTable getTable() throws AnalysisException {
if (table == null) {
diff --git a/regression-test/data/external_table_p2/tvf/test_tvf_view_p2.out
b/regression-test/data/external_table_p2/tvf/test_tvf_view_p2.out
new file mode 100644
index 0000000000..02304fb57a
--- /dev/null
+++ b/regression-test/data/external_table_p2/tvf/test_tvf_view_p2.out
@@ -0,0 +1,28 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !1 --
+852910
+
+-- !2 --
+199147091 plum blush violet orange bisque Manufacturer#5 Brand#51
MEDIUM ANODIZED NICKEL 28 SM DRUM 1128.14 nding, final decoy
+199147092 brown tan chocolate moccasin peru Manufacturer#4
Brand#44 STANDARD BRUSHED COPPER 40 JUMBO PKG 1129.14 ully
even acc
+199147093 white sandy burlywood orange powder Manufacturer#2
Brand#23 MEDIUM PLATED COPPER 15 MED PACK 1130.14
furiously special
+199147094 cyan almond olive steel navajo Manufacturer#1 Brand#15
ECONOMY BRUSHED STEEL 12 WRAP PACK 1131.14 dolites.
+199147095 linen moccasin snow deep dim Manufacturer#2 Brand#22
STANDARD POLISHED TIN 37 LG CASE 1132.14 furious
+199147096 dim violet ivory cream drab Manufacturer#4 Brand#44
MEDIUM ANODIZED COPPER 20 JUMBO CAN 1133.14 ions. sometime
+199147097 steel khaki smoke beige sienna Manufacturer#2 Brand#21
STANDARD BRUSHED BRASS 36 WRAP CASE 1134.14 und the blithely iron
+199147098 cornsilk red brown cyan moccasin Manufacturer#4
Brand#43 MEDIUM ANODIZED TIN 12 SM BOX 1135.14 hely across the
+199147099 slate wheat sienna almond spring Manufacturer#2
Brand#25 LARGE BURNISHED TIN 1 SM CAN 1136.14 uriously ironic
packag
+199147100 orange gainsboro chocolate ivory grey Manufacturer#4
Brand#45 PROMO POLISHED BRASS 42 MED DRUM 1137.15 sual req
+
+-- !3 --
+199147091
+199147092
+199147093
+199147094
+199147095
+199147096
+199147097
+199147098
+199147099
+199147100
+
diff --git
a/regression-test/suites/external_table_p2/tvf/test_tvf_view_p2.groovy
b/regression-test/suites/external_table_p2/tvf/test_tvf_view_p2.groovy
new file mode 100644
index 0000000000..2323fcaff8
--- /dev/null
+++ b/regression-test/suites/external_table_p2/tvf/test_tvf_view_p2.groovy
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_tvf_view_p2",
"p2,external,tvf,external_remote,external_remote_tvf") {
+ String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String nameNodeHost = context.config.otherConfigs.get("extHiveHmsHost")
+ String hdfsPort = context.config.otherConfigs.get("extHdfsPort")
+
+ sql """drop database if exists test_tvf_view_p2"""
+ sql """create database test_tvf_view_p2"""
+ sql """use test_tvf_view_p2"""
+ sql """set enable_fallback_to_original_planner=false"""
+ sql """create view tvf_view as select * from hdfs (
+
"uri"="hdfs://${nameNodeHost}:${hdfsPort}:/usr/hive/warehouse/tpch_1000_parquet.db/part/000091_0",
+ "fs.defaultFS"="hdfs://${nameNodeHost}:${hdfsPort}",
+ "hadoop.username" = "hadoop",
+ "format"="parquet");"""
+
+ qt_1 """select count(*) from tvf_view"""
+ qt_2 """select * from tvf_view order by p_partkey limit 10"""
+ qt_3 """select p_partkey from tvf_view order by p_partkey limit 10"""
+ explain{
+ sql("select * from tvf_view")
+ contains("_table_valued_function_hdfs.p_partkey")
+ contains("_table_valued_function_hdfs.p_name")
+ contains("_table_valued_function_hdfs.p_mfgr")
+ contains("_table_valued_function_hdfs.p_brand")
+ contains("_table_valued_function_hdfs.p_type")
+ contains("_table_valued_function_hdfs.p_size")
+ contains("_table_valued_function_hdfs.p_container")
+ contains("_table_valued_function_hdfs.p_retailprice")
+ contains("_table_valued_function_hdfs.p_comment")
+ }
+ explain{
+ sql("select * from hdfs (\n" +
+ "
\"uri\"=\"hdfs://${nameNodeHost}:${hdfsPort}:/usr/hive/warehouse/tpch_1000_parquet.db/part/000091_0\",\n"
+
+ "
\"fs.defaultFS\"=\"hdfs://${nameNodeHost}:${hdfsPort}\",\n" +
+ " \"hadoop.username\" = \"hadoop\",\n" +
+ " \"format\"=\"parquet\")")
+ contains("_table_valued_function_hdfs.p_partkey")
+ contains("_table_valued_function_hdfs.p_name")
+ contains("_table_valued_function_hdfs.p_mfgr")
+ contains("_table_valued_function_hdfs.p_brand")
+ contains("_table_valued_function_hdfs.p_type")
+ contains("_table_valued_function_hdfs.p_size")
+ contains("_table_valued_function_hdfs.p_container")
+ contains("_table_valued_function_hdfs.p_retailprice")
+ contains("_table_valued_function_hdfs.p_comment")
+ }
+
+ sql """drop database if exists test_tvf_view_p2"""
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
