This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2abda6c342c1e6c81b56fc5064820caa3c3afbdc Author: Jibing-Li <[email protected]> AuthorDate: Fri Aug 25 21:23:16 2023 +0800 [fix](tvf view)Support Table valued function view for nereids (#23317) Nereids doesn't support view based table value function, because tvf view doesn't contain the proper qualifier (catalog, db and table name). This pr is to support this function. Also, fix nereids table value function explain output exprs incorrect bug. --- .../doris/analysis/TableValuedFunctionRef.java | 4 +- .../glue/translator/PhysicalPlanTranslator.java | 4 +- .../trees/plans/logical/LogicalTVFRelation.java | 6 +- .../doris/tablefunction/TableValuedFunctionIf.java | 1 + .../external_table_p2/tvf/test_tvf_view_p2.out | 28 +++++++++ .../external_table_p2/tvf/test_tvf_view_p2.groovy | 69 ++++++++++++++++++++++ 6 files changed, 107 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java index 166b3297ea..b1e7c7c89e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java @@ -41,7 +41,7 @@ public class TableValuedFunctionRef extends TableRef { private Map<String, String> params; public TableValuedFunctionRef(String funcName, String alias, Map<String, String> params) throws AnalysisException { - super(new TableName(null, null, "_table_valued_function_" + funcName), alias); + super(new TableName(null, null, TableValuedFunctionIf.TVF_TABLE_PREFIX + funcName), alias); this.funcName = funcName; this.params = params; this.tableFunction = TableValuedFunctionIf.getTableFunction(funcName, params); @@ -49,7 +49,7 @@ public class TableValuedFunctionRef extends TableRef { if (hasExplicitAlias()) { return; } - aliases = new String[] { "_table_valued_function_" + funcName }; + aliases = new String[] { TableValuedFunctionIf.TVF_TABLE_PREFIX + funcName }; } public TableValuedFunctionRef(TableValuedFunctionRef other) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 048b4c8f11..1f58cd8d69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -662,8 +662,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla // TODO: it is weird update label in this way // set label for explain for (Slot slot : slots) { - String tableColumnName = "_table_valued_function_" + tvfRelation.getFunction().getName() - + "." + slots.get(0).getName(); + String tableColumnName = TableValuedFunctionIf.TVF_TABLE_PREFIX + tvfRelation.getFunction().getName() + + "." + slot.getName(); context.findSlotRef(slot.getExprId()).setLabel(tableColumnName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java index 6c5f554b2f..4527ffa316 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.tablefunction.TableValuedFunctionIf; import com.google.common.collect.ImmutableList; @@ -39,16 +40,19 @@ import java.util.Optional; public class LogicalTVFRelation extends LogicalRelation implements TVFRelation { private final TableValuedFunction function; + private final ImmutableList<String> qualifier; public LogicalTVFRelation(RelationId id, TableValuedFunction function) { super(id, PlanType.LOGICAL_TVF_RELATION); this.function = function; + qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName()); } public LogicalTVFRelation(RelationId id, TableValuedFunction function, Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) { super(id, PlanType.LOGICAL_TVF_RELATION, groupExpression, logicalProperties); this.function = function; + qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName()); } @Override @@ -94,7 +98,7 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation { public List<Slot> computeOutput() { return function.getTable().getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, ImmutableList.of())) + .map(col -> SlotReference.fromColumn(col, qualifier)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index 2c67178494..56e0328414 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -30,6 +30,7 @@ import java.util.Map; public abstract class TableValuedFunctionIf { private FunctionGenTable table = null; + public static final String TVF_TABLE_PREFIX = "_table_valued_function_"; public FunctionGenTable getTable() throws AnalysisException { if (table == null) { diff --git a/regression-test/data/external_table_p2/tvf/test_tvf_view_p2.out b/regression-test/data/external_table_p2/tvf/test_tvf_view_p2.out new file mode 100644 index 0000000000..02304fb57a --- /dev/null +++ b/regression-test/data/external_table_p2/tvf/test_tvf_view_p2.out @@ -0,0 +1,28 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +852910 + +-- !2 -- +199147091 plum blush violet orange bisque Manufacturer#5 Brand#51 MEDIUM ANODIZED NICKEL 28 SM DRUM 1128.14 nding, final decoy +199147092 brown tan chocolate moccasin peru Manufacturer#4 Brand#44 STANDARD BRUSHED COPPER 40 JUMBO PKG 1129.14 ully even acc +199147093 white sandy burlywood orange powder Manufacturer#2 Brand#23 MEDIUM PLATED COPPER 15 MED PACK 1130.14 furiously special +199147094 cyan almond olive steel navajo Manufacturer#1 Brand#15 ECONOMY BRUSHED STEEL 12 WRAP PACK 1131.14 dolites. +199147095 linen moccasin snow deep dim Manufacturer#2 Brand#22 STANDARD POLISHED TIN 37 LG CASE 1132.14 furious +199147096 dim violet ivory cream drab Manufacturer#4 Brand#44 MEDIUM ANODIZED COPPER 20 JUMBO CAN 1133.14 ions. sometime +199147097 steel khaki smoke beige sienna Manufacturer#2 Brand#21 STANDARD BRUSHED BRASS 36 WRAP CASE 1134.14 und the blithely iron +199147098 cornsilk red brown cyan moccasin Manufacturer#4 Brand#43 MEDIUM ANODIZED TIN 12 SM BOX 1135.14 hely across the +199147099 slate wheat sienna almond spring Manufacturer#2 Brand#25 LARGE BURNISHED TIN 1 SM CAN 1136.14 uriously ironic packag +199147100 orange gainsboro chocolate ivory grey Manufacturer#4 Brand#45 PROMO POLISHED BRASS 42 MED DRUM 1137.15 sual req + +-- !3 -- +199147091 +199147092 +199147093 +199147094 +199147095 +199147096 +199147097 +199147098 +199147099 +199147100 + diff --git a/regression-test/suites/external_table_p2/tvf/test_tvf_view_p2.groovy b/regression-test/suites/external_table_p2/tvf/test_tvf_view_p2.groovy new file mode 100644 index 0000000000..2323fcaff8 --- /dev/null +++ b/regression-test/suites/external_table_p2/tvf/test_tvf_view_p2.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_tvf_view_p2", "p2,external,tvf,external_remote,external_remote_tvf") { + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String nameNodeHost = context.config.otherConfigs.get("extHiveHmsHost") + String hdfsPort = context.config.otherConfigs.get("extHdfsPort") + + sql """drop database if exists test_tvf_view_p2""" + sql """create database test_tvf_view_p2""" + sql """use test_tvf_view_p2""" + sql """set enable_fallback_to_original_planner=false""" + sql """create view tvf_view as select * from hdfs ( + "uri"="hdfs://${nameNodeHost}:${hdfsPort}:/usr/hive/warehouse/tpch_1000_parquet.db/part/000091_0", + "fs.defaultFS"="hdfs://${nameNodeHost}:${hdfsPort}", + "hadoop.username" = "hadoop", + "format"="parquet");""" + + qt_1 """select count(*) from tvf_view""" + qt_2 """select * from tvf_view order by p_partkey limit 10""" + qt_3 """select p_partkey from tvf_view order by p_partkey limit 10""" + explain{ + sql("select * from tvf_view") + contains("_table_valued_function_hdfs.p_partkey") + contains("_table_valued_function_hdfs.p_name") + contains("_table_valued_function_hdfs.p_mfgr") + contains("_table_valued_function_hdfs.p_brand") + contains("_table_valued_function_hdfs.p_type") + contains("_table_valued_function_hdfs.p_size") + contains("_table_valued_function_hdfs.p_container") + contains("_table_valued_function_hdfs.p_retailprice") + contains("_table_valued_function_hdfs.p_comment") + } + explain{ + sql("select * from hdfs (\n" + + " \"uri\"=\"hdfs://${nameNodeHost}:${hdfsPort}:/usr/hive/warehouse/tpch_1000_parquet.db/part/000091_0\",\n" + + " \"fs.defaultFS\"=\"hdfs://${nameNodeHost}:${hdfsPort}\",\n" + + " \"hadoop.username\" = \"hadoop\",\n" + + " \"format\"=\"parquet\")") + contains("_table_valued_function_hdfs.p_partkey") + contains("_table_valued_function_hdfs.p_name") + contains("_table_valued_function_hdfs.p_mfgr") + contains("_table_valued_function_hdfs.p_brand") + contains("_table_valued_function_hdfs.p_type") + contains("_table_valued_function_hdfs.p_size") + contains("_table_valued_function_hdfs.p_container") + contains("_table_valued_function_hdfs.p_retailprice") + contains("_table_valued_function_hdfs.p_comment") + } + + sql """drop database if exists test_tvf_view_p2""" + } +} + --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
