This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit b744bd8eaebf351f7b250d85383229744f81f125 Author: Jibing-Li <[email protected]> AuthorDate: Tue Aug 22 13:58:25 2023 +0800 [fix])(nereids)Support select catalog.db.table.column from xxx for nereids planner. #23221 Nereids doesn't support select table.* from table, this pr is to fix this bug. Support three layer qualifier. (catalog.database.table) --- .../doris/nereids/rules/analysis/SlotBinder.java | 35 +++++++- .../plans/logical/LogicalCatalogRelation.java | 7 ++ .../hive/test_hive_star_qualifier.out | 97 ++++++++++++++++++++++ .../hive/test_hive_star_qualifier.groovy | 62 ++++++++++++++ 4 files changed, 200 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java index dfa2528793..ddbea3b3f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java @@ -144,6 +144,7 @@ class SlotBinder extends SubExprAnalyzer { return new BoundStar(slots); case 1: // select table.* case 2: // select db.table.* + case 3: // select catalog.db.table.* return bindQualifiedStar(qualifier, slots); default: throw new AnalysisException("Not supported qualifier: " @@ -167,6 +168,8 @@ class SlotBinder extends SubExprAnalyzer { return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)); case 2:// bound slot is `db`.`table`.`column` return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(1)); + case 3:// bound slot is `catalog`.`db`.`table`.`column` + return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(2)); default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifierStar, ".")); @@ -181,10 +184,29 @@ class SlotBinder extends SubExprAnalyzer { case 2:// bound slot is `db`.`table`.`column` return compareDbNameIgnoreClusterName(qualifierStar.get(0), boundSlotQualifier.get(0)) && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(1)); + case 3:// bound slot is `catalog`.`db`.`table`.`column` + return compareDbNameIgnoreClusterName(qualifierStar.get(0), boundSlotQualifier.get(1)) + && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(2)); default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifierStar, ".") + ".*"); } + case 3: // catalog.db.table.* + boundSlotQualifier = boundSlot.getQualifier(); + switch (boundSlotQualifier.size()) { + // bound slot is `column` and no qualified + case 0: + case 1: // bound slot is `table`.`column` + case 2: // bound slot is `db`.`table`.`column` + return false; + case 3:// bound slot is `catalog`.`db`.`table`.`column` + return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)) + && compareDbNameIgnoreClusterName(qualifierStar.get(1), boundSlotQualifier.get(1)) + && qualifierStar.get(2).equalsIgnoreCase(boundSlotQualifier.get(2)); + default: + throw new AnalysisException("Not supported qualifier: " + + StringUtils.join(qualifierStar, ".") + ".*"); + } default: throw new AnalysisException("Not supported name: " + StringUtils.join(qualifierStar, ".") + ".*"); @@ -221,13 +243,24 @@ class SlotBinder extends SubExprAnalyzer { String qualifierTableName = boundSlot.getQualifier().get(qualifierSize - 1); return qualifierTableName.equalsIgnoreCase(nameParts.get(0)) && boundSlot.getName().equalsIgnoreCase(nameParts.get(1)); - } else if (nameParts.size() == 3) { + } + if (nameParts.size() == 3) { String qualifierTableName = boundSlot.getQualifier().get(qualifierSize - 1); String qualifierDbName = boundSlot.getQualifier().get(qualifierSize - 2); return compareDbNameIgnoreClusterName(nameParts.get(0), qualifierDbName) && qualifierTableName.equalsIgnoreCase(nameParts.get(1)) && boundSlot.getName().equalsIgnoreCase(nameParts.get(2)); } + // catalog.db.table.column + if (nameParts.size() == 4) { + String qualifierTableName = boundSlot.getQualifier().get(qualifierSize - 1); + String qualifierDbName = boundSlot.getQualifier().get(qualifierSize - 2); + String qualifierCatalogName = boundSlot.getQualifier().get(qualifierSize - 3); + return qualifierCatalogName.equalsIgnoreCase(nameParts.get(0)) + && compareDbNameIgnoreClusterName(nameParts.get(1), qualifierDbName) + && qualifierTableName.equalsIgnoreCase(nameParts.get(2)) + && boundSlot.getName().equalsIgnoreCase(nameParts.get(3)); + } //TODO: handle name parts more than three. throw new AnalysisException("Not supported name: " + StringUtils.join(nameParts, ".")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 246c875f91..4dd5121ef8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Objects; @@ -97,6 +98,9 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements * Full qualified name parts, i.e., concat qualifier and name into a list. */ public List<String> qualified() { + if (qualifier.size() == 3) { + return qualifier; + } return Utils.qualifiedNameParts(qualifier, table.getName()); } @@ -104,6 +108,9 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements * Full qualified table name, concat qualifier and name with `.` as separator. */ public String qualifiedName() { + if (qualifier.size() == 3) { + return StringUtils.join(qualifier, "."); + } return Utils.qualifiedName(qualifier, table.getName()); } } diff --git a/regression-test/data/external_table_p2/hive/test_hive_star_qualifier.out b/regression-test/data/external_table_p2/hive/test_hive_star_qualifier.out new file mode 100644 index 0000000000..45457c1964 --- /dev/null +++ b/regression-test/data/external_table_p2/hive/test_hive_star_qualifier.out @@ -0,0 +1,97 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test1 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test2 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test3 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test4 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test5 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test6 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test7 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test8 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test9 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test10 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test11 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + +-- !test12 -- +1 1 +2 1 +3 2 +4 2 +5 \N +6 \N + diff --git a/regression-test/suites/external_table_p2/hive/test_hive_star_qualifier.groovy b/regression-test/suites/external_table_p2/hive/test_hive_star_qualifier.groovy new file mode 100644 index 0000000000..91f8a29331 --- /dev/null +++ b/regression-test/suites/external_table_p2/hive/test_hive_star_qualifier.groovy @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_star_qualifier", "p2,external,hive,external_remote,external_remote_hive") { + String catalog_name = "test_hive_star_qualifier" + + def test1 = """select * from ${catalog_name}.multi_catalog.one_partition order by id;""" + def test2 = """select * from multi_catalog.one_partition order by id;""" + def test3 = """select * from one_partition order by id;""" + def test4 = """select one_partition.* from ${catalog_name}.multi_catalog.one_partition order by id;""" + def test5 = """select one_partition.* from multi_catalog.one_partition order by id;""" + def test6 = """select one_partition.* from one_partition order by id;""" + def test7 = """select multi_catalog.one_partition.* from ${catalog_name}.multi_catalog.one_partition order by id;""" + def test8 = """select multi_catalog.one_partition.* from multi_catalog.one_partition order by id;""" + def test9 = """select multi_catalog.one_partition.* from one_partition order by id;""" + def test10 = """select ${catalog_name}.multi_catalog.one_partition.* from ${catalog_name}.multi_catalog.one_partition order by id;""" + def test11 = """select ${catalog_name}.multi_catalog.one_partition.* from multi_catalog.one_partition order by id;""" + def test12 = """select ${catalog_name}.multi_catalog.one_partition.* from one_partition order by id;""" + + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + sql """use ${catalog_name}.multi_catalog""" + qt_test1 test1 + qt_test2 test2 + qt_test3 test3 + qt_test4 test4 + qt_test5 test5 + qt_test6 test6 + qt_test7 test7 + qt_test8 test8 + qt_test9 test9 + qt_test10 test10 + qt_test11 test11 + qt_test12 test12 + sql """drop catalog if exists ${catalog_name};""" + } +} + --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
