This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4398b91576 [Fix](multi catalog)Change all partition names to lower
case (#19816)
4398b91576 is described below
commit 4398b9157605048d8dec44255ba3f5af760e56e7
Author: Jibing-Li <[email protected]>
AuthorDate: Tue May 23 09:31:31 2023 +0800
[Fix](multi catalog)Change all partition names to lower case (#19816)
Iceberg table partition name may contain upper case characters, for
example: City=xxx, Nation=xxx.
But in Doris, all column names are in lower case. Here we transfer the
partition name to lower case to keep consist with column name.
---
.../doris/planner/external/HiveScanNode.java | 2 +-
.../planner/external/iceberg/IcebergScanNode.java | 2 +-
.../iceberg_partition_upper_case_nereids.out | 109 +++++++++++++++++++++
.../iceberg_partition_upper_case_nereids.groovy | 84 ++++++++++++++++
4 files changed, 195 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanNode.java
index 560278cb48..956ad0d3e5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanNode.java
@@ -209,7 +209,7 @@ public class HiveScanNode extends FileQueryScanNode {
@Override
public List<String> getPathPartitionKeys() {
return hmsTable.getRemoteTable().getPartitionKeys()
-
.stream().map(FieldSchema::getName).collect(Collectors.toList());
+
.stream().map(FieldSchema::getName).map(String::toLowerCase).collect(Collectors.toList());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
index c149f24525..5038ecb476 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
@@ -272,7 +272,7 @@ public class IcebergScanNode extends FileQueryScanNode {
@Override
public List<String> getPathPartitionKeys() throws UserException {
- return
source.getIcebergTable().spec().fields().stream().map(PartitionField::name)
+ return
source.getIcebergTable().spec().fields().stream().map(PartitionField::name).map(String::toLowerCase)
.collect(Collectors.toList());
}
diff --git
a/regression-test/data/external_table_emr_p2/iceberg/iceberg_partition_upper_case_nereids.out
b/regression-test/data/external_table_emr_p2/iceberg/iceberg_partition_upper_case_nereids.out
new file mode 100644
index 0000000000..e286103bbd
--- /dev/null
+++
b/regression-test/data/external_table_emr_p2/iceberg/iceberg_partition_upper_case_nereids.out
@@ -0,0 +1,109 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !orcupper1 --
+1 k2_1 k3_1 Beijing
+2 k2_2 k3_2 Beijing
+3 k2_3 k3_3 Shanghai
+4 k2_4 k3_4 Shanghai
+
+-- !orcupper2 --
+1 Beijing
+2 Beijing
+3 Shanghai
+4 Shanghai
+
+-- !orcupper3 --
+1 k2_1
+2 k2_2
+3 k2_3
+4 k2_4
+
+-- !orcupper4 --
+Beijing
+Beijing
+Shanghai
+Shanghai
+
+-- !orcupper5 --
+2 k2_2 k3_2 Beijing
+
+-- !orclower1 --
+1 k2_1 k3_1 Beijing
+2 k2_2 k3_2 Beijing
+3 k2_3 k3_3 Shanghai
+4 k2_4 k3_4 Shanghai
+
+-- !orclower1 --
+1 Beijing
+2 Beijing
+3 Shanghai
+4 Shanghai
+
+-- !orclower1 --
+1 k2_1
+2 k2_2
+3 k2_3
+4 k2_4
+
+-- !orclower1 --
+Beijing
+Beijing
+Shanghai
+Shanghai
+
+-- !orclower1 --
+2 k2_2 k3_2 Beijing
+
+-- !parquetupper1 --
+1 k2_1 k3_1 Beijing
+2 k2_2 k3_2 Beijing
+3 k2_3 k3_3 Shanghai
+4 k2_4 k3_4 Shanghai
+
+-- !parquetupper2 --
+1 Beijing
+2 Beijing
+3 Shanghai
+4 Shanghai
+
+-- !parquetupper3 --
+1 k2_1
+2 k2_2
+3 k2_3
+4 k2_4
+
+-- !parquetupper4 --
+Beijing
+Beijing
+Shanghai
+Shanghai
+
+-- !parquetupper5 --
+2 k2_2 k3_2 Beijing
+
+-- !parquetlower1 --
+1 k2_1 k3_1 Beijing
+2 k2_2 k3_2 Beijing
+3 k2_3 k3_3 Shanghai
+4 k2_4 k3_4 Shanghai
+
+-- !parquetlower2 --
+1 Beijing
+2 Beijing
+3 Shanghai
+4 Shanghai
+
+-- !parquetlower3 --
+1 k2_1
+2 k2_2
+3 k2_3
+4 k2_4
+
+-- !parquetlower4 --
+Beijing
+Beijing
+Shanghai
+Shanghai
+
+-- !parquetlower5 --
+2 k2_2 k3_2 Beijing
+
diff --git
a/regression-test/suites/external_table_emr_p2/iceberg/iceberg_partition_upper_case_nereids.groovy
b/regression-test/suites/external_table_emr_p2/iceberg/iceberg_partition_upper_case_nereids.groovy
new file mode 100644
index 0000000000..3617c4620b
--- /dev/null
+++
b/regression-test/suites/external_table_emr_p2/iceberg/iceberg_partition_upper_case_nereids.groovy
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("iceberg_partition_upper_case_nereids", "p2") {
+ def orc_upper1 = """select * from iceberg_partition_upper_case_orc order
by k1;"""
+ def orc_upper2 = """select k1, city from iceberg_partition_upper_case_orc
order by k1;"""
+ def orc_upper3 = """select k1, k2 from iceberg_partition_upper_case_orc
order by k1;"""
+ def orc_upper4 = """select city from iceberg_partition_upper_case_orc
order by city;"""
+ def orc_upper5 = """select * from iceberg_partition_upper_case_orc where
k1>1 and city='Beijing' order by k1;"""
+
+ def orc_lower1 = """select * from iceberg_partition_lower_case_orc order
by k1;"""
+ def orc_lower2 = """select k1, city from iceberg_partition_lower_case_orc
order by k1;"""
+ def orc_lower3 = """select k1, k2 from iceberg_partition_lower_case_orc
order by k1;"""
+ def orc_lower4 = """select city from iceberg_partition_lower_case_orc
order by city;"""
+ def orc_lower5 = """select * from iceberg_partition_lower_case_orc where
k1>1 and city='Beijing' order by k1;"""
+
+ def parquet_upper1 = """select * from iceberg_partition_upper_case_parquet
order by k1;"""
+ def parquet_upper2 = """select k1, city from
iceberg_partition_upper_case_parquet order by k1;"""
+ def parquet_upper3 = """select k1, k2 from
iceberg_partition_upper_case_parquet order by k1;"""
+ def parquet_upper4 = """select city from
iceberg_partition_upper_case_parquet order by city;"""
+ def parquet_upper5 = """select * from iceberg_partition_upper_case_parquet
where k1>1 and city='Beijing' order by k1;"""
+
+ def parquet_lower1 = """select * from iceberg_partition_lower_case_parquet
order by k1;"""
+ def parquet_lower2 = """select k1, city from
iceberg_partition_lower_case_parquet order by k1;"""
+ def parquet_lower3 = """select k1, k2 from
iceberg_partition_lower_case_parquet order by k1;"""
+ def parquet_lower4 = """select city from
iceberg_partition_lower_case_parquet order by city;"""
+ def parquet_lower5 = """select * from iceberg_partition_lower_case_parquet
where k1>1 and city='Beijing' order by k1;"""
+
+ String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
+ String extHiveHmsPort =
context.config.otherConfigs.get("extHiveHmsPort")
+ String catalog_name = "iceberg_partition_nereids"
+ sql """drop catalog if exists ${catalog_name};"""
+ sql """
+ create catalog if not exists ${catalog_name} properties (
+ 'type'='hms',
+ 'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+ );
+ """
+ logger.info("catalog " + catalog_name + " created")
+ sql """switch ${catalog_name};"""
+ logger.info("switched to catalog " + catalog_name)
+ sql """use multi_catalog;"""
+ sql """set enable_nereids_planner=true;"""
+ sql """set enable_fallback_to_original_planner=false;"""
+ qt_orcupper1 orc_upper1
+ qt_orcupper2 orc_upper2
+ qt_orcupper3 orc_upper3
+ qt_orcupper4 orc_upper4
+ qt_orcupper5 orc_upper5
+ qt_orclower1 orc_lower1
+ qt_orclower1 orc_lower2
+ qt_orclower1 orc_lower3
+ qt_orclower1 orc_lower4
+ qt_orclower1 orc_lower5
+ qt_parquetupper1 parquet_upper1
+ qt_parquetupper2 parquet_upper2
+ qt_parquetupper3 parquet_upper3
+ qt_parquetupper4 parquet_upper4
+ qt_parquetupper5 parquet_upper5
+ qt_parquetlower1 parquet_lower1
+ qt_parquetlower2 parquet_lower2
+ qt_parquetlower3 parquet_lower3
+ qt_parquetlower4 parquet_lower4
+ qt_parquetlower5 parquet_lower5
+
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
