This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 3142b3225dc Revert "[enhancement](jdbc catalog) Add lowercase column
name mapping to Jdbc data source & optimize database and table mapping #27124
(#27130)" (#27230)
3142b3225dc is described below
commit 3142b3225dcf0725687b3fecd684a8fa31035f8e
Author: Kang <[email protected]>
AuthorDate: Sat Nov 18 09:49:36 2023 -0600
Revert "[enhancement](jdbc catalog) Add lowercase column name mapping to
Jdbc data source & optimize database and table mapping #27124 (#27130)" (#27230)
This reverts commit 087fccd9af38f4003b7cc78bf4cc04af87c561ad.
---
.../docker-compose/oracle/init/03-create-table.sql | 7 --
.../docker-compose/oracle/init/04-insert.sql | 2 -
docs/en/docs/lakehouse/multi-catalog/jdbc.md | 19 ++--
docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md | 35 +++----
.../java/org/apache/doris/catalog/JdbcTable.java | 41 +++-----
.../doris/catalog/external/JdbcExternalTable.java | 2 -
.../doris/datasource/jdbc/client/JdbcClient.java | 108 ++++++++-------------
.../datasource/jdbc/client/JdbcMySQLClient.java | 23 ++---
.../datasource/jdbc/client/JdbcOracleClient.java | 25 ++---
.../doris/planner/external/jdbc/JdbcScanNode.java | 16 ++-
.../doris/planner/external/odbc/OdbcScanNode.java | 2 +-
.../jdbc/test_oracle_jdbc_catalog.out | 9 --
.../jdbc/test_oracle_jdbc_catalog.groovy | 3 -
13 files changed, 96 insertions(+), 196 deletions(-)
diff --git a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql
b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql
index 24bf6a7a856..046f18c0dc6 100644
--- a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql
+++ b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql
@@ -132,13 +132,6 @@ age number(2),
score number(3,1)
);
-CREATE TABLE "DORIS_TEST"."student3"
-(
-"id" NUMBER(5,0),
-"NAME" VARCHAR2(20),
-"AGE" NUMBER(2,0),
-"SCORE" NUMBER(3,1)
-);
create table doris_test.test_all_types (
id int,
diff --git a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql
b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql
index 663851cfa4e..888cdf5bf7e 100644
--- a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql
+++ b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql
@@ -86,8 +86,6 @@ insert into doris_test."student2" values (2, 'bob', 21, 90.5);
insert into doris_test."student2" values (3, 'jerry', 23, 88.0);
insert into doris_test."student2" values (4, 'andy', 21, 93);
-insert into doris_test."student3" values(1, 'doris', 3, 1.0);
-
insert into doris_test.test_all_types values
(1, 111, 123, 7456123.89, 573, 34, 673.43, 34.1264, 56.2, 23.231,
99, 9999, 999999999, 999999999999999999, 999, 99999, 9999999999,
9999999999999999999,
diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md
b/docs/en/docs/lakehouse/multi-catalog/jdbc.md
index 5fa29b13065..13151bc2f8d 100644
--- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md
+++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md
@@ -52,7 +52,7 @@ PROPERTIES ("key"="value", ...)
| `driver_url ` | Yes | | JDBC Driver
Jar
|
| `driver_class ` | Yes | | JDBC Driver
Class
|
| `only_specified_database` | No | "false" | Whether only
the database specified to be synchronized.
|
-| `lower_case_table_names` | No | "false" | Whether to
synchronize the database name, table name and column name of jdbc external data
source in lowercase. |
+| `lower_case_table_names` | No | "false" | Whether to
synchronize jdbc external data source table names in lower case.
|
| `include_database_list` | No | "" | When
only_specified_database=true，only synchronize the specified databases. split
with ','. db name is case sensitive. |
| `exclude_database_list` | No | "" | When
only_specified_database=true，do not synchronize the specified databases. split
with ','. db name is case sensitive. |
@@ -68,7 +68,7 @@ PROPERTIES ("key"="value", ...)
### Lowercase table name synchronization
-When `lower_case_table_names` is set to `true`, Doris is able to query
non-lowercase databases and tables and columns by maintaining a mapping of
lowercase names to actual names on the remote system
+When `lower_case_table_names` is set to `true`, Doris is able to query
non-lowercase databases and tables by maintaining a mapping of lowercase names
to actual names on the remote system
**Notice:**
@@ -78,9 +78,9 @@ When `lower_case_table_names` is set to `true`, Doris is able
to query non-lower
For other databases, you still need to specify the real library name and
table name when querying.
-2. In Doris 2.0.3 and later versions, it is valid for all databases. When
querying, all database names and table names and columns will be converted into
real names and then queried. If you upgrade from an old version to 2.0. 3,
`Refresh <catalog_name>` is required to take effect.
+2. In Doris 2.0.3 and later versions, it is valid for all databases. When
querying, all library names and table names will be converted into real names
and then queried. If you upgrade from an old version to 2.0. 3, `Refresh
<catalog_name>` is required to take effect.
- However, if the database or table or column names differ only in case, such
as `Doris` and `doris`, Doris cannot query them due to ambiguity.
+ However, if the database or table names differ only in case, such as
`Doris` and `doris`, Doris cannot query them due to ambiguity.
3. When the FE parameter's `lower_case_table_names` is set to `1` or `2`, the
JDBC Catalog's `lower_case_table_names` parameter must be set to `true`. If the
FE parameter's `lower_case_table_names` is set to `0`, the JDBC Catalog
parameter can be `true` or `false` and defaults to `false`. This ensures
consistency and predictability in how Doris handles internal and external table
configurations.
@@ -113,8 +113,8 @@ In some cases, the keywords in the database might be used
as the field names. Fo
### Predicate Pushdown
1. When executing a query like `where dt = '2022-01-01'`, Doris can push down
these filtering conditions to the external data source, thereby directly
excluding data that does not meet the conditions at the data source level,
reducing the number of unqualified Necessary data acquisition and transfer.
This greatly improves query performance while also reducing the load on
external data sources.
-
-2. When `enable_func_pushdown` is set to true, the function conditions after
where will also be pushed down to the external data source. Currently, only
MySQL and ClickHouse are supported. If you encounter a function that is not
supported by MySQL or ClickHouse, you can set this parameter to false. ,
currently Doris will automatically identify some functions not supported by
MySQL and functions supported by CLickHouse for push-down condition filtering,
which can be viewed through explain sql.
+
+2. When `enable_func_pushdown` is set to true, the function condition after
where will also be pushed down to the external data source. Currently, only
MySQL is supported. If you encounter a function that MySQL does not support,
you can set this parameter to false, at present, Doris will automatically
identify some functions not supported by MySQL to filter the push-down
conditions, which can be checked by explain sql.
Functions that are currently not pushed down include:
@@ -123,13 +123,6 @@ Functions that are currently not pushed down include:
| DATE_TRUNC |
| MONEY_FORMAT |
-Functions that are currently pushed down include:
-
-| ClickHouse |
-|:--------------:|
-| FROM_UNIXTIME |
-| UNIX_TIMESTAMP |
-
### Line Limit
If there is a limit keyword in the query, Doris will translate it into
semantics suitable for different data sources.
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
index acc057caa54..545f5ada6e2 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
@@ -44,16 +44,16 @@ PROPERTIES ("key"="value", ...)
## 参数说明
-| 参数 | 必须 | 默认值 | 说明
|
-|---------------------------|-----|---------|-----------------------------------------------------------------------|
-| `user` | 是 | | 对应数据库的用户名
|
-| `password` | 是 | | 对应数据库的密码
|
-| `jdbc_url` | 是 | | JDBC 连接串
|
-| `driver_url` | 是 | | JDBC Driver Jar 包名称
|
-| `driver_class` | 是 | | JDBC Driver Class 名称
|
-| `lower_case_table_names` | 否 | "false" | 是否以小写的形式同步jdbc外部数据源的库名和表名以及列名
|
-| `only_specified_database` | 否 | "false" | 指定是否只同步指定的 database
|
-| `include_database_list` | 否 | "" |
当only_specified_database=true时，指定同步多个database，以','分隔。db名称是大小写敏感的。 |
+| 参数 | 必须 | 默认值 | 说明
|
+|---------------------------|-----|---------|---------------------------------------------------------------------------------------------|
+| `user` | 是 | | 对应数据库的用户名
|
+| `password` | 是 | | 对应数据库的密码
|
+| `jdbc_url` | 是 | | JDBC 连接串
|
+| `driver_url` | 是 | | JDBC Driver Jar 包名称
|
+| `driver_class` | 是 | | JDBC Driver Class 名称
|
+| `lower_case_table_names` | 否 | "false" | 是否以小写的形式同步jdbc外部数据源的库名和表名
|
+| `only_specified_database` | 否 | "false" | 指定是否只同步指定的 database
|
+| `include_database_list` | 否 | "" |
当only_specified_database=true时，指定同步多个database，以','分隔。db名称是大小写敏感的。 |
| `exclude_database_list` | 否 | "" |
当only_specified_database=true时，指定不需要同步的多个database，以','分割。db名称是大小写敏感的。 |
### 驱动包路径
@@ -68,7 +68,7 @@ PROPERTIES ("key"="value", ...)
### 小写表名同步
-当 `lower_case_table_names` 设置为 `true` 时，Doris
通过维护小写名称到远程系统中实际名称的映射，能够查询非小写的数据库和表以及列
+当 `lower_case_table_names` 设置为 `true` 时，Doris
通过维护小写名称到远程系统中实际名称的映射，能够查询非小写的数据库和表
**注意：**
@@ -78,9 +78,9 @@ PROPERTIES ("key"="value", ...)
对于其他数据库，仍需要在查询时指定真实的库名和表名。
-2. 在 Doris 2.0.3 及之后的版本，对所有的数据库都有效，在查询时，会将所有的库名和表名以及列名转换为真实的名称，再去查询，如果是从老版本升级到
2.0.3 ，需要 `Refresh <catalog_name>` 才能生效。
+2. 在 Doris 2.0.3 及之后的版本，对所有的数据库都有效，在查询时，会将所有的库名和表名转换为真实的名称，再去查询，如果是从老版本升级到
2.0.3 ，需要 `Refresh <catalog_name>` 才能生效。
- 但是，如果库名、表名或列名只有大小写不同，例如 `Doris` 和 `doris`，则 Doris 由于歧义而无法查询它们。
+ 但是，如果数据库或者表名只有大小写不同，例如 `Doris` 和 `doris`，则 Doris 由于歧义而无法查询它们。
3. 当 FE 参数的 `lower_case_table_names` 设置为 `1` 或 `2` 时，JDBC Catalog 的
`lower_case_table_names` 参数必须设置为 `true`。如果 FE 参数的 `lower_case_table_names` 设置为
`0`，则 JDBC Catalog 的参数可以为 `true` 或 `false`，默认为 `false`。这确保了 Doris
在处理内部和外部表配置时的一致性和可预测性。
@@ -114,7 +114,7 @@ select * from mysql_catalog.mysql_database.mysql_table
where k1 > 1000 and k3 ='
1. 当执行类似于 `where dt = '2022-01-01'` 这样的查询时，Doris
能够将这些过滤条件下推到外部数据源，从而直接在数据源层面排除不符合条件的数据，减少了不必要的数据获取和传输。这大大提高了查询性能，同时也降低了对外部数据源的负载。
-2. 当 `enable_func_pushdown` 设置为true，会将 where 之后的函数条件也下推到外部数据源，目前仅支持 MySQL 以及
ClickHouse，如遇到 MySQL 或 ClickHouse 不支持的函数，可以将此参数设置为 false，目前 Doris 会自动识别部分 MySQL
不支持的函数以及 CLickHouse 支持的函数进行下推条件过滤，可通过 explain sql 查看。
+2. 当 `enable_func_pushdown` 设置为true，会将 where 之后的函数条件也下推到外部数据源，目前仅支持 MySQL，如遇到
MySQL 不支持的函数，可以将此参数设置为 false，目前 Doris 会自动识别部分 MySQL 不支持的函数进行下推条件过滤，可通过 explain
sql 查看。
目前不会下推的函数有：
@@ -123,13 +123,6 @@ select * from mysql_catalog.mysql_database.mysql_table
where k1 > 1000 and k3 ='
| DATE_TRUNC |
| MONEY_FORMAT |
-目前会下推的函数有：
-
-| ClickHouse |
-|:--------------:|
-| FROM_UNIXTIME |
-| UNIX_TIMESTAMP |
-
### 行数限制
如果在查询中带有 limit 关键字，Doris 会将其转译成适合不同数据源的语义。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
index fe397929acc..795f00e7bfe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
@@ -27,8 +27,6 @@ import org.apache.doris.thrift.TOdbcTableType;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import lombok.Setter;
@@ -49,12 +47,9 @@ import java.util.stream.Collectors;
public class JdbcTable extends Table {
private static final Logger LOG = LogManager.getLogger(JdbcTable.class);
- private static final ObjectMapper objectMapper = new ObjectMapper();
-
private static final String TABLE = "table";
private static final String REAL_DATABASE = "real_database";
private static final String REAL_TABLE = "real_table";
- private static final String REAL_COLUMNS = "real_columns";
private static final String RESOURCE = "resource";
private static final String TABLE_TYPE = "table_type";
private static final String URL = "jdbc_url";
@@ -70,7 +65,6 @@ public class JdbcTable extends Table {
// real name only for jdbc catalog
private String realDatabaseName;
private String realTableName;
- private Map<String, String> realColumnNames;
private String jdbcTypeName;
@@ -116,7 +110,7 @@ public class JdbcTable extends Table {
sb.append(getProperRealFullTableName(TABLE_TYPE_MAP.get(getTableTypeName())));
sb.append("(");
List<String> transformedInsertCols = insertCols.stream()
- .map(col ->
getProperRealColumnName(TABLE_TYPE_MAP.get(getTableTypeName()), col))
+ .map(col ->
databaseProperName(TABLE_TYPE_MAP.get(getTableTypeName()), col))
.collect(Collectors.toList());
sb.append(String.join(",", transformedInsertCols));
sb.append(")");
@@ -206,7 +200,6 @@ public class JdbcTable extends Table {
serializeMap.put(CHECK_SUM, checkSum);
serializeMap.put(REAL_DATABASE, realDatabaseName);
serializeMap.put(REAL_TABLE, realTableName);
- serializeMap.put(REAL_COLUMNS,
objectMapper.writeValueAsString(realColumnNames));
int size = (int) serializeMap.values().stream().filter(v -> {
return v != null;
@@ -243,9 +236,6 @@ public class JdbcTable extends Table {
checkSum = serializeMap.get(CHECK_SUM);
realDatabaseName = serializeMap.get(REAL_DATABASE);
realTableName = serializeMap.get(REAL_TABLE);
- String realColumnNamesJson = serializeMap.get(REAL_COLUMNS);
- realColumnNames = objectMapper.readValue(realColumnNamesJson, new
TypeReference<Map<String, String>>() {
- });
}
public String getResourceName() {
@@ -273,14 +263,6 @@ public class JdbcTable extends Table {
}
}
- public String getProperRealColumnName(TOdbcTableType tableType, String
columnName) {
- if (realColumnNames == null || realColumnNames.isEmpty() ||
!realColumnNames.containsKey(columnName)) {
- return databaseProperName(tableType, columnName);
- } else {
- return properNameWithRealName(tableType,
realColumnNames.get(columnName));
- }
- }
-
public String getTableTypeName() {
return jdbcTypeName;
}
@@ -376,13 +358,14 @@ public class JdbcTable extends Table {
* @param wrapEnd The character(s) to be added at the end of each name
component.
* @param toUpperCase If true, convert the name to upper case.
* @param toLowerCase If true, convert the name to lower case.
- * <p>
- * Note: If both toUpperCase and toLowerCase are true, the name
will ultimately be converted to lower case.
- * <p>
- * The name is expected to be in the format of
'schemaName.tableName'. If there is no '.',
- * the function will treat the entire string as one name component.
- * If there is a '.', the function will treat the string before
the first '.' as the schema name
- * and the string after the '.' as the table name.
+ * <p>
+ * Note: If both toUpperCase and toLowerCase are true, the name will
ultimately be converted to lower case.
+ * <p>
+ * The name is expected to be in the format of 'schemaName.tableName'. If
there is no '.',
+ * the function will treat the entire string as one name component.
+ * If there is a '.', the function will treat the string before the first
'.' as the schema name
+ * and the string after the '.' as the table name.
+ *
* @return The formatted name.
*/
public static String formatName(String name, String wrapStart, String
wrapEnd, boolean toUpperCase,
@@ -403,18 +386,18 @@ public class JdbcTable extends Table {
/**
* Formats a database name according to the database type.
- * <p>
+ *
* Rules:
* - MYSQL, OCEANBASE: Wrap with backticks (`), case unchanged. Example:
mySchema.myTable -> `mySchema.myTable`
* - SQLSERVER: Wrap with square brackets ([]), case unchanged. Example:
mySchema.myTable -> [mySchema].[myTable]
* - POSTGRESQL, CLICKHOUSE, TRINO, OCEANBASE_ORACLE, SAP_HANA: Wrap with
double quotes ("), case unchanged.
- * Example: mySchema.myTable -> "mySchema"."myTable"
+ * Example: mySchema.myTable -> "mySchema"."myTable"
* - ORACLE: Wrap with double quotes ("), convert to upper case. Example:
mySchema.myTable -> "MYSCHEMA"."MYTABLE"
* For other types, the name is returned as is.
*
* @param tableType The database type.
* @param name The name to be formatted, expected in
'schemaName.tableName' format. If no '.', treats entire string
- * as one name component. If '.', treats string before first '.'
as schema name and after as table name.
+ * as one name component. If '.', treats string before first '.' as
schema name and after as table name.
* @return The formatted name.
*/
public static String databaseProperName(TOdbcTableType tableType, String
name) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java
index 9c5159a2bd8..a02c59080fc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java
@@ -91,8 +91,6 @@ public class JdbcExternalTable extends ExternalTable {
jdbcTable.setRealDatabaseName(((JdbcExternalCatalog)
catalog).getJdbcClient().getRealDatabaseName(this.dbName));
jdbcTable.setRealTableName(
((JdbcExternalCatalog)
catalog).getJdbcClient().getRealTableName(this.dbName, this.name));
- jdbcTable.setRealColumnNames(((JdbcExternalCatalog)
catalog).getJdbcClient().getRealColumnNames(this.dbName,
- this.name));
jdbcTable.setJdbcTypeName(jdbcCatalog.getDatabaseTypeName());
jdbcTable.setJdbcUrl(jdbcCatalog.getJdbcUrl());
jdbcTable.setJdbcUser(jdbcCatalog.getJdbcUser());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
index b4e2604bfc9..97052caccd6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
@@ -67,15 +67,10 @@ public abstract class JdbcClient {
// only used when isLowerCaseTableNames = true.
protected final ConcurrentHashMap<String, String> lowerDBToRealDB = new
ConcurrentHashMap<>();
// only used when isLowerCaseTableNames = true.
- protected final ConcurrentHashMap<String, ConcurrentHashMap<String,
String>> lowerTableToRealTable
- = new ConcurrentHashMap<>();
- // only used when isLowerCaseTableNames = true.
- protected final ConcurrentHashMap<String, ConcurrentHashMap<String,
ConcurrentHashMap<String, String>>>
- lowerColumnToRealColumn = new ConcurrentHashMap<>();
+ protected final ConcurrentHashMap<String, String> lowerTableToRealTable =
new ConcurrentHashMap<>();
private final AtomicBoolean dbNamesLoaded = new AtomicBoolean(false);
private final AtomicBoolean tableNamesLoaded = new AtomicBoolean(false);
- private final AtomicBoolean columnNamesLoaded = new AtomicBoolean(false);
public static JdbcClient createJdbcClient(JdbcClientConfig
jdbcClientConfig) {
String dbType = parseDbType(jdbcClientConfig.getJdbcUrl());
@@ -183,7 +178,7 @@ public abstract class JdbcClient {
if (closeable != null) {
try {
closeable.close();
- } catch (Exception e) {
+ } catch (Exception e) {
throw new JdbcClientException("Can not close : ", e);
}
}
@@ -191,10 +186,8 @@ public abstract class JdbcClient {
}
// This part used to process meta-information of database, table and
column.
-
/**
* get all database name through JDBC
- *
* @return list of database names
*/
public List<String> getDatabaseNameList() {
@@ -215,8 +208,6 @@ public abstract class JdbcClient {
if (isLowerCaseTableNames) {
lowerDBToRealDB.put(databaseName.toLowerCase(),
databaseName);
databaseName = databaseName.toLowerCase();
- } else {
- lowerDBToRealDB.put(databaseName, databaseName);
}
tempDatabaseNames.add(databaseName);
}
@@ -246,20 +237,20 @@ public abstract class JdbcClient {
* get all tables of one database
*/
public List<String> getTablesNameList(String dbName) {
+ String currentDbName = dbName;
List<String> tablesName = Lists.newArrayList();
String[] tableTypes = getTableTypes();
- String finalDbName = getRealDatabaseName(dbName);
+ if (isLowerCaseTableNames) {
+ currentDbName = getRealDatabaseName(dbName);
+ }
+ String finalDbName = currentDbName;
processTable(finalDbName, null, tableTypes, (rs) -> {
try {
while (rs.next()) {
String tableName = rs.getString("TABLE_NAME");
if (isLowerCaseTableNames) {
- lowerTableToRealTable.putIfAbsent(finalDbName, new
ConcurrentHashMap<>());
-
lowerTableToRealTable.get(finalDbName).put(tableName.toLowerCase(), tableName);
+ lowerTableToRealTable.put(tableName.toLowerCase(),
tableName);
tableName = tableName.toLowerCase();
- } else {
- lowerTableToRealTable.putIfAbsent(finalDbName, new
ConcurrentHashMap<>());
- lowerTableToRealTable.get(finalDbName).put(tableName,
tableName);
}
tablesName.add(tableName);
}
@@ -271,10 +262,16 @@ public abstract class JdbcClient {
}
public boolean isTableExist(String dbName, String tableName) {
+ String currentDbName = dbName;
+ String currentTableName = tableName;
final boolean[] isExist = {false};
+ if (isLowerCaseTableNames) {
+ currentDbName = getRealDatabaseName(dbName);
+ currentTableName = getRealTableName(dbName, tableName);
+ }
String[] tableTypes = getTableTypes();
- String finalDbName = getRealDatabaseName(dbName);
- String finalTableName = getRealTableName(dbName, tableName);
+ String finalTableName = currentTableName;
+ String finalDbName = currentDbName;
processTable(finalDbName, finalTableName, tableTypes, (rs) -> {
try {
if (rs.next()) {
@@ -295,25 +292,23 @@ public abstract class JdbcClient {
Connection conn = getConnection();
ResultSet rs = null;
List<JdbcFieldSchema> tableSchema = Lists.newArrayList();
- String finalDbName = getRealDatabaseName(dbName);
- String finalTableName = getRealTableName(dbName, tableName);
+ // if isLowerCaseTableNames == true, tableName is lower case
+ // but databaseMetaData.getColumns() is case sensitive
+ String currentDbName = dbName;
+ String currentTableName = tableName;
+ if (isLowerCaseTableNames) {
+ currentDbName = getRealDatabaseName(dbName);
+ currentTableName = getRealTableName(dbName, tableName);
+ }
+ String finalDbName = currentDbName;
+ String finalTableName = currentTableName;
try {
DatabaseMetaData databaseMetaData = conn.getMetaData();
String catalogName = getCatalogName(conn);
rs = getColumns(databaseMetaData, catalogName, finalDbName,
finalTableName);
while (rs.next()) {
- lowerColumnToRealColumn.putIfAbsent(finalDbName, new
ConcurrentHashMap<>());
-
lowerColumnToRealColumn.get(finalDbName).putIfAbsent(finalTableName, new
ConcurrentHashMap<>());
JdbcFieldSchema field = new JdbcFieldSchema();
- String columnName = rs.getString("COLUMN_NAME");
- if (isLowerCaseTableNames) {
-
lowerColumnToRealColumn.get(finalDbName).get(finalTableName)
- .put(columnName.toLowerCase(), columnName);
- columnName = columnName.toLowerCase();
- } else {
-
lowerColumnToRealColumn.get(finalDbName).get(finalTableName).put(columnName,
columnName);
- }
- field.setColumnName(columnName);
+ field.setColumnName(rs.getString("COLUMN_NAME"));
field.setDataType(rs.getInt("DATA_TYPE"));
field.setDataTypeName(rs.getString("TYPE_NAME"));
/*
@@ -357,9 +352,11 @@ public abstract class JdbcClient {
}
public String getRealDatabaseName(String dbname) {
- if (lowerDBToRealDB == null
- || lowerDBToRealDB.isEmpty()
- || !lowerDBToRealDB.containsKey(dbname)) {
+ if (!isLowerCaseTableNames) {
+ return dbname;
+ }
+
+ if (lowerDBToRealDB.isEmpty() || !lowerDBToRealDB.containsKey(dbname))
{
loadDatabaseNamesIfNeeded();
}
@@ -367,34 +364,15 @@ public abstract class JdbcClient {
}
public String getRealTableName(String dbName, String tableName) {
- String realDbName = getRealDatabaseName(dbName);
- if (lowerTableToRealTable == null
- || lowerTableToRealTable.isEmpty()
- || !lowerTableToRealTable.containsKey(realDbName)
- || lowerTableToRealTable.get(realDbName) == null
- || lowerTableToRealTable.get(realDbName).isEmpty()
- ||
!lowerTableToRealTable.get(realDbName).containsKey(tableName)
- || lowerTableToRealTable.get(realDbName).get(tableName) ==
null) {
- loadTableNamesIfNeeded(dbName);
+ if (!isLowerCaseTableNames) {
+ return tableName;
}
- return lowerTableToRealTable.get(realDbName).get(tableName);
- }
-
- public Map<String, String> getRealColumnNames(String dbName, String
tableName) {
- String realDbName = getRealDatabaseName(dbName);
- String realTableName = getRealTableName(dbName, tableName);
- if (lowerColumnToRealColumn == null
- || lowerColumnToRealColumn.isEmpty()
- || !lowerColumnToRealColumn.containsKey(realDbName)
- || lowerColumnToRealColumn.get(realDbName) == null
- || lowerColumnToRealColumn.get(realDbName).isEmpty()
- ||
!lowerColumnToRealColumn.get(realDbName).containsKey(realTableName)
- || lowerColumnToRealColumn.get(realDbName).get(realTableName)
== null
- ||
lowerColumnToRealColumn.get(realDbName).get(realTableName).isEmpty()) {
- loadColumnNamesIfNeeded(dbName, tableName);
+ if (lowerTableToRealTable.isEmpty() ||
!lowerTableToRealTable.containsKey(tableName)) {
+ loadTableNamesIfNeeded(dbName);
}
- return lowerColumnToRealColumn.get(realDbName).get(realTableName);
+
+ return lowerTableToRealTable.get(tableName);
}
private void loadDatabaseNamesIfNeeded() {
@@ -409,12 +387,6 @@ public abstract class JdbcClient {
}
}
- private void loadColumnNamesIfNeeded(String dbName, String tableName) {
- if (columnNamesLoaded.compareAndSet(false, true)) {
- getJdbcColumnsInfo(dbName, tableName);
- }
- }
-
// protected methods,for subclass to override
protected String getCatalogName(Connection conn) throws SQLException {
return null;
@@ -439,7 +411,7 @@ public abstract class JdbcClient {
}
protected void processTable(String dbName, String tableName, String[]
tableTypes,
- Consumer<ResultSet> resultSetConsumer) {
+ Consumer<ResultSet> resultSetConsumer) {
Connection conn = getConnection();
ResultSet rs = null;
try {
@@ -463,7 +435,7 @@ public abstract class JdbcClient {
}
protected ResultSet getColumns(DatabaseMetaData databaseMetaData, String
catalogName, String schemaName,
- String tableName) throws SQLException {
+ String tableName) throws SQLException {
return databaseMetaData.getColumns(catalogName, schemaName, tableName,
null);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
index 3273f4b7591..61ba2a0db47 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
@@ -35,7 +35,6 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
public class JdbcMySQLClient extends JdbcClient {
@@ -118,27 +117,21 @@ public class JdbcMySQLClient extends JdbcClient {
Connection conn = getConnection();
ResultSet rs = null;
List<JdbcFieldSchema> tableSchema =
com.google.common.collect.Lists.newArrayList();
- String finalDbName = getRealDatabaseName(dbName);
- String finalTableName = getRealTableName(dbName, tableName);
+ // if isLowerCaseTableNames == true, tableName is lower case
+ // but databaseMetaData.getColumns() is case sensitive
+ if (isLowerCaseTableNames) {
+ dbName = lowerDBToRealDB.get(dbName);
+ tableName = lowerTableToRealTable.get(tableName);
+ }
try {
DatabaseMetaData databaseMetaData = conn.getMetaData();
String catalogName = getCatalogName(conn);
- rs = getColumns(databaseMetaData, catalogName, finalDbName,
finalTableName);
+ rs = getColumns(databaseMetaData, catalogName, dbName, tableName);
List<String> primaryKeys = getPrimaryKeys(databaseMetaData,
catalogName, dbName, tableName);
Map<String, String> mapFieldtoType = null;
while (rs.next()) {
- lowerColumnToRealColumn.putIfAbsent(finalDbName, new
ConcurrentHashMap<>());
-
lowerColumnToRealColumn.get(finalDbName).putIfAbsent(finalTableName, new
ConcurrentHashMap<>());
JdbcFieldSchema field = new JdbcFieldSchema();
- String columnName = rs.getString("COLUMN_NAME");
- if (isLowerCaseTableNames) {
-
lowerColumnToRealColumn.get(finalDbName).get(finalTableName)
- .put(columnName.toLowerCase(), columnName);
- columnName = columnName.toLowerCase();
- } else {
-
lowerColumnToRealColumn.get(finalDbName).get(finalTableName).put(columnName,
columnName);
- }
- field.setColumnName(columnName);
+ field.setColumnName(rs.getString("COLUMN_NAME"));
field.setDataType(rs.getInt("DATA_TYPE"));
// in mysql-jdbc-connector-8.0.*, TYPE_NAME of the HLL column
in doris will be "UNKNOWN"
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java
index 270b5b4bdc8..d0a9f2c3de7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java
@@ -28,7 +28,6 @@ import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
public class JdbcOracleClient extends JdbcClient {
@@ -62,8 +61,6 @@ public class JdbcOracleClient extends JdbcClient {
if (isLowerCaseTableNames) {
lowerDBToRealDB.put(databaseName.toLowerCase(),
databaseName);
databaseName = databaseName.toLowerCase();
- } else {
- lowerDBToRealDB.put(databaseName, databaseName);
}
tempDatabaseNames.add(databaseName);
}
@@ -94,8 +91,14 @@ public class JdbcOracleClient extends JdbcClient {
Connection conn = getConnection();
ResultSet rs = null;
List<JdbcFieldSchema> tableSchema = Lists.newArrayList();
- String finalDbName = getRealDatabaseName(dbName);
- String finalTableName = getRealTableName(dbName, tableName);
+ String currentDbName = dbName;
+ String currentTableName = tableName;
+ if (isLowerCaseTableNames) {
+ currentDbName = getRealDatabaseName(dbName);
+ currentTableName = getRealTableName(dbName, tableName);
+ }
+ String finalDbName = currentDbName;
+ String finalTableName = currentTableName;
try {
DatabaseMetaData databaseMetaData = conn.getMetaData();
String catalogName = getCatalogName(conn);
@@ -116,18 +119,8 @@ public class JdbcOracleClient extends JdbcClient {
if (isModify && isTableModified(rs.getString("TABLE_NAME"),
finalTableName)) {
continue;
}
- lowerColumnToRealColumn.putIfAbsent(finalDbName, new
ConcurrentHashMap<>());
-
lowerColumnToRealColumn.get(finalDbName).putIfAbsent(finalTableName, new
ConcurrentHashMap<>());
JdbcFieldSchema field = new JdbcFieldSchema();
- String columnName = rs.getString("COLUMN_NAME");
- if (isLowerCaseTableNames) {
-
lowerColumnToRealColumn.get(finalDbName).get(finalTableName)
- .put(columnName.toLowerCase(), columnName);
- columnName = columnName.toLowerCase();
- } else {
-
lowerColumnToRealColumn.get(finalDbName).get(finalTableName).put(columnName,
columnName);
- }
- field.setColumnName(columnName);
+ field.setColumnName(rs.getString("COLUMN_NAME"));
field.setDataType(rs.getInt("DATA_TYPE"));
field.setDataTypeName(rs.getString("TYPE_NAME"));
/*
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java
index 4eccaff4774..23a44ed6432 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java
@@ -138,7 +138,7 @@ public class JdbcScanNode extends ExternalScanNode {
List<Expr> pushDownConjuncts =
collectConjunctsToPushDown(conjunctsList, errors);
for (Expr individualConjunct : pushDownConjuncts) {
- String filter = conjunctExprToString(jdbcType, individualConjunct,
tbl);
+ String filter = conjunctExprToString(jdbcType, individualConjunct);
filters.add(filter);
conjuncts.remove(individualConjunct);
}
@@ -169,9 +169,9 @@ public class JdbcScanNode extends ExternalScanNode {
continue;
}
Column col = slot.getColumn();
- columns.add(tbl.getProperRealColumnName(jdbcType, col.getName()));
+ columns.add(JdbcTable.databaseProperName(jdbcType, col.getName()));
}
- if (columns.isEmpty()) {
+ if (0 == columns.size()) {
columns.add("*");
}
}
@@ -324,12 +324,12 @@ public class JdbcScanNode extends ExternalScanNode {
return !fnExprList.isEmpty();
}
- public static String conjunctExprToString(TOdbcTableType tableType, Expr
expr, JdbcTable tbl) {
+ public static String conjunctExprToString(TOdbcTableType tableType, Expr
expr) {
if (expr instanceof CompoundPredicate) {
StringBuilder result = new StringBuilder();
CompoundPredicate compoundPredicate = (CompoundPredicate) expr;
for (Expr child : compoundPredicate.getChildren()) {
- result.append(conjunctExprToString(tableType, child, tbl));
+ result.append(conjunctExprToString(tableType, child));
result.append("
").append(compoundPredicate.getOp().toString()).append(" ");
}
// Remove the last operator
@@ -357,11 +357,7 @@ public class JdbcScanNode extends ExternalScanNode {
ArrayList<Expr> children = expr.getChildren();
String filter;
if (children.get(0) instanceof SlotRef) {
- if (tbl != null) {
- filter = tbl.getProperRealColumnName(tableType,
children.get(0).toMySql());
- } else {
- filter = JdbcTable.databaseProperName(tableType,
children.get(0).toMySql());
- }
+ filter = JdbcTable.databaseProperName(tableType,
children.get(0).toMySql());
} else {
filter = children.get(0).toMySql();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
index 68950ebab83..bf4e835e4f1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
@@ -184,7 +184,7 @@ public class OdbcScanNode extends ExternalScanNode {
ArrayList<Expr> odbcConjuncts = Expr.cloneList(conjuncts, sMap);
for (Expr p : odbcConjuncts) {
if (shouldPushDownConjunct(odbcType, p)) {
- String filter = JdbcScanNode.conjunctExprToString(odbcType, p,
null);
+ String filter = JdbcScanNode.conjunctExprToString(odbcType, p);
filters.add(filter);
conjuncts.remove(p);
}
diff --git
a/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out
b/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out
index 0545ddafac5..519c7ab131c 100644
--- a/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out
+++ b/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out
@@ -187,15 +187,6 @@ DORIS_TEST
3 jerry 23 88.0
4 andy 21 93.0
--- !lower_case_column_names1 --
-1 doris 3 1.0
-
--- !lower_case_column_names2 --
-1 doris 3 1.0
-
--- !lower_case_column_names3 --
-1 doris 3 1.0
-
-- !query_clob --
10010 liantong
10086 yidong
diff --git
a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
index f3463763b87..e762f2f4aaf 100644
---
a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
+++
b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
@@ -214,9 +214,6 @@ suite("test_oracle_jdbc_catalog",
"p0,external,oracle,external_docker,external_d
// test lower case name
order_qt_lower_case_table_names4 """ select * from student2 order by
id; """
- order_qt_lower_case_column_names1 """ select * from student3 order by
id; """
- order_qt_lower_case_column_names2 """ select * from student3 where
id = 1 order by id; """
- order_qt_lower_case_column_names3 """ select * from student3 where
id = 1 and name = 'doris' order by id; """
sql """drop catalog if exists ${catalog_name} """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
