This is an automated email from the ASF dual-hosted git repository.
zykkk pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 104284bd6f4 [Refactor](jdbc catalog) Enhance Field Handling in
JdbcFieldSchema Using Optional for Better Null Safety (#34730)
104284bd6f4 is described below
commit 104284bd6f4d0aa815b09abefce463182665a5cc
Author: zy-kkk <[email protected]>
AuthorDate: Mon May 13 22:37:35 2024 +0800
[Refactor](jdbc catalog) Enhance Field Handling in JdbcFieldSchema Using
Optional for Better Null Safety (#34730)
In some cases, using Resultset.getxxx cannot correctly handle null data, so
I use Optional to enhance the fields of JdbcFieldSchema for better type mapping
when some results are null.
---
.../hive/PostgreSQLJdbcHMSCachedClient.java | 1 +
.../jdbc/client/JdbcClickHouseClient.java | 7 +-
.../doris/datasource/jdbc/client/JdbcClient.java | 55 +------------
.../datasource/jdbc/client/JdbcDB2Client.java | 13 ++--
.../datasource/jdbc/client/JdbcMySQLClient.java | 60 +++++---------
.../jdbc/client/JdbcOceanBaseClient.java | 1 +
.../datasource/jdbc/client/JdbcOracleClient.java | 31 ++------
.../jdbc/client/JdbcPostgreSQLClient.java | 13 ++--
.../jdbc/client/JdbcSQLServerClient.java | 9 ++-
.../datasource/jdbc/client/JdbcSapHanaClient.java | 17 ++--
.../datasource/jdbc/client/JdbcTrinoClient.java | 11 ++-
.../datasource/jdbc/util/JdbcFieldSchema.java | 91 ++++++++++++++++++++++
12 files changed, 165 insertions(+), 144 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
index a0c2aa29ab5..932118001e5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
@@ -24,6 +24,7 @@ import org.apache.doris.datasource.DatabaseMetadata;
import org.apache.doris.datasource.TableMetadata;
import
org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException;
import org.apache.doris.datasource.jdbc.client.JdbcClientConfig;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import org.apache.doris.thrift.TOdbcTableType;
import com.google.common.base.Joiner;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
index 35e6c83af1f..bdf0cbbc934 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
@@ -20,6 +20,9 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
+
+import java.util.Optional;
public class JdbcClickHouseClient extends JdbcClient {
@@ -35,7 +38,7 @@ public class JdbcClickHouseClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
- String ckType = fieldSchema.getDataTypeName();
+ String ckType = fieldSchema.getDataTypeName().orElse("unknown");
if (ckType.startsWith("LowCardinality")) {
fieldSchema.setAllowNull(true);
@@ -81,7 +84,7 @@ public class JdbcClickHouseClient extends JdbcClient {
if (ckType.startsWith("Array")) {
String cktype = ckType.substring(6, ckType.length() - 1);
- fieldSchema.setDataTypeName(cktype);
+ fieldSchema.setDataTypeName(Optional.of(cktype));
Type type = jdbcTypeToDoris(fieldSchema);
return ArrayType.create(type, true);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
index 604e54277f1..0ae2966079c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
@@ -24,11 +24,11 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.jdbc.JdbcIdentifierMapping;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.zaxxer.hikari.HikariDataSource;
-import lombok.Data;
import lombok.Getter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -255,14 +255,7 @@ public abstract class JdbcClient {
public List<JdbcFieldSchema>
getSchemaFromResultSetMetaData(ResultSetMetaData metaData) throws SQLException {
List<JdbcFieldSchema> schemas = Lists.newArrayList();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
- JdbcFieldSchema field = new JdbcFieldSchema();
- field.setColumnName(metaData.getColumnName(i));
- field.setDataType(metaData.getColumnType(i));
- field.setDataTypeName(metaData.getColumnTypeName(i));
- field.setColumnSize(metaData.getColumnDisplaySize(i));
- field.setDecimalDigits(metaData.getScale(i));
- field.setNumPrecRadix(metaData.getPrecision(i));
- schemas.add(field);
+ schemas.add(new JdbcFieldSchema(metaData, i));
}
return schemas;
}
@@ -347,27 +340,7 @@ public abstract class JdbcClient {
String catalogName = getCatalogName(conn);
rs = getRemoteColumns(databaseMetaData, catalogName, remoteDbName,
remoteTableName);
while (rs.next()) {
- JdbcFieldSchema field = new JdbcFieldSchema();
- field.setColumnName(rs.getString("COLUMN_NAME"));
- field.setDataType(rs.getInt("DATA_TYPE"));
- field.setDataTypeName(rs.getString("TYPE_NAME"));
- /*
- We used this method to retrieve the key column of the JDBC
table, but since we only tested mysql,
- we kept the default key behavior in the parent class and
only overwrite it in the mysql subclass
- */
- field.setColumnSize(rs.getInt("COLUMN_SIZE"));
- field.setDecimalDigits(rs.getInt("DECIMAL_DIGITS"));
- field.setNumPrecRadix(rs.getInt("NUM_PREC_RADIX"));
- /*
- Whether it is allowed to be NULL
- 0 (columnNoNulls)
- 1 (columnNullable)
- 2 (columnNullableUnknown)
- */
- field.setAllowNull(rs.getInt("NULLABLE") != 0);
- field.setRemarks(rs.getString("REMARKS"));
- field.setCharOctetLength(rs.getInt("CHAR_OCTET_LENGTH"));
- tableSchema.add(field);
+ tableSchema.add(new JdbcFieldSchema(rs));
}
} catch (SQLException e) {
throw new JdbcClientException("failed to get jdbc columns info for
remote table `%s.%s`: %s",
@@ -478,28 +451,6 @@ public abstract class JdbcClient {
return jdbcLowerCaseMetaMatching.setColumnNameMapping(remoteDbName,
remoteTableName, remoteColumns);
}
- @Data
- protected static class JdbcFieldSchema {
- protected String columnName;
- // The SQL type of the corresponding java.sql.types (Type ID)
- protected int dataType;
- // The SQL type of the corresponding java.sql.types (Type Name)
- protected String dataTypeName;
- // For CHAR/DATA, columnSize means the maximum number of chars.
- // For NUMERIC/DECIMAL, columnSize means precision.
- protected int columnSize;
- protected int decimalDigits;
- // Base number (usually 10 or 2)
- protected int numPrecRadix;
- // column description
- protected String remarks;
- // This length is the maximum number of bytes for CHAR type
- // for utf8 encoding, if columnSize=10, then charOctetLength=30
- // because for utf8 encoding, a Chinese character takes up 3 bytes
- protected int charOctetLength;
- protected boolean isAllowNull;
- }
-
protected abstract Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema);
protected Type createDecimalOrStringType(int precision, int scale) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java
index af89f33eb9a..dafb00ca9e8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import com.google.common.collect.Lists;
@@ -63,7 +64,7 @@ public class JdbcDB2Client extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
- String db2Type = fieldSchema.getDataTypeName();
+ String db2Type = fieldSchema.getDataTypeName().orElse("unknown");
switch (db2Type) {
case "SMALLINT":
return Type.SMALLINT;
@@ -73,8 +74,8 @@ public class JdbcDB2Client extends JdbcClient {
return Type.BIGINT;
case "DECFLOAT":
case "DECIMAL": {
- int precision = fieldSchema.getColumnSize();
- int scale = fieldSchema.getDecimalDigits();
+ int precision = fieldSchema.getColumnSize().orElse(0);
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
return createDecimalOrStringType(precision, scale);
}
case "DOUBLE":
@@ -83,18 +84,18 @@ public class JdbcDB2Client extends JdbcClient {
return Type.FLOAT;
case "CHAR":
ScalarType charType =
ScalarType.createType(PrimitiveType.CHAR);
- charType.setLength(fieldSchema.columnSize);
+ charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
case "VARCHAR":
case "LONG VARCHAR":
ScalarType varcharType =
ScalarType.createType(PrimitiveType.VARCHAR);
- varcharType.setLength(fieldSchema.columnSize);
+ varcharType.setLength(fieldSchema.getColumnSize().orElse(0));
return varcharType;
case "DATE":
return ScalarType.createDateV2Type();
case "TIMESTAMP": {
// postgres can support microsecond
- int scale = fieldSchema.getDecimalDigits();
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
index 6ea5ea127d5..eee254482d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
@@ -22,6 +22,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@@ -122,40 +123,21 @@ public class JdbcMySQLClient extends JdbcClient {
public List<JdbcFieldSchema> getJdbcColumnsInfo(String localDbName, String
localTableName) {
Connection conn = getConnection();
ResultSet rs = null;
- List<JdbcFieldSchema> tableSchema =
com.google.common.collect.Lists.newArrayList();
+ List<JdbcFieldSchema> tableSchema = Lists.newArrayList();
String remoteDbName = getRemoteDatabaseName(localDbName);
String remoteTableName = getRemoteTableName(localDbName,
localTableName);
try {
DatabaseMetaData databaseMetaData = conn.getMetaData();
String catalogName = getCatalogName(conn);
rs = getRemoteColumns(databaseMetaData, catalogName, remoteDbName,
remoteTableName);
- Map<String, String> mapFieldtoType = null;
- while (rs.next()) {
- JdbcFieldSchema field = new JdbcFieldSchema();
- field.setColumnName(rs.getString("COLUMN_NAME"));
- field.setDataType(rs.getInt("DATA_TYPE"));
- // in mysql-jdbc-connector-8.0.*, TYPE_NAME of the HLL column
in doris will be "UNKNOWN"
- // in mysql-jdbc-connector-5.1.*, TYPE_NAME of the HLL column
in doris will be "HLL"
- // in mysql-jdbc-connector-8.0.*, TYPE_NAME of BITMAP column
in doris will be "BIT"
- // in mysql-jdbc-connector-5.1.*, TYPE_NAME of BITMAP column
in doris will be "BITMAP"
- field.setDataTypeName(rs.getString("TYPE_NAME"));
- if (isDoris) {
- mapFieldtoType = getColumnsDataTypeUseQuery(remoteDbName,
remoteTableName);
-
field.setDataTypeName(mapFieldtoType.get(rs.getString("COLUMN_NAME")));
- }
- field.setColumnSize(rs.getInt("COLUMN_SIZE"));
- field.setDecimalDigits(rs.getInt("DECIMAL_DIGITS"));
- field.setNumPrecRadix(rs.getInt("NUM_PREC_RADIX"));
- /*
- Whether it is allowed to be NULL
- 0 (columnNoNulls)
- 1 (columnNullable)
- 2 (columnNullableUnknown)
- */
- field.setAllowNull(rs.getInt("NULLABLE") != 0);
- field.setRemarks(rs.getString("REMARKS"));
- field.setCharOctetLength(rs.getInt("CHAR_OCTET_LENGTH"));
+ Map<String, String> mapFieldtoType = Maps.newHashMap();
+ if (isDoris) {
+ mapFieldtoType = getColumnsDataTypeUseQuery(remoteDbName,
remoteTableName);
+ }
+
+ while (rs.next()) {
+ JdbcFieldSchema field = new JdbcFieldSchema(rs,
mapFieldtoType);
tableSchema.add(field);
}
} catch (SQLException e) {
@@ -184,12 +166,12 @@ public class JdbcMySQLClient extends JdbcClient {
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
// For Doris type
if (isDoris) {
- return
dorisTypeToDoris(fieldSchema.getDataTypeName().toUpperCase());
+ return
dorisTypeToDoris(fieldSchema.getDataTypeName().orElse("unknown").toUpperCase());
}
// For mysql type: "INT UNSIGNED":
- // fieldSchema.getDataTypeName().split(" ")[0] == "INT"
- // fieldSchema.getDataTypeName().split(" ")[1] == "UNSIGNED"
- String[] typeFields = fieldSchema.getDataTypeName().split(" ");
+ // fieldSchema.getDataTypeName().orElse("unknown").split(" ")[0] ==
"INT"
+ // fieldSchema.getDataTypeName().orElse("unknown").split(" ")[1] ==
"UNSIGNED"
+ String[] typeFields =
fieldSchema.getDataTypeName().orElse("unknown").split(" ");
String mysqlType = typeFields[0];
// For unsigned int, should extend the type.
if (typeFields.length > 1 && "UNSIGNED".equals(typeFields[1])) {
@@ -204,8 +186,8 @@ public class JdbcMySQLClient extends JdbcClient {
case "BIGINT":
return Type.LARGEINT;
case "DECIMAL": {
- int precision = fieldSchema.getColumnSize() + 1;
- int scale = fieldSchema.getDecimalDigits();
+ int precision = fieldSchema.getColumnSize().orElse(0) + 1;
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
return createDecimalOrStringType(precision, scale);
}
case "DOUBLE":
@@ -242,7 +224,7 @@ public class JdbcMySQLClient extends JdbcClient {
case "DATETIME": {
// mysql can support microsecond
// use columnSize to calculate the precision of
timestamp/datetime
- int columnSize = fieldSchema.getColumnSize();
+ int columnSize = fieldSchema.getColumnSize().orElse(0);
int scale = columnSize > 19 ? columnSize - 20 : 0;
if (scale > 6) {
scale = 6;
@@ -257,18 +239,18 @@ public class JdbcMySQLClient extends JdbcClient {
case "DOUBLE":
return Type.DOUBLE;
case "DECIMAL": {
- int precision = fieldSchema.getColumnSize();
- int scale = fieldSchema.getDecimalDigits();
+ int precision = fieldSchema.getColumnSize().orElse(0);
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
return createDecimalOrStringType(precision, scale);
}
case "CHAR":
ScalarType charType =
ScalarType.createType(PrimitiveType.CHAR);
- charType.setLength(fieldSchema.columnSize);
+ charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
case "VARCHAR":
- return ScalarType.createVarcharType(fieldSchema.columnSize);
+ return
ScalarType.createVarcharType(fieldSchema.getColumnSize().orElse(0));
case "BIT":
- if (fieldSchema.getColumnSize() == 1) {
+ if (fieldSchema.getColumnSize().orElse(0) == 1) {
return Type.BOOLEAN;
} else {
return ScalarType.createStringType();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java
index 8628d62d70e..b8c5b61aea4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java
@@ -19,6 +19,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.JdbcResource;
import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import java.sql.Connection;
import java.sql.ResultSet;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java
index 484028ed92e..0efa94d9bff 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@@ -69,27 +70,7 @@ public class JdbcOracleClient extends JdbcClient {
if (isModify && isTableModified(rs.getString("TABLE_NAME"),
remoteTableName)) {
continue;
}
- JdbcFieldSchema field = new JdbcFieldSchema();
- field.setColumnName(rs.getString("COLUMN_NAME"));
- field.setDataType(rs.getInt("DATA_TYPE"));
- field.setDataTypeName(rs.getString("TYPE_NAME"));
- /*
- We used this method to retrieve the key column of the JDBC
table, but since we only tested mysql,
- we kept the default key behavior in the parent class and
only overwrite it in the mysql subclass
- */
- field.setColumnSize(rs.getInt("COLUMN_SIZE"));
- field.setDecimalDigits(rs.getInt("DECIMAL_DIGITS"));
- field.setNumPrecRadix(rs.getInt("NUM_PREC_RADIX"));
- /*
- Whether it is allowed to be NULL
- 0 (columnNoNulls)
- 1 (columnNullable)
- 2 (columnNullableUnknown)
- */
- field.setAllowNull(rs.getInt("NULLABLE") != 0);
- field.setRemarks(rs.getString("REMARKS"));
- field.setCharOctetLength(rs.getInt("CHAR_OCTET_LENGTH"));
- tableSchema.add(field);
+ tableSchema.add(new JdbcFieldSchema(rs));
}
} catch (SQLException e) {
throw new JdbcClientException("failed to get table name list from
jdbc for table %s:%s", e, remoteTableName,
@@ -126,7 +107,7 @@ public class JdbcOracleClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
- String oracleType = fieldSchema.getDataTypeName();
+ String oracleType = fieldSchema.getDataTypeName().orElse("unknown");
if (oracleType.startsWith("INTERVAL")) {
oracleType = oracleType.substring(0, 8);
} else if (oracleType.startsWith("TIMESTAMP")) {
@@ -134,7 +115,7 @@ public class JdbcOracleClient extends JdbcClient {
return Type.UNSUPPORTED;
}
// oracle can support nanosecond, will lose precision
- int scale = fieldSchema.getDecimalDigits();
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
@@ -160,8 +141,8 @@ public class JdbcOracleClient extends JdbcClient {
* In this case, doris can not determine p and s, so doris can
not determine data type.
*/
case "NUMBER":
- int precision = fieldSchema.getColumnSize();
- int scale = fieldSchema.getDecimalDigits();
+ int precision = fieldSchema.getColumnSize().orElse(0);
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale <= 0) {
precision -= scale;
if (precision < 3) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
index 3d0bbc4e561..e6694ffdc67 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
public class JdbcPostgreSQLClient extends JdbcClient {
@@ -34,7 +35,7 @@ public class JdbcPostgreSQLClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
- String pgType = fieldSchema.getDataTypeName();
+ String pgType = fieldSchema.getDataTypeName().orElse("unknown");
switch (pgType) {
case "int2":
case "smallserial":
@@ -46,8 +47,8 @@ public class JdbcPostgreSQLClient extends JdbcClient {
case "bigserial":
return Type.BIGINT;
case "numeric": {
- int precision = fieldSchema.getColumnSize();
- int scale = fieldSchema.getDecimalDigits();
+ int precision = fieldSchema.getColumnSize().orElse(0);
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
return createDecimalOrStringType(precision, scale);
}
case "float4":
@@ -56,12 +57,12 @@ public class JdbcPostgreSQLClient extends JdbcClient {
return Type.DOUBLE;
case "bpchar":
ScalarType charType =
ScalarType.createType(PrimitiveType.CHAR);
- charType.setLength(fieldSchema.columnSize);
+ charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
case "timestamp":
case "timestamptz": {
// postgres can support microsecond
- int scale = fieldSchema.getDecimalDigits();
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
@@ -72,7 +73,7 @@ public class JdbcPostgreSQLClient extends JdbcClient {
case "bool":
return Type.BOOLEAN;
case "bit":
- if (fieldSchema.getColumnSize() == 1) {
+ if (fieldSchema.getColumnSize().orElse(0) == 1) {
return Type.BOOLEAN;
} else {
return ScalarType.createStringType();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java
index e3d96421a13..1bb3ece7f09 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java
@@ -19,6 +19,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
public class JdbcSQLServerClient extends JdbcClient {
@@ -28,7 +29,7 @@ public class JdbcSQLServerClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
- String originSqlserverType = fieldSchema.getDataTypeName();
+ String originSqlserverType =
fieldSchema.getDataTypeName().orElse("unknown");
// For sqlserver IDENTITY type, such as 'INT IDENTITY'
// originSqlserverType is "int identity", so we only get "int".
String sqlserverType = originSqlserverType.split(" ")[0];
@@ -52,8 +53,8 @@ public class JdbcSQLServerClient extends JdbcClient {
return ScalarType.createDecimalV3Type(10, 4);
case "decimal":
case "numeric": {
- int precision = fieldSchema.getColumnSize();
- int scale = fieldSchema.getDecimalDigits();
+ int precision = fieldSchema.getColumnSize().orElse(0);
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
return ScalarType.createDecimalV3Type(precision, scale);
}
case "date":
@@ -62,7 +63,7 @@ public class JdbcSQLServerClient extends JdbcClient {
case "datetime2":
case "smalldatetime": {
// postgres can support microsecond
- int scale = fieldSchema.getDecimalDigits();
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java
index 34dde63c806..f10d082bada 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
public class JdbcSapHanaClient extends JdbcClient {
protected JdbcSapHanaClient(JdbcClientConfig jdbcClientConfig) {
@@ -38,7 +39,7 @@ public class JdbcSapHanaClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
- String hanaType = fieldSchema.getDataTypeName();
+ String hanaType = fieldSchema.getDataTypeName().orElse("unknown");
switch (hanaType) {
case "TINYINT":
return Type.TINYINT;
@@ -50,9 +51,13 @@ public class JdbcSapHanaClient extends JdbcClient {
return Type.BIGINT;
case "SMALLDECIMAL":
case "DECIMAL": {
- int precision = fieldSchema.getColumnSize();
- int scale = fieldSchema.getDecimalDigits();
- return createDecimalOrStringType(precision, scale);
+ if (!fieldSchema.getDecimalDigits().isPresent()) {
+ return Type.DOUBLE;
+ } else {
+ int precision = fieldSchema.getColumnSize().orElse(0);
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
+ return createDecimalOrStringType(precision, scale);
+ }
}
case "REAL":
return Type.FLOAT;
@@ -60,7 +65,7 @@ public class JdbcSapHanaClient extends JdbcClient {
return Type.DOUBLE;
case "TIMESTAMP": {
// postgres can support microsecond
- int scale = fieldSchema.getDecimalDigits();
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
@@ -76,7 +81,7 @@ public class JdbcSapHanaClient extends JdbcClient {
case "CHAR":
case "NCHAR":
ScalarType charType =
ScalarType.createType(PrimitiveType.CHAR);
- charType.setLength(fieldSchema.columnSize);
+ charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
case "TIME":
case "VARCHAR":
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java
index e922254dbc0..006bf5a44f9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java
@@ -21,6 +21,9 @@ import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
+
+import java.util.Optional;
public class JdbcTrinoClient extends JdbcClient {
protected JdbcTrinoClient(JdbcClientConfig jdbcClientConfig) {
@@ -29,7 +32,7 @@ public class JdbcTrinoClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
- String trinoType = fieldSchema.getDataTypeName();
+ String trinoType = fieldSchema.getDataTypeName().orElse("unknown");
switch (trinoType) {
case "integer":
return Type.INT;
@@ -61,12 +64,12 @@ public class JdbcTrinoClient extends JdbcClient {
if (trinoType.startsWith("char")) {
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
- charType.setLength(fieldSchema.columnSize);
+ charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
}
if (trinoType.startsWith("timestamp")) {
- int scale = fieldSchema.getDecimalDigits();
+ int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
@@ -75,7 +78,7 @@ public class JdbcTrinoClient extends JdbcClient {
if (trinoType.startsWith("array")) {
String trinoArrType = trinoType.substring(6, trinoType.length() -
1);
- fieldSchema.setDataTypeName(trinoArrType);
+ fieldSchema.setDataTypeName(Optional.of(trinoArrType));
Type type = jdbcTypeToDoris(fieldSchema);
return ArrayType.create(type, true);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java
new file mode 100644
index 00000000000..735de93e9eb
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.jdbc.util;
+
+import lombok.Data;
+
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.Optional;
+
+@Data
+public class JdbcFieldSchema {
+ protected String columnName;
+ // The SQL type of the corresponding java.sql.types (Type ID)
+ protected int dataType;
+ // The SQL type of the corresponding java.sql.types (Type Name)
+ protected Optional<String> dataTypeName;
+ // For CHAR/DATA, columnSize means the maximum number of chars.
+ // For NUMERIC/DECIMAL, columnSize means precision.
+ protected Optional<Integer> columnSize;
+ protected Optional<Integer> decimalDigits;
+ // Base number (usually 10 or 2)
+ protected int numPrecRadix;
+ // column description
+ protected String remarks;
+ // This length is the maximum number of bytes for CHAR type
+ // for utf8 encoding, if columnSize=10, then charOctetLength=30
+ // because for utf8 encoding, a Chinese character takes up 3 bytes
+ protected int charOctetLength;
+ protected boolean isAllowNull;
+
+ public JdbcFieldSchema(ResultSet rs) throws SQLException {
+ this.columnName = rs.getString("COLUMN_NAME");
+ this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new
IllegalStateException("DATA_TYPE is null"));
+ this.dataTypeName = Optional.ofNullable(rs.getString("TYPE_NAME"));
+ this.columnSize = getInteger(rs, "COLUMN_SIZE");
+ this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS");
+ this.numPrecRadix = rs.getInt("NUM_PREC_RADIX");
+ this.isAllowNull = rs.getInt("NULLABLE") !=
DatabaseMetaData.columnNoNulls;
+ this.remarks = rs.getString("REMARKS");
+ this.charOctetLength = rs.getInt("CHAR_OCTET_LENGTH");
+ }
+
+ public JdbcFieldSchema(ResultSet rs, Map<String, String>
dataTypeOverrides) throws SQLException {
+ this.columnName = rs.getString("COLUMN_NAME");
+ this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new
IllegalStateException("DATA_TYPE is null"));
+ this.dataTypeName =
Optional.ofNullable(dataTypeOverrides.getOrDefault(columnName,
rs.getString("TYPE_NAME")));
+ this.columnSize = getInteger(rs, "COLUMN_SIZE");
+ this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS");
+ this.numPrecRadix = rs.getInt("NUM_PREC_RADIX");
+ this.isAllowNull = rs.getInt("NULLABLE") != 0;
+ this.remarks = rs.getString("REMARKS");
+ this.charOctetLength = rs.getInt("CHAR_OCTET_LENGTH");
+ }
+
+ public JdbcFieldSchema(ResultSetMetaData metaData, int columnIndex) throws
SQLException {
+ this.columnName = metaData.getColumnName(columnIndex);
+ this.dataType = metaData.getColumnType(columnIndex);
+ this.dataTypeName =
Optional.ofNullable(metaData.getColumnTypeName(columnIndex));
+ this.columnSize = Optional.of(metaData.getPrecision(columnIndex));
+ this.decimalDigits = Optional.of(metaData.getScale(columnIndex));
+ }
+
+ protected static Optional<Integer> getInteger(ResultSet resultSet, String
columnLabel)
+ throws SQLException {
+ int value = resultSet.getInt(columnLabel);
+ if (resultSet.wasNull()) {
+ return Optional.empty();
+ }
+ return Optional.of(value);
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
