hailin0 commented on code in PR #5872:
URL: https://github.com/apache/seatunnel/pull/5872#discussion_r1397103027
##########
seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalog.java:
##########
@@ -120,61 +117,45 @@ protected List<ConstraintKey>
getConstraintKeys(DatabaseMetaData metaData, Table
@Override
protected Column buildColumn(ResultSet resultSet) throws SQLException {
String columnName = resultSet.getString("COLUMN_NAME");
- String sourceType = resultSet.getString("COLUMN_TYPE");
- String typeName = resultSet.getString("DATA_TYPE").toUpperCase();
- int precision = resultSet.getInt("NUMERIC_PRECISION");
- int scale = resultSet.getInt("NUMERIC_SCALE");
- long columnLength = resultSet.getLong("CHARACTER_MAXIMUM_LENGTH");
- long octetLength = resultSet.getLong("CHARACTER_OCTET_LENGTH");
- if (sourceType.toLowerCase(Locale.ROOT).contains("unsigned")) {
- typeName += "_UNSIGNED";
- }
- SeaTunnelDataType<?> type = fromJdbcType(columnName, typeName,
precision, scale);
+ // e.g. tinyint(1) unsigned
+ String columnType = resultSet.getString("COLUMN_TYPE");
+ // e.g. tinyint
+ String dataType = resultSet.getString("DATA_TYPE").toUpperCase();
String comment = resultSet.getString("COLUMN_COMMENT");
Object defaultValue = resultSet.getObject("COLUMN_DEFAULT");
String isNullableStr = resultSet.getString("IS_NULLABLE");
boolean isNullable = isNullableStr.equals("YES");
- long bitLen = 0;
- MysqlType mysqlType = MysqlType.valueOf(typeName);
- switch (mysqlType) {
- case BIT:
- bitLen = precision;
- break;
- case CHAR:
- case VARCHAR:
- columnLength = octetLength;
Review Comment:
use charlength
##########
seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalog.java:
##########
@@ -120,61 +117,45 @@ protected List<ConstraintKey>
getConstraintKeys(DatabaseMetaData metaData, Table
@Override
protected Column buildColumn(ResultSet resultSet) throws SQLException {
String columnName = resultSet.getString("COLUMN_NAME");
- String sourceType = resultSet.getString("COLUMN_TYPE");
- String typeName = resultSet.getString("DATA_TYPE").toUpperCase();
- int precision = resultSet.getInt("NUMERIC_PRECISION");
- int scale = resultSet.getInt("NUMERIC_SCALE");
- long columnLength = resultSet.getLong("CHARACTER_MAXIMUM_LENGTH");
- long octetLength = resultSet.getLong("CHARACTER_OCTET_LENGTH");
- if (sourceType.toLowerCase(Locale.ROOT).contains("unsigned")) {
- typeName += "_UNSIGNED";
- }
- SeaTunnelDataType<?> type = fromJdbcType(columnName, typeName,
precision, scale);
+ // e.g. tinyint(1) unsigned
+ String columnType = resultSet.getString("COLUMN_TYPE");
+ // e.g. tinyint
+ String dataType = resultSet.getString("DATA_TYPE").toUpperCase();
String comment = resultSet.getString("COLUMN_COMMENT");
Object defaultValue = resultSet.getObject("COLUMN_DEFAULT");
String isNullableStr = resultSet.getString("IS_NULLABLE");
boolean isNullable = isNullableStr.equals("YES");
- long bitLen = 0;
- MysqlType mysqlType = MysqlType.valueOf(typeName);
- switch (mysqlType) {
- case BIT:
- bitLen = precision;
- break;
- case CHAR:
- case VARCHAR:
- columnLength = octetLength;
- break;
- case BINARY:
- case VARBINARY:
- // Uniform conversion to bits
- bitLen = octetLength * 4 * 8L;
- break;
- case BLOB:
- case TINYBLOB:
- case MEDIUMBLOB:
- case LONGBLOB:
- bitLen = columnLength << 3;
- break;
- case JSON:
- columnLength = 4 * 1024 * 1024 * 1024L;
Review Comment:
json -> mapping to longtext
##########
seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalog.java:
##########
@@ -120,61 +117,45 @@ protected List<ConstraintKey>
getConstraintKeys(DatabaseMetaData metaData, Table
@Override
protected Column buildColumn(ResultSet resultSet) throws SQLException {
String columnName = resultSet.getString("COLUMN_NAME");
- String sourceType = resultSet.getString("COLUMN_TYPE");
- String typeName = resultSet.getString("DATA_TYPE").toUpperCase();
- int precision = resultSet.getInt("NUMERIC_PRECISION");
- int scale = resultSet.getInt("NUMERIC_SCALE");
- long columnLength = resultSet.getLong("CHARACTER_MAXIMUM_LENGTH");
- long octetLength = resultSet.getLong("CHARACTER_OCTET_LENGTH");
- if (sourceType.toLowerCase(Locale.ROOT).contains("unsigned")) {
- typeName += "_UNSIGNED";
- }
- SeaTunnelDataType<?> type = fromJdbcType(columnName, typeName,
precision, scale);
+ // e.g. tinyint(1) unsigned
+ String columnType = resultSet.getString("COLUMN_TYPE");
+ // e.g. tinyint
+ String dataType = resultSet.getString("DATA_TYPE").toUpperCase();
String comment = resultSet.getString("COLUMN_COMMENT");
Object defaultValue = resultSet.getObject("COLUMN_DEFAULT");
String isNullableStr = resultSet.getString("IS_NULLABLE");
boolean isNullable = isNullableStr.equals("YES");
- long bitLen = 0;
- MysqlType mysqlType = MysqlType.valueOf(typeName);
- switch (mysqlType) {
- case BIT:
- bitLen = precision;
- break;
- case CHAR:
- case VARCHAR:
- columnLength = octetLength;
- break;
- case BINARY:
- case VARBINARY:
- // Uniform conversion to bits
- bitLen = octetLength * 4 * 8L;
- break;
- case BLOB:
- case TINYBLOB:
- case MEDIUMBLOB:
- case LONGBLOB:
- bitLen = columnLength << 3;
- break;
- case JSON:
- columnLength = 4 * 1024 * 1024 * 1024L;
- break;
- default:
- break;
- }
-
- return PhysicalColumn.of(
- columnName,
- type,
- 0,
- isNullable,
- defaultValue,
- comment,
- sourceType,
- sourceType.contains("unsigned"),
- sourceType.contains("zerofill"),
- bitLen,
- null,
- columnLength);
+ // e.g. `decimal(10, 2)` is 10
+ long numberPrecision = resultSet.getInt("NUMERIC_PRECISION");
+ // e.g. `decimal(10, 2)` is 2
+ int numberScale = resultSet.getInt("NUMERIC_SCALE");
+ // e.g. `varchar(10)` is 10
+ long charLength = resultSet.getLong("CHARACTER_MAXIMUM_LENGTH");
+ long charOctetLength = resultSet.getLong("CHARACTER_OCTET_LENGTH");
+ // e.g. `timestamp(3)` is 3
+ int timePrecision = resultSet.getInt("DATETIME_PRECISION");
+
+ Preconditions.checkArgument(!(numberPrecision > 0 && charLength > 0));
+ Preconditions.checkArgument(!(numberScale > 0 && timePrecision > 0));
+
+ MysqlType mysqlType = MysqlType.getByName(columnType);
+ boolean unsigned =
columnType.toLowerCase(Locale.ROOT).contains("unsigned");
+
+ BasicTypeDefine<MysqlType> typeDefine =
+ BasicTypeDefine.<MysqlType>builder()
+ .name(columnName)
+ .columnType(columnType)
+ .dataType(dataType)
+ .nativeType(mysqlType)
+ .unsigned(unsigned)
+ .length(Math.max(charLength, numberPrecision))
+ .precision(numberPrecision)
+ .scale(Math.max(numberScale, timePrecision))
+ .nullable(isNullable)
+ .defaultValue(defaultValue)
+ .comment(comment)
+ .build();
+ return MySQLTypeConverter.INSTANCE.convert(typeDefine);
Review Comment:
ResultSet format
```
+-------------+------------+----------+--------------------+----------------+--------------+-----------+----------+------------------------+----------------------+-----------------+-------------+------------------+------------------+------------------+-----------------------------+----------+--------------+-------------------------------+--------------+---------------------+
|TABLE_CATALOG|TABLE_SCHEMA|TABLE_NAME|COLUMN_NAME
|ORDINAL_POSITION|COLUMN_DEFAULT|IS_NULLABLE|DATA_TYPE
|CHARACTER_MAXIMUM_LENGTH|CHARACTER_OCTET_LENGTH|NUMERIC_PRECISION|NUMERIC_SCALE|DATETIME_PRECISION|CHARACTER_SET_NAME|COLLATION_NAME
|COLUMN_TYPE |COLUMN_KEY|EXTRA |PRIVILEGES
|COLUMN_COMMENT|GENERATION_EXPRESSION|
+-------------+------------+----------+--------------------+----------------+--------------+-----------+----------+------------------------+----------------------+-----------------+-------------+------------------+------------------+------------------+-----------------------------+----------+--------------+-------------------------------+--------------+---------------------+
|def |wen_source |all_types |id |1
|null |NO |int |null |null
|10 |0 |null |null
|null |int(11) |PRI
|auto_increment|select,insert,update,references| |
|
|def |wen_source |all_types |f_boolean |2
|null |YES |tinyint |null |null
|3 |0 |null |null
|null |tinyint(1) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_tinyint |3
|null |YES |tinyint |null |null
|3 |0 |null |null
|null |tinyint(4) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_tinyint_unsigned |4
|null |YES |tinyint |null |null
|3 |0 |null |null
|null |tinyint(3) unsigned | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_smallint |5
|null |YES |smallint |null |null
|5 |0 |null |null
|null |smallint(6) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_smallint_unsigned |6
|null |YES |smallint |null |null
|5 |0 |null |null
|null |smallint(5) unsigned | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_mediumint |7
|null |YES |mediumint |null |null
|7 |0 |null |null
|null |mediumint(9) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_mediumint_unsigned|8
|null |YES |mediumint |null |null
|7 |0 |null |null
|null |mediumint(8) unsigned | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_int |9
|null |YES |int |null |null
|10 |0 |null |null
|null |int(11) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_int_unsigned |10
|null |YES |int |null |null
|10 |0 |null |null
|null |int(10) unsigned | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_integer |11
|null |YES |int |null |null
|10 |0 |null |null
|null |int(11) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_integer_unsigned |12
|null |YES |int |null |null
|10 |0 |null |null
|null |int(10) unsigned | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_bigint |13
|null |YES |bigint |null |null
|19 |0 |null |null
|null |bigint(20) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_bigint_unsigned |14
|null |YES |bigint |null |null
|20 |0 |null |null
|null |bigint(20) unsigned | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_float |15
|null |YES |float |null |null
|12 |null |null |null
|null |float | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_float_unsigned |16
|null |YES |float |null |null
|12 |null |null |null
|null |float unsigned | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_double |17
|null |YES |double |null |null
|22 |null |null |null
|null |double | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_double_unsigned |18
|null |YES |double |null |null
|22 |null |null |null
|null |double unsigned | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_double_precision |19
|null |YES |double |null |null
|22 |null |null |null
|null |double | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_numeric1 |20
|null |YES |decimal |null |null
|10 |0 |null |null
|null |decimal(10,0) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_decimal1 |21
|null |YES |decimal |null |null
|10 |0 |null |null
|null |decimal(10,0) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_decimal |22
|null |YES |decimal |null |null
|10 |2 |null |null
|null |decimal(10,2) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_decimal_unsigned |23
|null |YES |decimal |null |null
|10 |2 |null |null
|null |decimal(10,2) unsigned | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_char |24
|null |YES |char |1 |4
|null |null |null |utf8mb4
|utf8mb4_unicode_ci|char(1) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_varchar |25
|null |YES |varchar |100 |400
|null |null |null |utf8mb4
|utf8mb4_unicode_ci|varchar(100) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_tinytext |26
|null |YES |tinytext |255 |255
|null |null |null |utf8mb4
|utf8mb4_unicode_ci|tinytext | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_text |27
|null |YES |text |65535 |65535
|null |null |null |utf8mb4
|utf8mb4_unicode_ci|text | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_mediumtext |28
|null |YES |mediumtext|16777215 |16777215
|null |null |null |utf8mb4
|utf8mb4_unicode_ci|mediumtext | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_longtext |29
|null |YES |longtext |4294967295 |4294967295
|null |null |null |utf8mb4
|utf8mb4_unicode_ci|longtext | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_json |30
|null |YES |json |null |null
|null |null |null |null
|null |json | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_enum |31
|null |YES |enum |5 |20
|null |null |null |utf8mb4
|utf8mb4_unicode_ci|enum('enum1','enum2','enum3')| |
|select,insert,update,references| | |
|def |wen_source |all_types |f_bit11 |32
|null |YES |bit |null |null
|1 |null |null |null
|null |bit(1) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_bit1 |33
|null |YES |bit |null |null
|1 |null |null |null
|null |bit(1) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_bit64 |34
|null |YES |bit |null |null
|64 |null |null |null
|null |bit(64) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_binary1 |35
|null |YES |binary |1 |1
|null |null |null |null
|null |binary(1) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_binary |36
|null |YES |binary |64 |64
|null |null |null |null
|null |binary(64) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_varbinary |37
|null |YES |varbinary |100 |100
|null |null |null |null
|null |varbinary(100) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_tinyblob |38
|null |YES |tinyblob |255 |255
|null |null |null |null
|null |tinyblob | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_blob |39
|null |YES |blob |65535 |65535
|null |null |null |null
|null |blob | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_mediumblob |40
|null |YES |mediumblob|16777215 |16777215
|null |null |null |null
|null |mediumblob | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_longblob |41
|null |YES |longblob |4294967295 |4294967295
|null |null |null |null
|null |longblob | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_geometry |42
|null |YES |geometry |null |null
|null |null |null |null
|null |geometry | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_date |43
|null |YES |date |null |null
|null |null |null |null
|null |date | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_time |44
|null |YES |time |null |null
|null |null |3 |null
|null |time(3) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_year |45
|null |YES |year |null |null
|null |null |null |null
|null |year(4) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_datetime |46
|null |YES |datetime |null |null
|null |null |3 |null
|null |datetime(3) | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_timestamp1 |47
|null |YES |timestamp |null |null
|null |null |0 |null
|null |timestamp | |
|select,insert,update,references| | |
|def |wen_source |all_types |f_timestamp |48
|null |YES |timestamp |null |null
|null |null |3 |null
|null |timestamp(3) | |
|select,insert,update,references| | |
+-------------+------------+----------+--------------------+----------------+--------------+-----------+----------+------------------------+----------------------+-----------------+-------------+------------------+------------------+------------------+-----------------------------+----------+--------------+-------------------------------+--------------+---------------------+
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
