This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new e280bd5e0c [INLONG-8288][Manager] Add MySQL field type mapping
strategy to improve usability (#8289)
e280bd5e0c is described below
commit e280bd5e0ce98fe88806c74d3334d6b7a9d3dede
Author: chestnufang <[email protected]>
AuthorDate: Sun Jun 25 11:14:52 2023 +0800
[INLONG-8288][Manager] Add MySQL field type mapping strategy to improve
usability (#8289)
Co-authored-by: chestnufang <[email protected]>
---
...peStrategy.java => MySQLFieldTypeStrategy.java} | 10 +-
.../strategy/PostgreSQLFieldTypeStrategy.java | 4 +-
.../main/resources/mysql-field-type-mapping.yaml | 228 +++++++++++++++++++++
.../sort/node/provider/MySQLBinlogProvider.java | 7 +-
.../pojo/sort/node/provider/MySQLProvider.java | 7 +-
.../manager/pojo/sort/util/FieldInfoUtilsTest.java | 16 +-
6 files changed, 262 insertions(+), 10 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/MySQLFieldTypeStrategy.java
similarity index 82%
copy from
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
copy to
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/MySQLFieldTypeStrategy.java
index 304c88aa35..114519a6ac 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/MySQLFieldTypeStrategy.java
@@ -17,7 +17,7 @@
package org.apache.inlong.manager.common.fieldtype.strategy;
-import org.apache.inlong.manager.common.consts.StreamType;
+import org.apache.inlong.manager.common.consts.DataNodeType;
import org.apache.inlong.manager.common.fieldtype.FieldTypeMappingReader;
import org.apache.commons.lang3.StringUtils;
@@ -25,14 +25,14 @@ import org.apache.commons.lang3.StringUtils;
import static
org.apache.inlong.manager.common.consts.InlongConstants.LEFT_BRACKET;
/**
- * The postgresql field type mapping strategy
+ * The mysql field type mapping strategy
*/
-public class PostgreSQLFieldTypeStrategy implements FieldTypeMappingStrategy {
+public class MySQLFieldTypeStrategy implements FieldTypeMappingStrategy {
private final FieldTypeMappingReader reader;
- public PostgreSQLFieldTypeStrategy() {
- this.reader = new FieldTypeMappingReader(StreamType.POSTGRESQL);
+ public MySQLFieldTypeStrategy() {
+ this.reader = new FieldTypeMappingReader(DataNodeType.MYSQL);
}
@Override
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
index 304c88aa35..17e9c98d7b 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
@@ -17,7 +17,7 @@
package org.apache.inlong.manager.common.fieldtype.strategy;
-import org.apache.inlong.manager.common.consts.StreamType;
+import org.apache.inlong.manager.common.consts.DataNodeType;
import org.apache.inlong.manager.common.fieldtype.FieldTypeMappingReader;
import org.apache.commons.lang3.StringUtils;
@@ -32,7 +32,7 @@ public class PostgreSQLFieldTypeStrategy implements
FieldTypeMappingStrategy {
private final FieldTypeMappingReader reader;
public PostgreSQLFieldTypeStrategy() {
- this.reader = new FieldTypeMappingReader(StreamType.POSTGRESQL);
+ this.reader = new FieldTypeMappingReader(DataNodeType.POSTGRESQL);
}
@Override
diff --git
a/inlong-manager/manager-common/src/main/resources/mysql-field-type-mapping.yaml
b/inlong-manager/manager-common/src/main/resources/mysql-field-type-mapping.yaml
new file mode 100644
index 0000000000..0d2a0599cf
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/resources/mysql-field-type-mapping.yaml
@@ -0,0 +1,228 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source.type.to.target.type.converter:
+
+ - source.type: TINYINT
+ target.type: TINYINT
+
+ - source.type: SMALLINT
+ target.type: SMALLINT
+
+ - source.type: TINYINT UNSIGNED
+ target.type: SMALLINT
+
+ - source.type: TINYINT UNSIGNED ZEROFILL
+ target.type: SMALLINT
+
+ - source.type: INT
+ target.type: INT
+
+ - source.type: INTEGER
+ target.type: INT
+
+ - source.type: YEAR
+ target.type: INT
+
+ - source.type: SHORT
+ target.type: SHORT
+
+ - source.type: MEDIUMINT
+ target.type: INT
+
+ - source.type: SMALLINT UNSIGNED
+ target.type: INT
+
+ - source.type: SMALLINT UNSIGNED ZEROFILL
+ target.type: INT
+
+ - source.type: BIGINT
+ target.type: LONG
+
+ - source.type: INT UNSIGNED
+ target.type: LONG
+
+ - source.type: MEDIUMINT UNSIGNED
+ target.type: LONG
+
+ - source.type: MEDIUMINT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: INT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: BIGINT UNSIGNED
+ target.type: DECIMAL
+
+ - source.type: BIGINT UNSIGNED ZEROFILL
+ target.type: DECIMAL
+
+ - source.type: SERIAL
+ target.type: DECIMAL
+
+ - source.type: FLOAT
+ target.type: FLOAT
+
+ - source.type: FLOAT UNSIGNED
+ target.type: FLOAT
+
+ - source.type: FLOAT UNSIGNED ZEROFILL
+ target.type: FLOAT
+
+ - source.type: DOUBLE
+ target.type: DOUBLE
+
+ - source.type: DOUBLE UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: DOUBLE UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: DOUBLE PRECISION
+ target.type: DOUBLE
+
+ - source.type: DOUBLE PRECISION UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: REAL
+ target.type: DOUBLE
+
+ - source.type: REAL UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: REAL UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: NUMERIC
+ target.type: DECIMAL
+
+ - source.type: NUMERIC UNSIGNED
+ target.type: DECIMAL
+
+ - source.type: NUMERIC UNSIGNED ZEROFILL
+ target.type: DECIMAL
+
+ - source.type: DECIMAL
+ target.type: DECIMAL
+
+ - source.type: DECIMAL UNSIGNED
+ target.type: DECIMAL
+
+ - source.type: DECIMAL UNSIGNED ZEROFILL
+ target.type: DECIMAL
+
+ - source.type: FIXED
+ target.type: DECIMAL
+
+ - source.type: FIXED UNSIGNED
+ target.type: DECIMAL
+
+ - source.type: FIXED UNSIGNED ZEROFILL
+ target.type: DECIMAL
+
+ - source.type: BOOLEAN
+ target.type: BOOLEAN
+
+ - source.type: DATE
+ target.type: DATE
+
+ - source.type: TIME
+ target.type: TIME
+
+ - source.type: DATETIME
+ target.type: TIMESTAMP
+
+ - source.type: TIMESTAMP
+ target.type: TIMESTAMP
+
+ - source.type: CHAR
+ target.type: STRING
+
+ - source.type: JSON
+ target.type: STRING
+
+ - source.type: BIT
+ target.type: STRING
+
+ - source.type: VARCHAR
+ target.type: STRING
+
+ - source.type: TEXT
+ target.type: STRING
+
+ - source.type: BLOB
+ target.type: STRING
+
+ - source.type: TINYBLOB
+ target.type: STRING
+
+ - source.type: TINYTEXT
+ target.type: STRING
+
+ - source.type: MEDIUMBLOB
+ target.type: STRING
+
+ - source.type: MEDIUMTEXT
+ target.type: STRING
+
+ - source.type: LONGBLOB
+ target.type: STRING
+
+ - source.type: LONGTEXT
+ target.type: STRING
+
+ - source.type: VARBINARY
+ target.type: STRING
+
+ - source.type: GEOMETRY
+ target.type: STRING
+
+ - source.type: POINT
+ target.type: STRING
+
+ - source.type: LINESTRING
+ target.type: STRING
+
+ - source.type: POLYGON
+ target.type: STRING
+
+ - source.type: MULTIPOINT
+ target.type: STRING
+
+ - source.type: MULTILINESTRING
+ target.type: STRING
+
+ - source.type: MULTIPOLYGON
+ target.type: STRING
+
+ - source.type: GEOMETRYCOLLECTION
+ target.type: STRING
+
+ - source.type: ENUM
+ target.type: STRING
+
+ - source.type: STRING
+ target.type: STRING
+
+ - source.type: BINARY
+ target.type: BINARY
+
+ - source.type: BYTE
+ target.type: BYTE
\ No newline at end of file
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MySQLBinlogProvider.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MySQLBinlogProvider.java
index a95fc99008..894b83eb1c 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MySQLBinlogProvider.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MySQLBinlogProvider.java
@@ -18,6 +18,8 @@
package org.apache.inlong.manager.pojo.sort.node.provider;
import org.apache.inlong.manager.common.consts.SourceType;
+import
org.apache.inlong.manager.common.fieldtype.strategy.FieldTypeMappingStrategy;
+import
org.apache.inlong.manager.common.fieldtype.strategy.MySQLFieldTypeStrategy;
import org.apache.inlong.manager.pojo.sort.node.base.ExtractNodeProvider;
import org.apache.inlong.manager.pojo.source.mysql.MySQLBinlogSource;
import org.apache.inlong.manager.pojo.stream.StreamNode;
@@ -35,6 +37,8 @@ import java.util.Map;
*/
public class MySQLBinlogProvider implements ExtractNodeProvider {
+ private static final FieldTypeMappingStrategy FIELD_TYPE_MAPPING_STRATEGY
= new MySQLFieldTypeStrategy();
+
@Override
public Boolean accept(String sourceType) {
return SourceType.MYSQL_BINLOG.equals(sourceType);
@@ -43,7 +47,8 @@ public class MySQLBinlogProvider implements
ExtractNodeProvider {
@Override
public ExtractNode createExtractNode(StreamNode streamNodeInfo) {
MySQLBinlogSource binlogSource = (MySQLBinlogSource) streamNodeInfo;
- List<FieldInfo> fieldInfos =
parseStreamFieldInfos(binlogSource.getFieldList(),
binlogSource.getSourceName());
+ List<FieldInfo> fieldInfos =
parseStreamFieldInfos(binlogSource.getFieldList(), binlogSource.getSourceName(),
+ FIELD_TYPE_MAPPING_STRATEGY);
Map<String, String> properties =
parseProperties(binlogSource.getProperties());
final String database = binlogSource.getDatabaseWhiteList();
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MySQLProvider.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MySQLProvider.java
index 8a46d4bacb..a366688bc9 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MySQLProvider.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MySQLProvider.java
@@ -18,6 +18,8 @@
package org.apache.inlong.manager.pojo.sort.node.provider;
import org.apache.inlong.manager.common.consts.SinkType;
+import
org.apache.inlong.manager.common.fieldtype.strategy.FieldTypeMappingStrategy;
+import
org.apache.inlong.manager.common.fieldtype.strategy.MySQLFieldTypeStrategy;
import org.apache.inlong.manager.pojo.sink.mysql.MySQLSink;
import org.apache.inlong.manager.pojo.sink.mysql.MySQLSinkDTO;
import org.apache.inlong.manager.pojo.sort.node.base.LoadNodeProvider;
@@ -38,6 +40,8 @@ import java.util.Map;
*/
public class MySQLProvider implements LoadNodeProvider {
+ private static final FieldTypeMappingStrategy FIELD_TYPE_MAPPING_STRATEGY
= new MySQLFieldTypeStrategy();
+
@Override
public Boolean accept(String sinkType) {
return SinkType.MYSQL.equals(sinkType);
@@ -47,7 +51,8 @@ public class MySQLProvider implements LoadNodeProvider {
public LoadNode createLoadNode(StreamNode nodeInfo, Map<String,
StreamField> constantFieldMap) {
MySQLSink mysqlSink = (MySQLSink) nodeInfo;
Map<String, String> properties =
parseProperties(mysqlSink.getProperties());
- List<FieldInfo> fieldInfos =
parseSinkFieldInfos(mysqlSink.getSinkFieldList(), mysqlSink.getSinkName());
+ List<FieldInfo> fieldInfos =
parseSinkFieldInfos(mysqlSink.getSinkFieldList(), mysqlSink.getSinkName(),
+ FIELD_TYPE_MAPPING_STRATEGY);
List<FieldRelation> fieldRelations =
parseSinkFields(mysqlSink.getSinkFieldList(), constantFieldMap);
return new MySqlLoadNode(
diff --git
a/inlong-manager/manager-pojo/src/test/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtilsTest.java
b/inlong-manager/manager-pojo/src/test/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtilsTest.java
index e370c61f62..f556820f58 100644
---
a/inlong-manager/manager-pojo/src/test/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtilsTest.java
+++
b/inlong-manager/manager-pojo/src/test/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtilsTest.java
@@ -17,9 +17,11 @@
package org.apache.inlong.manager.pojo.sort.util;
+import
org.apache.inlong.manager.common.fieldtype.strategy.MySQLFieldTypeStrategy;
import
org.apache.inlong.manager.common.fieldtype.strategy.PostgreSQLFieldTypeStrategy;
import org.apache.inlong.manager.pojo.stream.StreamField;
import org.apache.inlong.sort.formats.common.IntTypeInfo;
+import org.apache.inlong.sort.formats.common.ShortTypeInfo;
import org.apache.inlong.sort.formats.common.TypeInfo;
import org.apache.inlong.sort.protocol.FieldInfo;
@@ -32,7 +34,7 @@ import org.junit.jupiter.api.Test;
public class FieldInfoUtilsTest {
@Test
- public void testPgIntTypeInfo() {
+ public void testPostgreSQLFieldTypeInfo() {
StreamField streamField = new StreamField();
streamField.setIsMetaField(0);
streamField.setFieldName("age");
@@ -42,4 +44,16 @@ public class FieldInfoUtilsTest {
TypeInfo typeInfo = fieldInfo.getFormatInfo().getTypeInfo();
Assertions.assertTrue(typeInfo instanceof IntTypeInfo);
}
+
+ @Test
+ public void testMySQLFieldTypeInfo() {
+ StreamField streamField = new StreamField();
+ streamField.setIsMetaField(0);
+ streamField.setFieldName("age");
+ streamField.setFieldType("tinyint unsigned");
+ FieldInfo fieldInfo = FieldInfoUtils.parseStreamFieldInfo(streamField,
+ "nodeId", new MySQLFieldTypeStrategy());
+ TypeInfo typeInfo = fieldInfo.getFormatInfo().getTypeInfo();
+ Assertions.assertTrue(typeInfo instanceof ShortTypeInfo);
+ }
}
