This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new f79b04eb29 [INLONG-8266][Manager] Optimize PostgreSQL field type
mapping strategy with the customized configuration file (#8268)
f79b04eb29 is described below
commit f79b04eb297b3df6f2ce612c4220ddd85e385c04
Author: chestnufang <[email protected]>
AuthorDate: Tue Jun 20 14:32:03 2023 +0800
[INLONG-8266][Manager] Optimize PostgreSQL field type mapping strategy with
the customized configuration file (#8268)
Co-authored-by: chestnufang <[email protected]>
---
inlong-manager/manager-common/pom.xml | 5 +-
.../common/fieldtype/FieldTypeMappingReader.java | 112 ++++++++++++++
.../fieldtype/datasource/BaseFieldTypeMapping.java | 38 -----
.../datasource/PostgreSQLFieldTypeMapping.java | 168 ---------------------
.../strategy/PostgreSQLFieldTypeStrategy.java | 16 +-
.../resources/postgresql-field-type-mapping.yaml | 105 +++++++++++++
licenses/inlong-manager/LICENSE | 1 -
7 files changed, 235 insertions(+), 210 deletions(-)
diff --git a/inlong-manager/manager-common/pom.xml
b/inlong-manager/manager-common/pom.xml
index bc6f8ecd10..f5590ca23d 100644
--- a/inlong-manager/manager-common/pom.xml
+++ b/inlong-manager/manager-common/pom.xml
@@ -123,6 +123,9 @@
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ </dependency>
</dependencies>
-
</project>
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/FieldTypeMappingReader.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/FieldTypeMappingReader.java
new file mode 100644
index 0000000000..24ae532df4
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/FieldTypeMappingReader.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.common.fieldtype;
+
+import com.google.common.collect.Maps;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * data field type mapping file reader
+ */
+@Slf4j
+public class FieldTypeMappingReader implements Serializable {
+
+ /**
+ * Field type mapping converter template file name.
+ */
+ private static final String FILE_TYPE_MAPPING_CONVERTER_TEMPLATE_NAME =
"%s-field-type-mapping.yaml";
+
+ /**
+ * Source type to target type key in converter file.
+ */
+ private static final String SOURCE_TO_TARGET_KEY =
"source.type.to.target.type.converter";
+
+ /**
+ * Field type mapping source type key
+ */
+ private static final String MAPPING_SOURCE_TYPE_KEY = "source.type";
+
+ /**
+ * Field type mapping target type key
+ */
+ private static final String MAPPING_TARGET_TYPE_KEY = "target.type";
+
+ @Getter
+ protected final String streamType;
+ @Getter
+ protected final Map<String, String> FIELD_TYPE_MAPPING_MAP =
Maps.newHashMap();
+
+ public FieldTypeMappingReader(String streamType) {
+ this.streamType = streamType;
+ log.info("Field type mapping reader stream type:{}.", streamType);
+ String converterMappingFileName = String
+ .format(FILE_TYPE_MAPPING_CONVERTER_TEMPLATE_NAME,
streamType.toLowerCase());
+ try {
+ URL resource =
Thread.currentThread().getContextClassLoader().getResource(converterMappingFileName);
+ if (Objects.isNull(resource)) {
+ throw new IllegalArgumentException(
+ String.format("Resource for the field type mapping
converter %s not found in classpath.",
+ converterMappingFileName));
+ }
+ Yaml yamlReader = new Yaml();
+ Map<?, ?> converterConf = yamlReader.loadAs(new InputStreamReader(
+ resource.openStream()), Map.class);
+ readerOption(converterConf, SOURCE_TO_TARGET_KEY,
FIELD_TYPE_MAPPING_MAP);
+ } catch (Exception e) {
+ log.error("Yaml reader read option error", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * reader option from converter file
+ *
+ * @param converterConf converterConf
+ * @param key key in converter file
+ * @param typeMapping type mapping map
+ */
+ @SuppressWarnings("unchecked")
+ private void readerOption(Map<?, ?> converterConf, String key, Map<String,
String> typeMapping) {
+ if (converterConf.containsKey(key)) {
+ List<Map<String, String>> typeMappings = (List<Map<String,
String>>) converterConf.get(key);
+ for (Map<String, String> mapping : typeMappings) {
+ String sourceType = mapping.get(MAPPING_SOURCE_TYPE_KEY);
+ String targetType = mapping.get(MAPPING_TARGET_TYPE_KEY);
+ if (StringUtils.isBlank(sourceType) ||
StringUtils.isBlank(targetType)) {
+ throw new IllegalArgumentException(
+ String.format("Reader source type: %s, target
type: %s are not valid.",
+ sourceType, targetType));
+ }
+ typeMapping.put(sourceType, targetType);
+ }
+ } else {
+ log.warn("Converter field type mapping conf can't find key: {} in
converter file type {}.", key,
+ streamType);
+ }
+ }
+}
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/datasource/BaseFieldTypeMapping.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/datasource/BaseFieldTypeMapping.java
deleted file mode 100644
index 9211914e91..0000000000
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/datasource/BaseFieldTypeMapping.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.common.fieldtype.datasource;
-
-/**
- * The interface of base field type mapping
- */
-public interface BaseFieldTypeMapping {
-
- /**
- * Get the source field type
- *
- * @return The source field type
- */
- String getSourceType();
-
- /**
- * Get the target field type of inlong field type mapping
- *
- * @return The target field type
- */
- String getTargetType();
-}
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/datasource/PostgreSQLFieldTypeMapping.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/datasource/PostgreSQLFieldTypeMapping.java
deleted file mode 100644
index 9f0e46655c..0000000000
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/datasource/PostgreSQLFieldTypeMapping.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.manager.common.fieldtype.datasource;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import static
org.apache.inlong.manager.common.consts.InlongConstants.LEFT_BRACKET;
-
-/**
- * The enum of PostgreSQL field type mapping.
- */
-public enum PostgreSQLFieldTypeMapping implements BaseFieldTypeMapping {
-
- /**
- * SMALLINT TYPE
- */
- SMALLINT("SMALLINT", "SMALLINT"),
-
- INT2("INT2", "SMALLINT"),
-
- SMALL_SERIAL("SMALLSERIAL", "SMALLINT"),
-
- SERIAL2("SERIAL2", "SMALLINT"),
-
- /**
- * INT TYPE
- */
- SERIAL("SERIAL", "INT"),
-
- INT4("INT4", "INT"),
-
- INT("INT", "INT"),
-
- INTEGER("INTEGER", "INT"),
-
- INT8("INT8", "BIGINT"),
-
- /**
- * BIGINT TYPE
- */
- BIGINT("BIGINT", "BIGINT"),
-
- BIGSERIAL("BIGSERIAL", "BIGINT"),
-
- /**
- * FLOAT TYPE
- */
- REAL("REAL", "FLOAT"),
-
- FLOAT4("FLOAT4", "FLOAT"),
-
- /**
- * DOUBLE TYPE
- */
- FLOAT8("FLOAT8", "DOUBLE"),
-
- DOUBLE("DOUBLE", "DOUBLE"),
-
- DOUBLE_PRECISION("DOUBLE PRECISION", "DOUBLE"),
-
- /**
- * DECIMAL TYPE
- */
- NUMERIC("NUMERIC", "DECIMAL"),
-
- DECIMAL("DECIMAL", "DECIMAL"),
-
- /**
- * BOOLEAN TYPE
- */
- BOOLEAN("BOOLEAN", "BOOLEAN"),
-
- /**
- * DATE TYPE
- */
- DATE("DATE", "DATE"),
-
- /**
- * TIME TYPE
- */
- TIME("TIME", "TIME"),
-
- TIMESTAMP("TIMESTAMP", "TIMESTAMP"),
-
- /**
- * STRING TYPE
- */
- CHAR("CHAR", "STRING"),
-
- CHARACTER("CHARACTER", "STRING"),
-
- VARCHAR("VARCHAR", "STRING"),
-
- CHARACTER_VARYING("CHARACTER VARYING", "STRING"),
-
- TEXT("TEXT", "STRING"),
-
- /**
- * BYTES TYPE
- */
- BYTEA("BYTEA", "VARBINARY"),
-
- /**
- * ARRAY TYPE
- */
- ARRAY("ARRAY", "ARRAY");
-
- /**
- * The source data field type
- */
- private final String sourceType;
-
- /**
- * The target data field type
- */
- private final String targetType;
-
- PostgreSQLFieldTypeMapping(String sourceType, String targetType) {
- this.sourceType = sourceType;
- this.targetType = targetType;
- }
-
- @Override
- public String getSourceType() {
- return sourceType;
- }
-
- @Override
- public String getTargetType() {
- return targetType;
- }
-
- private static final Map<String, String> FIELD_TYPE_MAPPING_MAP = new
HashMap<>();
-
- static {
- Stream.of(values()).forEach(v ->
FIELD_TYPE_MAPPING_MAP.put(v.getSourceType(), v.getTargetType()));
- }
-
- /**
- * Get the field type of inlong field type mapping by the source field
type.
- *
- * @param sourceType the source field type
- * @return the target field type of inlong field type mapping
- */
- public static String getFieldTypeMapping(String sourceType) {
- String dataType = StringUtils.substringBefore(sourceType,
LEFT_BRACKET).toUpperCase();
- return FIELD_TYPE_MAPPING_MAP.getOrDefault(dataType,
sourceType.toUpperCase());
- }
-}
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
index 235bdb5db5..304c88aa35 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/PostgreSQLFieldTypeStrategy.java
@@ -17,15 +17,27 @@
package org.apache.inlong.manager.common.fieldtype.strategy;
-import
org.apache.inlong.manager.common.fieldtype.datasource.PostgreSQLFieldTypeMapping;
+import org.apache.inlong.manager.common.consts.StreamType;
+import org.apache.inlong.manager.common.fieldtype.FieldTypeMappingReader;
+
+import org.apache.commons.lang3.StringUtils;
+
+import static
org.apache.inlong.manager.common.consts.InlongConstants.LEFT_BRACKET;
/**
* The postgresql field type mapping strategy
*/
public class PostgreSQLFieldTypeStrategy implements FieldTypeMappingStrategy {
+ private final FieldTypeMappingReader reader;
+
+ public PostgreSQLFieldTypeStrategy() {
+ this.reader = new FieldTypeMappingReader(StreamType.POSTGRESQL);
+ }
+
@Override
public String getFieldTypeMapping(String sourceType) {
- return PostgreSQLFieldTypeMapping.getFieldTypeMapping(sourceType);
+ String dataType = StringUtils.substringBefore(sourceType,
LEFT_BRACKET).toUpperCase();
+ return reader.getFIELD_TYPE_MAPPING_MAP().getOrDefault(dataType,
sourceType.toUpperCase());
}
}
diff --git
a/inlong-manager/manager-common/src/main/resources/postgresql-field-type-mapping.yaml
b/inlong-manager/manager-common/src/main/resources/postgresql-field-type-mapping.yaml
new file mode 100644
index 0000000000..256ebd0bed
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/resources/postgresql-field-type-mapping.yaml
@@ -0,0 +1,105 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source.type.to.target.type.converter:
+
+ - source.type: SMALLINT
+ target.type: SMALLINT
+
+ - source.type: INT2
+ target.type: SMALLINT
+
+ - source.type: SMALLSERIAL
+ target.type: SMALLINT
+
+ - source.type: SERIAL2
+ target.type: SMALLINT
+
+ - source.type: SERIAL
+ target.type: INT
+
+ - source.type: INT4
+ target.type: INT
+
+ - source.type: INT
+ target.type: INT
+
+ - source.type: INTEGER
+ target.type: INT
+
+ - source.type: INT8
+ target.type: BIGINT
+
+ - source.type: BIGINT
+ target.type: BIGINT
+
+ - source.type: BIGSERIAL
+ target.type: BIGINT
+
+ - source.type: REAL
+ target.type: FLOAT
+
+ - source.type: FLOAT4
+ target.type: FLOAT
+
+ - source.type: FLOAT8
+ target.type: DOUBLE
+
+ - source.type: DOUBLE
+ target.type: DOUBLE
+
+ - source.type: DOUBLE PRECISION
+ target.type: DOUBLE
+
+ - source.type: NUMERIC
+ target.type: DECIMAL
+
+ - source.type: DECIMAL
+ target.type: DECIMAL
+
+ - source.type: BOOLEAN
+ target.type: BOOLEAN
+
+ - source.type: DATE
+ target.type: DATE
+
+ - source.type: TIME
+ target.type: TIME
+
+ - source.type: TIMESTAMP
+ target.type: TIMESTAMP
+
+ - source.type: CHAR
+ target.type: STRING
+
+ - source.type: CHARACTER
+ target.type: STRING
+
+ - source.type: VARCHAR
+ target.type: STRING
+
+ - source.type: CHARACTER VARYING
+ target.type: STRING
+
+ - source.type: TEXT
+ target.type: STRING
+
+ - source.type: BYTEA
+ target.type: VARBINARY
+
+ - source.type: ARRAY
+ target.type: ARRAY
\ No newline at end of file
diff --git a/licenses/inlong-manager/LICENSE b/licenses/inlong-manager/LICENSE
index c57875baaf..a902502693 100644
--- a/licenses/inlong-manager/LICENSE
+++ b/licenses/inlong-manager/LICENSE
@@ -593,7 +593,6 @@ The text of each license is the standard Apache 2.0 license.
org.apache.zookeeper:zookeeper:3.6.3 - Apache ZooKeeper - Server
(https://github.com/apache/zookeeper/tree/release-3.6.3/zookeeper-server),
(Apache License, Version 2.0)
org.apache.zookeeper:zookeeper-jute:3.6.3 - Apache ZooKeeper - Jute
(https://github.com/apache/zookeeper/tree/release-3.6.3/zookeeper-jute),
(Apache License, Version 2.0)
-
========================================================================
Apache 2.0 License
========================================================================
