This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 5f544aaf27 [INLONG-8369][Manager] Add MongoDB field type mapping
strategy to improve usability (#8370)
5f544aaf27 is described below
commit 5f544aaf2752cc7065fa69c529bfff2f5791ba20
Author: chestnufang <[email protected]>
AuthorDate: Mon Jul 3 10:07:52 2023 +0800
[INLONG-8369][Manager] Add MongoDB field type mapping strategy to improve
usability (#8370)
Co-authored-by: chestnufang <[email protected]>
---
.../inlong/manager/common/consts/DataNodeType.java | 1 +
.../strategy/MongoDBFieldTypeStrategy.java | 43 ++++++++++++
.../main/resources/mongodb-field-type-mapping.yaml | 81 ++++++++++++++++++++++
.../pojo/sort/node/ExtractNodeProviderFactory.java | 4 +-
.../{MongoProvider.java => MongoDBProvider.java} | 11 ++-
.../manager/pojo/sort/util/FieldInfoUtilsTest.java | 13 ++++
6 files changed, 148 insertions(+), 5 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
index 05bf9ef281..09bb35344a 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
@@ -35,4 +35,5 @@ public class DataNodeType {
public static final String POSTGRESQL = "POSTGRESQL";
public static final String ORACLE = "ORACLE";
public static final String SQLSERVER = "SQLSERVER";
+ public static final String MONGODB = "MONGODB";
}
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/MongoDBFieldTypeStrategy.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/MongoDBFieldTypeStrategy.java
new file mode 100644
index 0000000000..0e760b87c0
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/MongoDBFieldTypeStrategy.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.common.fieldtype.strategy;
+
+import org.apache.inlong.manager.common.consts.DataNodeType;
+import org.apache.inlong.manager.common.fieldtype.FieldTypeMappingReader;
+
+import org.apache.commons.lang3.StringUtils;
+
+import static
org.apache.inlong.manager.common.consts.InlongConstants.LEFT_BRACKET;
+
+/**
+ * The mongoDB field type mapping strategy
+ */
+public class MongoDBFieldTypeStrategy implements FieldTypeMappingStrategy {
+
+ private final FieldTypeMappingReader reader;
+
+ public MongoDBFieldTypeStrategy() {
+ this.reader = new FieldTypeMappingReader(DataNodeType.MONGODB);
+ }
+
+ @Override
+ public String getFieldTypeMapping(String sourceType) {
+ String dataType = StringUtils.substringBefore(sourceType,
LEFT_BRACKET).toUpperCase();
+ return reader.getFIELD_TYPE_MAPPING_MAP().getOrDefault(dataType,
sourceType.toUpperCase());
+ }
+}
diff --git
a/inlong-manager/manager-common/src/main/resources/mongodb-field-type-mapping.yaml
b/inlong-manager/manager-common/src/main/resources/mongodb-field-type-mapping.yaml
new file mode 100644
index 0000000000..32bf65d23d
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/resources/mongodb-field-type-mapping.yaml
@@ -0,0 +1,81 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source.type.to.target.type.converter:
+
+ - source.type: INTEGER
+ target.type: INT
+
+ - source.type: INT
+ target.type: INT
+
+ - source.type: FLOAT
+ target.type: FLOAT
+
+ - source.type: DOUBLE
+ target.type: DOUBLE
+
+ - source.type: BOOLEAN
+ target.type: BOOLEAN
+
+ - source.type: DECIMAL
+ target.type: DECIMAL
+
+ - source.type: DECIMAL128
+ target.type: DECIMAL
+
+ - source.type: SHORT
+ target.type: SMALLINT
+
+ - source.type: LONG
+ target.type: LONG
+
+ - source.type: STRING
+ target.type: STRING
+
+ - source.type: OBJECTID
+ target.type: STRING
+
+ - source.type: UUID
+ target.type: STRING
+
+ - source.type: SYMBOL
+ target.type: STRING
+
+ - source.type: MD5
+ target.type: STRING
+
+ - source.type: JAVASCRIPT
+ target.type: STRING
+
+ - source.type: REGEX
+ target.type: STRING
+
+ - source.type: DATE
+ target.type: DATE
+
+ - source.type: TIMESTAMP
+ target.type: TIMESTAMP
+
+ - source.type: OBJECT
+ target.type: STRUCT
+
+ - source.type: ARRAY
+ target.type: ARRAY
+
+ - source.type: BINDATA
+ target.type: VARBINARY
\ No newline at end of file
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/ExtractNodeProviderFactory.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/ExtractNodeProviderFactory.java
index 6e95425ef2..189890e733 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/ExtractNodeProviderFactory.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/ExtractNodeProviderFactory.java
@@ -22,7 +22,7 @@ import
org.apache.inlong.manager.common.exceptions.BusinessException;
import org.apache.inlong.manager.pojo.sort.node.base.ExtractNodeProvider;
import org.apache.inlong.manager.pojo.sort.node.provider.HudiProvider;
import org.apache.inlong.manager.pojo.sort.node.provider.KafkaProvider;
-import org.apache.inlong.manager.pojo.sort.node.provider.MongoProvider;
+import org.apache.inlong.manager.pojo.sort.node.provider.MongoDBProvider;
import org.apache.inlong.manager.pojo.sort.node.provider.MySQLBinlogProvider;
import org.apache.inlong.manager.pojo.sort.node.provider.OracleProvider;
import org.apache.inlong.manager.pojo.sort.node.provider.PostgreSQLProvider;
@@ -48,7 +48,7 @@ public class ExtractNodeProviderFactory {
// The Providers Parsing SourceInfo to ExtractNode which sort needed
EXTRACT_NODE_PROVIDER_LIST.add(new HudiProvider());
EXTRACT_NODE_PROVIDER_LIST.add(new KafkaProvider());
- EXTRACT_NODE_PROVIDER_LIST.add(new MongoProvider());
+ EXTRACT_NODE_PROVIDER_LIST.add(new MongoDBProvider());
EXTRACT_NODE_PROVIDER_LIST.add(new OracleProvider());
EXTRACT_NODE_PROVIDER_LIST.add(new PulsarProvider());
EXTRACT_NODE_PROVIDER_LIST.add(new RedisProvider());
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MongoProvider.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MongoDBProvider.java
similarity index 81%
rename from
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MongoProvider.java
rename to
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MongoDBProvider.java
index 2b87c75221..f93db2b064 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MongoProvider.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/MongoDBProvider.java
@@ -18,6 +18,8 @@
package org.apache.inlong.manager.pojo.sort.node.provider;
import org.apache.inlong.manager.common.consts.SourceType;
+import
org.apache.inlong.manager.common.fieldtype.strategy.FieldTypeMappingStrategy;
+import
org.apache.inlong.manager.common.fieldtype.strategy.MongoDBFieldTypeStrategy;
import org.apache.inlong.manager.pojo.sort.node.base.ExtractNodeProvider;
import org.apache.inlong.manager.pojo.source.mongodb.MongoDBSource;
import org.apache.inlong.manager.pojo.stream.StreamNode;
@@ -29,9 +31,11 @@ import java.util.List;
import java.util.Map;
/**
- * The Provider for creating Mongo extract nodes.
+ * The Provider for creating MongoDB extract nodes.
*/
-public class MongoProvider implements ExtractNodeProvider {
+public class MongoDBProvider implements ExtractNodeProvider {
+
+ private static final FieldTypeMappingStrategy FIELD_TYPE_MAPPING_STRATEGY
= new MongoDBFieldTypeStrategy();
@Override
public Boolean accept(String sourceType) {
@@ -41,7 +45,8 @@ public class MongoProvider implements ExtractNodeProvider {
@Override
public ExtractNode createExtractNode(StreamNode streamNodeInfo) {
MongoDBSource source = (MongoDBSource) streamNodeInfo;
- List<FieldInfo> fieldInfos =
parseStreamFieldInfos(source.getFieldList(), source.getSourceName());
+ List<FieldInfo> fieldInfos =
parseStreamFieldInfos(source.getFieldList(), source.getSourceName(),
+ FIELD_TYPE_MAPPING_STRATEGY);
Map<String, String> properties =
parseProperties(source.getProperties());
return new MongoExtractNode(
diff --git
a/inlong-manager/manager-pojo/src/test/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtilsTest.java
b/inlong-manager/manager-pojo/src/test/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtilsTest.java
index 01b830eb20..5e7ad52789 100644
---
a/inlong-manager/manager-pojo/src/test/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtilsTest.java
+++
b/inlong-manager/manager-pojo/src/test/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtilsTest.java
@@ -17,6 +17,7 @@
package org.apache.inlong.manager.pojo.sort.util;
+import
org.apache.inlong.manager.common.fieldtype.strategy.MongoDBFieldTypeStrategy;
import
org.apache.inlong.manager.common.fieldtype.strategy.MySQLFieldTypeStrategy;
import
org.apache.inlong.manager.common.fieldtype.strategy.OracleFieldTypeStrategy;
import
org.apache.inlong.manager.common.fieldtype.strategy.PostgreSQLFieldTypeStrategy;
@@ -84,4 +85,16 @@ public class FieldInfoUtilsTest {
TypeInfo typeInfo = fieldInfo.getFormatInfo().getTypeInfo();
Assertions.assertTrue(typeInfo instanceof StringTypeInfo);
}
+
+ @Test
+ public void testMongoDBFieldTypeInfo() {
+ StreamField streamField = new StreamField();
+ streamField.setIsMetaField(0);
+ streamField.setFieldName("UUID");
+ streamField.setFieldType("uuid");
+ FieldInfo fieldInfo = FieldInfoUtils.parseStreamFieldInfo(streamField,
+ "nodeId", new MongoDBFieldTypeStrategy());
+ TypeInfo typeInfo = fieldInfo.getFormatInfo().getTypeInfo();
+ Assertions.assertTrue(typeInfo instanceof StringTypeInfo);
+ }
}
