This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/seatunnel-web.git
The following commit(s) were added to refs/heads/main by this push:
new b12506d2 [Feature][Seatunnel-web] Add FakeSource and Console data
sources into seatunnel-web (#176)
b12506d2 is described below
commit b12506d24579be36e13e611876960b7e9a66a09c
Author: Mohammad Arshad <[email protected]>
AuthorDate: Sat Aug 10 18:20:33 2024 +0530
[Feature][Seatunnel-web] Add FakeSource and Console data sources into
seatunnel-web (#176)
---
.../classloader/DatasourceLoadConfig.java | 12 ++-
.../{ => datasource-console}/pom.xml | 37 +++------
.../plugin/console/ConsoleDataSourceChannel.java | 72 +++++++++++++++++
.../plugin/console/ConsoleDataSourceConfig.java | 55 +++++++++++++
.../plugin/console/ConsoleDataSourceFactory.java | 48 +++++++++++
.../{ => datasource-fakesource}/pom.xml | 37 +++------
.../fakesource/FakeSourceDataSourceChannel.java | 94 ++++++++++++++++++++++
.../fakesource/FakeSourceDataSourceConfig.java | 52 ++++++++++++
.../fakesource/FakeSourceDataSourceFactory.java | 48 +++++++++++
.../plugin/api/DatasourcePluginTypeEnum.java | 3 +-
.../seatunnel-datasource-plugins/pom.xml | 2 +
.../src/main/bin/download_datasource.sh | 2 +
.../app/service/impl/ConnectorServiceImpl.java | 2 +-
.../impl/ConsoleDataSourceConfigSwitcher.java | 87 ++++++++++++++++++++
.../impl/FakeSourceDataSourceConfigSwitcher.java | 79 ++++++++++++++++++
.../resources/connector-datasource-mapper.yaml | 25 +++++-
.../src/main/resources/i18n_en.config | 12 +++
.../src/main/resources/i18n_zh.config | 12 +++
seatunnel-ui/src/hooks/use-source-type.ts | 3 +-
seatunnel-ui/src/locales/en_US/datasource.ts | 3 +-
seatunnel-ui/src/locales/en_US/project.ts | 1 +
seatunnel-ui/src/locales/zh_CN/datasource.ts | 3 +-
seatunnel-ui/src/locales/zh_CN/project.ts | 1 +
.../src/views/datasource/components/use-source.ts | 3 +-
.../src/views/datasource/list/use-source.ts | 3 +-
seatunnel-web-dist/pom.xml | 12 +++
26 files changed, 645 insertions(+), 63 deletions(-)
diff --git
a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java
b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java
index b2869691..b8ff93ce 100644
---
a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java
+++
b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java
@@ -97,6 +97,12 @@ public class DatasourceLoadConfig {
classLoaderFactoryName.put(
"JDBC-DB2",
"org.apache.seatunnel.datasource.plugin.db2.jdbc.Db2JdbcDataSourceFactory");
+ classLoaderFactoryName.put(
+ "FAKESOURCE",
+
"org.apache.seatunnel.datasource.plugin.fakesource.FakeSourceDataSourceFactory");
+ classLoaderFactoryName.put(
+ "CONSOLE",
+
"org.apache.seatunnel.datasource.plugin.console.ConsoleDataSourceFactory");
classLoaderJarName.put("JDBC-ORACLE", "datasource-jdbc-oracle-");
classLoaderJarName.put("JDBC-CLICKHOUSE",
"datasource-jdbc-clickhouse-");
@@ -118,6 +124,8 @@ public class DatasourceLoadConfig {
classLoaderJarName.put("JDBC-STARROCKS", "datasource-jdbc-starrocks-");
classLoaderJarName.put("MONGODB", "datasource-mongodb-");
classLoaderJarName.put("JDBC-DB2", "datasource-jdbc-db2-");
+ classLoaderJarName.put("FAKESOURCE", "datasource-fakesource-");
+ classLoaderJarName.put("CONSOLE", "datasource-console-");
}
public static final Set<String> pluginSet =
@@ -136,7 +144,9 @@ public class DatasourceLoadConfig {
"SqlServer-CDC",
"StarRocks",
"MongoDB",
- "JDBC-Db2");
+ "JDBC-Db2",
+ "FakeSource",
+ "Console");
public static Map<String, DatasourceClassLoader> datasourceClassLoaders =
new HashMap<>();
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/pom.xml
similarity index 52%
copy from seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
copy to
seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/pom.xml
index 13af3009..d97d155c 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/pom.xml
@@ -18,34 +18,17 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-datasource</artifactId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
<version>${revision}</version>
</parent>
- <artifactId>seatunnel-datasource-plugins</artifactId>
- <packaging>pom</packaging>
-
- <modules>
- <module>datasource-plugins-api</module>
- <module>datasource-all</module>
- <module>datasource-kafka</module>
- <module>datasource-elasticsearch</module>
- <module>datasource-jdbc-hive</module>
- <module>datasource-hive</module>
- <module>datasource-starrocks</module>
- <module>datasource-jdbc-clickhouse</module>
- <module>datasource-jdbc-mysql</module>
- <module>datasource-jdbc-oracle</module>
- <module>datasource-jdbc-postgresql</module>
- <module>datasource-jdbc-redshift</module>
- <module>datasource-jdbc-sqlserver</module>
- <module>datasource-jdbc-starrocks</module>
- <module>datasource-mysql-cdc</module>
- <module>datasource-s3</module>
- <module>datasource-sqlserver-cdc</module>
- <module>datasource-jdbc-tidb</module>
- <module>datasource-mongodb</module>
- <module>datasource-jdbc-db2</module>
- </modules>
-
+ <artifactId>datasource-console</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
</project>
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/src/main/java/org/apache/seatunnel/datasource/plugin/console/ConsoleDataSourceChannel.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/src/main/java/org/apache/seatunnel/datasource/plugin/console/ConsoleDataSourceChannel.java
new file mode 100644
index 00000000..63a997ca
--- /dev/null
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/src/main/java/org/apache/seatunnel/datasource/plugin/console/ConsoleDataSourceChannel.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.console;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import lombok.NonNull;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class ConsoleDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return ConsoleDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull
String pluginName) {
+ return ConsoleDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName,
+ Map<String, String> requestParams,
+ String database,
+ Map<String, String> options) {
+ return Arrays.asList("console_fake_table");
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String>
requestParams) {
+ return Arrays.asList("console_fake_database");
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String>
requestParams) {
+ return true;
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ return Collections.emptyList();
+ }
+}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/src/main/java/org/apache/seatunnel/datasource/plugin/console/ConsoleDataSourceConfig.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/src/main/java/org/apache/seatunnel/datasource/plugin/console/ConsoleDataSourceConfig.java
new file mode 100644
index 00000000..326790d8
--- /dev/null
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/src/main/java/org/apache/seatunnel/datasource/plugin/console/ConsoleDataSourceConfig.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.console;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+public class ConsoleDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "Console";
+
+ public static final DataSourcePluginInfo CONSOLE_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.FAKE_CONNECTION.getCode())
+ .build();
+
+ public static final Option<Boolean> LOG_PRINT_DATA =
+ Options.key("log.print.data")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "Flag to determine whether data should be printed
in the logs.");
+
+ public static final Option<Integer> LOG_PRINT_DELAY =
+ Options.key("log.print.delay.ms")
+ .intType()
+ .defaultValue(0)
+ .withDescription(
+ "Delay in milliseconds between printing each data
item to the logs.");
+
+ public static final OptionRule OPTION_RULE =
+
OptionRule.builder().required(LOG_PRINT_DATA).optional(LOG_PRINT_DELAY).build();
+ public static final OptionRule METADATA_RULE =
OptionRule.builder().build();
+}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/src/main/java/org/apache/seatunnel/datasource/plugin/console/ConsoleDataSourceFactory.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/src/main/java/org/apache/seatunnel/datasource/plugin/console/ConsoleDataSourceFactory.java
new file mode 100644
index 00000000..0b40fb14
--- /dev/null
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-console/src/main/java/org/apache/seatunnel/datasource/plugin/console/ConsoleDataSourceFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.console;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+
+@Slf4j
+@AutoService(DataSourceFactory.class)
+public class ConsoleDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return ConsoleDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return
Sets.newHashSet(ConsoleDataSourceConfig.CONSOLE_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new ConsoleDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/pom.xml
similarity index 52%
copy from seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
copy to
seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/pom.xml
index 13af3009..5c195d04 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/pom.xml
@@ -18,34 +18,17 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-datasource</artifactId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
<version>${revision}</version>
</parent>
- <artifactId>seatunnel-datasource-plugins</artifactId>
- <packaging>pom</packaging>
-
- <modules>
- <module>datasource-plugins-api</module>
- <module>datasource-all</module>
- <module>datasource-kafka</module>
- <module>datasource-elasticsearch</module>
- <module>datasource-jdbc-hive</module>
- <module>datasource-hive</module>
- <module>datasource-starrocks</module>
- <module>datasource-jdbc-clickhouse</module>
- <module>datasource-jdbc-mysql</module>
- <module>datasource-jdbc-oracle</module>
- <module>datasource-jdbc-postgresql</module>
- <module>datasource-jdbc-redshift</module>
- <module>datasource-jdbc-sqlserver</module>
- <module>datasource-jdbc-starrocks</module>
- <module>datasource-mysql-cdc</module>
- <module>datasource-s3</module>
- <module>datasource-sqlserver-cdc</module>
- <module>datasource-jdbc-tidb</module>
- <module>datasource-mongodb</module>
- <module>datasource-jdbc-db2</module>
- </modules>
-
+ <artifactId>datasource-fakesource</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
</project>
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/src/main/java/org/apache/seatunnel/datasource/plugin/fakesource/FakeSourceDataSourceChannel.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/src/main/java/org/apache/seatunnel/datasource/plugin/fakesource/FakeSourceDataSourceChannel.java
new file mode 100644
index 00000000..b99288ac
--- /dev/null
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/src/main/java/org/apache/seatunnel/datasource/plugin/fakesource/FakeSourceDataSourceChannel.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.fakesource;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.common.utils.JsonUtils;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import lombok.NonNull;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class FakeSourceDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return FakeSourceDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull
String pluginName) {
+ return FakeSourceDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName,
+ Map<String, String> requestParams,
+ String database,
+ Map<String, String> options) {
+ return Arrays.asList("fake_table");
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String>
requestParams) {
+ return Arrays.asList("fake_database");
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String>
requestParams) {
+ try {
+ JsonUtils.toMap(requestParams.get("fields"));
+ } catch (RuntimeException ex) {
+ throw new DataSourcePluginException(
+ "Failed to parse field parameter. " + ex.getMessage(), ex);
+ }
+ return true;
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ String fieldsJson = requestParams.get("fields");
+ if (fieldsJson == null) {
+ return Collections.emptyList();
+ }
+ Map<String, String> fields = JsonUtils.toMap(fieldsJson);
+ return fields.entrySet().stream()
+ .map(
+ entry -> {
+ TableField tableField = new TableField();
+ tableField.setName(entry.getKey());
+ tableField.setType(entry.getValue());
+ return tableField;
+ })
+ .collect(Collectors.toList());
+ }
+}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/src/main/java/org/apache/seatunnel/datasource/plugin/fakesource/FakeSourceDataSourceConfig.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/src/main/java/org/apache/seatunnel/datasource/plugin/fakesource/FakeSourceDataSourceConfig.java
new file mode 100644
index 00000000..b60c12b2
--- /dev/null
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/src/main/java/org/apache/seatunnel/datasource/plugin/fakesource/FakeSourceDataSourceConfig.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.fakesource;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+public class FakeSourceDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "FakeSource";
+
+ public static final DataSourcePluginInfo FAKESOURCE_DATASOURCE_PLUGIN_INFO
=
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.FAKE_CONNECTION.getCode())
+ .build();
+ public static final Option<String> FAKESOURCE_FIELDS =
+ Options.key("fields")
+ .stringType()
+ .defaultValue(
+ "{\n"
+ + " \"name\": \"string\",\n"
+ + " \"age\": \"int\"\n"
+ + " }")
+ .withDescription(
+ "This value will be used to populate field details
in UI, however, it won't be utilized during execution.");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder().required(FAKESOURCE_FIELDS).build();
+
+ public static final OptionRule METADATA_RULE =
OptionRule.builder().required().build();
+}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/src/main/java/org/apache/seatunnel/datasource/plugin/fakesource/FakeSourceDataSourceFactory.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/src/main/java/org/apache/seatunnel/datasource/plugin/fakesource/FakeSourceDataSourceFactory.java
new file mode 100644
index 00000000..52056130
--- /dev/null
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-fakesource/src/main/java/org/apache/seatunnel/datasource/plugin/fakesource/FakeSourceDataSourceFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.fakesource;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+
+@Slf4j
+@AutoService(DataSourceFactory.class)
+public class FakeSourceDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return FakeSourceDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return
Sets.newHashSet(FakeSourceDataSourceConfig.FAKESOURCE_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new FakeSourceDataSourceChannel();
+ }
+}
diff --git
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DatasourcePluginTypeEnum.java
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DatasourcePluginTypeEnum.java
index e5d8e004..47dc2ade 100644
---
a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DatasourcePluginTypeEnum.java
+++
b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DatasourcePluginTypeEnum.java
@@ -25,7 +25,8 @@ public enum DatasourcePluginTypeEnum {
FILE(2, "file", "文件"),
NO_STRUCTURED(3, "no_structured", "非结构化数据（NoSQLs）"),
STORAGE(4, "storage", "存储"),
- REMOTE_CONNECTION(5, "remote_connection", "远程连接");
+ REMOTE_CONNECTION(5, "remote_connection", "远程连接"),
+ FAKE_CONNECTION(6, "fake_connection", "假连接");
private final Integer code;
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
index 13af3009..7a456858 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
@@ -46,6 +46,8 @@
<module>datasource-jdbc-tidb</module>
<module>datasource-mongodb</module>
<module>datasource-jdbc-db2</module>
+ <module>datasource-fakesource</module>
+ <module>datasource-console</module>
</modules>
</project>
diff --git a/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh
b/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh
index 3ea36ab6..98544634 100644
--- a/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh
+++ b/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh
@@ -44,6 +44,8 @@ datasource_list=(
"datasource-sqlserver-cdc"
"datasource-starrocks"
"datasource-mongodb"
+ "datasource-fakesource"
+ "datasource-console"
)
# the datasource default version is 1.0.0, you can also choose a custom
version. eg: 1.1.2: sh install-datasource.sh 2.1.2
diff --git
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/ConnectorServiceImpl.java
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/ConnectorServiceImpl.java
index 92c60e72..307066a2 100644
---
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/ConnectorServiceImpl.java
+++
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/ConnectorServiceImpl.java
@@ -65,7 +65,7 @@ public class ConnectorServiceImpl extends
SeatunnelBaseServiceImpl implements IC
private static final List<String> SKIP_SOURCE = Collections.emptyList();
- private static final List<String> SKIP_SINK =
Collections.singletonList("Console");
+ private static final List<String> SKIP_SINK = Collections.emptyList();
@Autowired
public ConnectorServiceImpl(ConnectorCache connectorCache) {
diff --git
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ConsoleDataSourceConfigSwitcher.java
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ConsoleDataSourceConfigSwitcher.java
new file mode 100644
index 00000000..035cf0d1
--- /dev/null
+++
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ConsoleDataSourceConfigSwitcher.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.thirdparty.datasource.impl;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.configuration.util.RequiredOption;
+import org.apache.seatunnel.app.domain.request.connector.BusinessMode;
+import org.apache.seatunnel.app.domain.request.job.DataSourceOption;
+import org.apache.seatunnel.app.domain.request.job.SelectTableFields;
+import
org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes;
+import org.apache.seatunnel.app.dynamicforms.FormStructure;
+import
org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher;
+import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher;
+import org.apache.seatunnel.common.constants.PluginType;
+
+import com.google.auto.service.AutoService;
+
+import java.util.List;
+
+@AutoService(DataSourceConfigSwitcher.class)
+public class ConsoleDataSourceConfigSwitcher extends
AbstractDataSourceConfigSwitcher {
+
+ @Override
+ public String getDataSourceName() {
+ return "CONSOLE";
+ }
+
+ @Override
+ public FormStructure filterOptionRule(
+ String connectorName,
+ OptionRule dataSourceOptionRule,
+ OptionRule virtualTableOptionRule,
+ BusinessMode businessMode,
+ PluginType pluginType,
+ OptionRule connectorOptionRule,
+ List<RequiredOption> addRequiredOptions,
+ List<Option<?>> addOptionalOptions,
+ List<String> excludedKeys) {
+ return super.filterOptionRule(
+ connectorName,
+ dataSourceOptionRule,
+ virtualTableOptionRule,
+ businessMode,
+ pluginType,
+ connectorOptionRule,
+ addRequiredOptions,
+ addOptionalOptions,
+ excludedKeys);
+ }
+
+ @Override
+ public Config mergeDatasourceConfig(
+ Config dataSourceInstanceConfig,
+ VirtualTableDetailRes virtualTableDetail,
+ DataSourceOption dataSourceOption,
+ SelectTableFields selectTableFields,
+ BusinessMode businessMode,
+ PluginType pluginType,
+ Config connectorConfig) {
+ return super.mergeDatasourceConfig(
+ dataSourceInstanceConfig,
+ virtualTableDetail,
+ dataSourceOption,
+ selectTableFields,
+ businessMode,
+ pluginType,
+ connectorConfig);
+ }
+}
diff --git
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/FakeSourceDataSourceConfigSwitcher.java
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/FakeSourceDataSourceConfigSwitcher.java
new file mode 100644
index 00000000..29684dde
--- /dev/null
+++
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/FakeSourceDataSourceConfigSwitcher.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.thirdparty.datasource.impl;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.configuration.util.RequiredOption;
+import org.apache.seatunnel.app.domain.request.connector.BusinessMode;
+import org.apache.seatunnel.app.domain.request.job.DataSourceOption;
+import org.apache.seatunnel.app.domain.request.job.SelectTableFields;
+import
org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes;
+import org.apache.seatunnel.app.dynamicforms.FormStructure;
+import
org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher;
+import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher;
+import org.apache.seatunnel.common.constants.PluginType;
+
+import com.google.auto.service.AutoService;
+
+import java.util.List;
+
+@AutoService(DataSourceConfigSwitcher.class)
+public class FakeSourceDataSourceConfigSwitcher extends
AbstractDataSourceConfigSwitcher {
+ @Override
+ public String getDataSourceName() {
+ return "FAKESOURCE";
+ }
+
+ @Override
+ public FormStructure filterOptionRule(
+ String connectorName,
+ OptionRule dataSourceOptionRule,
+ OptionRule virtualTableOptionRule,
+ BusinessMode businessMode,
+ PluginType pluginType,
+ OptionRule connectorOptionRule,
+ List<RequiredOption> addRequiredOptions,
+ List<Option<?>> addOptionalOptions,
+ List<String> excludedKeys) {
+ return super.filterOptionRule(
+ connectorName,
+ dataSourceOptionRule,
+ virtualTableOptionRule,
+ businessMode,
+ pluginType,
+ connectorOptionRule,
+ addRequiredOptions,
+ addOptionalOptions,
+ excludedKeys);
+ }
+
+ @Override
+ public Config mergeDatasourceConfig(
+ Config dataSourceInstanceConfig,
+ VirtualTableDetailRes virtualTableDetail,
+ DataSourceOption dataSourceOption,
+ SelectTableFields selectTableFields,
+ BusinessMode businessMode,
+ PluginType pluginType,
+ Config connectorConfig) {
+ return connectorConfig;
+ }
+}
diff --git
a/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml
b/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml
index 4bcd6fd0..0ba9a780 100644
---
a/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml
+++
b/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml
@@ -59,6 +59,14 @@ connector-datasource-mapper:
dataSources:
- MongoDB
+ FakeSource:
+ dataSources:
+ - FakeSource
+
+ Console:
+ dataSources:
+ - Console
+
sourceDatasourceFeatures:
JDBC-Mysql:
businessMode:
@@ -140,6 +148,13 @@ connector-datasource-mapper:
- DATA_INTEGRATION
sceneMode:
- SINGLE_TABLE
+ FakeSource:
+ businessMode:
+ - DATA_INTEGRATION
+ - DATA_REPLICA
+ sceneMode:
+ - SINGLE_TABLE
+ - MULTIPLE_TABLE
sinkDatasourceFeatures:
JDBC-Mysql:
@@ -224,4 +239,12 @@ connector-datasource-mapper:
businessMode:
- DATA_INTEGRATION
sceneMode:
- - SINGLE_TABLE
\ No newline at end of file
+ - SINGLE_TABLE
+
+ Console:
+ businessMode:
+ - DATA_INTEGRATION
+ - DATA_REPLICA
+ sceneMode:
+ - SINGLE_TABLE
+ - MULTIPLE_TABLE
\ No newline at end of file
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/i18n_en.config
b/seatunnel-server/seatunnel-app/src/main/resources/i18n_en.config
index 750bacf0..a9b93482 100644
--- a/seatunnel-server/seatunnel-app/src/main/resources/i18n_en.config
+++ b/seatunnel-server/seatunnel-app/src/main/resources/i18n_en.config
@@ -294,3 +294,15 @@ PROXY {
username = "username"
password = "password"
}
+
+FakeSource {
+ fields = "Fields"
+ fields_description = "This value will be used to populate field details in
UI, however, it won't be utilized during execution."
+}
+
+Console {
+ log_print_data = "Log Print Data"
+ log_print_data_description = "Flag to determine whether data should be
printed in the logs."
+ log_print_delay_ms = "Log Print Delay(ms)"
+ log_print_delay_ms_description = "Delay in milliseconds between printing
each data item to the logs."
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/i18n_zh.config
b/seatunnel-server/seatunnel-app/src/main/resources/i18n_zh.config
index 7404a385..1dc5e5bb 100644
--- a/seatunnel-server/seatunnel-app/src/main/resources/i18n_zh.config
+++ b/seatunnel-server/seatunnel-app/src/main/resources/i18n_zh.config
@@ -293,3 +293,15 @@ PROXY {
username = "用户名"
password = "密码"
}
+
+FakeSource {
+ fields = "领域"
+ fields_description = "该值将用于填充 UI 中的字段详细信息，但是在执行期间不会使用它。"
+}
+
+Console {
+ log_print_data = "日志打印数据"
+ log_print_data_description = "决定是否应在日志中打印数据的标志。"
+ log_print_delay_ms = "日志打印延迟(毫秒)"
+ log_print_delay_ms_description = "打印每个数据项到日志之间的毫秒延迟。"
+}
diff --git a/seatunnel-ui/src/hooks/use-source-type.ts
b/seatunnel-ui/src/hooks/use-source-type.ts
index f5c6a098..fe71124b 100644
--- a/seatunnel-ui/src/hooks/use-source-type.ts
+++ b/seatunnel-ui/src/hooks/use-source-type.ts
@@ -34,7 +34,8 @@ export const useSourceType = (showVirtualDataSource = false)
=> {
2: 'file',
3: 'no_structured',
4: 'storage',
- 5: 'remote_connection'
+ 5: 'remote_connection',
+ 6: 'fake_connection'
}
const state = reactive({
loading: false,
diff --git a/seatunnel-ui/src/locales/en_US/datasource.ts
b/seatunnel-ui/src/locales/en_US/datasource.ts
index 27b2412c..69c9863c 100644
--- a/seatunnel-ui/src/locales/en_US/datasource.ts
+++ b/seatunnel-ui/src/locales/en_US/datasource.ts
@@ -86,5 +86,6 @@ export default {
no_structured: 'NoSQLs',
storage: 'Storage',
data_analysis: 'Data Analysis',
- remote_connection: 'Remote Connection'
+ remote_connection: 'Remote Connection',
+ fake_connection: 'Fake Connection'
}
diff --git a/seatunnel-ui/src/locales/en_US/project.ts
b/seatunnel-ui/src/locales/en_US/project.ts
index b28b497b..18e6c552 100644
--- a/seatunnel-ui/src/locales/en_US/project.ts
+++ b/seatunnel-ui/src/locales/en_US/project.ts
@@ -958,6 +958,7 @@ export default {
git_file: 'Git File',
break_continue: 'Break continue',
remote_connection: 'Remote Connection',
+ fake_connection: 'Fake Connection',
connection_type: 'Connection Type',
source_name: 'Source Name',
ssh: 'SSH',
diff --git a/seatunnel-ui/src/locales/zh_CN/datasource.ts
b/seatunnel-ui/src/locales/zh_CN/datasource.ts
index 6222fd05..ec3f0599 100644
--- a/seatunnel-ui/src/locales/zh_CN/datasource.ts
+++ b/seatunnel-ui/src/locales/zh_CN/datasource.ts
@@ -82,5 +82,6 @@ export default {
no_structured: '非结构化',
storage: '存储',
data_analysis: '数据分析',
- remote_connection: '远程连接'
+ remote_connection: '远程连接',
+ fake_connection: '假连接'
}
diff --git a/seatunnel-ui/src/locales/zh_CN/project.ts
b/seatunnel-ui/src/locales/zh_CN/project.ts
index a490f38d..83b98aa0 100644
--- a/seatunnel-ui/src/locales/zh_CN/project.ts
+++ b/seatunnel-ui/src/locales/zh_CN/project.ts
@@ -927,6 +927,7 @@ export default {
whale_seatunnel_task_tips: '请选择任务',
break_continue: '断点续传',
remote_connection: '远程连接',
+ fake_connection: '假连接',
connection_type: '连接类型',
source_name: '源名称',
ssh: 'SSH',
diff --git a/seatunnel-ui/src/views/datasource/components/use-source.ts
b/seatunnel-ui/src/views/datasource/components/use-source.ts
index a775cde6..f2a049f5 100644
--- a/seatunnel-ui/src/views/datasource/components/use-source.ts
+++ b/seatunnel-ui/src/views/datasource/components/use-source.ts
@@ -36,7 +36,8 @@ export const useSource = (showVirtualDataSource = false) => {
2: 'file',
3: 'no_structured',
4: 'storage',
- 5: 'remote_connection'
+ 5: 'remote_connection',
+ 6: 'fake_connection'
}
const state = reactive({
types: [] as IType[]
diff --git a/seatunnel-ui/src/views/datasource/list/use-source.ts
b/seatunnel-ui/src/views/datasource/list/use-source.ts
index c0e79cc4..02541281 100644
--- a/seatunnel-ui/src/views/datasource/list/use-source.ts
+++ b/seatunnel-ui/src/views/datasource/list/use-source.ts
@@ -34,7 +34,8 @@ export const useSource = (showVirtualDataSource = false) => {
2: 'file',
3: 'no_structured',
4: 'storage',
- 5: 'remote_connection'
+ 5: 'remote_connection',
+ 6: 'fake_connection'
}
const state = reactive({
loading: false,
diff --git a/seatunnel-web-dist/pom.xml b/seatunnel-web-dist/pom.xml
index 0623bd2f..372c24ce 100644
--- a/seatunnel-web-dist/pom.xml
+++ b/seatunnel-web-dist/pom.xml
@@ -375,6 +375,18 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-fakesource</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-console</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- jdbc driver -->
<dependency>
