This is an automated email from the ASF dual-hosted git repository.
lidongdai pushed a commit to branch 3.2.2-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/3.2.2-prepare by this push:
new fe96907fbb [Chore] Revert some incompatible change for 3.2.2 (#16214)
fe96907fbb is described below
commit fe96907fbb537c0b426a73d04d1aae56451115de
Author: xiangzihao <[email protected]>
AuthorDate: Wed Jun 26 13:17:51 2024 +0800
[Chore] Revert some incompatible change for 3.2.2 (#16214)
* revert pr 14977
* fix spotless
* revert pr 14977
* revert pr 14977
---
.../dolphinscheduler-datasource-all/pom.xml | 5 -
.../dolphinscheduler-datasource-k8s/pom.xml | 50 --------
.../plugin/datasource/k8s/K8sClientWrapper.java | 55 ---------
.../datasource/k8s/K8sDataSourceChannel.java | 37 ------
.../k8s/K8sDataSourceChannelFactory.java | 39 -------
.../datasource/k8s/param/K8sConnectionParam.java | 32 -----
.../k8s/param/K8sDataSourceParamDTO.java | 35 ------
.../k8s/param/K8sDataSourceProcessor.java | 129 ---------------------
.../datasource/k8s/K8sDataSourceProcessorTest.java | 107 -----------------
dolphinscheduler-datasource-plugin/pom.xml | 1 -
.../e2e/pages/datasource/DataSourcePage.java | 7 --
.../apache/dolphinscheduler/spi/enums/DbType.java | 3 +-
.../plugin/task/api/K8sTaskExecutionContext.java | 10 +-
.../plugin/task/api/k8s/K8sTaskMainParameters.java | 1 +
.../task/api/parameters/AbstractParameters.java | 13 ---
.../task/api/parameters/K8sTaskParameters.java | 15 +--
.../plugin/task/api/k8s/K8sTaskExecutorTest.java | 12 +-
.../dolphinscheduler/plugin/task/k8s/K8sTask.java | 30 ++---
.../plugin/task/k8s/K8sTaskChannel.java | 2 +-
.../plugin/task/k8s/K8sParametersTest.java | 2 +-
.../plugin/task/k8s/K8sTaskTest.java | 78 ++++---------
.../src/service/modules/data-source/types.ts | 4 -
.../src/views/datasource/list/detail.tsx | 39 -------
.../src/views/datasource/list/use-form.ts | 23 +---
.../projects/task/components/node/format-data.ts | 3 -
.../projects/task/components/node/tasks/use-k8s.ts | 7 +-
26 files changed, 55 insertions(+), 684 deletions(-)
diff --git
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
index effe3c9abb..987e419e7f 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
@@ -148,11 +148,6 @@
<artifactId>dolphinscheduler-datasource-sagemaker</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-k8s</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-hana</artifactId>
diff --git
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml
deleted file mode 100644
index 2d51b85fed..0000000000
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-plugin</artifactId>
- <version>dev-SNAPSHOT</version>
- </parent>
-
- <artifactId>dolphinscheduler-datasource-k8s</artifactId>
- <packaging>jar</packaging>
- <name>${project.artifactId}</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-spi</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-api</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>io.fabric8</groupId>
- <artifactId>kubernetes-client</artifactId>
- </dependency>
- </dependencies>
-
-</project>
diff --git
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sClientWrapper.java
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sClientWrapper.java
deleted file mode 100644
index 292eaf2f3d..0000000000
---
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sClientWrapper.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s;
-
-import lombok.extern.slf4j.Slf4j;
-import io.fabric8.kubernetes.api.model.NamespaceList;
-import io.fabric8.kubernetes.client.Config;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.KubernetesClientBuilder;
-
-@Slf4j
-public class K8sClientWrapper implements AutoCloseable {
-
- private KubernetesClient client;
-
- public K8sClientWrapper() {
- }
-
- public boolean checkConnect(String kubeConfigYaml, String namespace) {
- try {
- Config config = Config.fromKubeconfig(kubeConfigYaml);
- client = new KubernetesClientBuilder().withConfig(config).build();
- NamespaceList namespaceList = client.namespaces().list();
- if (!namespaceList.getItems().stream().anyMatch(ns ->
ns.getMetadata().getName().equals(namespace))) {
- log.info("failed to connect to the K8S cluster, namespace not
found\n");
- return false;
- }
- log.info("successfully connected to the K8S cluster");
- return true;
- } catch (Exception e) {
- log.info("failed to connect to the K8S cluster\n");
- return false;
- }
- }
-
- @Override
- public void close() throws Exception {
-
- }
-}
diff --git
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannel.java
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannel.java
deleted file mode 100644
index 3dac639259..0000000000
---
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannel.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s;
-
-import org.apache.dolphinscheduler.spi.datasource.AdHocDataSourceClient;
-import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
-import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
-import org.apache.dolphinscheduler.spi.datasource.PooledDataSourceClient;
-import org.apache.dolphinscheduler.spi.enums.DbType;
-
-public class K8sDataSourceChannel implements DataSourceChannel {
-
- @Override
- public AdHocDataSourceClient
createAdHocDataSourceClient(BaseConnectionParam baseConnectionParam, DbType
dbType) {
- throw new UnsupportedOperationException("K8S AdHocDataSourceClient is
not supported");
- }
-
- @Override
- public PooledDataSourceClient
createPooledDataSourceClient(BaseConnectionParam baseConnectionParam, DbType
dbType) {
- throw new UnsupportedOperationException("K8S AdHocDataSourceClient is
not supported");
- }
-}
diff --git
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java
deleted file mode 100644
index 6a4428b47b..0000000000
---
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s;
-
-import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
-import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
-import org.apache.dolphinscheduler.spi.enums.DbType;
-
-import com.google.auto.service.AutoService;
-
-@AutoService(DataSourceChannelFactory.class)
-public class K8sDataSourceChannelFactory implements DataSourceChannelFactory {
-
- @Override
- public DataSourceChannel create() {
- return new K8sDataSourceChannel();
- }
-
- @Override
- public String getName() {
- return DbType.K8S.getName();
- }
-
-}
diff --git
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sConnectionParam.java
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sConnectionParam.java
deleted file mode 100644
index 09e3e706a1..0000000000
---
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sConnectionParam.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s.param;
-
-import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
-
-import lombok.Data;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-
-@Data
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class K8sConnectionParam implements ConnectionParam {
-
- protected String kubeConfig;
- protected String namespace;
-}
diff --git
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceParamDTO.java
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceParamDTO.java
deleted file mode 100644
index f2ea50a40b..0000000000
---
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceParamDTO.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s.param;
-
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
-import org.apache.dolphinscheduler.spi.enums.DbType;
-
-import lombok.Data;
-
-@Data
-public class K8sDataSourceParamDTO extends BaseDataSourceParamDTO {
-
- protected String kubeConfig;
- protected String namespace;
-
- @Override
- public DbType getType() {
- return DbType.K8S;
- }
-}
diff --git
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java
deleted file mode 100644
index fd3b49469f..0000000000
---
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s.param;
-
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
-import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
-import org.apache.dolphinscheduler.plugin.datasource.k8s.K8sClientWrapper;
-import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
-import org.apache.dolphinscheduler.spi.enums.DbType;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.text.MessageFormat;
-
-import lombok.extern.slf4j.Slf4j;
-
-import com.google.auto.service.AutoService;
-
-@AutoService(DataSourceProcessor.class)
-@Slf4j
-public class K8sDataSourceProcessor extends AbstractDataSourceProcessor {
-
- @Override
- public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
- return JSONUtils.parseObject(paramJson, K8sDataSourceParamDTO.class);
-
- }
-
- @Override
- public void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam) {
- K8sDataSourceParamDTO k8sDataSourceParamDTO = (K8sDataSourceParamDTO)
datasourceParam;
- if (StringUtils.isEmpty(k8sDataSourceParamDTO.getKubeConfig())) {
- throw new IllegalArgumentException("sagemaker datasource param is
not valid");
- }
- }
-
- @Override
- public String getDatasourceUniqueId(ConnectionParam connectionParam,
DbType dbType) {
- K8sConnectionParam baseConnectionParam = (K8sConnectionParam)
connectionParam;
- return MessageFormat.format("{0}@{1}@{2}", dbType.getName(),
-
PasswordUtils.encodePassword(baseConnectionParam.getKubeConfig()),
baseConnectionParam.getNamespace());
- }
-
- @Override
- public BaseDataSourceParamDTO createDatasourceParamDTO(String
connectionJson) {
- K8sConnectionParam connectionParams = (K8sConnectionParam)
createConnectionParams(connectionJson);
- K8sDataSourceParamDTO k8sDataSourceParamDTO = new
K8sDataSourceParamDTO();
- k8sDataSourceParamDTO.setKubeConfig(connectionParams.getKubeConfig());
- k8sDataSourceParamDTO.setNamespace(connectionParams.getNamespace());
- return k8sDataSourceParamDTO;
- }
-
- @Override
- public K8sConnectionParam createConnectionParams(BaseDataSourceParamDTO
datasourceParam) {
- K8sDataSourceParamDTO k8sDataSourceParam = (K8sDataSourceParamDTO)
datasourceParam;
- K8sConnectionParam k8sConnectionParam = new K8sConnectionParam();
- k8sConnectionParam.setKubeConfig(k8sDataSourceParam.getKubeConfig());
- k8sConnectionParam.setNamespace(k8sDataSourceParam.getNamespace());
- return k8sConnectionParam;
- }
-
- @Override
- public ConnectionParam createConnectionParams(String connectionJson) {
- return JSONUtils.parseObject(connectionJson, K8sConnectionParam.class);
- }
-
- @Override
- public String getDatasourceDriver() {
- return "";
- }
-
- @Override
- public String getValidationQuery() {
- return "";
- }
-
- @Override
- public String getJdbcUrl(ConnectionParam connectionParam) {
- return "";
- }
-
- @Override
- public Connection getConnection(ConnectionParam connectionParam) throws
ClassNotFoundException, SQLException, IOException {
- return null;
- }
-
- @Override
- public boolean checkDataSourceConnectivity(ConnectionParam
connectionParam) {
- K8sConnectionParam baseConnectionParam = (K8sConnectionParam)
connectionParam;
- try (
- K8sClientWrapper k8sClientWrapper = new K8sClientWrapper()) {
- return
k8sClientWrapper.checkConnect(baseConnectionParam.kubeConfig,
baseConnectionParam.namespace);
- } catch (Exception e) {
- log.error("failed to connect to the K8S cluster", e);
- return false;
- }
- }
-
- @Override
- public DbType getDbType() {
- return DbType.K8S;
- }
-
- @Override
- public DataSourceProcessor create() {
- return new K8sDataSourceProcessor();
- }
-}
diff --git
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/test/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceProcessorTest.java
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/test/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceProcessorTest.java
deleted file mode 100644
index fe41f97686..0000000000
---
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/test/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceProcessorTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s;
-
-import
org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sConnectionParam;
-import
org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sDataSourceParamDTO;
-import
org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sDataSourceProcessor;
-import org.apache.dolphinscheduler.spi.enums.DbType;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.MockedConstruction;
-import org.mockito.Mockito;
-import org.mockito.junit.jupiter.MockitoExtension;
-
-@ExtendWith(MockitoExtension.class)
-public class K8sDataSourceProcessorTest {
-
- private K8sDataSourceProcessor k8sDataSourceProcessor;
-
- private String connectJson =
- "{\"namespace\":\"namespace\",\"kubeConfig\":\"kubeConfig\"}";
-
- @BeforeEach
- public void init() {
- k8sDataSourceProcessor = new K8sDataSourceProcessor();
- }
-
- @Test
- void testCheckDatasourceParam() {
- K8sDataSourceParamDTO k8sDataSourceParamDTO = new
K8sDataSourceParamDTO();
- Assertions.assertThrows(IllegalArgumentException.class,
- () ->
k8sDataSourceProcessor.checkDatasourceParam(k8sDataSourceParamDTO));
- k8sDataSourceParamDTO.setNamespace("namespace");
- Assertions.assertThrows(IllegalArgumentException.class,
- () ->
k8sDataSourceProcessor.checkDatasourceParam(k8sDataSourceParamDTO));
- k8sDataSourceParamDTO.setKubeConfig("kubeConfig");
- Assertions
- .assertDoesNotThrow(
- () ->
k8sDataSourceProcessor.checkDatasourceParam(k8sDataSourceParamDTO));
- }
-
- @Test
- void testGetDatasourceUniqueId() {
- K8sConnectionParam k8sConnectionParam = new K8sConnectionParam();
- k8sConnectionParam.setNamespace("namespace");
- k8sConnectionParam.setKubeConfig("kubeConfig");
- Assertions.assertEquals("k8s@kubeConfig@namespace",
-
k8sDataSourceProcessor.getDatasourceUniqueId(k8sConnectionParam, DbType.K8S));
-
- }
-
- @Test
- void testCreateDatasourceParamDTO() {
- K8sDataSourceParamDTO k8sDataSourceParamDTO =
- (K8sDataSourceParamDTO)
k8sDataSourceProcessor.createDatasourceParamDTO(connectJson);
- Assertions.assertEquals("namespace",
k8sDataSourceParamDTO.getNamespace());
- Assertions.assertEquals("kubeConfig",
k8sDataSourceParamDTO.getKubeConfig());
- }
-
- @Test
- void testCreateConnectionParams() {
- K8sDataSourceParamDTO k8sDataSourceParamDTO =
- (K8sDataSourceParamDTO)
k8sDataSourceProcessor.createDatasourceParamDTO(connectJson);
- K8sConnectionParam k8sConnectionParam =
-
k8sDataSourceProcessor.createConnectionParams(k8sDataSourceParamDTO);
- Assertions.assertEquals("namespace",
k8sConnectionParam.getNamespace());
- Assertions.assertEquals("kubeConfig",
k8sConnectionParam.getKubeConfig());
- }
-
- @Test
- void testTestConnection() {
- K8sDataSourceParamDTO k8sDataSourceParamDTO =
- (K8sDataSourceParamDTO)
k8sDataSourceProcessor.createDatasourceParamDTO(connectJson);
- K8sConnectionParam connectionParam =
-
k8sDataSourceProcessor.createConnectionParams(k8sDataSourceParamDTO);
-
Assertions.assertFalse(k8sDataSourceProcessor.checkDataSourceConnectivity(connectionParam));
-
- try (
- MockedConstruction<K8sClientWrapper>
k8sClientWrapperMockedConstruction =
- Mockito.mockConstruction(K8sClientWrapper.class,
(mock, context) -> {
- Mockito.when(
-
mock.checkConnect(connectionParam.getKubeConfig(),
connectionParam.getNamespace()))
- .thenReturn(true);
- })) {
-
Assertions.assertTrue(k8sDataSourceProcessor.checkDataSourceConnectivity(connectionParam));
- }
-
- }
-}
diff --git a/dolphinscheduler-datasource-plugin/pom.xml
b/dolphinscheduler-datasource-plugin/pom.xml
index c30a6b4258..a71b6ef683 100644
--- a/dolphinscheduler-datasource-plugin/pom.xml
+++ b/dolphinscheduler-datasource-plugin/pom.xml
@@ -54,7 +54,6 @@
<module>dolphinscheduler-datasource-vertica</module>
<module>dolphinscheduler-datasource-doris</module>
<module>dolphinscheduler-datasource-sagemaker</module>
- <module>dolphinscheduler-datasource-k8s</module>
<module>dolphinscheduler-datasource-hana</module>
</modules>
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java
index bd2f7e795b..f8db982d42 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java
@@ -192,12 +192,5 @@ public class DataSourcePage extends NavBarPage implements
NavBarPage.NavBarItem
@FindBy(tagName = "input"),
})
private WebElement inputZeppelinRestEndpoint;
-
- @FindBys({
- @FindBy(className = "input-kubeConfig"),
- @FindBy(tagName = "textarea"),
- })
- private WebElement inputKubeConfig;
-
}
}
diff --git
a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
index 882b170e11..f3e841a0e7 100644
---
a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
+++
b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
@@ -54,8 +54,7 @@ public enum DbType {
DORIS(23, "doris", "doris"),
ZEPPELIN(24, "zeppelin", "zeppelin"),
SAGEMAKER(25, "sagemaker", "sagemaker"),
-
- K8S(26, "k8s", "k8s");
+ ;
private static final Map<Integer, DbType> DB_TYPE_MAP =
Arrays.stream(DbType.values()).collect(toMap(DbType::getCode,
Functions.identity()));
@EnumValue
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java
index 4138e5465f..aa5bf62fe2 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java
@@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.plugin.task.api;
import java.io.Serializable;
-import lombok.Data;
+import lombok.Value;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -27,18 +27,13 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
* k8s Task ExecutionContext
*/
-@Data
+@Value
public class K8sTaskExecutionContext implements Serializable {
private String configYaml;
private String namespace;
- private String connectionParams;
-
- public K8sTaskExecutionContext() {
- }
-
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
public K8sTaskExecutionContext(
@JsonProperty("configYaml") String
configYaml,
@@ -52,7 +47,6 @@ public class K8sTaskExecutionContext implements Serializable {
return "K8sTaskExecutionContext{"
+ "namespace=" + namespace
+ ", configYaml='" + configYaml + '\''
- + ", connectionParams='" + connectionParams + '\''
+ '}';
}
}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
index 8adc912161..a6c4703f10 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
@@ -34,6 +34,7 @@ public class K8sTaskMainParameters {
private String args;
private String pullSecret;
private String namespaceName;
+ private String clusterName;
private String imagePullPolicy;
private double minCpuCores;
private double minMemorySpace;
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
index f99578d743..5d50038850 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
@@ -18,12 +18,9 @@
package org.apache.dolphinscheduler.plugin.task.api.parameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
-import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
-import
org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
import
org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.dolphinscheduler.plugin.task.api.utils.VarPoolUtils;
@@ -73,16 +70,6 @@ public abstract class AbstractParameters implements
IParameters {
return localParametersMaps;
}
- public K8sTaskExecutionContext
generateK8sTaskExecutionContext(ResourceParametersHelper parametersHelper,
- int
datasource) {
- DataSourceParameters dataSourceParameters =
- (DataSourceParameters)
parametersHelper.getResourceParameters(ResourceType.DATASOURCE, datasource);
- K8sTaskExecutionContext k8sTaskExecutionContext = new
K8sTaskExecutionContext();
- k8sTaskExecutionContext.setConnectionParams(
- Objects.nonNull(dataSourceParameters) ?
dataSourceParameters.getConnectionParams() : null);
- return k8sTaskExecutionContext;
- }
-
/**
* get input local parameters map if the param direct is IN
*
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
index 4f045abe19..69a190c023 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
@@ -17,11 +17,9 @@
package org.apache.dolphinscheduler.plugin.task.api.parameters;
-import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.task.api.model.Label;
import
org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
-import
org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.commons.lang3.StringUtils;
@@ -29,11 +27,13 @@ import java.util.ArrayList;
import java.util.List;
import lombok.Data;
+import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j;
/**
* k8s task parameters
*/
+@EqualsAndHashCode(callSuper = true)
@Data
@Slf4j
public class K8sTaskParameters extends AbstractParameters {
@@ -48,9 +48,7 @@ public class K8sTaskParameters extends AbstractParameters {
private double minMemorySpace;
private List<Label> customizedLabels;
private List<NodeSelectorExpression> nodeSelectors;
- private String kubeConfig;
- private int datasource;
- private String type;
+
@Override
public boolean checkParameters() {
return StringUtils.isNotEmpty(image);
@@ -60,11 +58,4 @@ public class K8sTaskParameters extends AbstractParameters {
public List<ResourceInfo> getResourceFilesList() {
return new ArrayList<>();
}
-
- @Override
- public ResourceParametersHelper getResources() {
- ResourceParametersHelper resources = super.getResources();
- resources.put(ResourceType.DATASOURCE, datasource);
- return resources;
- }
}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
index d93130caee..1afcc2204c 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
@@ -17,6 +17,10 @@
package org.apache.dolphinscheduler.plugin.task.api.k8s;
+import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER;
+import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAMESPACE_NAME;
+
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.k8s.impl.K8sTaskExecutor;
@@ -45,7 +49,7 @@ public class K8sTaskExecutorTest {
private K8sTaskMainParameters k8sTaskMainParameters = null;
private final String image = "ds-dev";
private final String imagePullPolicy = "IfNotPresent";
- private final String namespace = "namespace";
+ private final String namespace =
"{\"name\":\"default\",\"cluster\":\"lab\"}";
private final double minCpuCores = 2;
private final double minMemorySpace = 10;
private final int taskInstanceId = 1000;
@@ -56,6 +60,9 @@ public class K8sTaskExecutorTest {
TaskExecutionContext taskRequest = new TaskExecutionContext();
taskRequest.setTaskInstanceId(taskInstanceId);
taskRequest.setTaskName(taskName);
+ Map<String, String> namespace = JSONUtils.toMap(this.namespace);
+ String namespaceName = namespace.get(NAMESPACE_NAME);
+ String clusterName = namespace.get(CLUSTER);
Map<String, String> labelMap = new HashMap<>();
labelMap.put("test", "1234");
@@ -67,7 +74,8 @@ public class K8sTaskExecutorTest {
k8sTaskMainParameters = new K8sTaskMainParameters();
k8sTaskMainParameters.setImage(image);
k8sTaskMainParameters.setImagePullPolicy(imagePullPolicy);
- k8sTaskMainParameters.setNamespaceName(namespace);
+ k8sTaskMainParameters.setNamespaceName(namespaceName);
+ k8sTaskMainParameters.setClusterName(clusterName);
k8sTaskMainParameters.setMinCpuCores(minCpuCores);
k8sTaskMainParameters.setMinMemorySpace(minMemorySpace);
k8sTaskMainParameters.setCommand("[\"perl\" ,\"-Mbignum=bpi\",
\"-wle\", \"print bpi(2000)\"]");
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
index fdb39d7c28..7eb397425c 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
@@ -17,10 +17,10 @@
package org.apache.dolphinscheduler.plugin.task.k8s;
+import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER;
+import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAMESPACE_NAME;
+
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
-import
org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sConnectionParam;
-import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.k8s.AbstractK8sTask;
@@ -31,7 +31,6 @@ import
org.apache.dolphinscheduler.plugin.task.api.model.Property;
import
org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import
org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
-import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -53,33 +52,19 @@ public class K8sTask extends AbstractK8sTask {
private K8sTaskParameters k8sTaskParameters;
- private K8sTaskExecutionContext k8sTaskExecutionContext;
-
- private K8sConnectionParam k8sConnectionParam;
public K8sTask(TaskExecutionContext taskRequest) {
super(taskRequest);
this.taskExecutionContext = taskRequest;
+ this.k8sTaskParameters =
JSONUtils.parseObject(taskExecutionContext.getTaskParams(),
K8sTaskParameters.class);
+ log.info("Initialize k8s task parameters {}",
JSONUtils.toPrettyJsonString(k8sTaskParameters));
}
@Override
public void init() {
- String taskParams = taskExecutionContext.getTaskParams();
- k8sTaskParameters = JSONUtils.parseObject(taskParams,
K8sTaskParameters.class);
if (k8sTaskParameters == null || !k8sTaskParameters.checkParameters())
{
throw new TaskException("K8S task params is not valid");
}
- k8sTaskExecutionContext =
-
k8sTaskParameters.generateK8sTaskExecutionContext(taskExecutionContext.getResourceParametersHelper(),
- k8sTaskParameters.getDatasource());
- k8sConnectionParam =
- (K8sConnectionParam)
DataSourceUtils.buildConnectionParams(DbType.valueOf(k8sTaskParameters.getType()),
- k8sTaskExecutionContext.getConnectionParams());
- String kubeConfig = k8sConnectionParam.getKubeConfig();
- k8sTaskParameters.setNamespace(k8sConnectionParam.getNamespace());
- k8sTaskParameters.setKubeConfig(kubeConfig);
- k8sTaskExecutionContext.setConfigYaml(kubeConfig);
- taskRequest.setK8sTaskExecutionContext(k8sTaskExecutionContext);
log.info("Initialize k8s task params:{}",
JSONUtils.toPrettyJsonString(k8sTaskParameters));
}
@@ -97,10 +82,13 @@ public class K8sTask extends AbstractK8sTask {
protected String buildCommand() {
K8sTaskMainParameters k8sTaskMainParameters = new
K8sTaskMainParameters();
Map<String, Property> paramsMap =
taskExecutionContext.getPrepareParamsMap();
- String namespaceName = k8sTaskParameters.getNamespace();
+ Map<String, String> namespace =
JSONUtils.toMap(k8sTaskParameters.getNamespace());
+ String namespaceName = namespace.get(NAMESPACE_NAME);
+ String clusterName = namespace.get(CLUSTER);
k8sTaskMainParameters.setImage(k8sTaskParameters.getImage());
k8sTaskMainParameters.setPullSecret(k8sTaskParameters.getPullSecret());
k8sTaskMainParameters.setNamespaceName(namespaceName);
+ k8sTaskMainParameters.setClusterName(clusterName);
k8sTaskMainParameters.setMinCpuCores(k8sTaskParameters.getMinCpuCores());
k8sTaskMainParameters.setMinMemorySpace(k8sTaskParameters.getMinMemorySpace());
k8sTaskMainParameters.setParamsMap(ParameterUtils.convert(paramsMap));
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java
index 91c0faf684..f15ab4496e 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java
@@ -40,7 +40,7 @@ public class K8sTaskChannel implements TaskChannel {
@Override
public ResourceParametersHelper getResources(String parameters) {
- return JSONUtils.parseObject(parameters,
K8sTaskParameters.class).getResources();
+ return null;
}
@Override
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java
index 543ec3e4bc..eb65ede9d0 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java
@@ -33,7 +33,7 @@ public class K8sParametersTest {
private K8sTaskParameters k8sTaskParameters = null;
private final String image = "ds-dev";
private final String imagePullPolicy = "IfNotPresent";
- private final String namespace = "namespace";
+ private final String namespace =
"{\"name\":\"default\",\"cluster\":\"lab\"}";
private final double minCpuCores = 2;
private final double minMemorySpace = 10;
private final String command = "[\"/bin/bash\", \"-c\"]";
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
index 3895190cf2..241ac2ed10 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
@@ -17,13 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.k8s;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
-import
org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sConnectionParam;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
@@ -31,7 +25,6 @@ import
org.apache.dolphinscheduler.plugin.task.api.model.Label;
import
org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import
org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters;
-import
org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import java.util.ArrayList;
import java.util.Arrays;
@@ -39,12 +32,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
import com.google.common.collect.ImmutableMap;
import io.fabric8.kubernetes.api.model.NodeSelectorRequirement;
@@ -71,53 +61,14 @@ public class K8sTaskTest {
private final String date = "20220507";
private final String command = "[\"/bin/bash\", \"-c\"]";
private final String args = "[\"echo hello world\"]";
- private final String kubeConfig = "{}";
- private final String type = "K8S";
-
- private final Map<String, Property> prepareParamsMap = new HashMap<String,
Property>() {
-
- {
- put(DAY, new Property() {
- {
- setProp(DAY);
- setValue(date);
- }
- });
- }
- };
-
- private final int datasource = 0;
private final List<Label> labels = Arrays.asList(new Label("test",
"1234"));
private final List<NodeSelectorExpression> nodeSelectorExpressions =
Arrays.asList(new NodeSelectorExpression("node-label", "In",
"1234,12345"));
- private static MockedStatic<DataSourceUtils> dataSourceUtilsStaticMock =
null;
-
@BeforeEach
public void before() {
- String k8sTaskParameters = buildK8sTaskParameters();
- TaskExecutionContext taskExecutionContext =
mock(TaskExecutionContext.class);
- ResourceParametersHelper resourceParametersHelper =
mock(ResourceParametersHelper.class);
- K8sConnectionParam k8sConnectionParam = mock(K8sConnectionParam.class);
-
when(taskExecutionContext.getTaskParams()).thenReturn(k8sTaskParameters);
- when(k8sConnectionParam.getNamespace()).thenReturn(namespace);
- when(k8sConnectionParam.getKubeConfig()).thenReturn(kubeConfig);
-
when(taskExecutionContext.getPrepareParamsMap()).thenReturn(prepareParamsMap);
-
when(taskExecutionContext.getResourceParametersHelper()).thenReturn(resourceParametersHelper);
- dataSourceUtilsStaticMock = Mockito.mockStatic(DataSourceUtils.class);
- dataSourceUtilsStaticMock.when(() ->
DataSourceUtils.buildConnectionParams(Mockito.any(), Mockito.any()))
- .thenReturn(k8sConnectionParam);
- k8sTask = spy(new K8sTask(taskExecutionContext));
- k8sTask.init();
- }
-
- @AfterEach
- public void afterEach() {
- dataSourceUtilsStaticMock.close();
- }
- private String buildK8sTaskParameters() {
- K8sTaskParameters k8sTaskParameters = new K8sTaskParameters();
+ k8sTaskParameters = new K8sTaskParameters();
k8sTaskParameters.setImage(image);
k8sTaskParameters.setImagePullPolicy(imagePullPolicy);
k8sTaskParameters.setNamespace(namespace);
@@ -129,15 +80,32 @@ public class K8sTaskTest {
k8sTaskParameters.setNodeSelectors(nodeSelectorExpressions);
k8sTaskParameters.setLocalParams(new ArrayList<>());
k8sTaskParameters.setPullSecret(pullSecret);
- k8sTaskParameters.setType(type);
- k8sTaskParameters.setKubeConfig(kubeConfig);
- k8sTaskParameters.setDatasource(datasource);
- return JSONUtils.toJsonString(k8sTaskParameters);
+ TaskExecutionContext taskRequest = new TaskExecutionContext();
+ taskRequest.setTaskInstanceId(taskInstanceId);
+ taskRequest.setTaskName(taskName);
+ taskRequest.setTaskParams(JSONUtils.toJsonString(k8sTaskParameters));
+ Property property = new Property();
+ property.setProp(DAY);
+ property.setDirect(Direct.IN);
+ property.setType(DataType.VARCHAR);
+ property.setValue(date);
+ Map<String, Property> paramsMap = new HashMap<>();
+ paramsMap.put(DAY, property);
+ taskRequest.setParamsMap(paramsMap);
+
+ Map<String, Property> prepareParamsMap = new HashMap<>();
+ Property property1 = new Property();
+ property1.setProp("day");
+ property1.setValue("20220507");
+ prepareParamsMap.put("day", property1);
+ taskRequest.setPrepareParamsMap(prepareParamsMap);
+ k8sTask = new K8sTask(taskRequest);
}
+
@Test
public void testBuildCommandNormal() {
String expectedStr =
- "{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\",
\\\"-c\\\"]\",\"args\":\"[\\\"echo hello
world\\\"]\",\"pullSecret\":\"ds-secret\",\"namespaceName\":\"namespace\",\"imagePullPolicy\":\"IfNotPresent\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"},\"nodeSelectorRequirements\":[{\"key\":\"node-label\",\"operator\":\"In\",\"values\":[\"1234\",\"12345\"]}]}";
+ "{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\",
\\\"-c\\\"]\",\"args\":\"[\\\"echo hello
world\\\"]\",\"pullSecret\":\"ds-secret\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"imagePullPolicy\":\"IfNotPresent\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"},\"nodeSelectorRequirements\":[{\"key\":\"node-label\",\"operator\":\"In\",\"values\":[\"1234\",\"12345\"]}]}";
String commandStr = k8sTask.buildCommand();
Assertions.assertEquals(expectedStr, commandStr);
}
diff --git a/dolphinscheduler-ui/src/service/modules/data-source/types.ts
b/dolphinscheduler-ui/src/service/modules/data-source/types.ts
index 444f5293dd..a5fbb1a8a5 100644
--- a/dolphinscheduler-ui/src/service/modules/data-source/types.ts
+++ b/dolphinscheduler-ui/src/service/modules/data-source/types.ts
@@ -41,7 +41,6 @@ type IDataBase =
| 'KYUUBI'
| 'ZEPPELIN'
| 'SAGEMAKER'
- | 'K8S'
type IDataBaseLabel =
| 'MYSQL'
@@ -64,7 +63,6 @@ type IDataBaseLabel =
| 'KYUUBI'
| 'ZEPPELIN'
| 'SAGEMAKER'
- | 'K8S'
interface IDataSource {
id?: number
@@ -87,8 +85,6 @@ interface IDataSource {
other?: object
endpoint?: string
restEndpoint?: string
- kubeConfig?: string
- namespace?: string
MSIClientId?: string
dbUser?: string
compatibleMode?: string
diff --git a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx
b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx
index 4842651290..5986e35a42 100644
--- a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx
+++ b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx
@@ -163,8 +163,6 @@ const DetailModal = defineComponent({
showDataBaseName,
showJDBCConnectParameters,
showPublicKey,
- showNamespace,
- showKubeConfig,
modeOptions,
redShiftModeOptions,
sagemakerModeOption,
@@ -544,10 +542,6 @@ const DetailModal = defineComponent({
/>
</NFormItem>
<NFormItem
- v-show={
- (!showMode || detailForm.mode === 'password') &&
- detailForm.type != 'K8S'
- }
label={t('datasource.user_name')}
path='userName'
show-require-mark
@@ -562,10 +556,6 @@ const DetailModal = defineComponent({
/>
</NFormItem>
<NFormItem
- v-show={
- (!showMode || detailForm.mode === 'password') &&
- detailForm.type != 'K8S'
- }
label={t('datasource.user_password')}
path='password'
>
@@ -686,35 +676,6 @@ const DetailModal = defineComponent({
}}
/>
</NFormItem>
- <NFormItem
- v-show={showKubeConfig}
- label={t('datasource.kubeConfig')}
- path='kubeConfig'
- show-require-mark
- >
- <NInput
- allowInput={this.trim}
- class='input-kubeConfig'
- v-model={[detailForm.kubeConfig, 'value']}
- type='textarea'
- autosize={{
- minRows: 14
- }}
- placeholder={t('datasource.kubeConfig_tips')}
- />
- </NFormItem>
- <NFormItem
- v-show={showNamespace}
- label={t('datasource.namespace')}
- path='namespace'
- show-require-mark
- >
- <NInput
- allowInput={this.trim}
- v-model={[detailForm.namespace, 'value']}
- placeholder={t('datasource.namespace_tips')}
- />
- </NFormItem>
</NForm>
</NSpin>
),
diff --git a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts
b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts
index 21916667d2..ed383c3720 100644
--- a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts
+++ b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts
@@ -69,8 +69,6 @@ export function useForm(id?: number) {
showDataBaseName: true,
showJDBCConnectParameters: true,
showPublicKey: false,
- showNamespace: false,
- showKubeConfig: false,
rules: {
name: {
trigger: ['input'],
@@ -120,8 +118,7 @@ export function useForm(id?: number) {
validator() {
if (
!state.detailForm.userName &&
- state.detailForm.type !== 'AZURESQL' &&
- state.detailForm.type !== 'K8S'
+ state.detailForm.type !== 'AZURESQL'
) {
return new Error(t('datasource.user_name_tips'))
}
@@ -264,12 +261,7 @@ export function useForm(id?: number) {
} else {
state.showPrincipal = false
}
- if (
- type === 'SSH' ||
- type === 'ZEPPELIN' ||
- type === 'SAGEMAKER' ||
- type === 'K8S'
- ) {
+ if (type === 'SSH' || type === 'ZEPPELIN' || type === 'SAGEMAKER') {
state.showDataBaseName = false
state.requiredDataBase = false
state.showJDBCConnectParameters = false
@@ -282,14 +274,10 @@ export function useForm(id?: number) {
state.showPort = false
state.showRestEndpoint = true
}
- if (type === 'SAGEMAKER' || type === 'K8S') {
+ if (type === 'SAGEMAKER') {
state.showHost = false
state.showPort = false
}
- if (type === 'K8S') {
- state.showNamespace = true
- state.showKubeConfig = true
- }
} else {
state.showDataBaseName = true
state.requiredDataBase = true
@@ -453,11 +441,6 @@ export const datasourceType: IDataBaseOptionKeys = {
value: 'SAGEMAKER',
label: 'SAGEMAKER',
defaultPort: 0
- },
- K8S: {
- value: 'K8S',
- label: 'K8S',
- defaultPort: 6443
}
}
diff --git
a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
index 2ab1712b6f..3fb1e923eb 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
@@ -354,9 +354,6 @@ export function formatParams(data: INodeData): {
taskParams.args = data.args
taskParams.customizedLabels = data.customizedLabels
taskParams.nodeSelectors = data.nodeSelectors
- taskParams.datasource = data.datasource
- taskParams.type = data.type
- taskParams.kubeConfig = data.kubeConfig
taskParams.pullSecret = data.pullSecret
}
diff --git
a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts
b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts
index e20a506df6..df209702a2 100644
---
a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts
+++
b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts
@@ -46,11 +46,7 @@ export function useK8s({
workerGroup: 'default',
delayTime: 0,
timeout: 30,
- type: 'K8S',
- displayRows: 10,
- timeoutNotifyStrategy: ['WARN'],
- kubeConfig: '',
- namespace: ''
+ timeoutNotifyStrategy: ['WARN']
} as INodeData)
return {
@@ -67,7 +63,6 @@ export function useK8s({
...Fields.useFailed(),
Fields.useDelayTime(model),
...Fields.useTimeoutAlarm(model),
- ...Fields.useDatasource(model),
...Fields.useK8s(model),
Fields.usePreTasks()
] as IJsonItem[],
