This is an automated email from the ASF dual-hosted git repository.
wangyang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new f90eadd40f [feature-#14449] add secret in k8s Task (#14930)
f90eadd40f is described below
commit f90eadd40f4f660d251d7d3c722740d4951741dc
Author: fuchanghai <[email protected]>
AuthorDate: Thu Sep 28 11:06:35 2023 +0800
[feature-#14449] add secret in k8s Task (#14930)
* [feature-#14449] add secret
* [feature-#14449] upgrade UT and rename parameter ```secret```
* [feature-#14449] fix UT
* [feature-#14449] fix UT
* [feature-#14449] improvement
* [feature-#14449] format
---------
Co-authored-by: fuchanghai <‘[email protected]’>
---
.../plugin/task/api/k8s/K8sTaskMainParameters.java | 1 +
.../plugin/task/api/k8s/impl/K8sTaskExecutor.java | 15 ++++++++++-----
.../plugin/task/api/parameters/K8sTaskParameters.java | 1 +
.../plugin/task/api/k8s/K8sTaskExecutorTest.java | 3 ++-
.../apache/dolphinscheduler/plugin/task/k8s/K8sTask.java | 1 +
.../dolphinscheduler/plugin/task/k8s/K8sTaskTest.java | 7 +++++--
dolphinscheduler-ui/src/locales/en_US/project.ts | 2 ++
dolphinscheduler-ui/src/locales/zh_CN/project.ts | 2 ++
.../views/projects/task/components/node/fields/use-k8s.ts | 8 ++++++++
.../views/projects/task/components/node/format-data.ts | 1 +
.../src/views/projects/task/components/node/types.ts | 1 +
11 files changed, 34 insertions(+), 8 deletions(-)
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
index d70cf11a22..a6c4703f10 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
@@ -32,6 +32,7 @@ public class K8sTaskMainParameters {
private String image;
private String command;
private String args;
+ private String pullSecret;
private String namespaceName;
private String clusterName;
private String imagePullPolicy;
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
index ae78c66459..f973f5bd1b 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.api.k8s.impl;
+import static java.util.Collections.singletonList;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.API_VERSION;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CPU;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE;
@@ -67,6 +68,7 @@ import org.slf4j.Logger;
import io.fabric8.kubernetes.api.model.Affinity;
import io.fabric8.kubernetes.api.model.AffinityBuilder;
import io.fabric8.kubernetes.api.model.EnvVar;
+import io.fabric8.kubernetes.api.model.LocalObjectReference;
import io.fabric8.kubernetes.api.model.NodeSelectorTerm;
import io.fabric8.kubernetes.api.model.Quantity;
import io.fabric8.kubernetes.api.model.ResourceRequirements;
@@ -91,10 +93,11 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
super(logger, taskRequest);
}
- public Job buildK8sJob(K8sTaskMainParameters k8STaskMainParameters) {
+ public void buildK8sJob(K8sTaskMainParameters k8STaskMainParameters) {
String taskInstanceId =
String.valueOf(taskRequest.getTaskInstanceId());
String taskName = taskRequest.getTaskName().toLowerCase(Locale.ROOT);
String image = k8STaskMainParameters.getImage();
+ String pullSecret = k8STaskMainParameters.getPullSecret();
String namespaceName = k8STaskMainParameters.getNamespaceName();
String imagePullPolicy = k8STaskMainParameters.getImagePullPolicy();
Map<String, String> otherParams = k8STaskMainParameters.getParamsMap();
@@ -155,7 +158,7 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
.endRequiredDuringSchedulingIgnoredDuringExecution()
.endNodeAffinity().build();
- JobBuilder jobBuilder = new JobBuilder()
+ job = new JobBuilder()
.withApiVersion(API_VERSION)
.withNewMetadata()
.withName(k8sJobName)
@@ -178,14 +181,16 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
.withResources(new ResourceRequirements(limitRes, reqRes))
.withEnv(envVars)
.endContainer()
+ .withImagePullSecrets(
+ StringUtils.isEmpty(pullSecret) ? null :
singletonList(new LocalObjectReference(pullSecret)))
.withRestartPolicy(RESTART_POLICY)
.withAffinity(affinity)
.endSpec()
.endTemplate()
.withBackoffLimit(retryNum)
- .endSpec();
+ .endSpec()
+ .build();
- return jobBuilder.build();
}
public void registerBatchJobWatcher(Job job, String taskInstanceId,
TaskResponse taskResponse) {
@@ -322,7 +327,7 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
JSONUtils.parseObject(k8sParameterStr,
K8sTaskMainParameters.class);
try {
log.info("[K8sJobExecutor-{}-{}] start to submit job", taskName,
taskInstanceId);
- job = buildK8sJob(k8STaskMainParameters);
+ buildK8sJob(k8STaskMainParameters);
stopJobOnK8s(k8sParameterStr);
String namespaceName = k8STaskMainParameters.getNamespaceName();
k8sUtils.createJob(namespaceName, job);
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
index fd738fe736..ba9f6498af 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
@@ -38,6 +38,7 @@ public class K8sTaskParameters extends AbstractParameters {
private String namespace;
private String command;
private String args;
+ private String pullSecret;
private String imagePullPolicy;
private double minCpuCores;
private double minMemorySpace;
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
index 46b226268c..fac9a6292f 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
@@ -81,7 +81,8 @@ public class K8sTaskExecutorTest {
k8sTaskMainParameters.setCommand("[\"perl\" ,\"-Mbignum=bpi\",
\"-wle\", \"print bpi(2000)\"]");
k8sTaskMainParameters.setLabelMap(labelMap);
k8sTaskMainParameters.setNodeSelectorRequirements(Arrays.asList(requirement));
- job = k8sTaskExecutor.buildK8sJob(k8sTaskMainParameters);
+ k8sTaskExecutor.buildK8sJob(k8sTaskMainParameters);
+ job = k8sTaskExecutor.getJob();
}
@Test
public void testGetK8sJobStatusNormal() {
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
index 9c5e249b06..39ce7343d8 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
@@ -87,6 +87,7 @@ public class K8sTask extends AbstractK8sTask {
String namespaceName = namespace.get(NAMESPACE_NAME);
String clusterName = namespace.get(CLUSTER);
k8sTaskMainParameters.setImage(k8sTaskParameters.getImage());
+ k8sTaskMainParameters.setPullSecret(k8sTaskParameters.getPullSecret());
k8sTaskMainParameters.setNamespaceName(namespaceName);
k8sTaskMainParameters.setClusterName(clusterName);
k8sTaskMainParameters.setMinCpuCores(k8sTaskParameters.getMinCpuCores());
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
index 93b6f85e5a..3416d0288c 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
@@ -46,6 +46,8 @@ public class K8sTaskTest {
private final String image = "ds-dev";
private final String imagePullPolicy = "IfNotPresent";
+ private final String pullSecret = "ds-secret";
+
private final String namespace =
"{\"name\":\"default\",\"cluster\":\"lab\"}";
private final double minCpuCores = 2;
@@ -74,6 +76,7 @@ public class K8sTaskTest {
k8sTaskParameters.setArgs(args);
k8sTaskParameters.setCustomizedLabels(labels);
k8sTaskParameters.setNodeSelectors(nodeSelectorExpressions);
+ k8sTaskParameters.setPullSecret(pullSecret);
TaskExecutionContext taskRequest = new TaskExecutionContext();
taskRequest.setTaskInstanceId(taskInstanceId);
taskRequest.setTaskName(taskName);
@@ -99,7 +102,7 @@ public class K8sTaskTest {
@Test
public void testBuildCommandNormal() {
String expectedStr =
- "{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\",
\\\"-c\\\"]\",\"args\":\"[\\\"echo hello
world\\\"]\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"imagePullPolicy\":\"IfNotPresent\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"},\"nodeSelectorRequirements\":[{\"key\":\"node-label\",\"operator\":\"In\",\"values\":[\"1234\",\"12345\"]}]}";
+ "{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\",
\\\"-c\\\"]\",\"args\":\"[\\\"echo hello
world\\\"]\",\"pullSecret\":\"ds-secret\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"imagePullPolicy\":\"IfNotPresent\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"},\"nodeSelectorRequirements\":[{\"key\":\"node-label\",\"operator\":\"In\",\"values\":[\"1234\",\"12345\"]}]}";
String commandStr = k8sTask.buildCommand();
Assertions.assertEquals(expectedStr, commandStr);
}
@@ -107,7 +110,7 @@ public class K8sTaskTest {
@Test
public void testGetParametersNormal() {
String expectedStr =
- "K8sTaskParameters(image=ds-dev,
namespace={\"name\":\"default\",\"cluster\":\"lab\"}, command=[\"/bin/bash\",
\"-c\"], args=[\"echo hello world\"], imagePullPolicy=IfNotPresent,
minCpuCores=2.0, minMemorySpace=10.0, customizedLabels=[Label(label=test,
value=1234)], nodeSelectors=[NodeSelectorExpression(key=node-label,
operator=In, values=1234,12345)])";
+ "K8sTaskParameters(image=ds-dev,
namespace={\"name\":\"default\",\"cluster\":\"lab\"}, command=[\"/bin/bash\",
\"-c\"], args=[\"echo hello world\"], pullSecret=ds-secret,
imagePullPolicy=IfNotPresent, minCpuCores=2.0, minMemorySpace=10.0,
customizedLabels=[Label(label=test, value=1234)],
nodeSelectors=[NodeSelectorExpression(key=node-label, operator=In,
values=1234,12345)])";
String result = k8sTask.getParameters().toString();
Assertions.assertEquals(expectedStr, result);
}
diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts
b/dolphinscheduler-ui/src/locales/en_US/project.ts
index 1f146b6772..ee33434ad2 100644
--- a/dolphinscheduler-ui/src/locales/en_US/project.ts
+++ b/dolphinscheduler-ui/src/locales/en_US/project.ts
@@ -397,6 +397,8 @@ export default {
image_tips: 'Please enter image',
image_pull_policy: 'Image pull policy',
image_pull_policy_tips: 'Please select a image pull policy (required)',
+ pull_secret: 'Pull secret',
+ pull_secret_tips: 'Please enter pull secret',
command: 'Command',
command_tips:
'Please enter the container execution command, for example:
["printenv"]',
diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts
b/dolphinscheduler-ui/src/locales/zh_CN/project.ts
index 16f205accf..1470c6154e 100644
--- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts
+++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts
@@ -394,6 +394,8 @@ export default {
image_tips: '请输入镜像',
image_pull_policy: '镜像拉取策略',
image_pull_policy_tips: '请选择镜像拉取策略（必选）',
+ pull_secret: '仓库密钥',
+ pull_secret_tips: '请输入仓库密钥',
command: '容器执行命令',
command_tips: '请输入容器执行命令，例如：["printenv"]',
args: '执行命令参数',
diff --git
a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts
b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts
index 2002dcb260..66be87dfeb 100644
---
a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts
+++
b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts
@@ -79,6 +79,14 @@ export function useK8s(model: { [field: string]: any }):
IJsonItem[] {
},
value: 'IfNotPresent'
},
+ {
+ type: 'input',
+ field: 'pullSecret',
+ name: t('project.node.pull_secret'),
+ props: {
+ placeholder: t('project.node.pull_secret_tips')
+ }
+ },
{
type: 'input',
field: 'command',
diff --git
a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
index 2d03ea45d7..31398f98e6 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
@@ -359,6 +359,7 @@ export function formatParams(data: INodeData): {
taskParams.args = data.args
taskParams.customizedLabels = data.customizedLabels
taskParams.nodeSelectors = data.nodeSelectors
+ taskParams.pullSecret = data.pullSecret
}
if (data.taskType === 'JUPYTER') {
diff --git
a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
index 3f12a9e245..86eec11682 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
@@ -378,6 +378,7 @@ interface ITaskParams {
minMemorySpace?: string
image?: string
imagePullPolicy?: string
+ pullSecret?: string
command?: string
args?: string
customizedLabels?: ILabel[]
