This is an automated email from the ASF dual-hosted git repository.
FrankChen021 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 4f5a5802a80 fix: support task ids containing spaces in Kubernetes peon
tasks (#20833)
4f5a5802a80 is described below
commit 4f5a5802a808fc02c14bfc90332e65fb498c074d
Author: Frank Chen <[email protected]>
AuthorDate: Sat Nov 7 11:09:03 2026 +0800
fix: support task ids containing spaces in Kubernetes peon tasks (#20833)
* fix: support task ids containing spaces in Kubernetes peon tasks
Task ids embed the datasource name, which may contain a space. With the
Kubernetes task runner such tasks failed to start:
- peon.sh did not quote TASK_DIR, so writing task.json failed. Quote
TASK_DIR and TASK_JSON.
- K8sTaskAdapter flattened the peon command into a single "sh -c" string,
so --taskId was split and CliPeon fetched the payload from deep storage
with a truncated id when TASK_JSON was omitted for oversized payloads.
Pass the task id through the TASK_ID env variable instead, which peon.sh
forwards to CliPeon as a single quoted --taskId argument.
* docs: update stale TASK_ID comment in peon.sh
---
distribution/docker/peon.sh | 4 +--
.../k8s/overlord/common/PeonCommandContext.java | 22 +++++++++++++
.../k8s/overlord/taskadapter/K8sTaskAdapter.java | 16 ++++++++--
.../overlord/taskadapter/K8sTaskAdapterTest.java | 36 ++++++++++------------
4 files changed, 54 insertions(+), 24 deletions(-)
diff --git a/distribution/docker/peon.sh b/distribution/docker/peon.sh
index a3938733d91..28734493553 100755
--- a/distribution/docker/peon.sh
+++ b/distribution/docker/peon.sh
@@ -163,7 +163,7 @@ fi
# take the ${TASK_JSON} environment variable and base64 decode, unzip and
throw it in ${TASK_DIR}/task.json.
# If TASK_JSON is not set, CliPeon will pull the task.json file from deep
storage.
-mkdir -p ${TASK_DIR}; [ -n "$TASK_JSON" ] && echo ${TASK_JSON} | base64 -d |
gzip -d > ${TASK_DIR}/task.json;
+mkdir -p "${TASK_DIR}"; [ -n "$TASK_JSON" ] && echo "${TASK_JSON}" | base64 -d
| gzip -d > "${TASK_DIR}/task.json";
# Combine options from jvm.config and those given as JAVA_OPTS
# If a value is specified in both then JAVA_OPTS will take precedence when
using OpenJDK
@@ -174,7 +174,7 @@ fi
# Start peon using CliPeon, with variables `Main internal peon TASK_DIR
ATTEMPT_ID`
if [ -n "$TASK_ID" ]; then
- # TASK_ID is only set from PodTemplateTaskAdapter
+ # TASK_ID is set by all K8s pod adapters; older Overlords passed --taskId
in "$@" instead
exec bin/run-java ${JAVA_OPTS} -cp
$COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*:$DRUID_ADDITIONAL_CLASSPATH
org.apache.druid.cli.Main internal peon "${TASK_DIR}" 1 --taskId "${TASK_ID}"
"$@"
else
exec bin/run-java ${JAVA_OPTS} -cp
$COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*:$DRUID_ADDITIONAL_CLASSPATH
org.apache.druid.cli.Main internal peon "${TASK_DIR}" 1 "$@"
diff --git
a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/PeonCommandContext.java
b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/PeonCommandContext.java
index b1b5bf9ed31..4a46fe727c1 100644
---
a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/PeonCommandContext.java
+++
b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/PeonCommandContext.java
@@ -19,6 +19,7 @@
package org.apache.druid.k8s.overlord.common;
+import javax.annotation.Nullable;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
@@ -32,6 +33,8 @@ public class PeonCommandContext
private final File taskDir;
private final boolean enableTls;
private final int cpuMicroCore;
+ @Nullable
+ private final String taskId;
public PeonCommandContext(List<String> command, List<String> javaOpts, File
taskDir, int cpuMicroCore)
{
@@ -45,12 +48,25 @@ public class PeonCommandContext
int cpuMicroCore,
boolean enableTls
)
+ {
+ this(command, javaOpts, taskDir, cpuMicroCore, enableTls, null);
+ }
+
+ public PeonCommandContext(
+ List<String> command,
+ List<String> javaOpts,
+ File taskDir,
+ int cpuMicroCore,
+ boolean enableTls,
+ @Nullable String taskId
+ )
{
this.command = command;
this.javaOpts = javaOpts;
this.taskDir = taskDir;
this.cpuMicroCore = cpuMicroCore;
this.enableTls = enableTls;
+ this.taskId = taskId;
}
public List<String> getCommand()
@@ -83,4 +99,10 @@ public class PeonCommandContext
{
return enableTls;
}
+
+ @Nullable
+ public String getTaskId()
+ {
+ return taskId;
+ }
}
diff --git
a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java
b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java
index cef4588e08a..2fdfb395794 100644
---
a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java
+++
b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java
@@ -157,7 +157,8 @@ public abstract class K8sTaskAdapter implements TaskAdapter
javaOpts(task),
taskConfig.getTaskDir(task.getId()),
taskRunnerConfig.getCpuCoreInMicro(),
- node.isEnableTlsPort()
+ node.isEnableTlsPort(),
+ task.getId()
);
PodSpec podSpec = pod.getSpec();
massageSpec(podSpec, taskRunnerConfig.getPrimaryContainerName());
@@ -330,6 +331,17 @@ public abstract class K8sTaskAdapter implements TaskAdapter
)).build()).build()
);
+ // Pass the task id through the environment rather than as a command
argument: the command is flattened into a
+ // single "sh -c" string, which would split task ids containing spaces.
peon.sh forwards it to CliPeon as --taskId.
+ if (context.getTaskId() != null) {
+ envVars.add(
+ new EnvVarBuilder()
+ .withName(DruidK8sConstants.TASK_ID_ENV)
+ .withValue(context.getTaskId())
+ .build()
+ );
+ }
+
if (taskContents.length() < DruidK8sConstants.MAX_ENV_VARIABLE_KBS) {
envVars.add(
new EnvVarBuilder()
@@ -505,8 +517,6 @@ public abstract class K8sTaskAdapter implements TaskAdapter
command.add("--loadBroadcastDatasourceMode");
command.add(task.getBroadcastDatasourceLoadingSpec().getMode().toString());
- command.add("--taskId");
- command.add(task.getId());
log.info(
"Peon Command for K8s job: %s",
ForkingTaskRunner.getMaskedCommand(startupLoggingConfig.getMaskProperties(),
command)
diff --git
a/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java
b/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java
index 7275e1278cb..c2182b2f689 100644
---
a/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java
+++
b/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java
@@ -75,7 +75,6 @@ import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@@ -323,36 +322,35 @@ class K8sTaskAdapterTest
jsonMapper,
taskLogs
);
+ // task id contains a space, as it does for datasources whose name
contains a space
Task task = new NoopTask(
+ "noop_my ds_id",
"id",
- "id",
- "datasource",
+ "my ds",
0,
0,
ImmutableMap.of("context", RandomStringUtils.randomAlphanumeric((int)
DruidK8sConstants.MAX_ENV_VARIABLE_KBS * 20))
);
Job job = adapter.fromTask(task);
+ final Container mainContainer =
job.getSpec().getTemplate().getSpec().getContainers().get(0);
// TASK_JSON should not be set in env variables
Assertions.assertFalse(
- job.getSpec()
- .getTemplate()
- .getSpec()
- .getContainers()
- .get(0).getEnv()
- .stream().anyMatch(env ->
env.getName().equals(DruidK8sConstants.TASK_JSON_ENV))
+ mainContainer.getEnv()
+ .stream().anyMatch(env ->
env.getName().equals(DruidK8sConstants.TASK_JSON_ENV))
);
- // --taskId <TASK_ID> should be passed to the peon command args
- Assertions.assertTrue(
- Arrays.stream(job.getSpec()
- .getTemplate()
- .getSpec()
- .getContainers()
- .get(0)
- .getArgs()
- .get(0).split(" ")).collect(Collectors.toSet())
- .containsAll(ImmutableList.of("--taskId", task.getId()))
+ // TASK_ID should be passed through env, so that peon.sh can pass it to
CliPeon as a single --taskId argument
+ Assertions.assertEquals(
+ Collections.singletonList(task.getId()),
+ mainContainer.getEnv()
+ .stream()
+ .filter(env ->
env.getName().equals(DruidK8sConstants.TASK_ID_ENV))
+ .map(EnvVar::getValue)
+ .collect(Collectors.toList())
);
+
+ // --taskId should not be in the "sh -c" command string, where a task id
containing a space would be split
+
Assertions.assertFalse(mainContainer.getArgs().get(0).contains("--taskId"));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
