This is an automated email from the ASF dual-hosted git repository.
FrankChen021 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new b1dfe7cd257 test: stabilize KubernetesClusterDockerTest with readiness
probe and rich pod diagnostics (#20814)
b1dfe7cd257 is described below
commit b1dfe7cd2574f67056942fbce54471bad05b9e15
Author: Deep Shah <[email protected]>
AuthorDate: Wed Nov 4 23:55:52 2026 -0500
test: stabilize KubernetesClusterDockerTest with readiness probe and rich
pod diagnostics (#20814)
* test: stabilize KubernetesClusterDockerTest with readiness probe and rich
pod diagnostics
Fixes #20432.
KubernetesClusterDockerTest has intermittently failed in CI (docker-tests)
because k3s pods occasionally timed out waiting to become Ready within 300
s.
1. Added an HTTP GET /status/health readinessProbe with 5 s initial delay
and
5 s period to manifests/druid-service.yaml, matching the existing pattern
in druid-service-with-operator.yaml.
2. Bumped DRUID_XMX from 128m to 256m in manifests/druid-service.yaml to
give
Druid services sufficient memory headroom during startup in CI.
3. In K3sClusterResource, wait until all expected service pods are created
before
evaluating readiness conditions.
4. On pod readiness timeout or service health check failure, capture rich
pod
diagnostics (phase, conditions, container waiting/terminated reasons,
exit codes,
restart counts, Kubernetes cluster events, and log tail) in the
exception and
dump pod logs to druid-container-logs so CI can archive them as
artifacts.
5. Added K3sClusterResourceTest unit tests covering pod diagnostic
formatting
and manifest readinessProbe generation.
* Added the Test
* test: use target pod namespace when dumping timeout logs in
K3sClusterResource
* refactor: extract pod diagnostics logic into K3sPodDiagnostics helper
* refactor: simplify K3sPodDiagnostics constructor and test hygiene
---
.../testing/embedded/k8s/K3sClusterResource.java | 46 +++-
.../embedded/k8s/K3sClusterResourceTest.java | 41 +++
.../testing/embedded/k8s/K3sPodDiagnostics.java | 286 +++++++++++++++++++++
.../embedded/k8s/K3sPodDiagnosticsTest.java | 264 +++++++++++++++++++
.../test/resources/manifests/druid-service.yaml | 11 +-
5 files changed, 639 insertions(+), 9 deletions(-)
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sClusterResource.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sClusterResource.java
index f47bf3a7b32..6f028fa6c31 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sClusterResource.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sClusterResource.java
@@ -47,6 +47,8 @@ import org.testcontainers.k3s.K3sContainer;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.MountableFile;
+import javax.annotation.Nullable;
+
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -84,6 +86,7 @@ public class K3sClusterResource extends
TestcontainerResource<K3sContainer>
private final List<K3sDruidService> services = new ArrayList<>();
private KubernetesClient client;
+ private K3sPodDiagnostics podDiagnostics;
private String druidImageName;
private String druidManifestTemplate = DEFAULT_SERVICE_MANIFEST;
@@ -158,7 +161,7 @@ public class K3sClusterResource extends
TestcontainerResource<K3sContainer>
@Override
public void onStarted(EmbeddedDruidCluster cluster)
{
- initKubernetesClient();
+ initKubernetesClient(cluster);
loadLocalDockerImageIntoContainer(druidImageName, cluster.getTestFolder());
manifestFiles.forEach(this::applyManifest);
initializeDruidServices(cluster);
@@ -180,7 +183,13 @@ public class K3sClusterResource extends
TestcontainerResource<K3sContainer>
protected void waitUntilServicesAreHealthy()
{
- services.forEach(this::waitUntilServiceIsHealthy);
+ try {
+ services.forEach(this::waitUntilServiceIsHealthy);
+ }
+ catch (Exception e) {
+ dumpPodLogs(DRUID_NAMESPACE);
+ throw e;
+ }
}
protected List<K3sDruidService> getServices()
@@ -198,17 +207,33 @@ public class K3sClusterResource extends
TestcontainerResource<K3sContainer>
);
}
- private void initKubernetesClient()
+ private void initKubernetesClient(EmbeddedDruidCluster cluster)
{
client = new KubernetesClientBuilder()
.withConfig(Config.fromKubeconfig(getContainer().getKubeConfigYaml()))
.build();
closer.register(client);
+ final File podLogDir = new File(new File("druid-container-logs",
cluster.getTestClassName()), "k3s-pods");
+ podDiagnostics = new K3sPodDiagnostics(client, podLogDir);
}
- protected void waitUntilPodsAreReady(String namespace)
+ protected void waitUntilPodsAreReady(final String namespace)
{
-
client.pods().inNamespace(namespace).resources().forEach(this::waitUntilPodIsReady);
+ if (DRUID_NAMESPACE.equals(namespace) && !services.isEmpty()) {
+ try {
+ ITRetryUtil.retryUntil(
+ () ->
client.pods().inNamespace(namespace).list().getItems().size() >=
services.size(),
+ true,
+ 1_000L,
+ 60,
+ StringUtils.format("All [%d] service pods created in namespace
[%s]", services.size(), namespace)
+ );
+ }
+ catch (Exception e) {
+ log.warn(e, "Did not observe [%d] pods in namespace [%s]",
services.size(), namespace);
+ }
+ }
+ client.pods().inNamespace(namespace).resources().forEach(pod ->
waitUntilPodIsReady(pod, namespace));
}
/**
@@ -339,9 +364,9 @@ public class K3sClusterResource extends
TestcontainerResource<K3sContainer>
}
/**
- * Waits for the given pod to be ready.
+ * Waits for the given pod to be ready in the given namespace.
*/
- private void waitUntilPodIsReady(PodResource pod)
+ private void waitUntilPodIsReady(final PodResource pod, @Nullable final
String defaultNamespace)
{
try {
pod.waitUntilCondition(
@@ -355,10 +380,15 @@ public class K3sClusterResource extends
TestcontainerResource<K3sContainer>
);
}
catch (KubernetesClientTimeoutException e) {
- throw new ISE("Timed out waiting for pod[%s] to be ready",
pod.get().getMetadata().getName());
+ throw podDiagnostics.onReadyTimeout(pod, defaultNamespace);
}
}
+ public void dumpPodLogs(final String namespace)
+ {
+ podDiagnostics.dumpPodLogs(namespace);
+ }
+
/**
* Polls the health check endpoint of the given service until it is healthy.
*/
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sClusterResourceTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sClusterResourceTest.java
new file mode 100644
index 00000000000..58b569a0b58
--- /dev/null
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sClusterResourceTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.k8s;
+
+import org.apache.druid.testing.DruidCommand;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class K3sClusterResourceTest
+{
+ @Test
+ public void testDruidServiceManifestHasReadinessProbe()
+ {
+ final K3sDruidService coordinatorService = new
K3sDruidService(DruidCommand.Server.COORDINATOR);
+ final String yaml =
coordinatorService.createManifestYaml("manifests/druid-service.yaml",
"druid:test");
+
+ Assertions.assertTrue(yaml.contains("readinessProbe:"));
+ Assertions.assertTrue(yaml.contains("path: /status/health"));
+ Assertions.assertTrue(yaml.contains("port: " +
coordinatorService.getServicePort()));
+ Assertions.assertTrue(yaml.contains("initialDelaySeconds: 5"));
+ Assertions.assertTrue(yaml.contains("periodSeconds: 5"));
+ Assertions.assertTrue(yaml.contains("value: \"256m\""));
+ }
+}
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sPodDiagnostics.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sPodDiagnostics.java
new file mode 100644
index 00000000000..e5fe57da6f0
--- /dev/null
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sPodDiagnostics.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.k8s;
+
+import io.fabric8.kubernetes.api.model.ContainerState;
+import io.fabric8.kubernetes.api.model.ContainerStatus;
+import io.fabric8.kubernetes.api.model.Event;
+import io.fabric8.kubernetes.api.model.ObjectReference;
+import io.fabric8.kubernetes.api.model.ObjectReferenceBuilder;
+import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.api.model.PodCondition;
+import io.fabric8.kubernetes.api.model.PodStatus;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.dsl.PodResource;
+import org.apache.druid.java.util.common.FileUtils;
+import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.logger.Logger;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Diagnostic helper for K3s pods. Gathers pod status, container states,
+ * Kubernetes events, and log tails on timeout, and dumps pod logs to disk.
+ */
+public class K3sPodDiagnostics
+{
+ private static final Logger log = new Logger(K3sPodDiagnostics.class);
+
+ private final KubernetesClient client;
+ private final File podLogDirectory;
+
+ public K3sPodDiagnostics(final KubernetesClient client, final File
podLogDirectory)
+ {
+ this.client = Objects.requireNonNull(client, "client");
+ this.podLogDirectory = Objects.requireNonNull(podLogDirectory,
"podLogDirectory");
+ }
+
+ /**
+ * Handles pod readiness timeout in the given default namespace:
+ * dumps pod logs, formats detailed diagnostics, and returns an {@link ISE}.
+ */
+ public ISE onReadyTimeout(final PodResource pod, @Nullable final String
defaultNamespace)
+ {
+ final String resolvedNamespace = resolveNamespace(pod, defaultNamespace);
+ dumpPodLogs(resolvedNamespace);
+ final String diagnostics = buildPodDiagnostics(pod, resolvedNamespace);
+ return new ISE("%s", diagnostics);
+ }
+
+ public static String resolveNamespace(final PodResource podResource,
@Nullable final String fallbackNamespace)
+ {
+ try {
+ final Pod pod = podResource.get();
+ if (pod != null && pod.getMetadata() != null &&
pod.getMetadata().getNamespace() != null) {
+ return pod.getMetadata().getNamespace();
+ }
+ }
+ catch (Exception ignored) {
+ // ignore
+ }
+ return fallbackNamespace != null ? fallbackNamespace :
K3sClusterResource.DRUID_NAMESPACE;
+ }
+
+ public String buildPodDiagnostics(final PodResource podResource, final
String namespace)
+ {
+ Pod pod = null;
+ try {
+ pod = podResource.get();
+ }
+ catch (Exception e) {
+ log.warn(e, "Could not get pod resource");
+ }
+
+ final String podName = pod != null && pod.getMetadata() != null
+ ? pod.getMetadata().getName()
+ : null;
+
+ List<Event> events = null;
+ if (podName != null) {
+ try {
+ final ObjectReference objectRef = new ObjectReferenceBuilder()
+ .withApiVersion("v1")
+ .withKind("Pod")
+ .withName(podName)
+ .withNamespace(namespace)
+ .build();
+ events = client.v1()
+ .events()
+ .inNamespace(namespace)
+ .withInvolvedObject(objectRef)
+ .list()
+ .getItems();
+ }
+ catch (Exception e) {
+ log.warn(e, "Could not fetch events for pod [%s] in namespace [%s]",
podName, namespace);
+ }
+ }
+
+ String logTail = null;
+ try {
+ logTail = podResource.tailingLines(100).getLog();
+ }
+ catch (Exception e) {
+ log.warn(e, "Could not fetch log tail for pod [%s]", podName);
+ }
+
+ return formatPodDiagnostics(pod, events, logTail);
+ }
+
+ public static String formatPodDiagnostics(
+ @Nullable final Pod pod,
+ @Nullable final List<Event> events,
+ @Nullable final String logTail
+ )
+ {
+ final StringBuilder diagnostics = new StringBuilder();
+ if (pod == null || pod.getMetadata() == null) {
+ diagnostics.append("Timed out waiting for pod to be ready: Pod metadata
is unavailable (pod may have been deleted).");
+ if (events != null && !events.isEmpty()) {
+ appendEvents(diagnostics, events);
+ }
+ return diagnostics.toString();
+ }
+
+ final String podName = pod.getMetadata().getName();
+ final String namespace = pod.getMetadata().getNamespace();
+
+ diagnostics.append(StringUtils.format(
+ "Timed out waiting for pod [%s] in namespace [%s] to be ready.",
+ podName,
+ namespace
+ ));
+
+ final PodStatus status = pod.getStatus();
+ if (status != null) {
+ diagnostics.append(StringUtils.format("\nPhase: [%s]",
status.getPhase()));
+ if (status.getReason() != null || status.getMessage() != null) {
+ diagnostics.append(StringUtils.format(
+ ", Reason: [%s], Message: [%s]",
+ status.getReason(),
+ status.getMessage()
+ ));
+ }
+
+ final List<PodCondition> conditions = status.getConditions();
+ if (conditions != null && !conditions.isEmpty()) {
+ diagnostics.append("\nConditions:");
+ for (final PodCondition condition : conditions) {
+ diagnostics.append(StringUtils.format(
+ "\n - Type: [%s], Status: [%s], Reason: [%s], Message: [%s]",
+ condition.getType(),
+ condition.getStatus(),
+ condition.getReason(),
+ condition.getMessage()
+ ));
+ }
+ }
+
+ final List<ContainerStatus> containerStatuses =
status.getContainerStatuses();
+ if (containerStatuses != null && !containerStatuses.isEmpty()) {
+ diagnostics.append("\nContainer Statuses:");
+ for (final ContainerStatus containerStatus : containerStatuses) {
+ diagnostics.append(StringUtils.format(
+ "\n - Container: [%s], Ready: [%s], RestartCount: [%d]",
+ containerStatus.getName(),
+ containerStatus.getReady(),
+ containerStatus.getRestartCount()
+ ));
+ final ContainerState state = containerStatus.getState();
+ if (state != null) {
+ if (state.getWaiting() != null) {
+ diagnostics.append(StringUtils.format(
+ ", State: Waiting(reason=[%s], message=[%s])",
+ state.getWaiting().getReason(),
+ state.getWaiting().getMessage()
+ ));
+ } else if (state.getTerminated() != null) {
+ diagnostics.append(StringUtils.format(
+ ", State: Terminated(exitCode=[%d], reason=[%s],
message=[%s])",
+ state.getTerminated().getExitCode(),
+ state.getTerminated().getReason(),
+ state.getTerminated().getMessage()
+ ));
+ } else if (state.getRunning() != null) {
+ diagnostics.append(StringUtils.format(
+ ", State: Running(startedAt=[%s])",
+ state.getRunning().getStartedAt()
+ ));
+ }
+ }
+ final ContainerState lastState = containerStatus.getLastState();
+ if (lastState != null && lastState.getTerminated() != null) {
+ diagnostics.append(StringUtils.format(
+ ", LastState: Terminated(exitCode=[%d], reason=[%s],
message=[%s])",
+ lastState.getTerminated().getExitCode(),
+ lastState.getTerminated().getReason(),
+ lastState.getTerminated().getMessage()
+ ));
+ }
+ }
+ }
+ }
+
+ if (events != null && !events.isEmpty()) {
+ appendEvents(diagnostics, events);
+ }
+
+ if (logTail != null && !logTail.isBlank()) {
+ diagnostics.append("\nLog Tail (last lines):\n").append(logTail);
+ }
+
+ return diagnostics.toString();
+ }
+
+ private static void appendEvents(final StringBuilder diagnostics, final
List<Event> events)
+ {
+ diagnostics.append("\nEvents:");
+ for (final Event event : events) {
+ diagnostics.append(StringUtils.format(
+ "\n - [%s] %s: %s (count: [%d], lastObserved: [%s])",
+ event.getType(),
+ event.getReason(),
+ event.getMessage(),
+ event.getCount() != null ? event.getCount() : 1,
+ event.getLastTimestamp()
+ ));
+ }
+ }
+
+ public void dumpPodLogs(final String namespace)
+ {
+ try {
+ FileUtils.mkdirp(podLogDirectory);
+ }
+ catch (Exception e) {
+ log.warn(e, "Failed to create pod log directory [%s]",
podLogDirectory.getAbsolutePath());
+ }
+
+ try {
+ for (final Pod pod :
client.pods().inNamespace(namespace).list().getItems()) {
+ if (pod.getMetadata() == null || pod.getMetadata().getName() == null) {
+ continue;
+ }
+ final String podName = pod.getMetadata().getName();
+ final File podLogFile = new File(podLogDirectory, podName + ".log");
+ try {
+ final String podLogs =
client.pods().inNamespace(namespace).withName(podName).getLog();
+ if (podLogs != null) {
+ Files.writeString(podLogFile.toPath(), podLogs,
StandardCharsets.UTF_8);
+ log.info("Dumped pod [%s] logs to [%s]", podName,
podLogFile.getAbsolutePath());
+ }
+ }
+ catch (Exception e) {
+ log.warn(e, "Could not dump logs for pod [%s]", podName);
+ }
+ }
+ }
+ catch (Exception e) {
+ log.warn(e, "Could not list pods in namespace [%s] for dumping logs",
namespace);
+ }
+ }
+}
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sPodDiagnosticsTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sPodDiagnosticsTest.java
new file mode 100644
index 00000000000..32ea6d03990
--- /dev/null
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/k8s/K3sPodDiagnosticsTest.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.k8s;
+
+import io.fabric8.kubernetes.api.model.ContainerStateBuilder;
+import io.fabric8.kubernetes.api.model.ContainerStateTerminatedBuilder;
+import io.fabric8.kubernetes.api.model.ContainerStateWaitingBuilder;
+import io.fabric8.kubernetes.api.model.ContainerStatus;
+import io.fabric8.kubernetes.api.model.ContainerStatusBuilder;
+import io.fabric8.kubernetes.api.model.Event;
+import io.fabric8.kubernetes.api.model.EventBuilder;
+import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
+import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.api.model.PodBuilder;
+import io.fabric8.kubernetes.api.model.PodCondition;
+import io.fabric8.kubernetes.api.model.PodConditionBuilder;
+import io.fabric8.kubernetes.api.model.PodStatus;
+import io.fabric8.kubernetes.api.model.PodStatusBuilder;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.dsl.PodResource;
+import org.apache.druid.java.util.common.ISE;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.List;
+
+public class K3sPodDiagnosticsTest
+{
+ @Test
+ public void testFormatPodDiagnosticsWithNullPod()
+ {
+ final String diagnostics = K3sPodDiagnostics.formatPodDiagnostics(null,
null, null);
+ Assertions.assertNotNull(diagnostics);
+ Assertions.assertTrue(diagnostics.contains("Pod metadata is unavailable"));
+ }
+
+ @Test
+ public void testFormatPodDiagnosticsWithRunningAndReadyPod()
+ {
+ final PodCondition readyCondition = new PodConditionBuilder()
+ .withType("Ready")
+ .withStatus("True")
+ .withReason("PodReady")
+ .withMessage("All containers are ready")
+ .build();
+
+ final ContainerStatus containerStatus = new ContainerStatusBuilder()
+ .withName("druid-coordinator")
+ .withReady(true)
+ .withRestartCount(0)
+ .withState(new
ContainerStateBuilder().withNewRunning().withStartedAt("2026-11-01T12:00:00Z").endRunning().build())
+ .build();
+
+ final PodStatus status = new PodStatusBuilder()
+ .withPhase("Running")
+ .withConditions(Collections.singletonList(readyCondition))
+ .withContainerStatuses(Collections.singletonList(containerStatus))
+ .build();
+
+ final Pod pod = new PodBuilder()
+ .withMetadata(new
ObjectMetaBuilder().withName("druid-coordinator-1").withNamespace("druid").build())
+ .withStatus(status)
+ .build();
+
+ final String diagnostics = K3sPodDiagnostics.formatPodDiagnostics(pod,
null, null);
+ Assertions.assertTrue(diagnostics.contains("pod [druid-coordinator-1] in
namespace [druid]"));
+ Assertions.assertTrue(diagnostics.contains("Phase: [Running]"));
+ Assertions.assertTrue(diagnostics.contains("Type: [Ready], Status: [True],
Reason: [PodReady]"));
+ Assertions.assertTrue(diagnostics.contains("Container:
[druid-coordinator], Ready: [true], RestartCount: [0]"));
+ Assertions.assertTrue(diagnostics.contains("State:
Running(startedAt=[2026-11-01T12:00:00Z])"));
+ }
+
+ @Test
+ public void testFormatPodDiagnosticsWithCrashLoopBackOffPod()
+ {
+ final PodCondition condition = new PodConditionBuilder()
+ .withType("Ready")
+ .withStatus("False")
+ .withReason("ContainersNotReady")
+ .withMessage("containers with unready status: [druid-coordinator]")
+ .build();
+
+ final ContainerStatus containerStatus = new ContainerStatusBuilder()
+ .withName("druid-coordinator")
+ .withReady(false)
+ .withRestartCount(3)
+ .withState(
+ new ContainerStateBuilder()
+ .withWaiting(
+ new ContainerStateWaitingBuilder()
+ .withReason("CrashLoopBackOff")
+ .withMessage("back-off 40s restarting failed
container")
+ .build()
+ )
+ .build()
+ )
+ .withLastState(
+ new ContainerStateBuilder()
+ .withTerminated(
+ new ContainerStateTerminatedBuilder()
+ .withExitCode(137)
+ .withReason("OOMKilled")
+ .withMessage("Command terminated")
+ .build()
+ )
+ .build()
+ )
+ .build();
+
+ final PodStatus status = new PodStatusBuilder()
+ .withPhase("Running")
+ .withConditions(Collections.singletonList(condition))
+ .withContainerStatuses(Collections.singletonList(containerStatus))
+ .build();
+
+ final Pod pod = new PodBuilder()
+ .withMetadata(new
ObjectMetaBuilder().withName("druid-coordinator-1").withNamespace("druid").build())
+ .withStatus(status)
+ .build();
+
+ final Event event = new EventBuilder()
+ .withType("Warning")
+ .withReason("BackOff")
+ .withMessage("Back-off restarting failed container")
+ .withCount(3)
+ .withLastTimestamp("2026-11-01T12:05:00Z")
+ .build();
+
+ final String logTail = "java.lang.OutOfMemoryError: Java heap space\n\tat
org.apache.druid.server.Main.start(Main.java:42)";
+
+ final String diagnostics = K3sPodDiagnostics.formatPodDiagnostics(pod,
List.of(event), logTail);
+ Assertions.assertTrue(diagnostics.contains("Timed out waiting for pod
[druid-coordinator-1] in namespace [druid] to be ready."));
+ Assertions.assertTrue(diagnostics.contains("Phase: [Running]"));
+ Assertions.assertTrue(diagnostics.contains("Type: [Ready], Status:
[False], Reason: [ContainersNotReady]"));
+ Assertions.assertTrue(diagnostics.contains("Container:
[druid-coordinator], Ready: [false], RestartCount: [3]"));
+ Assertions.assertTrue(diagnostics.contains("State:
Waiting(reason=[CrashLoopBackOff], message=[back-off 40s restarting failed
container])"));
+ Assertions.assertTrue(diagnostics.contains("LastState:
Terminated(exitCode=[137], reason=[OOMKilled], message=[Command terminated])"));
+ Assertions.assertTrue(diagnostics.contains("Events:"));
+ Assertions.assertTrue(diagnostics.contains("[Warning] BackOff: Back-off
restarting failed container (count: [3], lastObserved:
[2026-11-01T12:05:00Z])"));
+ Assertions.assertTrue(diagnostics.contains("Log Tail (last lines):"));
+ Assertions.assertTrue(diagnostics.contains("OutOfMemoryError: Java heap
space"));
+ }
+
+ @Test
+ public void testFormatPodDiagnosticsWithUnhealthyReadinessProbeEvent()
+ {
+ final Pod pod = new PodBuilder()
+ .withMetadata(new
ObjectMetaBuilder().withName("druid-router-1").withNamespace("druid").build())
+ .withStatus(new PodStatusBuilder().withPhase("Running").build())
+ .build();
+
+ final Event event = new EventBuilder()
+ .withType("Warning")
+ .withReason("Unhealthy")
+ .withMessage("Readiness probe failed: HTTP probe failed with
statuscode: 503")
+ .withCount(5)
+ .withLastTimestamp("2026-11-01T12:03:00Z")
+ .build();
+
+ final String diagnostics = K3sPodDiagnostics.formatPodDiagnostics(pod,
List.of(event), null);
+ Assertions.assertTrue(diagnostics.contains("pod [druid-router-1] in
namespace [druid]"));
+ Assertions.assertTrue(diagnostics.contains("[Warning] Unhealthy: Readiness
probe failed: HTTP probe failed with statuscode: 503 (count: [5]"));
+ }
+
+ @Test
+ public void testFormatPodDiagnosticsWithCustomNamespace()
+ {
+ final Pod pod = new PodBuilder()
+ .withMetadata(new
ObjectMetaBuilder().withName("druid-operator-1").withNamespace("druid-operator-system").build())
+ .withStatus(new PodStatusBuilder().withPhase("Pending").build())
+ .build();
+
+ final String diagnostics = K3sPodDiagnostics.formatPodDiagnostics(pod,
null, null);
+ Assertions.assertTrue(diagnostics.contains("pod [druid-operator-1] in
namespace [druid-operator-system]"));
+ Assertions.assertTrue(diagnostics.contains("Phase: [Pending]"));
+ }
+
+ @Test
+ public void testResolveNamespaceFromPodMetadata()
+ {
+ final PodResource podResource = Mockito.mock(PodResource.class);
+ final Pod pod = new PodBuilder()
+ .withMetadata(new
ObjectMetaBuilder().withName("operator").withNamespace("druid-operator-system").build())
+ .build();
+ Mockito.when(podResource.get()).thenReturn(pod);
+
+ Assertions.assertEquals("druid-operator-system",
K3sPodDiagnostics.resolveNamespace(podResource, "druid"));
+ }
+
+ @Test
+ public void testResolveNamespaceFallback()
+ {
+ final PodResource podResourceWithoutNamespace =
Mockito.mock(PodResource.class);
+ final Pod podWithoutNamespace = new PodBuilder()
+ .withMetadata(new ObjectMetaBuilder().withName("operator").build())
+ .build();
+
Mockito.when(podResourceWithoutNamespace.get()).thenReturn(podWithoutNamespace);
+
+ Assertions.assertEquals(
+ "druid-operator-system",
+ K3sPodDiagnostics.resolveNamespace(podResourceWithoutNamespace,
"druid-operator-system")
+ );
+ Assertions.assertEquals(
+ K3sClusterResource.DRUID_NAMESPACE,
+ K3sPodDiagnostics.resolveNamespace(podResourceWithoutNamespace, null)
+ );
+
+ final PodResource brokenPodResource = Mockito.mock(PodResource.class);
+ Mockito.when(brokenPodResource.get()).thenThrow(new RuntimeException("API
error"));
+ Assertions.assertEquals(
+ "fallback-ns",
+ K3sPodDiagnostics.resolveNamespace(brokenPodResource, "fallback-ns")
+ );
+ }
+
+ @Test
+ public void testOnReadyTimeout(@TempDir final File tempDir)
+ {
+ final PodResource podResource = Mockito.mock(PodResource.class);
+ final Pod pod = new PodBuilder()
+ .withMetadata(new
ObjectMetaBuilder().withName("druid-broker-1").withNamespace("druid").build())
+ .withStatus(new PodStatusBuilder().withPhase("Pending").build())
+ .build();
+ Mockito.when(podResource.get()).thenReturn(pod);
+
+ final KubernetesClient client = Mockito.mock(KubernetesClient.class,
Mockito.RETURNS_DEEP_STUBS);
+ final K3sPodDiagnostics diagnostics = new K3sPodDiagnostics(client,
tempDir);
+
+ final ISE ise = diagnostics.onReadyTimeout(podResource, "druid");
+ Assertions.assertNotNull(ise);
+ Assertions.assertTrue(ise.getMessage().contains("Timed out waiting for pod
[druid-broker-1] in namespace [druid] to be ready."));
+ Assertions.assertTrue(ise.getMessage().contains("Phase: [Pending]"));
+ }
+
+ @Test
+ public void testDumpPodLogs(@TempDir final File tempDir)
+ {
+ final KubernetesClient client = Mockito.mock(KubernetesClient.class,
Mockito.RETURNS_DEEP_STUBS);
+ final K3sPodDiagnostics diagnostics = new K3sPodDiagnostics(client,
tempDir);
+ Assertions.assertDoesNotThrow(() -> diagnostics.dumpPodLogs("druid"));
+ Assertions.assertTrue(tempDir.exists());
+ }
+}
diff --git a/embedded-tests/src/test/resources/manifests/druid-service.yaml
b/embedded-tests/src/test/resources/manifests/druid-service.yaml
index 329298feaa7..c53ad76da56 100644
--- a/embedded-tests/src/test/resources/manifests/druid-service.yaml
+++ b/embedded-tests/src/test/resources/manifests/druid-service.yaml
@@ -28,7 +28,16 @@ spec:
- name: DRUID_XMS
value: "128m"
- name: DRUID_XMX
- value: "128m"
+ value: "256m"
+ readinessProbe:
+ failureThreshold: 30
+ httpGet:
+ path: /status/health
+ port: ${port}
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ successThreshold: 1
+ timeoutSeconds: 5
volumeMounts:
- name: druid-common-props
mountPath:
/opt/druid/conf/druid/cluster/_common/common.runtime.properties
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
