This is an automated email from the ASF dual-hosted git repository.
georgew5656 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 4152f1d1471 Fix empty logs and status messages for mmless ingestion
(#15527)
4152f1d1471 is described below
commit 4152f1d1471e68417bc0a9b92f5c230a5f771d11
Author: George Shiqi Wu <[email protected]>
AuthorDate: Mon Dec 11 13:20:45 2023 -0500
Fix empty logs and status messages for mmless ingestion (#15527)
* Fix empty logs and status messages for mmless ingestion
* Add tests
---
.../druid/k8s/overlord/KubernetesPeonLifecycle.java | 15 ++++++++++++++-
.../druid/k8s/overlord/KubernetesPeonLifecycleTest.java | 13 +++++--------
2 files changed, 19 insertions(+), 9 deletions(-)
diff --git
a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java
b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java
index 5c6c7c6b3eb..eaef0cba6a1 100644
---
a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java
+++
b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java
@@ -288,7 +288,11 @@ public class KubernetesPeonLifecycle
TaskStatus.class
);
} else {
- taskStatus = TaskStatus.failure(taskId.getOriginalTaskId(), "task
status not found");
+ log.info(
+ "Peon for task [%s] did not push its task status. Check k8s logs
and events for the pod to see what happened.",
+ taskId
+ );
+ taskStatus = TaskStatus.failure(taskId.getOriginalTaskId(), "Peon did
not report status successfully.");
}
}
catch (IOException e) {
@@ -329,6 +333,15 @@ public class KubernetesPeonLifecycle
FileUtils.copyInputStreamToFile(logWatch.getOutput(), file.toFile());
} else {
log.debug("Log stream not found for %s", taskId.getOriginalTaskId());
+ FileUtils.writeStringToFile(
+ file.toFile(),
+ StringUtils.format(
+ "Peon for task [%s] did not report any logs. Check k8s
metrics and events for the pod to see what happened.",
+ taskId
+ ),
+ Charset.defaultCharset()
+ );
+
}
taskLogs.pushTaskLog(taskId.getOriginalTaskId(), file.toFile());
}
diff --git
a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java
b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java
index 1c6e429a3dc..59c3700b1fc 100644
---
a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java
+++
b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java
@@ -276,16 +276,15 @@ public class KubernetesPeonLifecycleTest extends
EasyMockSupport
EasyMock.anyLong(),
EasyMock.eq(TimeUnit.MILLISECONDS)
)).andReturn(new JobResponse(null, PeonPhase.FAILED));
-
EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.of(logWatch));
+
EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.absent());
EasyMock.expect(taskLogs.streamTaskStatus(ID)).andReturn(Optional.absent());
+
taskLogs.pushTaskLog(EasyMock.eq(ID), EasyMock.anyObject(File.class));
EasyMock.expectLastCall();
stateListener.stateChanged(KubernetesPeonLifecycle.State.RUNNING, ID);
EasyMock.expectLastCall().once();
stateListener.stateChanged(KubernetesPeonLifecycle.State.STOPPED, ID);
EasyMock.expectLastCall().once();
- logWatch.close();
- EasyMock.expectLastCall();
replayAll();
@@ -295,7 +294,7 @@ public class KubernetesPeonLifecycleTest extends
EasyMockSupport
Assert.assertTrue(taskStatus.isFailure());
Assert.assertEquals(ID, taskStatus.getId());
- Assert.assertEquals("task status not found", taskStatus.getErrorMsg());
+ Assert.assertEquals("Peon did not report status successfully.",
taskStatus.getErrorMsg());
Assert.assertEquals(KubernetesPeonLifecycle.State.STOPPED,
peonLifecycle.getState());
}
@@ -436,7 +435,7 @@ public class KubernetesPeonLifecycleTest extends
EasyMockSupport
EasyMock.anyLong(),
EasyMock.eq(TimeUnit.MILLISECONDS)
)).andReturn(new JobResponse(job, PeonPhase.SUCCEEDED));
-
EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.of(logWatch));
+
EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.absent());
EasyMock.expect(taskLogs.streamTaskStatus(ID)).andReturn(Optional.absent());
taskLogs.pushTaskLog(EasyMock.eq(ID), EasyMock.anyObject(File.class));
EasyMock.expectLastCall();
@@ -444,8 +443,6 @@ public class KubernetesPeonLifecycleTest extends
EasyMockSupport
EasyMock.expectLastCall().once();
stateListener.stateChanged(KubernetesPeonLifecycle.State.STOPPED, ID);
EasyMock.expectLastCall().once();
- logWatch.close();
- EasyMock.expectLastCall();
Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED,
peonLifecycle.getState());
@@ -457,7 +454,7 @@ public class KubernetesPeonLifecycleTest extends
EasyMockSupport
Assert.assertTrue(taskStatus.isFailure());
Assert.assertEquals(ID, taskStatus.getId());
- Assert.assertEquals("task status not found", taskStatus.getErrorMsg());
+ Assert.assertEquals("Peon did not report status successfully.",
taskStatus.getErrorMsg());
Assert.assertEquals(KubernetesPeonLifecycle.State.STOPPED,
peonLifecycle.getState());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
