yyj8 commented on code in PR #23634:
URL: https://github.com/apache/pulsar/pull/23634#discussion_r1860267230
##########
pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/VipStatus.java:
##########
@@ -38,25 +44,71 @@ public class VipStatus {
public static final String ATTRIBUTE_STATUS_FILE_PATH = "statusFilePath";
public static final String ATTRIBUTE_IS_READY_PROBE = "isReadyProbe";
+ // log a full thread dump when a deadlock is detected in status check once
every 10 minutes
+ // to prevent excessive logging
+ private static final long LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED =
600000L;
+ private static volatile long lastCheckStatusTimestamp;
+
+ // Rate limit status checks to every 500ms to prevent DoS
+ private static final long CHECK_STATUS_INTERVAL = 500L;
+ private static volatile boolean lastCheckStatusResult;
+
@Context
protected ServletContext servletContext;
@GET
public String checkStatus() {
- String statusFilePath = (String)
servletContext.getAttribute(ATTRIBUTE_STATUS_FILE_PATH);
- @SuppressWarnings("unchecked")
- Supplier<Boolean> isReadyProbe = (Supplier<Boolean>)
servletContext.getAttribute(ATTRIBUTE_IS_READY_PROBE);
+ // Locking classes to avoid deadlock detection in multi-thread
concurrent requests.
+ synchronized (VipStatus.class) {
+ if (System.currentTimeMillis() - lastCheckStatusTimestamp <
CHECK_STATUS_INTERVAL) {
+ lastCheckStatusTimestamp = System.currentTimeMillis();
+ if (lastCheckStatusResult) {
+ return "OK";
+ } else {
+ throw new
WebApplicationException(Status.SERVICE_UNAVAILABLE);
+ }
+ }
+ lastCheckStatusTimestamp = System.currentTimeMillis();
Review Comment:
The final global code logic content is as follows:
```java
public class VipStatus {
public static final String ATTRIBUTE_STATUS_FILE_PATH = "statusFilePath";
public static final String ATTRIBUTE_IS_READY_PROBE = "isReadyProbe";
// log a full thread dump when a deadlock is detected in status check
once every 10 minutes
// to prevent excessive logging
private static final long LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED
= 600000L;
private static volatile long lastCheckStatusTimestamp;
// Rate limit status checks to every 500ms to prevent DoS
private static final long CHECK_STATUS_INTERVAL = 500L;
private static volatile boolean lastCheckStatusResult;
@Context
protected ServletContext servletContext;
@GET
public String checkStatus() {
// Locking classes to avoid deadlock detection in multi-thread
concurrent requests.
synchronized (VipStatus.class) {
if (System.currentTimeMillis() - lastCheckStatusTimestamp <
CHECK_STATUS_INTERVAL) {
if (lastCheckStatusResult) {
return "OK";
} else {
throw new
WebApplicationException(Status.SERVICE_UNAVAILABLE);
}
}
lastCheckStatusTimestamp = System.currentTimeMillis();
String statusFilePath = (String)
servletContext.getAttribute(ATTRIBUTE_STATUS_FILE_PATH);
@SuppressWarnings("unchecked")
Supplier<Boolean> isReadyProbe = (Supplier<Boolean>)
servletContext.getAttribute(ATTRIBUTE_IS_READY_PROBE);
boolean isReady = isReadyProbe != null ? isReadyProbe.get() :
true;
if (statusFilePath != null) {
File statusFile = new File(statusFilePath);
if (isReady && statusFile.exists() && statusFile.isFile()) {
// check deadlock
ThreadMXBean threadBean =
ManagementFactory.getThreadMXBean();
long[] threadIds = threadBean.findDeadlockedThreads();
if (threadIds != null && threadIds.length > 0) {
ThreadInfo[] threadInfos =
threadBean.getThreadInfo(threadIds, false,
false);
String threadNames = Arrays.stream(threadInfos)
.map(threadInfo -> threadInfo.getThreadName()
+ "(tid=" + threadInfo.getThreadId()
+ ")")
.collect(Collectors.joining(", "));
if (System.currentTimeMillis() -
lastCheckStatusTimestamp
>
LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED) {
String diagnosticResult =
ThreadDumpUtil.buildThreadDiagnosticString();
log.error("Deadlock detected, service may be
unavailable, "
+ "thread stack details are as follows:
{}.", diagnosticResult);
} else {
log.error("Deadlocked threads detected. {}",
threadNames);
}
lastCheckStatusResult = false;
throw new
WebApplicationException(Status.SERVICE_UNAVAILABLE);
} else {
lastCheckStatusResult = true;
return "OK";
}
}
}
lastCheckStatusResult = false;
log.warn("Failed to access \"status.html\". The service is not
ready");
throw new WebApplicationException(Status.NOT_FOUND);
}
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
