This is an automated email from the ASF dual-hosted git repository.
siddteotia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c54070d53c Metric for HTTP Thread Utilization (Controller) (#15716)
c54070d53c is described below
commit c54070d53c241101d3dd661b1f317e94bd1bfb81
Author: Praveen <[email protected]>
AuthorDate: Fri May 9 12:01:52 2025 -0700
Metric for HTTP Thread Utilization (Controller) (#15716)
* Metric for HTTP Thread Utilization
* review comments 1
---
.../pinot/common/metrics/ControllerGauge.java | 4 +-
.../pinot/controller/BaseControllerStarter.java | 2 +-
.../api/ControllerAdminApiApplication.java | 63 +++++++++++++++++++++-
3 files changed, 66 insertions(+), 3 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
index ddbb67a049..0e95f1915e 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
@@ -219,7 +219,9 @@ public enum ControllerGauge implements
AbstractMetrics.Gauge {
DEEP_STORE_WRITE_OPS_IN_PROGRESS("deepStoreWriteOpsInProgress", true),
// The progress of a certain table rebalance job of a table
- TABLE_REBALANCE_JOB_PROGRESS_PERCENT("percent", false);
+ TABLE_REBALANCE_JOB_PROGRESS_PERCENT("percent", false),
+ // HTTP thread utilization
+ HTTP_THREAD_UTILIZATION("httpThreadUtilization", true);
private final String _gaugeName;
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
index 8722f80487..6b9b88f88e 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
@@ -607,7 +607,7 @@ public abstract class BaseControllerStarter implements
ServiceStartable {
});
LOGGER.info("Starting controller admin application on: {}",
ListenerConfigUtil.toString(_listenerConfigs));
- _adminApp.start(_listenerConfigs);
+ _adminApp.start(_listenerConfigs, _controllerMetrics);
enforceTableConfigAndSchema();
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java
index 68d02fbaef..55c101a5d9 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java
@@ -21,10 +21,14 @@ package org.apache.pinot.controller.api;
import io.swagger.jaxrs.listing.SwaggerSerializers;
import java.io.IOException;
import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerResponseContext;
import javax.ws.rs.container.ContainerResponseFilter;
+import org.apache.pinot.common.metrics.ControllerGauge;
+import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.swagger.SwaggerApiListingResource;
import org.apache.pinot.common.swagger.SwaggerSetupUtils;
import org.apache.pinot.controller.ControllerConf;
@@ -36,6 +40,12 @@ import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.PinotReflectionUtils;
import org.glassfish.grizzly.http.server.CLStaticHttpHandler;
import org.glassfish.grizzly.http.server.HttpServer;
+import org.glassfish.grizzly.http.server.NetworkListener;
+import org.glassfish.grizzly.monitoring.MonitoringAware;
+import org.glassfish.grizzly.monitoring.MonitoringConfig;
+import org.glassfish.grizzly.threadpool.AbstractThreadPool;
+import org.glassfish.grizzly.threadpool.ThreadPoolConfig;
+import org.glassfish.grizzly.threadpool.ThreadPoolProbe;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.jackson.JacksonFeature;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
@@ -79,7 +89,7 @@ public class ControllerAdminApiApplication extends
ResourceConfig {
register(binder);
}
- public void start(List<ListenerConfig> listenerConfigs) {
+ public void start(List<ListenerConfig> listenerConfigs, ControllerMetrics
controllerMetrics) {
_httpServer = ListenerConfigUtil.buildHttpServer(this, listenerConfigs);
try {
@@ -104,6 +114,7 @@ public class ControllerAdminApiApplication extends
ResourceConfig {
_httpServer.getServerConfiguration()
.addHttpHandler(new CLStaticHttpHandler(classLoader,
"/webapp/images/"), "/images/");
_httpServer.getServerConfiguration().addHttpHandler(new
CLStaticHttpHandler(classLoader, "/webapp/js/"), "/js/");
+ registerHttpThreadUtilizationGauge(controllerMetrics);
}
public void stop() {
@@ -130,4 +141,54 @@ public class ControllerAdminApiApplication extends
ResourceConfig {
public HttpServer getHttpServer() {
return _httpServer;
}
+
+ /**
+ * Registers a gauge that tracks HTTP thread pool utilization without using
reflection.
+ * Instead, it uses a custom ThreadPoolProbe to count active threads.
+ */
+ private void registerHttpThreadUtilizationGauge(ControllerMetrics metrics) {
+ NetworkListener listener = _httpServer.getListeners().iterator().next();
+ ExecutorService executor = listener.getTransport().getWorkerThreadPool();
+ ThreadPoolConfig poolCfg =
listener.getTransport().getWorkerThreadPoolConfig();
+
+ ActiveThreadProbe probe = new ActiveThreadProbe();
+ // Try to attach probe to the executor if it supports monitoring
+ if (executor instanceof MonitoringAware) {
+ @SuppressWarnings("unchecked")
+ MonitoringConfig<ThreadPoolProbe> mc =
((MonitoringAware<ThreadPoolProbe>) executor).getMonitoringConfig();
+ mc.addProbes(probe);
+ }
+
+
metrics.setOrUpdateGauge(ControllerGauge.HTTP_THREAD_UTILIZATION.getGaugeName(),
() -> {
+ int max = poolCfg.getMaxPoolSize();
+ if (max <= 0) {
+ return 0L;
+ }
+ return Math.round(probe.getActiveCount() * 100.0 / max);
+ });
+ }
+
+ /**
+ * Custom probe to track busy threads in Grizzly thread pools without using
reflection.
+ */
+ public static final class ActiveThreadProbe extends ThreadPoolProbe.Adapter {
+ private final AtomicInteger _active = new AtomicInteger();
+
+ @Override
+ public void onTaskDequeueEvent(AbstractThreadPool pool, Runnable task) {
+ // one more thread just got real work
+ _active.incrementAndGet();
+ }
+
+ @Override
+ public void onTaskCompleteEvent(AbstractThreadPool pool, Runnable task) {
+ // work finished, thread is idle again
+ _active.decrementAndGet();
+ }
+
+ /** Current number of active threads. */
+ public int getActiveCount() {
+ return _active.get();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
