This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch branch-lance-namepspace-dev
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-lance-namepspace-dev by
this push:
new 67179e6c28 [#8909] feat(lance): Add metric systems to lance server
(#8904)
67179e6c28 is described below
commit 67179e6c283b39c6d198fe7d5a73eb7e9ea6f0de
Author: Junda Yang <[email protected]>
AuthorDate: Fri Oct 24 05:17:39 2025 -0700
[#8909] feat(lance): Add metric systems to lance server (#8904)
### What changes were proposed in this pull request?
1. Added Lance REST server metrics constant (MetricsSource.java)
2. Added MapperConfig rule to transform Dropwizard metrics to Prometheus
format
3. Integrated metrics collection in Lance REST service
(LanceRESTService.java)
4. Enables automatic exposure of metrics at /metrics and
/prometheus/metrics endpoints
### Why are the changes needed?
Observability is needed for the new lance rest service.
Fix: #8909
### Does this PR introduce _any_ user-facing change?
Yes - New metrics endpoints
Standalone mode:
- http://localhost:9101/metrics (Dropwizard JSON format)
- http://localhost:9101/prometheus/metrics (Prometheus text format)
Auxiliary mode:
- Metrics exposed on all service ports (8090, 9001, 9101) showing
unified metrics from all services
### How was this patch tested?
Unit tests
---------
Co-authored-by: Jerry Shao <[email protected]>
---
.../java/org/apache/gravitino/metrics/MetricsSystem.java | 4 ++++
.../apache/gravitino/metrics/source/MetricsSource.java | 1 +
.../gravitino/metrics/TestExtractMetricNameAndLabel.java | 16 ++++++++++++++++
.../org/apache/gravitino/lance/LanceRESTService.java | 12 ++++++++++++
.../lance/service/rest/LanceNamespaceOperations.java | 5 +++++
.../lance/service/rest/LanceTableOperations.java | 9 ++++++++-
6 files changed, 46 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/gravitino/metrics/MetricsSystem.java
b/core/src/main/java/org/apache/gravitino/metrics/MetricsSystem.java
index dde401bfd1..27ae051a7d 100644
--- a/core/src/main/java/org/apache/gravitino/metrics/MetricsSystem.java
+++ b/core/src/main/java/org/apache/gravitino/metrics/MetricsSystem.java
@@ -203,6 +203,10 @@ public class MetricsSystem implements Closeable {
MetricsSource.ICEBERG_REST_SERVER_METRIC_NAME + ".*.*",
MetricsSource.ICEBERG_REST_SERVER_METRIC_NAME + "_${1}",
ImmutableMap.of("operation", "${0}")),
+ new MapperConfig(
+ MetricsSource.LANCE_REST_SERVER_METRIC_NAME + ".*.*",
+ MetricsSource.LANCE_REST_SERVER_METRIC_NAME + "_${1}",
+ ImmutableMap.of("operation", "${0}")),
new MapperConfig(
MetricsSource.GRAVITINO_SERVER_METRIC_NAME + ".*.*",
MetricsSource.GRAVITINO_SERVER_METRIC_NAME + "_${1}",
diff --git
a/core/src/main/java/org/apache/gravitino/metrics/source/MetricsSource.java
b/core/src/main/java/org/apache/gravitino/metrics/source/MetricsSource.java
index cee2d6afe5..1d079f49ae 100644
--- a/core/src/main/java/org/apache/gravitino/metrics/source/MetricsSource.java
+++ b/core/src/main/java/org/apache/gravitino/metrics/source/MetricsSource.java
@@ -40,6 +40,7 @@ public abstract class MetricsSource {
// metrics source name
public static final String ICEBERG_REST_SERVER_METRIC_NAME =
"iceberg-rest-server";
+ public static final String LANCE_REST_SERVER_METRIC_NAME =
"lance-rest-server";
public static final String GRAVITINO_SERVER_METRIC_NAME = "gravitino-server";
public static final String GRAVITINO_RELATIONAL_STORE_METRIC_NAME =
"gravitino-relational-store";
public static final String GRAVITINO_CATALOG_METRIC_PREFIX =
"gravitino-catalog";
diff --git
a/core/src/test/java/org/apache/gravitino/metrics/TestExtractMetricNameAndLabel.java
b/core/src/test/java/org/apache/gravitino/metrics/TestExtractMetricNameAndLabel.java
index 4f3a1feba2..c6a57f3c87 100644
---
a/core/src/test/java/org/apache/gravitino/metrics/TestExtractMetricNameAndLabel.java
+++
b/core/src/test/java/org/apache/gravitino/metrics/TestExtractMetricNameAndLabel.java
@@ -62,6 +62,13 @@ public class TestExtractMetricNameAndLabel {
+ Collector.sanitizeMetricName(MetricNames.SERVER_IDLE_THREAD_NUM),
ImmutableMap.of());
+ checkResult(
+ MetricsSource.LANCE_REST_SERVER_METRIC_NAME + "." +
MetricNames.SERVER_IDLE_THREAD_NUM,
+
Collector.sanitizeMetricName(MetricsSource.LANCE_REST_SERVER_METRIC_NAME)
+ + "_"
+ + Collector.sanitizeMetricName(MetricNames.SERVER_IDLE_THREAD_NUM),
+ ImmutableMap.of());
+
checkResult(
MetricsSource.ICEBERG_REST_SERVER_METRIC_NAME
+ ".update-table."
@@ -71,6 +78,15 @@ public class TestExtractMetricNameAndLabel {
+ Collector.sanitizeMetricName(MetricNames.HTTP_PROCESS_DURATION),
ImmutableMap.of("operation", "update-table"));
+ checkResult(
+ MetricsSource.LANCE_REST_SERVER_METRIC_NAME
+ + ".load-namespace."
+ + MetricNames.HTTP_PROCESS_DURATION,
+
Collector.sanitizeMetricName(MetricsSource.LANCE_REST_SERVER_METRIC_NAME)
+ + "_"
+ + Collector.sanitizeMetricName(MetricNames.HTTP_PROCESS_DURATION),
+ ImmutableMap.of("operation", "load-namespace"));
+
checkResult(
MetricsSource.GRAVITINO_SERVER_METRIC_NAME
+ ".update-table."
diff --git
a/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/LanceRESTService.java
b/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/LanceRESTService.java
index 123781262f..d1409c8e12 100644
---
a/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/LanceRESTService.java
+++
b/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/LanceRESTService.java
@@ -23,10 +23,14 @@ import static
org.apache.gravitino.lance.common.config.LanceConfig.NAMESPACE_BAC
import java.lang.reflect.Constructor;
import java.util.Map;
import javax.servlet.Servlet;
+import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.auxiliary.GravitinoAuxiliaryService;
import org.apache.gravitino.lance.common.config.LanceConfig;
import org.apache.gravitino.lance.common.ops.LanceNamespaceBackend;
import org.apache.gravitino.lance.common.ops.NamespaceWrapper;
+import org.apache.gravitino.metrics.MetricsSystem;
+import org.apache.gravitino.metrics.source.MetricsSource;
+import org.apache.gravitino.server.web.HttpServerMetricsSource;
import org.apache.gravitino.server.web.JettyServer;
import org.apache.gravitino.server.web.JettyServerConfig;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -58,6 +62,8 @@ public class LanceRESTService implements
GravitinoAuxiliaryService {
JettyServerConfig serverConfig = JettyServerConfig.fromConfig(lanceConfig);
server = new JettyServer();
+ // Get MetricsSystem directly from GravitinoEnv for zero-overhead access
+ MetricsSystem metricsSystem = GravitinoEnv.getInstance().metricsSystem();
server.initialize(serverConfig, SERVICE_NAME, false);
this.lanceNamespace = loadNamespaceImpl(lanceConfig);
@@ -73,6 +79,12 @@ public class LanceRESTService implements
GravitinoAuxiliaryService {
}
});
+ // Register metrics with shared MetricsSystem
+ HttpServerMetricsSource httpServerMetricsSource =
+ new HttpServerMetricsSource(
+ MetricsSource.LANCE_REST_SERVER_METRIC_NAME, resourceConfig,
server);
+ metricsSystem.register(httpServerMetricsSource);
+
Servlet container = new ServletContainer(resourceConfig);
server.addServlet(container, LANCE_SPEC);
server.addCustomFilters(LANCE_SPEC);
diff --git
a/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceNamespaceOperations.java
b/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceNamespaceOperations.java
index 6c0477a51c..2d07357f30 100644
---
a/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceNamespaceOperations.java
+++
b/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceNamespaceOperations.java
@@ -20,6 +20,8 @@ package org.apache.gravitino.lance.service.rest;
import static
org.apache.gravitino.lance.common.ops.NamespaceWrapper.NAMESPACE_DELIMITER_DEFAULT;
+import com.codahale.metrics.annotation.ResponseMetered;
+import com.codahale.metrics.annotation.Timed;
import com.lancedb.lance.namespace.model.ListNamespacesResponse;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
@@ -34,6 +36,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.gravitino.lance.common.ops.NamespaceWrapper;
import org.apache.gravitino.lance.service.LanceExceptionMapper;
+import org.apache.gravitino.metrics.MetricNames;
@Path("/v1/namespace")
@Consumes(MediaType.APPLICATION_JSON)
@@ -49,6 +52,8 @@ public class LanceNamespaceOperations {
@GET
@Path("/{id}/list")
+ @Timed(name = "list-namespaces." + MetricNames.HTTP_PROCESS_DURATION,
absolute = true)
+ @ResponseMetered(name = "list-namespaces", absolute = true)
public Response listNamespaces(
@Encoded @PathParam("id") String namespaceId,
@DefaultValue(NAMESPACE_DELIMITER_DEFAULT) @QueryParam("delimiter")
String delimiter,
diff --git
a/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceTableOperations.java
b/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceTableOperations.java
index 10f7399c40..1f30d1b326 100644
---
a/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceTableOperations.java
+++
b/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceTableOperations.java
@@ -18,6 +18,10 @@
*/
package org.apache.gravitino.lance.service.rest;
+import static
org.apache.gravitino.lance.common.ops.NamespaceWrapper.NAMESPACE_DELIMITER_DEFAULT;
+
+import com.codahale.metrics.annotation.ResponseMetered;
+import com.codahale.metrics.annotation.Timed;
import com.lancedb.lance.namespace.model.ListTablesResponse;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
@@ -32,6 +36,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.gravitino.lance.common.ops.NamespaceWrapper;
import org.apache.gravitino.lance.service.LanceExceptionMapper;
+import org.apache.gravitino.metrics.MetricNames;
@Path("/v1/namespace/{id}/table")
@Consumes(MediaType.APPLICATION_JSON)
@@ -47,9 +52,11 @@ public class LanceTableOperations {
@GET
@Path("/list")
+ @Timed(name = "list-tables." + MetricNames.HTTP_PROCESS_DURATION, absolute =
true)
+ @ResponseMetered(name = "list-tables", absolute = true)
public Response listTables(
@Encoded @PathParam("id") String namespaceId,
- @DefaultValue("$") @QueryParam("delimiter") String delimiter,
+ @DefaultValue(NAMESPACE_DELIMITER_DEFAULT) @QueryParam("delimiter")
String delimiter,
@QueryParam("page_token") String pageToken,
@QueryParam("limit") Integer limit) {
try {
