This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 136d1bd1f22 SOLR-18442: Fix OpenTelemetry Observable gauge memory leak
136d1bd1f22 is described below
commit 136d1bd1f22ee1aa3566b85d5bd177f886a8c2ee
Author: Chris Hostetter <[email protected]>
AuthorDate: Thu Sep 17 15:50:55 2026 -0700
SOLR-18442: Fix OpenTelemetry Observable gauge memory leak
---
.../SOLR-18442-observable-gauge-leak.yml | 13 ++++
.../apache/solr/metrics/SolrMetricsContext.java | 40 +++++------
.../solr/metrics/SolrMetricsContextTest.java | 77 ++++++++++++++++++++++
3 files changed, 111 insertions(+), 19 deletions(-)
diff --git a/changelog/unreleased/SOLR-18442-observable-gauge-leak.yml
b/changelog/unreleased/SOLR-18442-observable-gauge-leak.yml
new file mode 100644
index 00000000000..1479c33371b
--- /dev/null
+++ b/changelog/unreleased/SOLR-18442-observable-gauge-leak.yml
@@ -0,0 +1,13 @@
+title: >
+ Fix a memory leak where every SolrIndexSearcher ever opened was retained by
the OpenTelemetry
+ metric registry via its "index.commit_size" observable gauge, along with its
DirectoryReader and
+ segment live-docs bitsets. Nodes with a high commit cadence would eventually
run out of heap.
+ Observable metrics registered through SolrMetricsContext with an explicit
unit are now closed
+ together with the context.
+type: fixed
+authors:
+ - name: Jan Høydahl
+ - name: Mikhail Khludnev
+links:
+ - name: SOLR-18442
+ url: https://issues.apache.org/jira/browse/SOLR-18442
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java
b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java
index 720bbeb301d..45d7c591ef9 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java
@@ -160,9 +160,7 @@ public class SolrMetricsContext implements AutoCloseable {
public ObservableLongGauge observableLongGauge(
String metricName, String description,
Consumer<ObservableLongMeasurement> callback) {
- var observableLongGauge = observableLongGauge(metricName, description,
callback, null);
- closeables.add(observableLongGauge);
- return observableLongGauge;
+ return observableLongGauge(metricName, description, callback, null);
}
public ObservableLongGauge observableLongGauge(
@@ -170,14 +168,15 @@ public class SolrMetricsContext implements AutoCloseable {
String description,
Consumer<ObservableLongMeasurement> callback,
OtelUnit unit) {
- return metricManager.observableLongGauge(registryName, metricName,
description, callback, unit);
+ var observableLongGauge =
+ metricManager.observableLongGauge(registryName, metricName,
description, callback, unit);
+ closeables.add(observableLongGauge);
+ return observableLongGauge;
}
public ObservableDoubleGauge observableDoubleGauge(
String metricName, String description,
Consumer<ObservableDoubleMeasurement> callback) {
- var observableDoubleGauge = observableDoubleGauge(metricName, description,
callback, null);
- closeables.add(observableDoubleGauge);
- return observableDoubleGauge;
+ return observableDoubleGauge(metricName, description, callback, null);
}
public ObservableDoubleGauge observableDoubleGauge(
@@ -185,15 +184,15 @@ public class SolrMetricsContext implements AutoCloseable {
String description,
Consumer<ObservableDoubleMeasurement> callback,
OtelUnit unit) {
- return metricManager.observableDoubleGauge(
- registryName, metricName, description, callback, unit);
+ var observableDoubleGauge =
+ metricManager.observableDoubleGauge(registryName, metricName,
description, callback, unit);
+ closeables.add(observableDoubleGauge);
+ return observableDoubleGauge;
}
public ObservableLongCounter observableLongCounter(
String metricName, String description,
Consumer<ObservableLongMeasurement> callback) {
- var observableLongCounter = observableLongCounter(metricName, description,
callback, null);
- closeables.add(observableLongCounter);
- return observableLongCounter;
+ return observableLongCounter(metricName, description, callback, null);
}
public ObservableLongCounter observableLongCounter(
@@ -201,15 +200,15 @@ public class SolrMetricsContext implements AutoCloseable {
String description,
Consumer<ObservableLongMeasurement> callback,
OtelUnit unit) {
- return metricManager.observableLongCounter(
- registryName, metricName, description, callback, unit);
+ var observableLongCounter =
+ metricManager.observableLongCounter(registryName, metricName,
description, callback, unit);
+ closeables.add(observableLongCounter);
+ return observableLongCounter;
}
public ObservableDoubleCounter observableDoubleCounter(
String metricName, String description,
Consumer<ObservableDoubleMeasurement> callback) {
- var observableDoubleCounter = observableDoubleCounter(metricName,
description, callback, null);
- closeables.add(observableDoubleCounter);
- return observableDoubleCounter;
+ return observableDoubleCounter(metricName, description, callback, null);
}
public ObservableDoubleCounter observableDoubleCounter(
@@ -217,8 +216,11 @@ public class SolrMetricsContext implements AutoCloseable {
String description,
Consumer<ObservableDoubleMeasurement> callback,
OtelUnit unit) {
- return metricManager.observableDoubleCounter(
- registryName, metricName, description, callback, unit);
+ var observableDoubleCounter =
+ metricManager.observableDoubleCounter(
+ registryName, metricName, description, callback, unit);
+ closeables.add(observableDoubleCounter);
+ return observableDoubleCounter;
}
public ObservableLongMeasurement longGaugeMeasurement(String metricName,
String description) {
diff --git
a/solr/core/src/test/org/apache/solr/metrics/SolrMetricsContextTest.java
b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsContextTest.java
new file mode 100644
index 00000000000..048962f7a80
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsContextTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.metrics;
+
+import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;
+import io.opentelemetry.sdk.testing.exporter.InMemoryMetricExporter;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.metrics.otel.OtelUnit;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SolrMetricsContextTest extends SolrTestCase {
+ private static final String REGISTRY = "test_context_registry";
+ private SolrMetricManager metricManager;
+ private PrometheusMetricReader reader;
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ metricManager = new SolrMetricManager(InMemoryMetricExporter.create());
+ metricManager.meterProvider(REGISTRY);
+ reader = metricManager.getPrometheusMetricReader(REGISTRY);
+ }
+
+ @After
+ @Override
+ public void tearDown() throws Exception {
+ metricManager.closeAllRegistries();
+ super.tearDown();
+ }
+
+ /** Callbacks registered through a context must stop firing once the context
is closed. */
+ @Test
+ public void testCloseUnregistersObservableCallbacks() {
+ SolrMetricsContext ctx = new SolrMetricsContext(metricManager, REGISTRY);
+ AtomicInteger invocations = new AtomicInteger();
+
+ ctx.observableLongGauge("long_gauge", "d", m ->
invocations.incrementAndGet());
+ ctx.observableLongGauge(
+ "long_gauge_unit", "d", m -> invocations.incrementAndGet(),
OtelUnit.BYTES);
+ ctx.observableDoubleGauge("double_gauge", "d", m ->
invocations.incrementAndGet());
+ ctx.observableDoubleGauge(
+ "double_gauge_unit", "d", m -> invocations.incrementAndGet(),
OtelUnit.BYTES);
+ ctx.observableLongCounter("long_counter", "d", m ->
invocations.incrementAndGet());
+ ctx.observableLongCounter(
+ "long_counter_unit", "d", m -> invocations.incrementAndGet(),
OtelUnit.BYTES);
+ ctx.observableDoubleCounter("double_counter", "d", m ->
invocations.incrementAndGet());
+ ctx.observableDoubleCounter(
+ "double_counter_unit", "d", m -> invocations.incrementAndGet(),
OtelUnit.BYTES);
+
+ reader.collect();
+ assertEquals("all 8 callbacks should fire while the context is open", 8,
invocations.get());
+
+ ctx.close();
+ invocations.set(0);
+ reader.collect();
+ assertEquals("no callback should fire after the context is closed", 0,
invocations.get());
+ }
+}
