This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch branch_10_1 in repository https://gitbox.apache.org/repos/asf/solr.git
commit 4c6dd37120240b5f85d88217e83eeadaf13af554 Author: David Smiley <[email protected]> AuthorDate: Mon Oct 5 14:21:58 2026 -0400 SOLR-17432: OpenTelemetry init refactor; OTel Java agent support (#2687) Solr now detects a loaded OpenTelemetry Java agent and uses the OpenTelemetry it registered, with no need for a <tracerConfig> in solr.xml or the opentelemetry module. Previously the agent worked only incidentally (with an ignorable ClassNotFoundException). Underneath, GlobalOpenTelemetry is now set in exactly one place, which was previously scattered across several classes and test utilities and hard to reason about. - OpenTelemetryConfigurator owns initialization, invoked first thing in the CoreContainer constructor. - API change: configurators now implement createOpenTelemetry() and must not set GlobalOpenTelemetry themselves; getTracer() and getOpenTelemetrySdk() are deprecated. Existing configurators keep working. OtelTracerConfigurator was updated accordingly, building the SDK without registering it globally. - Test infrastructure: MiniSolrCloudCluster no longer touches tracing; nodes initialize OTel the same way production does. SolrTestCase resets OTel after each class and randomizes the span-recording check for all tests, not just SolrCloud ones. Span-examining tests use OTel's own OpenTelemetryRule instead of the custom configurator that worked around tests racing to set the global, which is removed. - Ref guide: new dedicated OpenTelemetry page covering the Java agent, the module, <tracerConfig>, and how to verify which integration is active, so the tracing and metrics pages don't each re-explain setup. (cherry picked from commit 22a8cfebbbdb026437d10b0aa47bb66ffdb23a9a) (cherry picked from commit 5effc04b67e5cd51a9fbc9517d0c2fbdf74b0d98) --- .../unreleased/SOLR-17432-otel-java-agent.yml | 12 ++ .../java/org/apache/solr/core/CoreContainer.java | 7 +- .../solr/core/OpenTelemetryConfigurator.java | 122 +++++++++---- .../apache/solr/core/TestTracerConfigurator.java | 35 ++++ .../solr/opentelemetry/OtelTracerConfigurator.java | 25 +-- .../opentelemetry/src/test-files/solr/solr.xml | 2 - .../BasicAuthIntegrationTracingTest.java | 20 +-- .../CustomTestOtelTracerConfigurator.java | 102 ----------- .../solr/opentelemetry/GoldFileTraceVerifier.java | 31 ++-- .../solr/opentelemetry/TestDistributedTracing.java | 68 ++------ .../solr/opentelemetry/TestMetricExemplars.java | 23 +-- .../apache/solr/opentelemetry/TracingTestUtil.java | 71 ++++++++ .../modules/deployment-guide/deployment-nav.adoc | 7 +- .../pages/distributed-tracing.adoc | 127 ++++++-------- .../deployment-guide/pages/metrics-reporting.adoc | 10 +- .../deployment-guide/pages/opentelemetry.adoc | 191 +++++++++++++++++++++ .../src/java/org/apache/solr/SolrTestCase.java | 23 +++ .../src/java/org/apache/solr/SolrTestCaseJ4.java | 2 - .../apache/solr/cloud/MiniSolrCloudCluster.java | 48 ------ 19 files changed, 535 insertions(+), 391 deletions(-) diff --git a/changelog/unreleased/SOLR-17432-otel-java-agent.yml b/changelog/unreleased/SOLR-17432-otel-java-agent.yml new file mode 100644 index 00000000000..94ff9d7f9b4 --- /dev/null +++ b/changelog/unreleased/SOLR-17432-otel-java-agent.yml @@ -0,0 +1,12 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: >- + Solr now detects a loaded OpenTelemetry Java agent and uses the OpenTelemetry it registered, + with no need for a <tracerConfig> in solr.xml or the opentelemetry module. + OpenTelemetryConfigurator subclasses should now implement createOpenTelemetry() and not set + GlobalOpenTelemetry themselves; getTracer()/getOpenTelemetrySdk() are deprecated. +type: changed +authors: + - name: David Smiley +links: + - name: SOLR-17432 + url: https://issues.apache.org/jira/browse/SOLR-17432 diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index b1f550fc9bc..566dc699b84 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -401,12 +401,13 @@ public class CoreContainer { public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrCoreLoad) { this.cfg = requireNonNull(config); this.loader = config.getSolrResourceLoader(); + OpenTelemetryConfigurator.initializeOpenTelemetrySdk(cfg, loader); // early as possible! + this.tracer = TraceUtils.getGlobalTracer(); + this.metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig().isEnabled()); + this.solrHome = config.getSolrHome(); this.solrCores = SolrCores.newSolrCores(this); this.nodeKeyPair = new SolrNodeKeyPair(cfg.getCloudConfig()); - OpenTelemetryConfigurator.initializeOpenTelemetrySdk(cfg, loader); - this.metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig().isEnabled()); - this.tracer = TraceUtils.getGlobalTracer(); containerHandlers.put(PublicKeyHandler.PATH, new PublicKeyHandler(nodeKeyPair)); if (null != this.cfg.getBooleanQueryMaxClauseCount()) { diff --git a/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java b/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java index b134877c8e7..9d5e76833c8 100644 --- a/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java +++ b/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java @@ -33,12 +33,13 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; +import org.apache.solr.logging.DeprecationLog; import org.apache.solr.util.plugin.NamedListInitializedPlugin; import org.apache.solr.util.tracing.SimplePropagator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Configures and loads/sets {@link GlobalOpenTelemetry} from a {@link OpenTelemetrySdk}. */ +/** Configures and sets {@link GlobalOpenTelemetry}. */ public abstract class OpenTelemetryConfigurator implements NamedListInitializedPlugin { public static final boolean TRACE_ID_GEN_ENABLED = @@ -52,68 +53,113 @@ public abstract class OpenTelemetryConfigurator implements NamedListInitializedP private static volatile boolean loaded = false; - /** - * Initializes the {@link io.opentelemetry.api.GlobalOpenTelemetry} instance by configuring the - * {@link io.opentelemetry.sdk.OpenTelemetrySdk} through custom plugin, auto-configure or default - * SDK. - */ - public static synchronized void initializeOpenTelemetrySdk( - NodeConfig cfg, SolrResourceLoader loader) { - PluginInfo info = (cfg != null) ? cfg.getTracerConfiguratorPluginInfo() : null; + /** Is the OpenTelemetry Java agent present? */ + protected static final boolean OTEL_AGENT_PRESENT; - if (info != null && info.isEnabled()) { - OpenTelemetryConfigurator.configureCustomOpenTelemetrySdk( - loader, cfg.getTracerConfiguratorPluginInfo()); - } else if (OpenTelemetryConfigurator.shouldAutoConfigOTEL()) { - OpenTelemetryConfigurator.autoConfigureOpenTelemetrySdk(loader); - } else { - OpenTelemetryConfigurator.configureOpenTelemetrySdk(); + static { + // https://github.com/open-telemetry/opentelemetry-java-instrumentation/discussions/9173 + boolean b; + try { + Class.forName("io.opentelemetry.javaagent.OpenTelemetryAgent"); + b = true; + } catch (ClassNotFoundException e) { + b = false; } + OTEL_AGENT_PRESENT = b; } - private static void configureOpenTelemetrySdk() { + /** + * Initializes {@link io.opentelemetry.api.GlobalOpenTelemetry} from a custom plugin, + * auto-configuration, or simple trace ID propagation. Does nothing if the OpenTelemetry Java + * agent is present or {@link GlobalOpenTelemetry} is already set (e.g. by tests). + */ + public static synchronized void initializeOpenTelemetrySdk( + NodeConfig cfg, SolrResourceLoader loader) { + // synchronized & "loaded" to avoid races in tests starting Solr nodes concurrently if (loaded) return; + loaded = true; - if (TRACE_ID_GEN_ENABLED) { - log.info("OpenTelemetry tracer enabled with simple propagation only."); - ExecutorUtil.addThreadLocalProvider(new ContextThreadLocalProvider()); + // Check for the agent explicitly; older agents don't instrument isSet(), so it returns false + if (OTEL_AGENT_PRESENT) { + log.info("OpenTelemetry Java agent is installed; using the OpenTelemetry it registered."); + } else if (GlobalOpenTelemetry.isSet()) { + // e.g. a 3rd party agent, or an early GlobalOpenTelemetry.get() call which sets a no-op + log.info("GlobalOpenTelemetry was already initialized by something else; using that."); + } else { + PluginInfo info = (cfg != null) ? cfg.getTracerConfiguratorPluginInfo() : null; + OpenTelemetry otel = null; + if (info != null && info.isEnabled()) { + OpenTelemetryConfigurator configurator = + loader.newInstance(info.className, OpenTelemetryConfigurator.class); + configurator.init(info.initArgs); + otel = configurator.createOpenTelemetry(); + log.info("OpenTelemetry loaded via {}", info.className); + } else if (shouldAutoConfigOTEL()) { + otel = autoConfigOTEL(loader); // null if it failed to load + } + if (otel == null && TRACE_ID_GEN_ENABLED) { + otel = OpenTelemetry.propagating(ContextPropagators.create(SimplePropagator.getInstance())); + log.info("OpenTelemetry loaded with simple propagation only."); + } + + try { + // throws IllegalStateException if already set + GlobalOpenTelemetry.set(otel == null ? OpenTelemetry.noop() : otel); + if (otel == null) { + return; // no point in the thread local provider below + } + } catch (IllegalStateException e) { // deprecated; remove in Solr 11. + // e.g. a legacy configurator that set it in init() + log.info("GlobalOpenTelemetry was already initialized by the configurator; using that."); + } } - OpenTelemetry otel = - OpenTelemetry.propagating(ContextPropagators.create(SimplePropagator.getInstance())); - GlobalOpenTelemetry.set(otel); - loaded = true; + ExecutorUtil.addThreadLocalProvider(new ContextThreadLocalProvider()); } - private static void autoConfigureOpenTelemetrySdk(SolrResourceLoader loader) { - if (loaded) return; + private static OpenTelemetry autoConfigOTEL(SolrResourceLoader loader) { try { OpenTelemetryConfigurator configurator = loader.newInstance(DEFAULT_CLASS_NAME, OpenTelemetryConfigurator.class); configurator.init(new NamedList<>()); - ExecutorUtil.addThreadLocalProvider(new ContextThreadLocalProvider()); - loaded = true; + log.info("OpenTelemetry loaded via auto configuration."); + return configurator.createOpenTelemetry(); } catch (SolrException e) { log.error( "Unable to auto-config OpenTelemetry with class {}. Make sure you have enabled the 'opentelemetry' module", DEFAULT_CLASS_NAME, e); + return null; } } - private static void configureCustomOpenTelemetrySdk(SolrResourceLoader loader, PluginInfo info) { - if (loaded) return; - - OpenTelemetryConfigurator configurator = - loader.newInstance(info.className, OpenTelemetryConfigurator.class); - configurator.init(info.initArgs); - ExecutorUtil.addThreadLocalProvider(new ContextThreadLocalProvider()); - loaded = true; + /** + * Creates the {@link OpenTelemetry} to install as {@link GlobalOpenTelemetry}; called after + * {@link #init(NamedList)}. Implementations must not set {@link GlobalOpenTelemetry} themselves. + * Subclasses should override this; the default returns {@link #getOpenTelemetrySdk()} for + * backwards compatibility. + */ + protected OpenTelemetry createOpenTelemetry() { + DeprecationLog.log( + "createOpenTelemetry", getClass() + " should implement createOpenTelemetry()"); + return getOpenTelemetrySdk(); } - protected abstract Tracer getTracer(); + /** + * @deprecated Not used by Solr; override {@link #createOpenTelemetry()} instead. + */ + @Deprecated(since = "10.2") + protected Tracer getTracer() { + return null; + } - protected abstract OpenTelemetrySdk getOpenTelemetrySdk(); + /** + * @deprecated override {@link #createOpenTelemetry()} instead. + */ + @Deprecated(since = "10.2") + protected OpenTelemetrySdk getOpenTelemetrySdk() { + return null; + } private static class ContextThreadLocalProvider implements ExecutorUtil.InheritableThreadLocalProvider { diff --git a/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java b/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java index fa344deb0cc..cdcf9d1b5fb 100644 --- a/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java +++ b/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java @@ -16,9 +16,15 @@ */ package org.apache.solr.core; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import java.util.Map; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.util.tracing.TraceUtils; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -30,6 +36,35 @@ public class TestTracerConfigurator extends SolrTestCaseJ4 { System.setProperty("solr.otelDefaultConfigurator", "configuratorClassDoesNotExistTest"); } + @Before + public void resetOtel() { + OpenTelemetryConfigurator.resetForTest(); + } + + /** A {@code <tracerConfig>} in solr.xml takes precedence and supplies the OpenTelemetry. */ + @Test + public void customConfiguratorTest() { + SolrResourceLoader loader = new SolrResourceLoader(TEST_PATH().resolve("collection1")); + NodeConfig cfg = + new NodeConfig.NodeConfigBuilder("testNode", TEST_PATH()) + .setTracerConfig( + new PluginInfo("tracerConfig", Map.of("class", Custom.class.getName()))) + .build(); + OpenTelemetryConfigurator.initializeOpenTelemetrySdk(cfg, loader); + // not SimplePropagator, which is what the other code paths would have installed + assertSame( + W3CTraceContextPropagator.getInstance(), + GlobalOpenTelemetry.getPropagators().getTextMapPropagator()); + } + + public static class Custom extends OpenTelemetryConfigurator { + @Override + protected OpenTelemetry createOpenTelemetry() { + return OpenTelemetry.propagating( + ContextPropagators.create(W3CTraceContextPropagator.getInstance())); + } + } + @Test public void configuratorClassDoesNotExistTest() { assertTrue(OpenTelemetryConfigurator.shouldAutoConfigOTEL()); diff --git a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java b/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java index d3cb53f5f74..fe52c43f7b9 100644 --- a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java +++ b/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java @@ -16,8 +16,7 @@ */ package org.apache.solr.opentelemetry; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; import java.lang.invoke.MethodHandles; import java.util.Arrays; @@ -29,20 +28,18 @@ import java.util.stream.Collectors; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.OpenTelemetryConfigurator; -import org.apache.solr.util.tracing.TraceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Tracing TracerConfigurator implementation which exports spans to OpenTelemetry in OTLP format. + * An {@link OpenTelemetryConfigurator} implementation which exports spans to OpenTelemetry in OTLP + * format. */ public class OtelTracerConfigurator extends OpenTelemetryConfigurator { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final Map<String, String> currentEnv; - private OpenTelemetrySdk openTelemetrySdk; - public OtelTracerConfigurator() { this(System.getenv()); } @@ -52,19 +49,15 @@ public class OtelTracerConfigurator extends OpenTelemetryConfigurator { } @Override - public Tracer getTracer() { - return TraceUtils.getGlobalTracer(); - } - - @Override - public OpenTelemetrySdk getOpenTelemetrySdk() { - return this.openTelemetrySdk; + public void init(NamedList<?> args) { + prepareConfiguration(args); } @Override - public void init(NamedList<?> args) { - prepareConfiguration(args); - this.openTelemetrySdk = AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk(); + public OpenTelemetry createOpenTelemetry() { + // deliberately build() and not initialize(); the latter sets GlobalOpenTelemetry, which is + // OpenTelemetryConfigurator's job + return AutoConfiguredOpenTelemetrySdk.builder().build().getOpenTelemetrySdk(); } void prepareConfiguration(NamedList<?> args) { diff --git a/solr/modules/opentelemetry/src/test-files/solr/solr.xml b/solr/modules/opentelemetry/src/test-files/solr/solr.xml index 42f330816fd..65d7f48c5a2 100644 --- a/solr/modules/opentelemetry/src/test-files/solr/solr.xml +++ b/solr/modules/opentelemetry/src/test-files/solr/solr.xml @@ -33,8 +33,6 @@ <int name="connTimeout">${connTimeout:15000}</int> </shardHandlerFactory> - <tracerConfig name="tracerConfig" class="org.apache.solr.opentelemetry.CustomTestOtelTracerConfigurator" /> - <solrcloud> <str name="host">127.0.0.1</str> <int name="hostPort">${hostPort:8983}</int> diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/BasicAuthIntegrationTracingTest.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/BasicAuthIntegrationTracingTest.java index 797f46bda54..1d05381ba40 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/BasicAuthIntegrationTracingTest.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/BasicAuthIntegrationTracingTest.java @@ -16,10 +16,11 @@ */ package org.apache.solr.opentelemetry; -import static org.apache.solr.opentelemetry.TestDistributedTracing.getAndClearSpans; +import static org.apache.solr.opentelemetry.TracingTestUtil.getAndClearSpans; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; import java.util.List; import java.util.Map; import org.apache.solr.client.solrj.SolrRequest; @@ -31,23 +32,21 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.security.BasicAuthPlugin; import org.apache.solr.util.SecurityJson; import org.apache.solr.util.tracing.TraceUtils; -import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; public class BasicAuthIntegrationTracingTest extends SolrCloudTestCase { private static final String COLLECTION = "collection1"; + @ClassRule public static OpenTelemetryRule otelRule = OpenTelemetryRule.create(); + @BeforeClass public static void setupCluster() throws Exception { - // force early init - CustomTestOtelTracerConfigurator.prepareForTest(); - configureCluster(4) .addConfig("config", TEST_PATH().resolve("collection1").resolve("conf")) .withSolrXml(TEST_PATH().resolve("solr.xml")) - .withTraceIdGenerationDisabled() .withSecurityJson(SecurityJson.SIMPLE) .configure(); @@ -62,15 +61,10 @@ public class BasicAuthIntegrationTracingTest extends SolrCloudTestCase { cluster.waitForActiveCollection(COLLECTION, 2, 4); } - @AfterClass - public static void afterClass() { - CustomTestOtelTracerConfigurator.resetForTest(); - } - /** See SOLR-16955 */ @Test public void testSetupBasicAuth() throws Exception { - getAndClearSpans(); // reset + getAndClearSpans(otelRule); // reset CloudSolrClient cloudClient = cluster.getSolrClient(); Map<String, Object> ops = @@ -85,7 +79,7 @@ public class BasicAuthIntegrationTracingTest extends SolrCloudTestCase { req.setBasicAuthCredentials(SecurityJson.USER, SecurityJson.PASS); assertEquals(0, req.process(cloudClient, COLLECTION).getStatus()); - var finishedSpans = getAndClearSpans(); + var finishedSpans = getAndClearSpans(otelRule); assertEquals(1, finishedSpans.size()); var span = finishedSpans.get(0); assertEquals("post:/cluster/security/authentication", span.getName()); diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java deleted file mode 100644 index 3790bc7c61b..00000000000 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.opentelemetry; - -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; -import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; -import java.lang.invoke.MethodHandles; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.NamedList; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CustomTestOtelTracerConfigurator extends OtelTracerConfigurator { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - static { - if (EnvUtils.getProperty("solr.host.advertise") == null) { - System.setProperty("solr.host.advertise", "localhost"); - } - } - - private static InMemorySpanExporter exporter; - private static volatile boolean isRegistered = false; - private static OpenTelemetrySdk otelSdk = null; - - @Override - public synchronized Tracer getTracer() { - if (!isRegistered) { - throw new IllegalStateException( - "Tracer is not initialized. you need to call #prepareForTest for correct setup"); - } - return super.getTracer(); - } - - @Override - public void init(NamedList<?> args) { - // prevent parent from init otel - } - - public static synchronized void prepareForTest() { - CustomTestOtelTracerConfigurator.resetForTest(); - isRegistered = true; - System.setProperty("otel.traces.exporter", "none"); - - // force early init - CustomTestOtelTracerConfigurator tracer = new CustomTestOtelTracerConfigurator(); - tracer.prepareConfiguration(new NamedList<>()); - - bootOtel(); - } - - private static void bootOtel() { - try { - exporter = InMemorySpanExporter.create(); - otelSdk = - AutoConfiguredOpenTelemetrySdk.builder() - .setResultAsGlobal() - .addTracerProviderCustomizer( - (builder, props) -> - builder.addSpanProcessor(SimpleSpanProcessor.create(exporter))) - .build() - .getOpenTelemetrySdk(); - } catch (RuntimeException e) { - log.error("Error on OTEL init ", e); - } - } - - public static InMemorySpanExporter getInMemorySpanExporter() { - return exporter; - } - - public static synchronized void resetForTest() { - if (isRegistered) { - isRegistered = false; - if (otelSdk != null) { - otelSdk.close(); - } - if (exporter != null) { - exporter.close(); - exporter = null; - } - } - } -} diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java index d3667009605..7542bf8503e 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/GoldFileTraceVerifier.java @@ -17,7 +17,7 @@ package org.apache.solr.opentelemetry; import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; import io.opentelemetry.sdk.trace.data.SpanData; import java.io.IOException; import java.io.UncheckedIOException; @@ -49,7 +49,7 @@ import org.junit.Assert; * <h2>Usage</h2> * * <pre> - * var verifier = new GoldFileTraceVerifier("testV2Api"); + * var verifier = new GoldFileTraceVerifier(otelRule, getClass(), "testV2Api"); * // ... perform operations ... * verifier.verifyPhase(); // waits for spans, normalizes, compares * // ... more operations ... @@ -89,13 +89,16 @@ public class GoldFileTraceVerifier { Comparator.comparing((Map<String, Object> s) -> (String) s.get("name")) .thenComparing(s -> s.getOrDefault(TraceUtils.TAG_DB.getKey(), "").toString()); + private final OpenTelemetryRule otelRule; private final String testName; private final Path goldFilePath; private final Map<String, Object> goldFile; // parsed gold file (phases list) private final List<List<Map<String, Object>>> recordedPhases = new ArrayList<>(); private int currentPhaseIndex = 0; - public GoldFileTraceVerifier(Class<?> testClass, String testMethodName) { + public GoldFileTraceVerifier( + OpenTelemetryRule otelRule, Class<?> testClass, String testMethodName) { + this.otelRule = otelRule; this.testName = testMethodName; this.goldFilePath = resolveGoldFilePath(testClass, testMethodName); if (REGENERATE) { @@ -111,20 +114,18 @@ public class GoldFileTraceVerifier { */ @SuppressWarnings("unchecked") public void verifyPhase() { - InMemorySpanExporter exporter = CustomTestOtelTracerConfigurator.getInMemorySpanExporter(); - List<SpanData> spans; if (REGENERATE) { - spans = waitForStableSpans(exporter); + spans = waitForStableSpans(); } else { List<Map<String, Object>> phases = (List<Map<String, Object>>) goldFile.get("phases"); Map<String, Object> phase = phases.get(currentPhaseIndex); List<Map<String, Object>> expectedSpans = (List<Map<String, Object>>) phase.get("spans"); int expectedCount = countSpansRecursive(expectedSpans); - spans = waitForSpans(exporter, expectedCount); + spans = waitForSpans(expectedCount); } - exporter.reset(); + otelRule.clearSpans(); if (DUMP_TRACES) { dumpRawSpans(spans); @@ -155,32 +156,32 @@ public class GoldFileTraceVerifier { // --- Span collection --- - private List<SpanData> waitForSpans(InMemorySpanExporter exporter, int expectedCount) { + private List<SpanData> waitForSpans(int expectedCount) { try { RetryUtil.retryUntil( "Timed out waiting for " + expectedCount + " span(s) in phase " + currentPhaseIndex, 500, 20, TimeUnit.MILLISECONDS, - () -> exporter.getFinishedSpanItems().size() >= expectedCount); + () -> otelRule.getSpans().size() >= expectedCount); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - return new ArrayList<>(exporter.getFinishedSpanItems()); + return new ArrayList<>(otelRule.getSpans()); } - private List<SpanData> waitForStableSpans(InMemorySpanExporter exporter) { + private List<SpanData> waitForStableSpans() { try { RetryUtil.retryUntil( "Timed out waiting for any spans in phase " + currentPhaseIndex, 500, 20, TimeUnit.MILLISECONDS, - () -> !exporter.getFinishedSpanItems().isEmpty()); + () -> !otelRule.getSpans().isEmpty()); int lastCount = -1; for (int i = 0; i < 10; i++) { TimeUnit.MILLISECONDS.sleep(500); - int current = exporter.getFinishedSpanItems().size(); + int current = otelRule.getSpans().size(); if (current == lastCount) { break; } @@ -189,7 +190,7 @@ public class GoldFileTraceVerifier { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - return new ArrayList<>(exporter.getFinishedSpanItems()); + return new ArrayList<>(otelRule.getSpans()); } // --- Raw dump (for debugging, no normalization) --- diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java index 22e4f9d87d6..448d35bc5e6 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java @@ -20,15 +20,12 @@ package org.apache.solr.opentelemetry; import com.carrotsearch.randomizedtesting.annotations.Seed; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.TracerProvider; -import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; import io.opentelemetry.sdk.trace.data.SpanData; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.MetricsRequest; @@ -41,12 +38,11 @@ import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.RetryUtil; import org.apache.solr.util.stats.MetricUtils; import org.apache.solr.util.tracing.TraceUtils; -import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; @Seed("0") // don't want randomization when testing observability @@ -54,17 +50,16 @@ public class TestDistributedTracing extends SolrCloudTestCase { private static final String COLLECTION = "collection1"; + @ClassRule public static OpenTelemetryRule otelRule = OpenTelemetryRule.create(); + @BeforeClass public static void setupCluster() throws Exception { - // force early init - CustomTestOtelTracerConfigurator.prepareForTest(); // HTTP 2 clients can do things more asynchronously, leading to less determinism in what we test System.setProperty("solr.http1", "true"); configureCluster(4) .addConfig("config", TEST_PATH().resolve("collection1").resolve("conf")) .withSolrXml(TEST_PATH().resolve("solr.xml")) - .withTraceIdGenerationDisabled() .withOverseer(true) // some assertions assume overseer .configure(); @@ -99,11 +94,6 @@ public class TestDistributedTracing extends SolrCloudTestCase { cluster.waitForActiveCollection(COLLECTION, 2, 4); } - @AfterClass - public static void afterClass() { - CustomTestOtelTracerConfigurator.resetForTest(); - } - @Before private void resetSpanData() { getAndClearSpans(); @@ -111,7 +101,7 @@ public class TestDistributedTracing extends SolrCloudTestCase { @Test public void test() throws Exception { - var verifier = new GoldFileTraceVerifier(getClass(), "test"); + var verifier = new GoldFileTraceVerifier(otelRule, getClass(), "test"); // TODO use a CloudSolrClient. However it's not yet deterministic due to use of random not // aligned to the test seed. var client = cluster.getJettySolrRunner(0).getSolrClient(); @@ -131,7 +121,7 @@ public class TestDistributedTracing extends SolrCloudTestCase { @Test public void testAdminApi() throws Exception { - var verifier = new GoldFileTraceVerifier(getClass(), "testAdminApi"); + var verifier = new GoldFileTraceVerifier(otelRule, getClass(), "testAdminApi"); // TODO use a CloudSolrClient. However it's not yet deterministic due to use of random not // aligned to the test seed. var client = cluster.getJettySolrRunner(0).getSolrClient(); @@ -150,7 +140,7 @@ public class TestDistributedTracing extends SolrCloudTestCase { @Test public void testV2Api() throws Exception { - var verifier = new GoldFileTraceVerifier(getClass(), "testV2Api"); + var verifier = new GoldFileTraceVerifier(otelRule, getClass(), "testV2Api"); // TODO use a CloudSolrClient. However it's not yet deterministic due to use of random not // aligned to the test seed. var client = cluster.getJettySolrRunner(0).getSolrClient(); @@ -226,9 +216,9 @@ public class TestDistributedTracing extends SolrCloudTestCase { Map<String, Integer> ops = new HashMap<>(); assertEquals(11, finishedSpans.size()); - var parentTraceId = getRootTraceId(finishedSpans); + var parentTraceId = TracingTestUtil.getRootTraceId(finishedSpans); for (var span : finishedSpans) { - if (isRootSpan(span)) { + if (TracingTestUtil.isRootSpan(span)) { assertCollectionName(span, collection); } else { assertEquals(span.getParentSpanContext().getTraceId(), parentTraceId); @@ -267,9 +257,9 @@ public class TestDistributedTracing extends SolrCloudTestCase { Map<String, Integer> ops = new HashMap<>(); assertEquals(5, finishedSpans.size()); - var parentTraceId = getRootTraceId(finishedSpans); + var parentTraceId = TracingTestUtil.getRootTraceId(finishedSpans); for (var span : finishedSpans) { - if (isRootSpan(span)) { + if (TracingTestUtil.isRootSpan(span)) { assertCollectionName(span, collection); } else { assertEquals(span.getParentSpanContext().getTraceId(), parentTraceId); @@ -282,10 +272,6 @@ public class TestDistributedTracing extends SolrCloudTestCase { assertEquals(expectedOps, ops); } - private static boolean isRootSpan(SpanData span) { - return !span.getParentSpanContext().isValid(); - } - private static void assertCollectionName(SpanData span, String collection) { assertEquals(collection, span.getAttributes().get(TraceUtils.TAG_DB)); } @@ -294,35 +280,11 @@ public class TestDistributedTracing extends SolrCloudTestCase { assertTrue(span.getAttributes().get(TraceUtils.TAG_DB).startsWith(collection + "_")); } - static List<SpanData> getAndClearSpans() { - return getAndClearSpans(0); - } - - static List<SpanData> getAndClearSpans(int minExpected) { - InMemorySpanExporter exporter = CustomTestOtelTracerConfigurator.getInMemorySpanExporter(); - try { - RetryUtil.retryUntil( - "Timed out waiting for " + minExpected + " span(s)", - 250, - 20, - TimeUnit.MILLISECONDS, - () -> exporter.getFinishedSpanItems().size() >= minExpected); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - List<SpanData> result = new ArrayList<>(exporter.getFinishedSpanItems()); - Collections.reverse(result); // nicer to see spans chronologically - exporter.reset(); - return result; + private static List<SpanData> getAndClearSpans() { + return TracingTestUtil.getAndClearSpans(otelRule); } - static String getRootTraceId(List<SpanData> finishedSpans) { - assertEquals(1, finishedSpans.stream().filter(TestDistributedTracing::isRootSpan).count()); - return finishedSpans.stream() - .filter(TestDistributedTracing::isRootSpan) - .findFirst() - .get() - .getSpanContext() - .getTraceId(); + private static List<SpanData> getAndClearSpans(int minExpected) { + return TracingTestUtil.getAndClearSpans(otelRule, minExpected); } } diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java index 0c8ba50078f..32885f9b1db 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java @@ -16,11 +16,12 @@ */ package org.apache.solr.opentelemetry; -import static org.apache.solr.opentelemetry.TestDistributedTracing.getAndClearSpans; -import static org.apache.solr.opentelemetry.TestDistributedTracing.getRootTraceId; +import static org.apache.solr.opentelemetry.TracingTestUtil.getAndClearSpans; +import static org.apache.solr.opentelemetry.TracingTestUtil.getRootTraceId; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; import java.io.InputStream; import java.nio.charset.StandardCharsets; import org.apache.solr.client.solrj.impl.CloudSolrClient; @@ -29,23 +30,20 @@ import org.apache.solr.client.solrj.request.MetricsRequest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; -import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; public class TestMetricExemplars extends SolrCloudTestCase { + @ClassRule public static OpenTelemetryRule otelRule = OpenTelemetryRule.create(); + @BeforeClass public static void setupCluster() throws Exception { - System.setProperty("otel.traces.sampler", "always_on"); - // force early init - CustomTestOtelTracerConfigurator.prepareForTest(); - configureCluster(1) .addConfig("config", TEST_PATH().resolve("collection1").resolve("conf")) .withSolrXml(TEST_PATH().resolve("solr.xml")) - .withTraceIdGenerationDisabled() .configure(); assertNotEquals( @@ -58,14 +56,9 @@ public class TestMetricExemplars extends SolrCloudTestCase { cluster.waitForActiveCollection("collection1", 1, 1); } - @AfterClass - public static void afterClass() { - CustomTestOtelTracerConfigurator.resetForTest(); - } - @Before private void resetSpanData() { - getAndClearSpans(); + getAndClearSpans(otelRule); } @Test @@ -74,7 +67,7 @@ public class TestMetricExemplars extends SolrCloudTestCase { // Generate exemplars cloudClient.add("collection1", sdoc("id", "1")); - var spans = getAndClearSpans(); + var spans = getAndClearSpans(otelRule); var expectedTrace = getRootTraceId(spans); var req = new MetricsRequest(new ModifiableSolrParams().set("wt", "openmetrics")); diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TracingTestUtil.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TracingTestUtil.java new file mode 100644 index 00000000000..bc7c56f7366 --- /dev/null +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TracingTestUtil.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.opentelemetry; + +import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.solr.common.util.RetryUtil; +import org.junit.Assert; + +/** Helpers for tests that collect spans via an {@link OpenTelemetryRule}. */ +final class TracingTestUtil { + + private TracingTestUtil() {} + + static List<SpanData> getAndClearSpans(OpenTelemetryRule otelRule) { + return getAndClearSpans(otelRule, 0); + } + + /** + * Waits for at least {@code minExpected} spans, then returns and clears the collected spans, most + * recently finished first. + */ + static List<SpanData> getAndClearSpans(OpenTelemetryRule otelRule, int minExpected) { + try { + RetryUtil.retryUntil( + "Timed out waiting for " + minExpected + " span(s)", + 250, + 20, + TimeUnit.MILLISECONDS, + () -> otelRule.getSpans().size() >= minExpected); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + List<SpanData> result = new ArrayList<>(otelRule.getSpans()); + Collections.reverse(result); // nicer to see spans chronologically + otelRule.clearSpans(); + return result; + } + + static boolean isRootSpan(SpanData span) { + return !span.getParentSpanContext().isValid(); + } + + static String getRootTraceId(List<SpanData> finishedSpans) { + Assert.assertEquals(1, finishedSpans.stream().filter(TracingTestUtil::isRootSpan).count()); + return finishedSpans.stream() + .filter(TracingTestUtil::isRootSpan) + .findFirst() + .orElseThrow() + .getSpanContext() + .getTraceId(); + } +} diff --git a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc index 9b81acaf3e7..3049ad1cd5e 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc @@ -56,15 +56,16 @@ *** xref:user-managed-distributed-search.adoc[] * Monitoring Solr -** xref:configuring-logging.adoc[] ** xref:ping.adoc[] ** xref:healthcheck.adoc[] +** xref:configuring-logging.adoc[] +** xref:opentelemetry.adoc[] ** xref:metrics-reporting.adoc[] +** xref:distributed-tracing.adoc[] ** xref:performance-statistics-reference.adoc[] -** xref:plugins-stats-screen.adoc[] ** xref:jmx-with-solr.adoc[] +** xref:plugins-stats-screen.adoc[] ** xref:thread-dump.adoc[] -** xref:distributed-tracing.adoc[] ** xref:circuit-breakers.adoc[] ** xref:rate-limiters.adoc[] ** xref:task-management.adoc[] diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc index fb60329ee03..976a0787a05 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc @@ -16,112 +16,89 @@ // specific language governing permissions and limitations // under the License. -Solr includes a general tracing framework based on OpenTracing that can be used to trace the lifecycle of a request for performance monitoring. +Solr uses https://opentelemetry.io/[OpenTelemetry] APIs/standards for its https://opentelemetry.io/docs/concepts/signals/traces/[distributed tracing] support. +Tracing is an observability signal focused on diagnosing performance of requests across distributed systems. -Tracing data can be configured to send to various backends, either directly or via an agent. -Solr can send traces directly in OTLP/HTTP format with the opentelemetry module, out-of-the-box. -Other protocols, destinations and cloud providers may be supported. -For example, gRPC can be supported by adding more JARs and tweaking configuration. - -A sampled distributed tracing query request displayed in Jaeger UI looks like this: +The power of tracing is best appreciated in visualizing a distributed request over a time axis as a tree of layered temporal spans emitted by systems and sub-systems/components. +A sampled distributed tracing query request displayed in https://www.jaegertracing.io/[Jaeger] looks like this: .Tracing of a Solr query image::distributed-tracing/query-request-tracing.png[] -== Always-on Trace Id generation - -Solr includes an always-on trace id generator. This will inject trace id headers for every Solr request, propagating the client supplied value or generating a new id as needed and replaces the existing `rid` mechanism. - -The header name it uses for propagation is `X-Trace-Id`, this can be changed by updating the system property `solr.traceIdHeader`. - -This plugin will not be enabled if another Tracing mechanism is setup via Solr config. -This plugin can be disabled by setting the `solr.tracing.always.on.enabled` system property to `false`. - -== Modules and Configuration - -The `TracerConfigurator` is a class used to provide an instance of `io.opentracing.Tracer` based on configuration in `solr.xml`. -These are pluggable, and each plugin is shipped as a xref:configuration-guide:solr-modules.adoc[Solr Module]: - -**opentelemetry**: The `opentelemetry` module supports OTLP over HTTP. - -A setup of a `TracerConfigurator` in `solr.xml` looks like this: - -[source,xml] ----- -<solr> - <tracerConfig name="tracerConfig" class="org.apache.solr.opentelemetry.OtelTracerConfigurator" /> -</solr> ----- - -If `<tracerConfig>` is absent, `TracerConfigurator` will try to pick up the `Tracer` instance that was registered in `io.opentracing.util.GlobalTracer`. -By doing this, some backends like DataDog are supported out of the box since https://docs.datadoghq.com/tracing/setup/java/[datadog-java-agent] uses Javaagent to register a `Tracer` in `io.opentracing.util.GlobalTracer`. +While logs and metrics can be observed directly in Solr, tracing requires that you integrate with an OTEL compliant server that includes visualization capabilities or is part of a larger system plumbed into one. +See xref:opentelemetry.adoc[] for the ways to do that -- a Java agent, Solr's `opentelemetry` module, or a custom configurator -- and how each is configured. +This page describes what Solr emits once one of them is in place. -== Open Telemetry Module +== Spans -This module brings support for the industry standard https://opentelemetry.io[OpenTelemetry] ("OTEL"). -OTEL is broader than tracing, but nonetheless, Solr's focal configuration element for it in Solr is `<tracerConfig>` in `solr.xml` as follows: +Solr names a request span after the operation and the path it resolves to, as `_verb_:_path_`. +The verb is the `action` request parameter when there is one, otherwise the lowercased HTTP method. +The path is prefixed with `/\{core}` or `/\{collection}` when the request targets one, so that spans group by operation rather than by which core happened to serve them. +For example, `get:/\{collection}/select`, `post:/\{core}/update`, `list:/admin/collections`, or `reload:/admin/cores`. +These spans have kind `SERVER`. -[source,xml] ----- -<tracerConfig name="tracerConfig" class="org.apache.solr.opentelemetry.OtelTracerConfigurator"/> ----- +Collection API commands that Solr runs internally produce a span named after the command, such as `CreateCollectionCmd` or `ReloadCollectionCmd`, of kind `CLIENT` -- or `PRODUCER` when the command was submitted asynchronously. +These are the parent of the per-node work the command fans out. -As an alternative to changing the `solr.xml` file, the `OTEL` tracer will be enabled if the system property `otel.service.name` or environment variable `OTEL_SERVICE_NAME` is present. The `opentelemetry` module still needs to be enabled for the tracer to work. +Spans carry these attributes, where applicable: -Enable the module with either system property `-Dsolr.modules=opentelemetry` or environment variable `SOLR_MODULES=opentelemetry`. +[cols="2,4",options="header"] +|=== +| Attribute | Description -=== Configuration +| `db.instance` | The core or collection the request targets. +| `db.type` | Always `solr`. +| `db.user` | The authenticated user, when a security plugin is in use. +| `http.request.method` | The HTTP method. +| `http.response.status_code` | The HTTP status code. +| `http.url` | The request URL. +| `http.params` | The query string. +| `ops` | On a security configuration request, the operation names in the payload. +| `class` | The security plugin class that handled those operations. +|=== -The tracer can be configured through environment variables or Java system properties, see https://opentelemetry.io/docs/reference/specification/sdk-environment-variables/[OTEL SDK Environment Variables] and https://github.com/open-telemetry/opentelemetry-java/blob/v{dep-version-opentelemetry}/sdk-extensions/autoconfigure/README.md[Java SDK Autoconfigure]. +Solr injects the trace context into the requests it makes to other Solr nodes, so a client request and all the internal requests it causes share one trace. +The header used depends on the configured propagator; by default that is https://www.w3.org/TR/trace-context/[W3C TraceContext], which uses `traceparent`. -The default configuration will ship traces to a collector running on localhost, using https://opentelemetry.io/docs/reference/specification/protocol/[OTLP] over HTTP (Protobuf), and will propagate trace IDs using https://www.w3.org/TR/trace-context/[W3C TraceContext]. Here are the environment settings that are enabled by default: +== Sampling -[source,bash] ----- -OTEL_SDK_DISABLED=false -OTEL_SERVICE_NAME=solr -OTEL_TRACES_EXPORTER=otlp -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 -OTEL_TRACES_SAMPLER=parentbased_always_on -OTEL_PROPAGATORS=tracecontext,baggage ----- - -NOTE: gRPC transport is not supported out of the box. -To use gRPC, add the required gRPC and Netty JARs to the `opentelemetry` module's lib directory and set `OTEL_EXPORTER_OTLP_PROTOCOL=grpc`. +Tracing every request is rarely affordable in production. +Sampling is an SDK setting, so it applies when you use the `opentelemetry` module; with a Java agent, configure sampling through the agent. -Say you want to send traces to a remote https://opentelemetry.io/docs/collector/[OTEL Collector], with 10% sampling. That could be done with this configuration: +The module defaults to `parentbased_always_on`, which records every trace that Solr starts and honors the decision of an upstream caller when there is one. +To record 10% of traces instead: [source,bash] ---- -OTEL_EXPORTER_OTLP_ENDPOINT=http://my-remote-collector:4318 OTEL_TRACES_SAMPLER=parentbased_traceidratio OTEL_TRACES_SAMPLER_ARG=0.1 ---- -An equivalent configuration using system properties would be: +The equivalent with system properties: [source,bash] ---- -SOLR_OPTS=-Dotel.exporter.otlp.endpoint=http://my-remote-collector:4318 -Dotel.traces.sampler=parentbased_traceidratio -Dotel.traces.sampler.arg=0.1 +SOLR_OPTS="-Dotel.traces.sampler=parentbased_traceidratio -Dotel.traces.sampler.arg=0.1" ---- -To add custom tags to the trace, use `OTEL_RESOURCE_ATTRIBUTES`: +Because the default is parent-based, a sampling decision made by the client that called Solr is respected either way. -[source,bash] ----- -OTEL_RESOURCE_ATTRIBUTES="application=OnlineBanking,exampleKey=exampleValue" ----- +== Always-on Trace Id generation -Consult the https://opentelemetry.io/docs/reference/specification/sdk-environment-variables/[OTEL documentation] for details on all the configuration options. -This version of Solr uses https://github.com/open-telemetry/opentelemetry-java/tree/v{dep-version-opentelemetry}[OpenTelemetry SDK v{dep-version-opentelemetry}]. +If no other OTEL configuration/integration is active, Solr includes a simple "always-on" trace id generator. +This will create an internal trace ID and propagate them in an HTTP header on every request Solr makes, thus appearing in Solr's logs. -=== Using other exporters +The header name it uses for propagation is `X-Trace-Id`, which can be changed by updating the system property `solr.traceIdHeader`. -The OTEL tracer also supports other exporters like Jaeger and Zipkin. However, Solr does not ship with those 3rd party libraries, thus you need to add them to Solr yourself if you have this as a requirement. Say you add the Zipkin exporter jar file(s) to `$SOLR_TIP/lib/`, then you would enable OpenTelemetry module to send traces on the Zipkin format using the Protobuf encoding with a configuration like this: +Solr puts the trace ID into the logging MDC as `trace_id`, and the default `log4j2.xml` already includes it in the log pattern as `t:%X\{trace_id}`, so a trace ID looks like this in a log line: -[source,bash] +[source,text] ---- -OTEL_TRACES_EXPORTER=zipkin -OTEL_EXPORTER_ZIPKIN_ENDPOINT=http://localhost:9411/api/v2/spans +2026-09-21 10:15:42.123 INFO (qtp1234-25) [c:techproducts s:shard1 r:core_node2 x:techproducts_shard1_replica_n1 t:solr-node1-17] o.a.s.c.S.Request webapp=/solr path=/select params={q=*:*} ---- + +This means you can follow one request across every node's log without any tracing backend at all. +See xref:configuring-logging.adoc[] to change the pattern. + +This mechanism is a default for when no other tracing configuration is in effect. Those mechanisms will generally _at least_ do what this does anyway. +This mechanism can be disabled by setting the `solr.tracing.always.on.enabled` system property to `false`. diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc index adab8021580..1be5df0b61b 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc @@ -200,7 +200,7 @@ OpenMetrics can be used to pull/scrape metrics to a Prometheus server or any Ope ==== Prometheus setup with exemplars -OpenMetrics includes `exemplars` that provide additional information and allow users to leverage Solr's xref:deployment-guide:distributed-tracing.adoc[OpenTelemetry distributed tracing module] and metrics in a cohesive view for correlating traces and metrics. +OpenMetrics includes `exemplars` that provide additional information and allow users to leverage Solr's xref:deployment-guide:distributed-tracing.adoc[distributed tracing] and metrics in a cohesive view for correlating traces and metrics. Distributed tracing must be enabled to see exemplars. Exemplars will never appear in OpenMetrics format otherwise. You can then scrape OpenMetrics format to a Prometheus server or OpenMetrics-compatible backend. @@ -312,15 +312,13 @@ http://localhost:8983/api/metrics?name=solr_core_index_size_bytes&collection=foo == OTLP -For users who do not use or support pulling metrics in Prometheus format with the `/metrics` API, Solr also supports pushing metrics natively with https://opentelemetry.io/docs/specs/otlp/[OTLP], which is a vendor-agnostic protocol for pushing metrics. +For users who do not use or support pulling metrics in Prometheus format with the `/metrics` API, Solr also supports pushing metrics natively with https://opentelemetry.io/docs/specs/otlp/[OTLP], which is a vendor-agnostic protocol for pushing telemetry data. OTLP is widely supported by many tools, vendors, and pipelines. See the OpenTelemetry https://opentelemetry.io/ecosystem/vendors/[vendors list] for more details on available and compatible options. -=== OTLP properties - -Solr's internal OTLP exporter is disabled by default and is packaged with the OpenTelemetry module. +This exporter is disabled by default and is packaged with the `opentelemetry` module; see xref:opentelemetry.adoc[] for enabling that module. *It does NOT work with the OTEL Java Agent in any way*. -The module can be enabled with either the system property `-Dsolr.modules=opentelemetry` or the environment variable `SOLR_MODULES=opentelemetry`, similar to distributed tracing. +=== OTLP properties The OTLP exporter can be configured with the supported system properties below. These can also be set as environment variables by following these mapping rules: diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/opentelemetry.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/opentelemetry.adoc new file mode 100644 index 00000000000..3662487a1f5 --- /dev/null +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/opentelemetry.adoc @@ -0,0 +1,191 @@ += OpenTelemetry +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +https://opentelemetry.io/[OpenTelemetry] ("OTEL") is the vendor-neutral standard that Solr uses. +Solr uses its Java APIs for Metrics and Tracing (not logging), and to emit observability data via OTLP (if desired). +This page covers how OpenTelemetry gets into a Solr JVM and how it is configured; what Solr emits through it is covered on the pages for each signal. + +== Traces and Metrics + +Solr uses OpenTelemetry for two signals, configured independently: + +xref:distributed-tracing.adoc[]:: +Spans describing requests as they move between nodes. +Exported by the OpenTelemetry SDK itself, so it is configured with the standard `OTEL_*` environment variables (or `otel.*` system properties). + +xref:metrics-reporting.adoc#otlp[Metrics over OTLP]:: +Solr's own metrics, pushed periodically. +Solr builds and owns the meter providers these come from -- it has to, so that the `/metrics` endpoints can read them -- so they are configured with `solr.metrics.otlp*` properties, with their own endpoint and protocol settings. + +Only tracing goes through `GlobalOpenTelemetry`. +A Java agent therefore takes over tracing but not Solr's metrics, which keep their own configuration and exporter either way. + +NOTE: The `opentelemetry` module turns the autoconfigured SDK's metrics and logs exporters off (`otel.metrics.exporter=none`, `otel.logs.exporter=none`) so they cannot duplicate or conflict with the above. +Setting `OTEL_METRICS_EXPORTER` or `OTEL_LOGS_EXPORTER` has no effect and logs a warning. + +== Integration Options + +[cols="2,5",options="header"] +|=== +| Option | When to use it + +| <<opentelemetry-java-agent,OpenTelemetry Java agent>> +| You want automatic instrumentation of third-party libraries (e.g. for backups), `@WithSpan` support, wide exporter format support, or a vendor's agent distribution. Requires a JVM argument and an external special JAR that you must install. + +| <<opentelemetry-module,`opentelemetry` module>> +| You want OTLP metrics push, or prefer the convenience of using software that ships with Solr. + +| <<custom-configurator,Custom configurator>> +| Neither of the above fits, for example an OTEL SDK setup your organization standardizes on. + +| _(none)_ +| Solr falls back to xref:distributed-tracing.adoc#always-on-trace-id-generation[always-on trace ID generation], which propagates a trace ID but exports nothing. +|=== + +Solr resolves these in a fixed order, using the first that applies: a Java agent, then a `<tracerConfig>` in `solr.xml`, then auto-activation from `OTEL_SERVICE_NAME`, then always-on trace ID generation. +A Java agent always wins; when one is present Solr configures nothing and `<tracerConfig>` is ignored. + +== OpenTelemetry Java Agent + +Run Solr with the https://opentelemetry.io/docs/zero-code/java/agent/[OpenTelemetry Java agent] and it takes over OpenTelemetry entirely: + +[source,bash] +---- +SOLR_OPTS="-javaagent:/path/to/opentelemetry-javaagent.jar" +---- + +The agent loads its dependencies in an isolated classloader, so it cannot conflict with Solr's. The `opentelemetry` module should not be enabled. +All configuration is the agent's; see its documentation. + +Solr's own instrumentation continues to work as long as the agent's `opentelemetry-api` instrumentation stays enabled. +That matters if you disable instrumentation by default in order to opt in selectively: + +[source,properties] +---- +otel.instrumentation.common.default-enabled=false +otel.instrumentation.opentelemetry-api.enabled=true +otel.instrumentation.opentelemetry-instrumentation-annotations.enabled=true +---- + +Agents from observability vendors generally work the same way, provided they register an OpenTelemetry `GlobalOpenTelemetry` instance. + +== OpenTelemetry Module + +The `opentelemetry` xref:configuration-guide:solr-modules.adoc[module] bundles the OpenTelemetry SDK and an OTLP exporter. +Enable it with either the system property `-Dsolr.modules=opentelemetry` or the environment variable `SOLR_MODULES=opentelemetry`. + +Then activate it in one of two ways. +Either declare it in `solr.xml`: + +[source,xml] +---- +<solr> + <tracerConfig name="tracerConfig" class="org.apache.solr.opentelemetry.OtelTracerConfigurator"/> +</solr> +---- + +Or, without touching `solr.xml`, set the system property `otel.service.name` or the environment variable `OTEL_SERVICE_NAME`; Solr then loads the module's configurator automatically. +Setting `OTEL_SDK_DISABLED=true` suppresses that auto-activation. + +=== Configuration + +The SDK is configured through environment variables or Java system properties -- see https://opentelemetry.io/docs/reference/specification/sdk-environment-variables/[OTEL SDK Environment Variables] and https://github.com/open-telemetry/opentelemetry-java/blob/v{dep-version-opentelemetry}/sdk-extensions/autoconfigure/README.md[Java SDK Autoconfigure]. +The effective defaults, some set by Solr and the rest by the SDK, are: + +[source,bash] +---- +OTEL_SDK_DISABLED=false +OTEL_SERVICE_NAME=solr +OTEL_TRACES_EXPORTER=otlp +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +OTEL_TRACES_SAMPLER=parentbased_always_on +OTEL_PROPAGATORS=tracecontext,baggage +---- + +So out of the box, traces go to a collector on localhost using https://opentelemetry.io/docs/reference/specification/protocol/[OTLP] over HTTP (Protobuf), and trace IDs propagate using https://www.w3.org/TR/trace-context/[W3C TraceContext]. + +To send to a remote https://opentelemetry.io/docs/collector/[OTEL Collector]: + +[source,bash] +---- +OTEL_EXPORTER_OTLP_ENDPOINT=http://my-remote-collector:4318 +---- + +The equivalent with system properties: + +[source,bash] +---- +SOLR_OPTS=-Dotel.exporter.otlp.endpoint=http://my-remote-collector:4318 +---- + +=== Exporters and Transports + +The module ships a deliberately minimal set of dependencies: OTLP over HTTP, and nothing else. +Anything beyond that means adding JARs yourself (or use the Java agent). + +gRPC:: Add the required gRPC and Netty JARs to the module's lib directory, then set `OTEL_EXPORTER_OTLP_PROTOCOL=grpc`. + +Other backends:: Exporters such as Jaeger and Zipkin are supported by the SDK but not shipped. Add the exporter JAR(s) to `$SOLR_TIP/lib/` and configure the exporter, for example: ++ +[source,bash] +---- +OTEL_TRACES_EXPORTER=zipkin +OTEL_EXPORTER_ZIPKIN_ENDPOINT=http://localhost:9411/api/v2/spans +---- + +A Java agent avoids the need to add more JARs and risking version conflicts, since it carries its own exporters in an isolated classloader. + +== Custom Configurator + +`org.apache.solr.core.OpenTelemetryConfigurator` is the plugin API behind `<tracerConfig>`. +A subclass implements `createOpenTelemetry()`, returning the `io.opentelemetry.api.OpenTelemetry` that Solr will install as `GlobalOpenTelemetry`; implementations must not install it themselves. +Configure it exactly as the module's own configurator is configured, with your class name: + +[source,xml] +---- +<solr> + <tracerConfig name="tracerConfig" class="com.example.MyOpenTelemetryConfigurator"/> +</solr> +---- + +NOTE: The `<tracerConfig>` element name and the module's `OtelTracerConfigurator` class name predate OpenTelemetry covering more than tracing; both now configure OpenTelemetry as a whole. + +== Verifying Which Integration Is Active + +Each mode logs a distinct line at startup, which is the quickest way to confirm what Solr actually picked up: + +[cols="3,2",options="header"] +|=== +| Log message | Meaning + +| `OpenTelemetry Java agent is installed; using the OpenTelemetry it registered.` +| A Java agent was detected; Solr configured nothing. + +| `OpenTelemetry loaded via <class name>` +| The configurator named in `<tracerConfig>` was loaded. + +| `OpenTelemetry loaded via auto configuration.` +| `OTEL_SERVICE_NAME` is set, so the `opentelemetry` module's configurator was loaded automatically. + +| `OpenTelemetry loaded with simple propagation only.` +| No integration; always-on trace ID generation is in effect. + +| `Unable to auto-config OpenTelemetry with class ...` +| `OTEL_SERVICE_NAME` is set but the `opentelemetry` module is not enabled. +|=== diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java index ad0857dee74..554f65b8d49 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java @@ -36,11 +36,13 @@ import org.apache.lucene.tests.util.VerifyTestClassNamingConvention; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.ObjectReleaseTracker; import org.apache.solr.core.ConfigSetService; +import org.apache.solr.core.OpenTelemetryConfigurator; import org.apache.solr.util.ExternalPaths; import org.apache.solr.util.LogLevelTestRule; import org.apache.solr.util.QueryLimitsTestInjectionRule; import org.apache.solr.util.RevertDefaultThreadHandlerRule; import org.apache.solr.util.StartupLoggingUtils; +import org.apache.solr.util.tracing.TraceUtils; import org.hamcrest.Matcher; import org.hamcrest.MatcherAssert; import org.junit.AfterClass; @@ -161,6 +163,22 @@ public class SolrTestCase extends LuceneTestCase { System.setProperty("zookeeper.forceSync", "no"); System.setProperty("zookeeper.nio.shutdownTimeout", "100"); } + + injectRandomTraceRecordingFlag(); + } + + /** + * Randomizes the tracing {@link TraceUtils#ifNotNoop(io.opentelemetry.api.trace.Span, + * java.util.function.Consumer)} check. + * + * <p>Rationale: to have better coverage of all methods that deal with span creation without + * having to enable tracing. + */ + private static void injectRandomTraceRecordingFlag() { + TraceUtils.IS_RECORDING = + LuceneTestCase.rarely() + ? (ignored) -> true + : TraceUtils.DEFAULT_IS_RECORDING; // honors Span::isRecording } /** @@ -195,6 +213,11 @@ public class SolrTestCase extends LuceneTestCase { assumeFalse(PROP + " == true", systemPropertyAsBoolean(PROP, false)); } + @AfterClass + public static void afterSolrTestCase() { + OpenTelemetryConfigurator.resetForTest(); + } + // UTILITY METHODS FOLLOW public static void assertJSONEquals(String expected, String actual) { diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index e77fa41fcad..fc9416924fa 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -102,7 +102,6 @@ import org.apache.solr.common.util.XML; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoresLocator; import org.apache.solr.core.NodeConfig; -import org.apache.solr.core.OpenTelemetryConfigurator; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrXmlConfig; @@ -285,7 +284,6 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase { } ExecutorUtil.resetThreadLocalProviders(); - OpenTelemetryConfigurator.resetForTest(); } @AfterClass diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index 1338cf6c94b..f9acdbfc0bf 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -79,11 +79,9 @@ import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.common.util.TimeSource; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.OpenTelemetryConfigurator; import org.apache.solr.embedded.JettyConfig; import org.apache.solr.embedded.JettySolrRunner; import org.apache.solr.util.TimeOut; -import org.apache.solr.util.tracing.TraceUtils; import org.apache.zookeeper.KeeperException; import org.eclipse.jetty.ee10.servlet.ServletHolder; import org.slf4j.Logger; @@ -644,7 +642,6 @@ public class MiniSolrCloudCluster implements SolrBackend { if (!externalZkServer) { zkServer.shutdown(); } - resetRecordingFlag(); } } @@ -1017,7 +1014,6 @@ public class MiniSolrCloudCluster implements SolrBackend { private boolean overseerEnabled = EnvUtils.getPropertyAsBool("solr.cloud.overseer.enabled", true); private boolean formatZkServer = true; - private boolean disableTraceIdGeneration = false; /** * Create a builder @@ -1148,11 +1144,6 @@ public class MiniSolrCloudCluster implements SolrBackend { public MiniSolrCloudCluster build() throws Exception { System.setProperty("solr.cloud.overseer.enabled", Boolean.toString(overseerEnabled)); - if (!disableTraceIdGeneration && OpenTelemetryConfigurator.TRACE_ID_GEN_ENABLED) { - OpenTelemetryConfigurator.initializeOpenTelemetrySdk(null, null); - injectRandomRecordingFlag(); - } - JettyConfig jettyConfig = jettyConfigBuilder.build(); MiniSolrCloudCluster cluster = new MiniSolrCloudCluster( @@ -1189,44 +1180,5 @@ public class MiniSolrCloudCluster implements SolrBackend { cluster.put(key, value); return this; } - - /** - * Disables the default/built-in simple trace ID generation/propagation. - * - * <p>Tracers are registered as global singletons and if for example a test needs to use a - * MockTracer or a "real" Tracer, it needs to call this method so that the test setup doesn't - * accidentally reset the Tracer it wants to use. - */ - public Builder withTraceIdGenerationDisabled() { - this.disableTraceIdGeneration = true; - return this; - } - } - - /** - * Randomizes the tracing Span::isRecording check. - * - * <p>This will randomize the Span::isRecording check so we have better coverage of all methods - * that deal with span creation without having to enable otel module. - * - * <p>It only makes sense to call this if we are using the alwaysOn tracer, the OTEL tracer - * already has this flag turned on and randomizing it would risk not recording trace data. - * - * <p>Note. Tracing is not a SolrCloud only feature. this method is placed here for convenience - * only, any test can make use of this example for more complete coverage of the tracing - * mechanics. - */ - private static void injectRandomRecordingFlag() { - try { - boolean isRecording = LuceneTestCase.rarely(); - TraceUtils.IS_RECORDING = (ignored) -> isRecording; - } catch (IllegalStateException e) { - // This can happen in benchmarks or other places that aren't in a randomized test - log.warn("Unable to inject random recording flag due to outside randomized context", e); - } - } - - private static void resetRecordingFlag() { - TraceUtils.IS_RECORDING = TraceUtils.DEFAULT_IS_RECORDING; } }
