This is an automated email from the ASF dual-hosted git repository. luigidemasi pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel.git
commit 77cc5be66fcef2666d62473e4c0613bae8d0f2a5 Author: Luigi De Masi <[email protected]> AuthorDate: Fri Nov 13 10:23:44 2026 +0100 CAMEL-25648: Correct Prometheus score names and improve metric diagnostics Avoid the reserved bucket suffix for score counters, align camelContext with Camel's other metrics, and log the first collection failure using only its exception class. Document and verify bounded timer histogram settings. Protect the actual Prometheus exposition name and labels, warning privacy and suppression, and timer counts with configured buckets. Co-authored-by: OpenAI Codex <[email protected]> Signed-off-by: Luigi De Masi <[email protected]> --- .../camel/catalog/docs/semantic-language.adoc | 38 ++++++++++-- .../src/main/docs/semantic-language.adoc | 38 ++++++++++-- .../micrometer/MicrometerSemanticMetrics.java | 16 +++-- .../micrometer/MicrometerSemanticMetricsTest.java | 72 ++++++++++++++++++++-- 4 files changed, 148 insertions(+), 16 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-language.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-language.adoc index 29a0a531b1ab..445e1ccc4955 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-language.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-language.adoc @@ -1546,6 +1546,8 @@ Each completed assessment member is counted once, including individual batch mem and locally aggregated `typed_results` snapshots are not counted again. Provider attempts/retries are not inferred from semantic completions. Audit capture, input capture, JMX, the local monitoring cache and UI polling are not required. Collection failures are isolated from the assessment and reported by the collection health meters. +The first failure since collector start also logs a warning containing only the exception class, without its +message, stack trace or event payload; subsequent failures only increment the error count. [cols="2,3",options="header"] |=== @@ -1561,19 +1563,47 @@ selected label once per assessment, so membership percentages need not sum to 10 |`camel.semantic.score.observations` |Count of available finite numeric score observations. |`camel.semantic.score.sum`, `.minimum`, `.maximum` |Gauges preserving signed, fractional scores since collector start. The sum can decrease; do not treat it as a monotonic counter. -|`camel.semantic.score.bucket` |Cumulative counters with inclusive `le` bounds supplied to the constructor and a +|`camel.semantic.score.distribution` |Cumulative counters with inclusive `le` bounds supplied to the constructor and a `+Inf` bucket. Bounds describe numeric values, not severity or policy decisions. |`camel.semantic.collection.loss` |Counter by fixed `reason`: `population_limit`, `answer_limit`, `invalid_event`, or `unsupported_measurement`. Loss is excluded from the corresponding result distribution. |`camel.semantic.collection.errors` |Number of metric collection failures since collector start. |=== -Assessment meters have `camel.context`, `expert`, `definition`, `operation`, `origin` and `result.type` tags. +Assessment meters have `camelContext`, `expert`, `definition`, `operation`, `origin` and `result.type` tags. Missing identity is the empty string; direct calls remain distinguishable by origin and the absence of a definition. The listener retains at most 100 distinct combinations and 64 answer labels of at most 256 characters per combination. Score histograms accept at most 64 finite, strictly increasing bounds; an empty array keeps count, sum, minimum, maximum and the overflow bucket. Configure identical bounds on instances you aggregate together. -Micrometer's timer histogram configuration can be adjusted with the application's usual meter filters. +Each population can have both successful and failed duration timers. Default percentile histograms can add about +70 bucket series per timer: the 100-population limit alone can therefore allow roughly 14,000 duration bucket +series. Configure the registry's meter filter before registering the listener to choose fewer boundaries, for example: + +[source,java] +---- +import java.time.Duration; + +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.config.MeterFilter; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; + +registry.config().meterFilter(new MeterFilter() { + @Override + public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { + if (id.getName().equals("camel.semantic.duration")) { + return DistributionStatisticConfig.builder().percentilesHistogram(false) + .serviceLevelObjectives(Duration.ofMillis(10).toNanos(), Duration.ofMillis(100).toNanos(), + Duration.ofSeconds(1).toNanos(), Duration.ofSeconds(10).toNanos()) + .build().merge(config); + } + return config; + } +}); +---- + +This example exports five duration buckets per timer in Prometheus, including `+Inf`, while retaining every +observation in the count and sum. Choose boundaries for your latency goals: fewer buckets reduce percentile +resolution. The `camelContext` tag matches Camel's other Micrometer metrics. Prompts, input, interpretation, exception messages and per-execution identifiers are never metric labels. Choose stable expert/definition names and result labels. Bounds limit retained series, but collection loss still @@ -1624,7 +1654,7 @@ histogram_quantile(0.95, sum by (expert, le) (rate(camel_semantic_duration_seconds_bucket[5m]))) # Available score distribution using application-declared bounds. -sum by (expert, definition, le) (rate(camel_semantic_score_bucket_total[5m])) +sum by (expert, definition, le) (rate(camel_semantic_score_distribution_total[5m])) # Counter resets observed on individual application series. resets(camel_semantic_evaluations_total[1h]) diff --git a/components/camel-ai/camel-semantic/src/main/docs/semantic-language.adoc b/components/camel-ai/camel-semantic/src/main/docs/semantic-language.adoc index 29a0a531b1ab..445e1ccc4955 100644 --- a/components/camel-ai/camel-semantic/src/main/docs/semantic-language.adoc +++ b/components/camel-ai/camel-semantic/src/main/docs/semantic-language.adoc @@ -1546,6 +1546,8 @@ Each completed assessment member is counted once, including individual batch mem and locally aggregated `typed_results` snapshots are not counted again. Provider attempts/retries are not inferred from semantic completions. Audit capture, input capture, JMX, the local monitoring cache and UI polling are not required. Collection failures are isolated from the assessment and reported by the collection health meters. +The first failure since collector start also logs a warning containing only the exception class, without its +message, stack trace or event payload; subsequent failures only increment the error count. [cols="2,3",options="header"] |=== @@ -1561,19 +1563,47 @@ selected label once per assessment, so membership percentages need not sum to 10 |`camel.semantic.score.observations` |Count of available finite numeric score observations. |`camel.semantic.score.sum`, `.minimum`, `.maximum` |Gauges preserving signed, fractional scores since collector start. The sum can decrease; do not treat it as a monotonic counter. -|`camel.semantic.score.bucket` |Cumulative counters with inclusive `le` bounds supplied to the constructor and a +|`camel.semantic.score.distribution` |Cumulative counters with inclusive `le` bounds supplied to the constructor and a `+Inf` bucket. Bounds describe numeric values, not severity or policy decisions. |`camel.semantic.collection.loss` |Counter by fixed `reason`: `population_limit`, `answer_limit`, `invalid_event`, or `unsupported_measurement`. Loss is excluded from the corresponding result distribution. |`camel.semantic.collection.errors` |Number of metric collection failures since collector start. |=== -Assessment meters have `camel.context`, `expert`, `definition`, `operation`, `origin` and `result.type` tags. +Assessment meters have `camelContext`, `expert`, `definition`, `operation`, `origin` and `result.type` tags. Missing identity is the empty string; direct calls remain distinguishable by origin and the absence of a definition. The listener retains at most 100 distinct combinations and 64 answer labels of at most 256 characters per combination. Score histograms accept at most 64 finite, strictly increasing bounds; an empty array keeps count, sum, minimum, maximum and the overflow bucket. Configure identical bounds on instances you aggregate together. -Micrometer's timer histogram configuration can be adjusted with the application's usual meter filters. +Each population can have both successful and failed duration timers. Default percentile histograms can add about +70 bucket series per timer: the 100-population limit alone can therefore allow roughly 14,000 duration bucket +series. Configure the registry's meter filter before registering the listener to choose fewer boundaries, for example: + +[source,java] +---- +import java.time.Duration; + +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.config.MeterFilter; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; + +registry.config().meterFilter(new MeterFilter() { + @Override + public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { + if (id.getName().equals("camel.semantic.duration")) { + return DistributionStatisticConfig.builder().percentilesHistogram(false) + .serviceLevelObjectives(Duration.ofMillis(10).toNanos(), Duration.ofMillis(100).toNanos(), + Duration.ofSeconds(1).toNanos(), Duration.ofSeconds(10).toNanos()) + .build().merge(config); + } + return config; + } +}); +---- + +This example exports five duration buckets per timer in Prometheus, including `+Inf`, while retaining every +observation in the count and sum. Choose boundaries for your latency goals: fewer buckets reduce percentile +resolution. The `camelContext` tag matches Camel's other Micrometer metrics. Prompts, input, interpretation, exception messages and per-execution identifiers are never metric labels. Choose stable expert/definition names and result labels. Bounds limit retained series, but collection loss still @@ -1624,7 +1654,7 @@ histogram_quantile(0.95, sum by (expert, le) (rate(camel_semantic_duration_seconds_bucket[5m]))) # Available score distribution using application-declared bounds. -sum by (expert, definition, le) (rate(camel_semantic_score_bucket_total[5m])) +sum by (expert, definition, le) (rate(camel_semantic_score_distribution_total[5m])) # Counter resets observed on individual application series. resets(camel_semantic_evaluations_total[1h]) diff --git a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/micrometer/MicrometerSemanticMetrics.java b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/micrometer/MicrometerSemanticMetrics.java index 39348968f8b6..988d99282729 100644 --- a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/micrometer/MicrometerSemanticMetrics.java +++ b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/micrometer/MicrometerSemanticMetrics.java @@ -37,6 +37,8 @@ import org.apache.camel.semantic.SemanticExpert.ResultType; import org.apache.camel.semantic.SemanticMonitoringEvent; import org.apache.camel.spi.CamelEvent; import org.apache.camel.support.EventNotifierSupport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Optional Micrometer instrumentation of completed semantic assessment members. Register one notifier with the Camel @@ -46,6 +48,8 @@ import org.apache.camel.support.EventNotifierSupport; * @since 4.24 */ public final class MicrometerSemanticMetrics extends EventNotifierSupport { + private static final Logger LOG = LoggerFactory.getLogger(MicrometerSemanticMetrics.class); + private static final String CAMEL_CONTEXT_TAG = "camelContext"; private static final int MAX_POPULATIONS = 100; private static final int MAX_ANSWERS = 64; private static final Set<String> STATUSES = Set.of("success", "failed", "not_executed"); @@ -104,7 +108,11 @@ public final class MicrometerSemanticMetrics extends EventNotifierSupport { } } catch (Exception | AssertionError | LinkageError failure) { // Instrumentation must not change the assessment or expose payloads through exception logging. - errors.incrementAndGet(); + if (errors.incrementAndGet() == 1) { + LOG.warn( + "Semantic metric collection failed ({}); further failures are counted in camel.semantic.collection.errors", + failure.getClass().getName()); + } } } @@ -235,7 +243,7 @@ public final class MicrometerSemanticMetrics extends EventNotifierSupport { throw new IllegalStateException("Only one Semantic Micrometer collector may be registered per CamelContext"); } existing = new HashSet<>(registry.getMeters()); - contextTags = Tags.of("camel.context", getCamelContext().getName()); + contextTags = Tags.of(CAMEL_CONTEXT_TAG, getCamelContext().getName()); errors.set(0); own(Gauge.builder("camel.semantic.collection.errors", errors, AtomicLong::doubleValue) .description("Metric collection failures since this collector started").tags(contextTags).register(registry)); @@ -295,9 +303,9 @@ public final class MicrometerSemanticMetrics extends EventNotifierSupport { own(Gauge.builder("camel.semantic.score.maximum", this, p -> p.maximum).tags(tags).register(registry)); List<Counter> registeredBuckets = new ArrayList<>(); for (double bound : scoreBounds) { - registeredBuckets.add(counter("score.bucket", tags.and("le", Double.toString(bound)))); + registeredBuckets.add(counter("score.distribution", tags.and("le", Double.toString(bound)))); } - registeredBuckets.add(counter("score.bucket", tags.and("le", "+Inf"))); + registeredBuckets.add(counter("score.distribution", tags.and("le", "+Inf"))); buckets.addAll(registeredBuckets); scores = observations; } diff --git a/components/camel-ai/camel-semantic/src/test/java/org/apache/camel/semantic/micrometer/MicrometerSemanticMetricsTest.java b/components/camel-ai/camel-semantic/src/test/java/org/apache/camel/semantic/micrometer/MicrometerSemanticMetricsTest.java index 3acca7ddb481..a1d1a3a8cb09 100644 --- a/components/camel-ai/camel-semantic/src/test/java/org/apache/camel/semantic/micrometer/MicrometerSemanticMetricsTest.java +++ b/components/camel-ai/camel-semantic/src/test/java/org/apache/camel/semantic/micrometer/MicrometerSemanticMetricsTest.java @@ -19,6 +19,7 @@ package org.apache.camel.semantic.micrometer; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -29,6 +30,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import io.micrometer.core.instrument.Meter; import io.micrometer.core.instrument.config.MeterFilter; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.micrometer.prometheusmetrics.PrometheusConfig; import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; @@ -51,6 +53,12 @@ import org.apache.camel.semantic.SemanticResultMetrics; import org.apache.camel.semantic.internal.SemanticAuditService; import org.apache.camel.support.DefaultExchange; import org.apache.camel.support.EventHelper; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -178,7 +186,7 @@ class MicrometerSemanticMetricsTest { assertThat(registry.get("camel.semantic.collection.errors").gauge().value()).isEqualTo(1); assertThat(registry.get("camel.semantic.score.observations").counter().count()).isEqualTo(1); assertThat(registry.get("camel.semantic.score.sum").gauge().value()).isEqualTo(1.6); - assertThat(registry.get("camel.semantic.score.bucket").tag("le", "+Inf").counter().count()).isEqualTo(1); + assertThat(registry.get("camel.semantic.score.distribution").tag("le", "+Inf").counter().count()).isEqualTo(1); } } @@ -244,16 +252,50 @@ class MicrometerSemanticMetricsTest { assertThat(registry.get("camel.semantic.score.minimum").gauge().value()).isEqualTo(-0.4); assertThat(registry.get("camel.semantic.score.maximum").gauge().value()).isEqualTo(1.6); assertThat(registry.get("camel.semantic.score.observations").counter().count()).isEqualTo(2); - assertThat(registry.get("camel.semantic.score.bucket").tag("le", "0.0").counter().count()).isEqualTo(1); - assertThat(registry.get("camel.semantic.score.bucket").tag("le", "+Inf").counter().count()).isEqualTo(2); + assertThat(registry.get("camel.semantic.score.distribution").tag("le", "0.0").counter().count()).isEqualTo(1); + assertThat(registry.get("camel.semantic.score.distribution").tag("le", "+Inf").counter().count()).isEqualTo(2); String scrape = registry.scrape(); assertThat(scrape).contains("camel_semantic_duration_seconds_bucket", "camel_semantic_duration_seconds_count", "camel_semantic_duration_seconds_sum", "camel_semantic_score_observations_total") .doesNotContain("private-input", "eventId", "breadcrumbId", "invocationId"); + assertThat(scrape.lines().filter(line -> line.startsWith("camel_semantic_score_distribution_total{"))) + .anySatisfy(line -> assertThat(line).contains("le=\"0.0\"", "camelContext=\"" + context.getName() + "\"") + .endsWith(" 1.0")); assertThat(registry.get("camel.semantic.duration").timer().totalTime(TimeUnit.MILLISECONDS)).isEqualTo(10); } } + @Test + void applicationCanLimitDurationHistogramBuckets() throws Exception { + var registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + registry.config().meterFilter(new MeterFilter() { + @Override + public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { + if (id.getName().equals("camel.semantic.duration")) { + return DistributionStatisticConfig.builder().percentilesHistogram(false) + .serviceLevelObjectives(Duration.ofMillis(10).toNanos(), Duration.ofMillis(100).toNanos(), + Duration.ofSeconds(1).toNanos(), Duration.ofSeconds(10).toNanos()) + .build().merge(config); + } + return config; + } + }); + try (AutoCloseable closeRegistry = registry::close; var context = new DefaultCamelContext()) { + context.getManagementStrategy().addEventNotifier(new MicrometerSemanticMetrics(registry)); + context.start(); + send(context, event(ResultType.SCORE, Map.of("value", 1), "success", 50000000)); + send(context, event(ResultType.SCORE, Map.of("value", 2), "success", 15000000000L)); + String scrape = registry.scrape(); + assertThat(scrape.lines().filter(line -> line.startsWith("camel_semantic_duration_seconds_bucket{"))) + .hasSize(5); + assertThat(scrape.lines().filter(line -> line.startsWith("camel_semantic_duration_seconds_bucket{"))) + .anySatisfy(line -> assertThat(line).contains("le=\"0.1\"").endsWith(" 1")) + .anySatisfy(line -> assertThat(line).contains("le=\"+Inf\"").endsWith(" 2")); + assertThat(registry.get("camel.semantic.duration").timer().count()).isEqualTo(2); + assertThat(registry.get("camel.semantic.duration").timer().totalTime(TimeUnit.MILLISECONDS)).isEqualTo(15050); + } + } + @Test void cardinalityIsBoundedAndLossIsVisible() throws Exception { var registry = new SimpleMeterRegistry(); @@ -298,6 +340,17 @@ class MicrometerSemanticMetricsTest { @Test void registryFailureIsVisibleAndDoesNotEscapeTheCallback() throws Exception { + Logger logger = (Logger) LogManager.getLogger(MicrometerSemanticMetrics.class); + Level previous = logger.getLevel(); + List<LogEvent> messages = new ArrayList<>(); + var appender = new AbstractAppender("SemanticMetricsTest", null, null, true, Property.EMPTY_ARRAY) { + public void append(LogEvent event) { + messages.add(event.toImmutable()); + } + }; + appender.start(); + logger.addAppender(appender); + logger.setLevel(Level.WARN); var registry = new SimpleMeterRegistry(); try (AutoCloseable closeRegistry = registry::close; var context = new DefaultCamelContext()) { registry.config().meterFilter(new MeterFilter() { @@ -313,7 +366,18 @@ class MicrometerSemanticMetricsTest { context.getManagementStrategy().addEventNotifier(metrics); context.start(); metrics.notify(event(ResultType.BOOLEAN, Map.of("value", true), "success", 1)); - assertThat(registry.get("camel.semantic.collection.errors").gauge().value()).isEqualTo(1); + metrics.notify(event(ResultType.BOOLEAN, Map.of("value", false), "success", 1)); + assertThat(registry.get("camel.semantic.collection.errors").gauge().value()).isEqualTo(2); + assertThat(messages).singleElement().satisfies(message -> { + assertThat(message.getLevel()).isEqualTo(Level.WARN); + assertThat(message.getMessage().getFormattedMessage()).contains("java.lang.IllegalStateException") + .doesNotContain("private-provider-error", "private-input"); + assertThat(message.getThrown()).isNull(); + }); + } finally { + logger.setLevel(previous); + logger.removeAppender(appender); + appender.stop(); } }
