This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch feature/CAMEL-23872-jfr-old-objects in repository https://gitbox.apache.org/repos/asf/camel.git
commit d78673dbf6184215aab6e3f1954c29e7a1ab3a49 Author: Claus Ibsen <[email protected]> AuthorDate: Wed Jul 1 23:32:45 2026 +0200 CAMEL-23872: Add JFR Old Object Sample panel for memory leak diagnosis Adds JFR-based old object sampling across dev console, CLI connector, CLI command, TUI tab, and MCP tool. JFR OldObjectSample tracks objects surviving multiple GC cycles and captures reference chains back to GC roots, complementing the existing heap histogram. Key features: - Dev console manages JFR recording lifecycle (start/stop/query) - Aggregates samples by class + stack trace fingerprint (count, totalSize) - Converts JVM array descriptors to human-readable names - CLI supports --stacktrace, --min-size, --top options - TUI tab with recording progress, sortable table, detail panel with reference chains and allocation stack traces, min-size filter - MCP tool for AI agent access - JDK/Jakarta stack trace frames dimmed in TUI detail panel Co-Authored-By: Claude <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../apache/camel/catalog/dev-consoles.properties | 1 + .../catalog/dev-consoles/jfr-old-objects.json | 15 + .../JfrOldObjectSampleDevConsoleConfigurer.java | 63 ++ .../apache/camel/dev-console/jfr-old-objects.json | 15 + ...camel.impl.console.JfrOldObjectSampleDevConsole | 2 + .../org/apache/camel/dev-console/jfr-old-objects | 2 + .../org/apache/camel/dev-consoles.properties | 2 +- .../impl/console/JfrOldObjectSampleDevConsole.java | 608 ++++++++++++ .../camel-jbang-cmd-jfr-old-objects.adoc | 33 + .../ROOT/pages/jbang-commands/camel-jbang-cmd.adoc | 1 + .../camel/cli/connector/LocalCliConnector.java | 27 + .../META-INF/camel-jbang-commands-metadata.json | 2 +- .../dsl/jbang/core/commands/CamelJBangMain.java | 1 + .../core/commands/action/CamelJfrOldObjects.java | 350 +++++++ .../dsl/jbang/core/commands/mcp/RuntimeTools.java | 22 + .../core/commands/tui/JfrOldObjectSampleTab.java | 1030 ++++++++++++++++++++ .../dsl/jbang/core/commands/tui/McpFacade.java | 6 +- .../dsl/jbang/core/commands/tui/PopupManager.java | 26 +- .../dsl/jbang/core/commands/tui/TabRegistry.java | 24 +- 19 files changed, 2205 insertions(+), 25 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles.properties b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles.properties index b714441d0ead..ae3fef97c516 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles.properties +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles.properties @@ -24,6 +24,7 @@ heap-histogram inflight internal-tasks java-security +jfr-old-objects jvm kafka kubernetes-configmaps diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles/jfr-old-objects.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles/jfr-old-objects.json new file mode 100644 index 000000000000..51cd112feba4 --- /dev/null +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles/jfr-old-objects.json @@ -0,0 +1,15 @@ +{ + "console": { + "kind": "console", + "group": "camel", + "name": "jfr-old-objects", + "title": "JFR Old Object Samples", + "description": "JFR-based old object sampling for memory leak diagnosis", + "deprecated": false, + "javaType": "org.apache.camel.impl.console.JfrOldObjectSampleDevConsole", + "groupId": "org.apache.camel", + "artifactId": "camel-console", + "version": "4.21.0-SNAPSHOT" + } +} + diff --git a/core/camel-console/src/generated/java/org/apache/camel/impl/console/JfrOldObjectSampleDevConsoleConfigurer.java b/core/camel-console/src/generated/java/org/apache/camel/impl/console/JfrOldObjectSampleDevConsoleConfigurer.java new file mode 100644 index 000000000000..ad9a16765e5e --- /dev/null +++ b/core/camel-console/src/generated/java/org/apache/camel/impl/console/JfrOldObjectSampleDevConsoleConfigurer.java @@ -0,0 +1,63 @@ +/* Generated by camel build tools - do NOT edit this file! */ +package org.apache.camel.impl.console; + +import javax.annotation.processing.Generated; +import java.util.Map; + +import org.apache.camel.CamelContext; +import org.apache.camel.spi.ExtendedPropertyConfigurerGetter; +import org.apache.camel.spi.PropertyConfigurerGetter; +import org.apache.camel.spi.ConfigurerStrategy; +import org.apache.camel.spi.GeneratedPropertyConfigurer; +import org.apache.camel.util.CaseInsensitiveMap; +import org.apache.camel.impl.console.JfrOldObjectSampleDevConsole; + +/** + * Generated by camel build tools - do NOT edit this file! + */ +@Generated("org.apache.camel.maven.packaging.GenerateConfigurerMojo") +@SuppressWarnings("unchecked") +public class JfrOldObjectSampleDevConsoleConfigurer extends org.apache.camel.support.component.PropertyConfigurerSupport implements GeneratedPropertyConfigurer, ExtendedPropertyConfigurerGetter { + + private static final Map<String, Object> ALL_OPTIONS; + static { + Map<String, Object> map = new CaseInsensitiveMap(); + map.put("CamelContext", org.apache.camel.CamelContext.class); + ALL_OPTIONS = map; + } + + @Override + public boolean configure(CamelContext camelContext, Object obj, String name, Object value, boolean ignoreCase) { + org.apache.camel.impl.console.JfrOldObjectSampleDevConsole target = (org.apache.camel.impl.console.JfrOldObjectSampleDevConsole) obj; + switch (ignoreCase ? name.toLowerCase() : name) { + case "camelcontext": + case "camelContext": target.setCamelContext(property(camelContext, org.apache.camel.CamelContext.class, value)); return true; + default: return false; + } + } + + @Override + public Map<String, Object> getAllOptions(Object target) { + return ALL_OPTIONS; + } + + @Override + public Class<?> getOptionType(String name, boolean ignoreCase) { + switch (ignoreCase ? name.toLowerCase() : name) { + case "camelcontext": + case "camelContext": return org.apache.camel.CamelContext.class; + default: return null; + } + } + + @Override + public Object getOptionValue(Object obj, String name, boolean ignoreCase) { + org.apache.camel.impl.console.JfrOldObjectSampleDevConsole target = (org.apache.camel.impl.console.JfrOldObjectSampleDevConsole) obj; + switch (ignoreCase ? name.toLowerCase() : name) { + case "camelcontext": + case "camelContext": return target.getCamelContext(); + default: return null; + } + } +} + diff --git a/core/camel-console/src/generated/resources/META-INF/org/apache/camel/dev-console/jfr-old-objects.json b/core/camel-console/src/generated/resources/META-INF/org/apache/camel/dev-console/jfr-old-objects.json new file mode 100644 index 000000000000..51cd112feba4 --- /dev/null +++ b/core/camel-console/src/generated/resources/META-INF/org/apache/camel/dev-console/jfr-old-objects.json @@ -0,0 +1,15 @@ +{ + "console": { + "kind": "console", + "group": "camel", + "name": "jfr-old-objects", + "title": "JFR Old Object Samples", + "description": "JFR-based old object sampling for memory leak diagnosis", + "deprecated": false, + "javaType": "org.apache.camel.impl.console.JfrOldObjectSampleDevConsole", + "groupId": "org.apache.camel", + "artifactId": "camel-console", + "version": "4.21.0-SNAPSHOT" + } +} + diff --git a/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/configurer/org.apache.camel.impl.console.JfrOldObjectSampleDevConsole b/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/configurer/org.apache.camel.impl.console.JfrOldObjectSampleDevConsole new file mode 100644 index 000000000000..29273029b845 --- /dev/null +++ b/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/configurer/org.apache.camel.impl.console.JfrOldObjectSampleDevConsole @@ -0,0 +1,2 @@ +# Generated by camel build tools - do NOT edit this file! +class=org.apache.camel.impl.console.JfrOldObjectSampleDevConsoleConfigurer diff --git a/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/dev-console/jfr-old-objects b/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/dev-console/jfr-old-objects new file mode 100644 index 000000000000..fa238f8e4377 --- /dev/null +++ b/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/dev-console/jfr-old-objects @@ -0,0 +1,2 @@ +# Generated by camel build tools - do NOT edit this file! +class=org.apache.camel.impl.console.JfrOldObjectSampleDevConsole diff --git a/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/dev-consoles.properties b/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/dev-consoles.properties index c429fc6982f4..2107cf01e0b6 100644 --- a/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/dev-consoles.properties +++ b/core/camel-console/src/generated/resources/META-INF/services/org/apache/camel/dev-consoles.properties @@ -1,5 +1,5 @@ # Generated by camel build tools - do NOT edit this file! -dev-consoles=bean blocked browse circuit-breaker consumer context datasource debug endpoint errors eval-language event gc health heap-histogram inflight internal-tasks java-security jvm log memory message-history processor producer properties receive reload rest rest-spec route route-controller route-dump route-group route-structure route-topology send service simple-language source sql-query sql-trace startup-recorder system-properties thread top trace transformers type-converters variables +dev-consoles=bean blocked browse circuit-breaker consumer context datasource debug endpoint errors eval-language event gc health heap-histogram inflight internal-tasks java-security jfr-old-objects jvm log memory message-history processor producer properties receive reload rest rest-spec route route-controller route-dump route-group route-structure route-topology send service simple-language source sql-query sql-trace startup-recorder system-properties thread top trace transformers type- [...] groupId=org.apache.camel artifactId=camel-console version=4.21.0-SNAPSHOT diff --git a/core/camel-console/src/main/java/org/apache/camel/impl/console/JfrOldObjectSampleDevConsole.java b/core/camel-console/src/main/java/org/apache/camel/impl/console/JfrOldObjectSampleDevConsole.java new file mode 100644 index 000000000000..bf8614734654 --- /dev/null +++ b/core/camel-console/src/main/java/org/apache/camel/impl/console/JfrOldObjectSampleDevConsole.java @@ -0,0 +1,608 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.impl.console; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import jdk.jfr.Recording; +import jdk.jfr.consumer.RecordedClass; +import jdk.jfr.consumer.RecordedEvent; +import jdk.jfr.consumer.RecordedFrame; +import jdk.jfr.consumer.RecordedObject; +import jdk.jfr.consumer.RecordedStackTrace; +import jdk.jfr.consumer.RecordingFile; +import org.apache.camel.spi.Configurer; +import org.apache.camel.spi.annotations.DevConsole; +import org.apache.camel.support.console.AbstractDevConsole; +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Dev console for JFR-based old object sampling. Captures objects surviving multiple GC cycles and their reference + * chains back to GC roots, enabling memory leak diagnosis. + */ +@DevConsole(name = "jfr-old-objects", displayName = "JFR Old Object Samples", + description = "JFR-based old object sampling for memory leak diagnosis") +@Configurer(extended = true) +public class JfrOldObjectSampleDevConsole extends AbstractDevConsole { + + private static final Logger LOG = LoggerFactory.getLogger(JfrOldObjectSampleDevConsole.class); + + private static final int DEFAULT_LIMIT = 100; + private static final int MAX_STACK_FRAMES = 10; + private static final int MAX_CHAIN_DEPTH = 20; + + private volatile Recording activeRecording; + private volatile JsonObject cachedResults; + private volatile long recordingStartTime; + private volatile int requestedDurationSeconds; + private ScheduledExecutorService scheduler; + private ScheduledFuture<?> autoStopFuture; + + public JfrOldObjectSampleDevConsole() { + super("jvm", "jfr-old-objects", "JFR Old Object Samples", + "JFR-based old object sampling for memory leak diagnosis"); + } + + @Override + protected String doCallText(Map<String, Object> options) { + JsonObject json = doCallJson(options); + return json.toJson(); + } + + @Override + protected JsonObject doCallJson(Map<String, Object> options) { + String command = optionString(options, "command"); + if (command == null) { + command = "status"; + } + + return switch (command) { + case "start" -> doStart(options); + case "stop" -> doStop(options); + case "status" -> doStatus(); + case "query" -> doQuery(options); + default -> errorJson("Unknown command: " + command); + }; + } + + private JsonObject doStart(Map<String, Object> options) { + if (activeRecording != null) { + return errorJson("A JFR recording is already active. Stop it first."); + } + + try { + Recording rec = new Recording(); + rec.setName("Camel OldObjectSample"); + rec.enable("jdk.OldObjectSample").withStackTrace().withPeriod(Duration.ofSeconds(1)); + + int duration = optionInt(options, "duration", 0); + requestedDurationSeconds = duration; + if (duration > 0) { + rec.setMaxAge(Duration.ofSeconds(duration + 10)); + } + + rec.start(); + activeRecording = rec; + recordingStartTime = System.currentTimeMillis(); + + if (duration > 0) { + ensureScheduler(); + autoStopFuture = scheduler.schedule(() -> { + try { + doStopRecordingAndParse(DEFAULT_LIMIT); + } catch (Exception e) { + LOG.warn("Error auto-stopping JFR recording: {}", e.getMessage(), e); + } + }, duration, TimeUnit.SECONDS); + } + + JsonObject result = new JsonObject(); + result.put("status", "recording"); + result.put("startTime", recordingStartTime); + if (duration > 0) { + result.put("durationSeconds", duration); + } + return result; + } catch (Exception e) { + LOG.warn("Failed to start JFR recording: {}", e.getMessage(), e); + return errorJson("Failed to start JFR recording: " + e.getMessage()); + } + } + + private JsonObject doStop(Map<String, Object> options) { + if (activeRecording == null) { + if (cachedResults != null) { + return cachedResults; + } + return errorJson("No active JFR recording to stop."); + } + + cancelAutoStop(); + int limit = optionInt(options, "limit", DEFAULT_LIMIT); + + try { + return doStopRecordingAndParse(limit); + } catch (Exception e) { + LOG.warn("Error stopping JFR recording: {}", e.getMessage(), e); + return errorJson("Error stopping JFR recording: " + e.getMessage()); + } + } + + private synchronized JsonObject doStopRecordingAndParse(int limit) { + Recording rec = activeRecording; + if (rec == null) { + return cachedResults != null ? cachedResults : errorJson("No active recording."); + } + + Path tempFile = null; + try { + tempFile = Files.createTempFile("camel-jfr-old-objects-", ".jfr"); + // trigger GC before stopping to flush objects into the recording + System.gc(); + rec.stop(); + rec.dump(tempFile); + + long endTime = System.currentTimeMillis(); + long durationMs = endTime - recordingStartTime; + JsonObject result = parseRecordingFile(tempFile, limit); + result.put("status", "completed"); + result.put("recordingDurationMs", durationMs); + result.put("recordingEndTime", endTime); + + cachedResults = result; + return result; + } catch (IOException e) { + LOG.warn("Error parsing JFR recording: {}", e.getMessage(), e); + return errorJson("Error parsing JFR recording: " + e.getMessage()); + } finally { + rec.close(); + activeRecording = null; + recordingStartTime = 0; + requestedDurationSeconds = 0; + if (tempFile != null) { + try { + Files.deleteIfExists(tempFile); + } catch (IOException e) { + // ignore + } + } + } + } + + private JsonObject doStatus() { + JsonObject result = new JsonObject(); + if (activeRecording != null) { + result.put("status", "recording"); + result.put("startTime", recordingStartTime); + long elapsed = System.currentTimeMillis() - recordingStartTime; + result.put("elapsedMs", elapsed); + if (requestedDurationSeconds > 0) { + result.put("durationSeconds", requestedDurationSeconds); + long remaining = (requestedDurationSeconds * 1000L) - elapsed; + result.put("remainingMs", Math.max(0, remaining)); + } + } else if (cachedResults != null) { + result.put("status", "completed"); + result.put("hasCachedResults", true); + result.put("sampleCount", cachedResults.getIntegerOrDefault("sampleCount", 0)); + } else { + result.put("status", "idle"); + } + return result; + } + + private JsonObject doQuery(Map<String, Object> options) { + if (cachedResults != null) { + return cachedResults; + } + if (activeRecording != null) { + return doStatus(); + } + JsonObject result = new JsonObject(); + result.put("status", "idle"); + result.put("sampleCount", 0); + result.put("samples", new JsonArray()); + result.put("note", "No results available. Start a recording first."); + return result; + } + + private JsonObject parseRecordingFile(Path file, int limit) throws IOException { + // parse all raw samples first + List<JsonObject> rawSamples = new ArrayList<>(); + try (RecordingFile rf = new RecordingFile(file)) { + while (rf.hasMoreEvents()) { + RecordedEvent event = rf.readEvent(); + if (!"jdk.OldObjectSample".equals(event.getEventType().getName())) { + continue; + } + JsonObject sample = parseOldObjectSampleEvent(event); + if (sample != null) { + rawSamples.add(sample); + } + } + } + + // aggregate by class + stack trace fingerprint + Map<String, JsonObject> groups = new LinkedHashMap<>(); + for (JsonObject sample : rawSamples) { + String key = sampleGroupKey(sample); + JsonObject existing = groups.get(key); + if (existing == null) { + sample.put("count", 1); + long size = sample.getLongOrDefault("allocationSize", 0); + sample.put("totalSize", size); + groups.put(key, sample); + } else { + existing.put("count", existing.getIntegerOrDefault("count", 1) + 1); + long prevTotal = existing.getLongOrDefault("totalSize", 0); + long curSize = sample.getLongOrDefault("allocationSize", 0); + existing.put("totalSize", prevTotal + curSize); + long prevAge = existing.getLongOrDefault("objectAge", 0); + long curAge = sample.getLongOrDefault("objectAge", 0); + if (curAge > prevAge) { + existing.put("objectAge", curAge); + } + } + } + + JsonArray samples = new JsonArray(); + int count = 0; + for (JsonObject group : groups.values()) { + if (limit > 0 && count >= limit) { + break; + } + samples.add(group); + count++; + } + + JsonObject root = new JsonObject(); + root.put("samples", samples); + root.put("sampleCount", count); + root.put("rawSampleCount", rawSamples.size()); + return root; + } + + private static String sampleGroupKey(JsonObject sample) { + StringBuilder sb = new StringBuilder(); + sb.append(sample.getStringOrDefault("allocationClass", "")); + JsonArray st = (JsonArray) sample.get("stackTrace"); + if (st != null) { + for (int i = 0; i < st.size(); i++) { + JsonObject frame = (JsonObject) st.get(i); + sb.append('|').append(frame.getStringOrDefault("method", "")) + .append(':').append(frame.getIntegerOrDefault("line", 0)); + } + } + return sb.toString(); + } + + private JsonObject parseOldObjectSampleEvent(RecordedEvent event) { + JsonObject sample = new JsonObject(); + + // extract the OldObject reference (contains the sampled object's class and reference chain) + RecordedObject objectRef = null; + if (event.hasField("object")) { + try { + objectRef = event.getValue("object"); + } catch (Exception e) { + // ignore + } + } + + // allocation class — primary source is object.type, fallback to objectClass on event + if (objectRef != null && objectRef.hasField("type")) { + try { + RecordedClass type = objectRef.getClass("type"); + if (type != null) { + sample.put("allocationClass", readableClassName(type.getName())); + } + } catch (Exception e) { + // ignore + } + } + if (!sample.containsKey("allocationClass") && event.hasField("objectClass")) { + try { + RecordedClass objectClass = event.getClass("objectClass"); + if (objectClass != null) { + sample.put("allocationClass", readableClassName(objectClass.getName())); + } + } catch (Exception e) { + // ignore + } + } + + // allocation size — try multiple field names across JDK versions + if (event.hasField("allocationSize")) { + sample.put("allocationSize", event.getLong("allocationSize")); + } else if (event.hasField("objectSize")) { + sample.put("allocationSize", event.getLong("objectSize")); + } + + // last known heap usage + if (event.hasField("lastKnownHeapUsage")) { + sample.put("lastKnownHeapUsage", event.getLong("lastKnownHeapUsage")); + } + + // array elements + if (event.hasField("arrayElements")) { + int arrayElements = event.getInt("arrayElements"); + if (arrayElements > 0) { + sample.put("arrayElements", arrayElements); + } + } + + // object age + if (event.hasField("objectAge")) { + try { + sample.put("objectAge", event.getDuration("objectAge").toMillis()); + } catch (Exception e) { + // some JDK versions may not support this field as Duration + } + } + + // allocation time + sample.put("allocationTime", event.getStartTime().toEpochMilli()); + + // stack trace (where the object was allocated) + RecordedStackTrace stackTrace = event.getStackTrace(); + if (stackTrace != null) { + JsonArray frames = new JsonArray(); + for (RecordedFrame frame : stackTrace.getFrames()) { + JsonObject f = new JsonObject(); + f.put("method", frame.getMethod().getType().getName() + "." + frame.getMethod().getName()); + f.put("line", frame.getLineNumber()); + frames.add(f); + if (frames.size() >= MAX_STACK_FRAMES) { + break; + } + } + sample.put("stackTrace", frames); + } + + // reference chain (path from object to GC root) + if (objectRef != null) { + JsonArray chain = extractReferenceChain(objectRef); + appendGcRoot(event, chain); + if (!chain.isEmpty()) { + sample.put("referenceChain", chain); + } + } + + return sample; + } + + private JsonArray extractReferenceChain(RecordedObject objectRef) { + JsonArray chain = new JsonArray(); + try { + RecordedObject obj = objectRef; + int depth = 0; + while (obj != null && depth < MAX_CHAIN_DEPTH) { + JsonObject link = new JsonObject(); + + if (obj.hasField("type")) { + try { + RecordedClass type = obj.getClass("type"); + if (type != null) { + link.put("type", readableClassName(type.getName())); + } + } catch (Exception e) { + // ignore + } + } + + if (obj.hasField("field")) { + try { + RecordedObject field = obj.getValue("field"); + if (field != null && field.hasField("name")) { + link.put("field", field.getString("name")); + } + } catch (Exception e) { + try { + String fieldName = obj.getString("field"); + if (fieldName != null) { + link.put("field", fieldName); + } + } catch (Exception ex) { + // ignore + } + } + } + + if (obj.hasField("description")) { + try { + String desc = obj.getString("description"); + if (desc != null && !desc.isEmpty()) { + link.put("description", desc); + } + } catch (Exception e) { + // ignore + } + } + + if (!link.isEmpty()) { + chain.add(link); + } + + // walk to next referrer + if (obj.hasField("referrer")) { + try { + obj = obj.getValue("referrer"); + } catch (Exception e) { + break; + } + } else { + break; + } + depth++; + } + } catch (Exception e) { + LOG.debug("Error extracting reference chain: {}", e.getMessage()); + } + return chain; + } + + private void appendGcRoot(RecordedEvent event, JsonArray chain) { + if (!event.hasField("root")) { + return; + } + try { + RecordedObject root = event.getValue("root"); + if (root == null) { + return; + } + JsonObject link = new JsonObject(); + if (root.hasField("type")) { + RecordedClass type = root.getClass("type"); + if (type != null) { + link.put("type", readableClassName(type.getName())); + } + } + if (root.hasField("description")) { + String desc = root.getString("description"); + if (desc != null && !desc.isEmpty()) { + link.put("description", desc); + } + } + if (root.hasField("system")) { + try { + String system = root.getString("system"); + if (system != null && !system.isEmpty()) { + link.put("description", + link.getStringOrDefault("description", "") + " [GC Root: " + system + "]"); + } + } catch (Exception e) { + // ignore + } + } + if (!link.isEmpty()) { + chain.add(link); + } + } catch (Exception e) { + LOG.debug("Error extracting GC root: {}", e.getMessage()); + } + } + + static String readableClassName(String name) { + if (name == null) { + return null; + } + // JVM array descriptors → human-readable + if (name.startsWith("[")) { + int dims = 0; + int i = 0; + while (i < name.length() && name.charAt(i) == '[') { + dims++; + i++; + } + String suffix = "[]".repeat(dims); + if (i < name.length()) { + String element = switch (name.charAt(i)) { + case 'B' -> "byte"; + case 'C' -> "char"; + case 'D' -> "double"; + case 'F' -> "float"; + case 'I' -> "int"; + case 'J' -> "long"; + case 'S' -> "short"; + case 'Z' -> "boolean"; + case 'L' -> name.substring(i + 1, name.endsWith(";") ? name.length() - 1 : name.length()); + default -> name.substring(i); + }; + return element + suffix; + } + } + return name; + } + + private void cancelAutoStop() { + if (autoStopFuture != null) { + autoStopFuture.cancel(false); + autoStopFuture = null; + } + } + + private void ensureScheduler() { + if (scheduler == null || scheduler.isShutdown()) { + scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "JfrOldObjectSampleAutoStop"); + t.setDaemon(true); + return t; + }); + } + } + + @Override + protected void doStop() throws Exception { + super.doStop(); + + cancelAutoStop(); + + Recording rec = activeRecording; + if (rec != null) { + try { + rec.stop(); + rec.close(); + } catch (Exception e) { + // ignore + } + activeRecording = null; + } + + if (scheduler != null) { + scheduler.shutdownNow(); + scheduler = null; + } + } + + private static String optionString(Map<String, Object> options, String key) { + Object val = options.get(key); + return val != null ? val.toString() : null; + } + + private static int optionInt(Map<String, Object> options, String key, int defaultValue) { + Object val = options.get(key); + if (val != null) { + try { + return Integer.parseInt(val.toString()); + } catch (NumberFormatException e) { + // use default + } + } + return defaultValue; + } + + private static JsonObject errorJson(String message) { + JsonObject result = new JsonObject(); + result.put("status", "error"); + result.put("error", message); + return result; + } +} diff --git a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-jfr-old-objects.adoc b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-jfr-old-objects.adoc new file mode 100644 index 000000000000..ec86b875a1de --- /dev/null +++ b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-jfr-old-objects.adoc @@ -0,0 +1,33 @@ + +// AUTO-GENERATED by camel-package-maven-plugin - DO NOT EDIT THIS FILE += camel cmd jfr-old-objects + +Diagnose memory leaks using JFR OldObjectSample events in a running Camel integration + + +== Usage + +[source,bash] +---- +camel cmd jfr-old-objects [options] +---- + + + +== Options + +[cols="2,5,1,2",options="header"] +|=== +| Option | Description | Default | Type +| `--duration` | Recording duration in seconds (auto-stops after this time) | 0 | int +| `--min-size` | Only show samples with total size above this value (e.g. 1024, 10KB, 1MB) | 0 | String +| `--query` | Query cached results from the last recording | | boolean +| `--stacktrace` | Show allocation stack trace for each sample | | boolean +| `--start` | Start a JFR recording for OldObjectSample events | | boolean +| `--status` | Check recording status | | boolean +| `--stop` | Stop the active recording and display results | | boolean +| `--top` | Show only the top N samples | 50 | int +| `-h,--help` | Display the help and sub-commands | | boolean +|=== + + diff --git a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd.adoc b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd.adoc index c5cfec7993db..08caa171d180 100644 --- a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd.adoc +++ b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd.adoc @@ -23,6 +23,7 @@ camel cmd [options] | xref:jbang-commands/camel-jbang-cmd-enable-processor.adoc[enable-processor] | Enable Camel processor | xref:jbang-commands/camel-jbang-cmd-gc.adoc[gc] | Trigger Java Memory Garbage Collector | xref:jbang-commands/camel-jbang-cmd-heap-histogram.adoc[heap-histogram] | Display class-level heap memory usage in a running Camel integration +| xref:jbang-commands/camel-jbang-cmd-jfr-old-objects.adoc[jfr-old-objects] | Diagnose memory leaks using JFR OldObjectSample events in a running Camel integration | xref:jbang-commands/camel-jbang-cmd-load.adoc[load] | Loads new source files into an existing Camel | xref:jbang-commands/camel-jbang-cmd-logger.adoc[logger] | List or change logging levels | xref:jbang-commands/camel-jbang-cmd-receive.adoc[receive] | Receive and dump messages from remote endpoints diff --git a/dsl/camel-cli-connector/src/main/java/org/apache/camel/cli/connector/LocalCliConnector.java b/dsl/camel-cli-connector/src/main/java/org/apache/camel/cli/connector/LocalCliConnector.java index 2e336aec1cf7..4ae986a5260c 100644 --- a/dsl/camel-cli-connector/src/main/java/org/apache/camel/cli/connector/LocalCliConnector.java +++ b/dsl/camel-cli-connector/src/main/java/org/apache/camel/cli/connector/LocalCliConnector.java @@ -377,6 +377,8 @@ public class LocalCliConnector extends ServiceSupport implements CliConnector, C doActionSqlQueryTask(root); } else if ("sql-update-row".equals(action)) { doActionSqlUpdateRowTask(root); + } else if ("jfr-old-objects".equals(action)) { + doActionJfrOldObjectsTask(root); } else if ("cli-debug".equals(action)) { doActionCliDebug(root); } @@ -850,6 +852,31 @@ public class LocalCliConnector extends ServiceSupport implements CliConnector, C } } + private void doActionJfrOldObjectsTask(JsonObject root) throws IOException { + DevConsole dc = camelContext.getCamelContextExtension().getContextPlugin(DevConsoleRegistry.class) + .resolveById("jfr-old-objects"); + if (dc != null) { + Map<String, Object> params = new HashMap<>(); + String command = root.getString("command"); + if (command != null) { + params.put("command", command); + } + String duration = root.getString("duration"); + if (duration != null) { + params.put("duration", duration); + } + String limit = root.getString("limit"); + if (limit != null) { + params.put("limit", limit); + } + JsonObject json = (JsonObject) dc.call(DevConsole.MediaType.JSON, params); + LOG.trace("Updating output file: {}", outputFile); + IOHelper.writeText(json.toJson(), outputFile); + } else { + IOHelper.writeText("{}", outputFile); + } + } + private void doActionKafkaTask() throws IOException { DevConsole dc = camelContext.getCamelContextExtension().getContextPlugin(DevConsoleRegistry.class) .resolveById("kafka"); diff --git a/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json b/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json index 0a05c64223ab..648542848ff6 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json +++ b/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json @@ -3,7 +3,7 @@ { "name": "ask", "fullName": "ask", "description": "Ask a question about a running Camel application using AI", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Ask", "options": [ { "names": "--api-key", "description": "API key. Also reads ANTHROPIC_API_KEY, OPENAI_API_KEY, or LLM_API_KEY env vars", "javaType": "java.lang.String", "type": "string" }, { "names": "--api-type", "description": "API type: 'ollama', 'openai', or 'anthropic'", "javaType": "LlmClient.ApiType", "type" [...] { "name": "bind", "fullName": "bind", "description": "DEPRECATED: Bind source and sink Kamelets as a new Camel integration", "deprecated": true, "sourceClass": "org.apache.camel.dsl.jbang.core.commands.bind.Bind", "options": [ { "names": "--error-handler", "description": "Add error handler (none|log|sink:<endpoint>). Sink endpoints are expected in the format [[apigroup\/]version:]kind:[namespace\/]name, plain Camel URIs or Kamelet name.", "javaType": "java.lang.String", "type": "stri [...] { "name": "catalog", "fullName": "catalog", "description": "List artifacts from Camel Catalog", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.catalog.CatalogCommand", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "component", "fullName": "catalog component", "description": "List components from the Camel Catalog", "sourceClass": "org.apache.camel.dsl.jbang.co [...] - { "name": "cmd", "fullName": "cmd", "description": "Performs commands in the running Camel integrations, such as start\/stop route, or change logging levels.", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.action.CamelAction", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "browse", "fullName": "cmd browse", "description": "Browse pending messages on endpoints [...] + { "name": "cmd", "fullName": "cmd", "description": "Performs commands in the running Camel integrations, such as start\/stop route, or change logging levels.", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.action.CamelAction", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "browse", "fullName": "cmd browse", "description": "Browse pending messages on endpoints [...] { "name": "completion", "fullName": "completion", "description": "Generate completion script for bash\/zsh", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Complete", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ] }, { "name": "config", "fullName": "config", "description": "Get and set user configuration values", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.config.ConfigCommand", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "get", "fullName": "config get", "description": "Display user configuration value", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.config. [...] { "name": "debug", "fullName": "debug", "description": "Debug local Camel integration", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Debug", "options": [ { "names": "--ago", "description": "Use ago instead of yyyy-MM-dd HH:mm:ss in timestamp.", "javaType": "boolean", "type": "boolean" }, { "names": "--background", "description": "Run in the background", "defaultValue": "false", "javaType": "boolean", "type": "boolean" }, { "names": "--background-wait", "description": "To [...] diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/CamelJBangMain.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/CamelJBangMain.java index c09e343495f5..a3ef865d8413 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/CamelJBangMain.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/CamelJBangMain.java @@ -111,6 +111,7 @@ public class CamelJBangMain implements Callable<Integer> { .addSubcommand("enable-processor", new CommandLine(new CamelProcessorEnableAction(this))) .addSubcommand("gc", new CommandLine(new CamelGCAction(this))) .addSubcommand("heap-histogram", new CommandLine(new CamelHeapHistogram(this))) + .addSubcommand("jfr-old-objects", new CommandLine(new CamelJfrOldObjects(this))) .addSubcommand("load", new CommandLine(new CamelLoadAction(this))) .addSubcommand("logger", new CommandLine(new LoggerAction(this))) .addSubcommand("receive", new CommandLine(new CamelReceiveAction(this))) diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelJfrOldObjects.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelJfrOldObjects.java new file mode 100644 index 000000000000..4ab294c9bf50 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelJfrOldObjects.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.action; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import com.github.freva.asciitable.AsciiTable; +import com.github.freva.asciitable.Column; +import com.github.freva.asciitable.HorizontalAlign; +import com.github.freva.asciitable.OverflowBehaviour; +import org.apache.camel.dsl.jbang.core.commands.CamelJBangMain; +import org.apache.camel.dsl.jbang.core.common.PathUtils; +import org.apache.camel.dsl.jbang.core.common.TerminalWidthHelper; +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +@Command(name = "jfr-old-objects", + description = "Diagnose memory leaks using JFR OldObjectSample events in a running Camel integration", + sortOptions = false, showDefaultValues = true, + footer = { + "%nExamples:", + " camel cmd jfr-old-objects --start", + " camel cmd jfr-old-objects --start --duration 30", + " camel cmd jfr-old-objects --stop", + " camel cmd jfr-old-objects --status", + " camel cmd jfr-old-objects --query" }) +public class CamelJfrOldObjects extends ActionBaseCommand { + + @CommandLine.Parameters(description = "Name or pid of running Camel integration", arity = "0..1") + String name = "*"; + + @CommandLine.Option(names = { "--start" }, description = "Start a JFR recording for OldObjectSample events") + boolean start; + + @CommandLine.Option(names = { "--stop" }, description = "Stop the active recording and display results") + boolean stop; + + @CommandLine.Option(names = { "--status" }, description = "Check recording status") + boolean status; + + @CommandLine.Option(names = { "--query" }, description = "Query cached results from the last recording") + boolean query; + + @CommandLine.Option(names = { "--duration" }, + description = "Recording duration in seconds (auto-stops after this time)", defaultValue = "0") + int duration; + + @CommandLine.Option(names = { "--top" }, + description = "Show only the top N samples", defaultValue = "50") + int top; + + @CommandLine.Option(names = { "--min-size" }, + description = "Only show samples with total size above this value (e.g. 1024, 10KB, 1MB)", + defaultValue = "0") + String minSize; + + @CommandLine.Option(names = { "--stacktrace" }, + description = "Show allocation stack trace for each sample") + boolean stacktrace; + + public CamelJfrOldObjects(CamelJBangMain main) { + super(main); + } + + @Override + public Integer doCall() throws Exception { + List<Long> pids = findPids(name); + if (pids.isEmpty()) { + return 1; + } else if (pids.size() > 1) { + printer().println("Name or pid " + name + " matches " + pids.size() + + " running Camel integrations. Specify a name or PID that matches exactly one."); + return 1; + } + + long pid = pids.get(0); + + String command; + if (start) { + command = "start"; + } else if (stop) { + command = "stop"; + } else if (query) { + command = "query"; + } else { + command = "status"; + } + + Path outputFile = getOutputFile(Long.toString(pid)); + PathUtils.deleteFile(outputFile); + + JsonObject root = new JsonObject(); + root.put("action", "jfr-old-objects"); + root.put("command", command); + if (start && duration > 0) { + root.put("duration", String.valueOf(duration)); + } + + Path f = getActionFile(Long.toString(pid)); + try { + Files.writeString(f, root.toJson()); + } catch (Exception e) { + // ignore + } + + JsonObject jo = getJsonObject(outputFile, start ? 15000 : 30000); + if (jo != null) { + String responseStatus = jo.getString("status"); + + if ("error".equals(responseStatus)) { + printer().println("Error: " + jo.getString("error")); + return 1; + } + + if ("recording".equals(responseStatus)) { + printer().println("JFR recording started for PID: " + pid); + if (jo.containsKey("durationSeconds")) { + printer().println("Duration: " + jo.getInteger("durationSeconds") + " seconds (auto-stop)"); + } else { + printer().println("Duration: manual (use --stop to end recording)"); + } + return 0; + } + + if ("idle".equals(responseStatus)) { + printer().println("No active JFR recording for PID: " + pid); + if (jo.containsKey("note")) { + printer().println(jo.getString("note")); + } + return 0; + } + + if ("completed".equals(responseStatus)) { + int sampleCount = jo.getIntegerOrDefault("sampleCount", 0); + long durationMs = jo.getLongOrDefault("recordingDurationMs", 0); + printer().printf("PID: %s\tSamples: %d\tRecording duration: %s%n", + pid, sampleCount, formatDuration(durationMs)); + + JsonArray samples = (JsonArray) jo.get("samples"); + if (samples != null && !samples.isEmpty()) { + long minBytes = parseSize(minSize); + List<Row> rows = new ArrayList<>(); + int num = 0; + for (int i = 0; i < samples.size(); i++) { + JsonObject sample = (JsonObject) samples.get(i); + long totalSize = sample.getLongOrDefault("totalSize", 0); + if (totalSize < minBytes) { + continue; + } + num++; + if (num > top) { + break; + } + Row row = new Row(); + row.num = num; + row.className = sample.getStringOrDefault("allocationClass", "unknown"); + row.count = sample.getIntegerOrDefault("count", 1); + row.totalSize = totalSize; + row.objectAge = sample.getLongOrDefault("objectAge", 0); + row.chainSummary = buildChainSummary(sample); + row.stackTrace = (JsonArray) sample.get("stackTrace"); + rows.add(row); + } + printTable(rows); + } else { + printer().println("No old object samples captured."); + printer().println("Tip: Try a longer recording duration or ensure GC occurs during recording."); + } + return 0; + } + + // status response with cached results info + if (jo.containsKey("hasCachedResults") && jo.getBooleanOrDefault("hasCachedResults", false)) { + printer().println("JFR recording completed. Use --query to view results."); + printer().println("Cached samples: " + jo.getIntegerOrDefault("sampleCount", 0)); + } else if (jo.containsKey("elapsedMs")) { + long elapsed = jo.getLongOrDefault("elapsedMs", 0); + printer().println("JFR recording in progress for PID: " + pid); + printer().println("Elapsed: " + formatDuration(elapsed)); + if (jo.containsKey("remainingMs")) { + printer().println("Remaining: " + formatDuration(jo.getLongOrDefault("remainingMs", 0))); + } + } + } else { + printer().println("Response from running Camel with PID " + pid + " not received within timeout"); + return 1; + } + + PathUtils.deleteFile(outputFile); + + return 0; + } + + private void printTable(List<Row> rows) { + if (stacktrace) { + printTableWithStacktrace(rows); + } else { + printTableCompact(rows); + } + } + + private void printTableCompact(List<Row> rows) { + int tw = terminalWidth(); + int fixedWidth = 6 + 8 + 12 + 12; + int borderOverhead = TerminalWidthHelper.noBorderOverhead(5); + int classWidth = TerminalWidthHelper.flexWidth(tw, fixedWidth + 30, borderOverhead, 20, 50); + int chainWidth = TerminalWidthHelper.flexWidth(tw, fixedWidth + classWidth, borderOverhead, 20, 60); + + printer().println(AsciiTable.getTable(AsciiTable.NO_BORDERS, rows, Arrays.asList( + new Column().header("#").headerAlign(HorizontalAlign.RIGHT).dataAlign(HorizontalAlign.RIGHT) + .with(r -> Integer.toString(r.num)), + new Column().header("CLASS").dataAlign(HorizontalAlign.LEFT) + .maxWidth(classWidth, OverflowBehaviour.ELLIPSIS_LEFT) + .with(r -> r.className), + new Column().header("COUNT").headerAlign(HorizontalAlign.RIGHT).dataAlign(HorizontalAlign.RIGHT) + .with(r -> Integer.toString(r.count)), + new Column().header("TOTAL").headerAlign(HorizontalAlign.RIGHT).dataAlign(HorizontalAlign.RIGHT) + .with(r -> r.totalSize > 0 ? formatBytes(r.totalSize) : "-"), + new Column().header("AGE").headerAlign(HorizontalAlign.RIGHT).dataAlign(HorizontalAlign.RIGHT) + .with(r -> formatDuration(r.objectAge)), + new Column().header("REFERENCE CHAIN").dataAlign(HorizontalAlign.LEFT) + .maxWidth(chainWidth, OverflowBehaviour.ELLIPSIS_RIGHT) + .with(r -> r.chainSummary)))); + } + + private void printTableWithStacktrace(List<Row> rows) { + for (Row r : rows) { + String sizeStr = r.totalSize > 0 ? formatBytes(r.totalSize) : "-"; + printer().printf("%d) %s count:%d total:%s age:%s%n", + r.num, r.className, r.count, sizeStr, formatDuration(r.objectAge)); + if (r.chainSummary != null && !r.chainSummary.isEmpty()) { + printer().println(" chain: " + r.chainSummary); + } + if (r.stackTrace != null) { + for (int i = 0; i < r.stackTrace.size(); i++) { + JsonObject frame = (JsonObject) r.stackTrace.get(i); + printer().printf(" at %s:%s%n", + frame.getStringOrDefault("method", "?"), + frame.getIntegerOrDefault("line", 0)); + } + } + printer().println(); + } + } + + private static String buildChainSummary(JsonObject sample) { + JsonArray chain = (JsonArray) sample.get("referenceChain"); + if (chain == null || chain.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < chain.size() && i < 3; i++) { + JsonObject link = (JsonObject) chain.get(i); + if (i > 0) { + sb.append(" -> "); + } + String type = link.getString("type"); + if (type != null) { + int dot = type.lastIndexOf('.'); + sb.append(dot >= 0 ? type.substring(dot + 1) : type); + } + String field = link.getString("field"); + if (field != null) { + sb.append('.').append(field); + } + } + if (chain.size() > 3) { + sb.append(" -> ..."); + } + return sb.toString(); + } + + static String formatBytes(long bytes) { + if (bytes < 1024) { + return bytes + " B"; + } else if (bytes < 1024 * 1024) { + return String.format(Locale.US, "%.1f KB", bytes / 1024.0); + } else if (bytes < 1024L * 1024 * 1024) { + return String.format(Locale.US, "%.1f MB", bytes / (1024.0 * 1024)); + } else { + return String.format(Locale.US, "%.1f GB", bytes / (1024.0 * 1024 * 1024)); + } + } + + static String formatDuration(long ms) { + if (ms < 1000) { + return ms + "ms"; + } else if (ms < 60000) { + return String.format(Locale.US, "%.1fs", ms / 1000.0); + } else { + long min = ms / 60000; + long sec = (ms % 60000) / 1000; + return min + "m " + sec + "s"; + } + } + + static long parseSize(String value) { + if (value == null || value.isEmpty() || "0".equals(value)) { + return 0; + } + String v = value.trim().toUpperCase(Locale.US); + long multiplier = 1; + if (v.endsWith("GB") || v.endsWith("G")) { + multiplier = 1024L * 1024 * 1024; + v = v.replaceAll("[GMK]B?$", ""); + } else if (v.endsWith("MB") || v.endsWith("M")) { + multiplier = 1024L * 1024; + v = v.replaceAll("[GMK]B?$", ""); + } else if (v.endsWith("KB") || v.endsWith("K")) { + multiplier = 1024; + v = v.replaceAll("[GMK]B?$", ""); + } + try { + return (long) (Double.parseDouble(v) * multiplier); + } catch (NumberFormatException e) { + return 0; + } + } + + private static class Row { + int num; + String className; + int count; + long totalSize; + long objectAge; + String chainSummary; + JsonArray stackTrace; + } +} diff --git a/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java b/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java index 3f86c4ecd9a6..5d6596a04aeb 100644 --- a/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java +++ b/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java @@ -327,6 +327,28 @@ public class RuntimeTools { return runtimeService.executeAction(p.pid(), "heap-histogram", null); } + @Tool(annotations = @Tool.Annotations(readOnlyHint = false, destructiveHint = false, openWorldHint = false), + description = """ + Manage JFR OldObjectSample recording for memory leak diagnosis. \ + Captures objects surviving multiple GC cycles and their reference chains back to GC roots. \ + Use command 'start' to begin recording, 'stop' to stop and get results, \ + 'status' to check recording state, and 'query' to retrieve cached results from the last recording.""") + public JsonObject camel_runtime_jfr_old_objects( + @ToolArg(description = NAME_OR_PID_DESC) String nameOrPid, + @ToolArg(description = "Command: start, stop, status, or query") String command, + @ToolArg(description = "Recording duration in seconds (only for start command, 0 = manual stop)") String duration) { + if (command == null || command.isBlank()) { + throw new ToolCallException("command is required (start, stop, status, or query)", null); + } + RuntimeService.ProcessInfo p = runtimeService.findSingleProcess(nameOrPid); + return runtimeService.executeAction(p.pid(), "jfr-old-objects", root -> { + root.put("command", command); + if ("start".equals(command) && duration != null && !duration.isBlank()) { + root.put("duration", duration); + } + }); + } + @Tool(annotations = @Tool.Annotations(readOnlyHint = true, destructiveHint = false, openWorldHint = false), description = """ Get the message history trace of the last completed exchange. \ diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/JfrOldObjectSampleTab.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/JfrOldObjectSampleTab.java new file mode 100644 index 000000000000..8d0039e2c577 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/JfrOldObjectSampleTab.java @@ -0,0 +1,1030 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; + +import dev.tamboui.layout.Constraint; +import dev.tamboui.layout.Layout; +import dev.tamboui.layout.Rect; +import dev.tamboui.style.Color; +import dev.tamboui.style.Style; +import dev.tamboui.terminal.Frame; +import dev.tamboui.text.Line; +import dev.tamboui.text.Span; +import dev.tamboui.text.Text; +import dev.tamboui.tui.event.KeyCode; +import dev.tamboui.tui.event.KeyEvent; +import dev.tamboui.widgets.block.Block; +import dev.tamboui.widgets.block.BorderType; +import dev.tamboui.widgets.block.Borders; +import dev.tamboui.widgets.paragraph.Paragraph; +import dev.tamboui.widgets.table.Cell; +import dev.tamboui.widgets.table.Row; +import dev.tamboui.widgets.table.Table; +import dev.tamboui.widgets.table.TableState; +import org.apache.camel.dsl.jbang.core.common.PathUtils; +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; + +import static org.apache.camel.dsl.jbang.core.commands.tui.MonitorContext.*; + +class JfrOldObjectSampleTab implements MonitorTab { + + private enum State { + IDLE, + RECORDING, + LOADING_RESULTS, + HAS_RESULTS + } + + private static final String[] SORT_COLUMNS = { "allocationClass", "totalSize", "count", "objectAge" }; + private static final long[] MIN_SIZE_PRESETS = { + 0, 1024, 10 * 1024, 100 * 1024, 1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024 }; + private static final String[] MIN_SIZE_LABELS = { + "off", "1 KB", "10 KB", "100 KB", "1 MB", "10 MB", "100 MB" }; + + private final MonitorContext ctx; + private final TableState tableState = new TableState(); + private final AtomicBoolean loading = new AtomicBoolean(false); + + private State state = State.IDLE; + private int duration = 30; + private long recordingStartTime; + + private String sort = "totalSize"; + private int sortIndex = 1; + private boolean sortReversed; + + private List<SampleEntry> samples = Collections.emptyList(); + private int sampleCount; + private long recordingDurationMs; + private long recordingEndTime; + private String lastPid; + private int detailScroll; + private int minSizeIndex; + + JfrOldObjectSampleTab(MonitorContext ctx) { + this.ctx = ctx; + } + + @Override + public void onTabSelected() { + String pid = ctx.selectedPid; + if (pid != null && !pid.equals(lastPid)) { + lastPid = pid; + state = State.IDLE; + samples = Collections.emptyList(); + } + if (state == State.IDLE && samples.isEmpty()) { + checkStatus(); + } + } + + @Override + public void onIntegrationChanged() { + state = State.IDLE; + samples = Collections.emptyList(); + lastPid = null; + } + + @Override + public boolean handleKeyEvent(KeyEvent ke) { + if (state == State.IDLE || state == State.HAS_RESULTS) { + if (ke.isCharIgnoreCase('r')) { + startRecording(); + return true; + } + } + if (state == State.IDLE || state == State.HAS_RESULTS) { + if (ke.isChar('+') || ke.isChar('=')) { + duration = Math.min(300, duration + 10); + return true; + } + if (ke.isChar('-')) { + duration = Math.max(10, duration - 10); + return true; + } + } + if (state == State.RECORDING) { + if (ke.isCharIgnoreCase('x')) { + stopRecording(); + return true; + } + } + if (state == State.HAS_RESULTS) { + if (ke.isChar('s')) { + sortIndex = (sortIndex + 1) % SORT_COLUMNS.length; + sort = SORT_COLUMNS[sortIndex]; + sortReversed = false; + return true; + } + if (ke.isChar('S')) { + sortReversed = !sortReversed; + return true; + } + if (ke.isChar('m')) { + minSizeIndex = (minSizeIndex + 1) % MIN_SIZE_PRESETS.length; + tableState.select(0); + return true; + } + if (ke.isPageUp() || ke.isKey(KeyCode.PAGE_UP)) { + detailScroll = Math.max(0, detailScroll - 10); + return true; + } + if (ke.isPageDown() || ke.isKey(KeyCode.PAGE_DOWN)) { + detailScroll += 10; + return true; + } + } + return false; + } + + @Override + public boolean handleEscape() { + return false; + } + + @Override + public void navigateUp() { + if (state == State.HAS_RESULTS) { + tableState.selectPrevious(); + detailScroll = 0; + } + } + + @Override + public void navigateDown() { + if (state == State.HAS_RESULTS) { + List<SampleEntry> visible = sortedSamples(); + tableState.selectNext(visible.size()); + detailScroll = 0; + } + } + + @Override + public void render(Frame frame, Rect area) { + IntegrationInfo info = ctx.findSelectedIntegration(); + if (info == null) { + renderNoSelection(frame, area); + return; + } + + switch (state) { + case IDLE -> renderIdle(frame, area); + case RECORDING -> renderRecording(frame, area); + case LOADING_RESULTS -> renderLoading(frame, area); + case HAS_RESULTS -> renderResults(frame, area); + } + } + + private void renderIdle(Frame frame, Rect area) { + List<Line> lines = new ArrayList<>(); + lines.add(Line.from(Span.raw(""))); + + if (loading.get()) { + lines.add(Line.from( + Span.styled(" Checking for existing JFR results...", Style.EMPTY.dim()))); + } else { + lines.add(Line.from( + Span.styled(" No JFR recording active.", Style.EMPTY.dim()))); + lines.add(Line.from(Span.raw(""))); + lines.add(Line.from( + Span.styled(" Press ", Style.EMPTY.dim()), + Span.styled("R", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled(" to start recording OldObjectSample events.", Style.EMPTY.dim()))); + lines.add(Line.from(Span.raw(""))); + lines.add(Line.from( + Span.styled(" Duration: ", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled(duration + "s", Style.EMPTY.fg(Color.WHITE)), + Span.styled(" (use ", Style.EMPTY.dim()), + Span.styled("+", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled("/", Style.EMPTY.dim()), + Span.styled("-", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled(" to adjust)", Style.EMPTY.dim()))); + lines.add(Line.from(Span.raw(""))); + lines.add(Line.from( + Span.styled(" JFR OldObjectSample tracks objects surviving multiple GC cycles", + Style.EMPTY.dim()))); + lines.add(Line.from( + Span.styled(" and captures reference chains back to GC roots, helping", + Style.EMPTY.dim()))); + lines.add(Line.from( + Span.styled(" diagnose memory leaks.", Style.EMPTY.dim()))); + } + + frame.renderWidget( + Paragraph.builder() + .text(Text.from(lines)) + .block(Block.builder().borderType(BorderType.ROUNDED).borders(Borders.ALL) + .title(" JFR Old Object Samples ").build()) + .build(), + area); + } + + private void renderRecording(Frame frame, Rect area) { + long elapsed = System.currentTimeMillis() - recordingStartTime; + long remaining = Math.max(0, (duration * 1000L) - elapsed); + + List<Line> lines = new ArrayList<>(); + lines.add(Line.from(Span.raw(""))); + lines.add(Line.from( + Span.styled(" JFR recording in progress...", Style.EMPTY.fg(Color.GREEN).bold()))); + lines.add(Line.from(Span.raw(""))); + lines.add(Line.from( + Span.styled(" Elapsed: ", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled(formatDuration(elapsed), Style.EMPTY.fg(Color.WHITE)))); + lines.add(Line.from( + Span.styled(" Remaining: ", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled(formatDuration(remaining), Style.EMPTY.fg(Color.WHITE)))); + lines.add(Line.from( + Span.styled(" Duration: ", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled(duration + "s", Style.EMPTY.fg(Color.WHITE)))); + lines.add(Line.from(Span.raw(""))); + lines.add(Line.from( + Span.styled(" Press ", Style.EMPTY.dim()), + Span.styled("X", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled(" to stop recording early and view results.", Style.EMPTY.dim()))); + + // progress bar + int barWidth = Math.max(10, area.width() - 6); + double pct = duration > 0 ? Math.min(1.0, elapsed / (duration * 1000.0)) : 0; + int filled = (int) (pct * barWidth); + StringBuilder bar = new StringBuilder(" "); + for (int i = 0; i < barWidth; i++) { + bar.append(i < filled ? '█' : '░'); + } + lines.add(Line.from(Span.raw(""))); + lines.add(Line.from(Span.styled(bar.toString(), Style.EMPTY.fg(Color.GREEN)))); + + frame.renderWidget( + Paragraph.builder() + .text(Text.from(lines)) + .block(Block.builder().borderType(BorderType.ROUNDED).borders(Borders.ALL) + .title(" JFR Recording ").build()) + .build(), + area); + } + + private void renderLoading(Frame frame, Rect area) { + frame.renderWidget( + Paragraph.builder() + .text(Text.from( + Line.from(Span.styled(" Stopping recording and analyzing results...", + Style.EMPTY.dim())))) + .block(Block.builder().borderType(BorderType.ROUNDED).borders(Borders.ALL) + .title(" JFR Old Object Samples ").build()) + .build(), + area); + } + + private void renderResults(Frame frame, Rect area) { + List<SampleEntry> visible = sortedSamples(); + + List<Rect> chunks = Layout.vertical() + .constraints(Constraint.percentage(40), Constraint.fill()) + .split(area); + renderSampleTable(frame, chunks.get(0), visible); + renderDetail(frame, chunks.get(1), visible); + } + + private void renderSampleTable(Frame frame, Rect area, List<SampleEntry> visible) { + List<Row> rows = new ArrayList<>(); + for (SampleEntry e : visible) { + String totalStr = e.totalSize > 0 ? formatBytes(e.totalSize) : "-"; + rows.add(Row.from( + rightCell(String.valueOf(e.num), 6), + Cell.from(Span.styled(e.className != null ? e.className : "", Style.EMPTY.fg(Color.CYAN))), + rightCell(String.valueOf(e.count), 8), + rightCell(totalStr, 12), + rightCell(formatDuration(e.objectAge), 12))); + } + + if (rows.isEmpty()) { + String msg = minSizeIndex > 0 ? "No samples above " + MIN_SIZE_LABELS[minSizeIndex] : "No samples captured"; + rows.add(Row.from( + Cell.from(""), Cell.from(Span.styled(msg, Style.EMPTY.dim())), + Cell.from(""), Cell.from(""), Cell.from(""))); + } + + long minSize = MIN_SIZE_PRESETS[minSizeIndex]; + String minLabel = minSize > 0 ? " min:" + MIN_SIZE_LABELS[minSizeIndex] : ""; + String agoLabel = ""; + if (recordingEndTime > 0) { + long agoMin = (System.currentTimeMillis() - recordingEndTime) / 60000; + if (agoMin >= 1) { + agoLabel = " (" + agoMin + "m ago)"; + } + } + String title = String.format(" JFR Old Objects [%d] duration:%s sort:%s%s%s ", + visible.size(), formatDuration(recordingDurationMs), sort, minLabel, agoLabel); + + Table table = Table.builder() + .rows(rows) + .header(Row.from( + rightCell("#", 6, Style.EMPTY.bold()), + Cell.from(Span.styled(sortLabel("CLASS", "allocationClass"), + sortStyle("allocationClass"))), + rightCell(sortLabel("COUNT", "count"), 8, + sortStyle("count")), + rightCell(sortLabel("TOTAL", "totalSize"), 12, + sortStyle("totalSize")), + rightCell(sortLabel("AGE", "objectAge"), 12, + sortStyle("objectAge")))) + .widths( + Constraint.length(6), + Constraint.fill(), + Constraint.length(8), + Constraint.length(12), + Constraint.length(12)) + .highlightStyle(Style.EMPTY.fg(Color.WHITE).bold().onBlue()) + .highlightSpacing(Table.HighlightSpacing.ALWAYS) + .block(Block.builder().borderType(BorderType.ROUNDED).borders(Borders.ALL).title(title).build()) + .build(); + + frame.renderStatefulWidget(table, area, tableState); + } + + private void renderDetail(Frame frame, Rect area, List<SampleEntry> visible) { + Integer sel = tableState.selected(); + if (sel == null || sel < 0 || sel >= visible.size()) { + frame.renderWidget( + Paragraph.builder() + .text(Text.from(Line.from( + Span.styled(" Select a sample to see reference chain details", + Style.EMPTY.dim())))) + .block(Block.builder().borderType(BorderType.ROUNDED).borders(Borders.ALL) + .title(" Detail ").build()) + .build(), + area); + return; + } + + SampleEntry entry = visible.get(sel); + List<Line> lines = new ArrayList<>(); + + // Sample info + lines.add(Line.from( + Span.styled(" Class: ", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled(entry.className != null ? entry.className : "unknown", Style.EMPTY.fg(Color.CYAN)))); + + List<Span> infoSpans = new ArrayList<>(); + if (entry.count > 1) { + infoSpans.add(Span.styled(" Count: ", Style.EMPTY.fg(Color.YELLOW).bold())); + infoSpans.add(Span.styled(String.valueOf(entry.count), Style.EMPTY.fg(Color.WHITE))); + } + if (entry.totalSize > 0) { + infoSpans.add(Span.styled(infoSpans.isEmpty() ? " Total: " : " Total: ", + Style.EMPTY.fg(Color.YELLOW).bold())); + infoSpans.add(Span.styled(formatBytes(entry.totalSize), Style.EMPTY.fg(Color.WHITE))); + } + if (!infoSpans.isEmpty()) { + lines.add(Line.from(infoSpans)); + } + lines.add(Line.from( + Span.styled(" Age: ", Style.EMPTY.fg(Color.YELLOW).bold()), + Span.styled(formatDuration(entry.objectAge), Style.EMPTY.fg(Color.WHITE)))); + + // Reference chain + if (entry.referenceChain != null && !entry.referenceChain.isEmpty()) { + lines.add(Line.from(Span.raw(""))); + lines.add(Line.from( + Span.styled(" Reference Chain (Object → GC Root):", Style.EMPTY.fg(Color.YELLOW).bold()))); + + for (int i = 0; i < entry.referenceChain.size(); i++) { + ChainLink link = entry.referenceChain.get(i); + String prefix = i == entry.referenceChain.size() - 1 ? " └─ " : " ├─ "; + String typeName = link.type != null ? abbreviateType(link.type) : "?"; + String fieldInfo = link.field != null ? " (field: " + link.field + ")" : ""; + String descInfo = link.description != null ? " [" + link.description + "]" : ""; + + lines.add(Line.from( + Span.styled(prefix, Style.EMPTY.fg(Color.BLUE)), + Span.styled(typeName, Style.EMPTY.fg(Color.CYAN)), + Span.styled(fieldInfo, Style.EMPTY.fg(Color.GREEN)), + Span.styled(descInfo, Style.EMPTY.dim()))); + } + } + + // Stack trace + if (entry.stackTrace != null && !entry.stackTrace.isEmpty()) { + lines.add(Line.from(Span.raw(""))); + lines.add(Line.from( + Span.styled(" Allocation Stack Trace:", Style.EMPTY.fg(Color.YELLOW).bold()))); + + for (StackEntry frame2 : entry.stackTrace) { + Style methodStyle = isJdkFrame(frame2.method) ? Style.EMPTY.dim() : Style.EMPTY.fg(Color.WHITE); + lines.add(Line.from( + Span.styled(" at ", Style.EMPTY.dim()), + Span.styled(frame2.method, methodStyle), + Span.styled(":" + frame2.line, Style.EMPTY.dim()))); + } + } + + // apply scroll offset + if (detailScroll > 0 && detailScroll < lines.size()) { + lines = new ArrayList<>(lines.subList(detailScroll, lines.size())); + } else if (detailScroll >= lines.size()) { + detailScroll = Math.max(0, lines.size() - 1); + if (!lines.isEmpty()) { + lines = new ArrayList<>(lines.subList(detailScroll, lines.size())); + } + } + + String title = " Detail "; + frame.renderWidget( + Paragraph.builder() + .text(Text.from(lines)) + .block(Block.builder().borderType(BorderType.ROUNDED).borders(Borders.ALL).title(title).build()) + .build(), + area); + } + + @Override + public void renderFooter(List<Span> spans) { + switch (state) { + case IDLE -> { + hint(spans, "R", "record"); + hint(spans, "+/-", "duration [" + duration + "s]"); + hintLast(spans, "Esc", "back"); + } + case RECORDING -> { + hint(spans, "X", "stop"); + hintLast(spans, "Esc", "back"); + } + case HAS_RESULTS -> { + hint(spans, "Esc", "back"); + hint(spans, "s", "sort"); + hint(spans, "m", "min-size [" + MIN_SIZE_LABELS[minSizeIndex] + "]"); + hint(spans, "R", "new recording"); + hint(spans, "+/-", "duration [" + duration + "s]"); + hintLast(spans, "PgUp/Dn", "scroll detail"); + } + default -> hintLast(spans, "Esc", "back"); + } + } + + @Override + public SelectionContext getSelectionContext() { + if (state != State.HAS_RESULTS) { + return null; + } + List<SampleEntry> visible = sortedSamples(); + if (visible.isEmpty()) { + return null; + } + List<String> items = visible.stream() + .map(e -> e.className != null ? e.className : "").toList(); + Integer sel = tableState.selected(); + return new SelectionContext("table", items, sel != null ? sel : -1, items.size(), "JFR Old Objects"); + } + + @Override + public JsonObject getTableDataAsJson() { + if (state != State.HAS_RESULTS || samples.isEmpty()) { + return null; + } + List<SampleEntry> visible = sortedSamples(); + JsonObject result = new JsonObject(); + result.put("tab", "JFR Old Objects"); + JsonArray rows = new JsonArray(); + for (SampleEntry e : visible) { + JsonObject row = new JsonObject(); + row.put("num", e.num); + row.put("className", e.className); + row.put("count", e.count); + row.put("totalSize", e.totalSize); + row.put("allocationSize", e.allocationSize); + row.put("objectAge", e.objectAge); + row.put("chainSummary", e.chainSummary); + if (e.referenceChain != null) { + JsonArray chain = new JsonArray(); + for (ChainLink link : e.referenceChain) { + JsonObject cl = new JsonObject(); + if (link.type != null) { + cl.put("type", link.type); + } + if (link.field != null) { + cl.put("field", link.field); + } + if (link.description != null) { + cl.put("description", link.description); + } + chain.add(cl); + } + row.put("referenceChain", chain); + } + rows.add(row); + } + result.put("rows", rows); + result.put("totalRows", sampleCount); + result.put("recordingDurationMs", recordingDurationMs); + Integer sel = tableState.selected(); + result.put("selectedIndex", sel != null ? sel : -1); + return result; + } + + @Override + public String getHelpText() { + return """ + # JFR Old Object Samples + + The JFR Old Object Samples tab uses Java Flight Recorder to capture objects + that have survived multiple GC cycles (long-lived objects) and traces their + reference chains back to GC roots. This helps diagnose memory leaks by + answering "why is this object still alive?" + + ## How To Use + + 1. Press **R** to start a JFR recording (default 30 seconds) + 2. Use **+**/**-** to adjust the duration before starting + 3. Wait for the recording to complete (or press **X** to stop early) + 4. Browse the samples table and select entries to see reference chains + + ## Table Columns + + Samples from the same class and allocation site (stack trace) are + grouped together automatically. + + - **#** — Group number + - **CLASS** — The class of the sampled long-lived object + - **COUNT** — Number of samples from the same allocation site + - **TOTAL** — Total allocation size across all samples in the group + - **AGE** — Maximum age across samples in the group + + ## Detail Panel + + The detail panel shows the full reference chain and allocation stack trace + for the selected sample: + + - **Reference Chain** — Path from the object to its GC root, showing + each referencing type and field name + - **Allocation Stack Trace** — Where the object was originally allocated + + ## What To Look For + + - **Objects with very long ages**: These have survived many GC cycles + - **Unexpected reference chains**: Objects held by caches, maps, or + static fields that prevent garbage collection + - **Growing collections**: HashMap, ArrayList, ConcurrentHashMap entries + that keep accumulating + - **Thread-local references**: Objects held by thread-local variables + + ## Comparison With Heap Histogram + + The Heap Histogram tab shows WHAT is using memory (class instance counts + and sizes). This tab shows WHY objects are still alive (reference chains + to GC roots). Use both together: find suspicious classes in Heap Histogram, + then use JFR Old Objects to trace why they are not being collected. + + ## Keys + + | Key | Action | + |-----|--------| + | R | Start/restart JFR recording | + | X | Stop recording early | + | +/- | Adjust recording duration | + | Up/Down | Select sample | + | s | Cycle sort column (class, size, age) | + | S | Reverse sort order | + | m | Cycle minimum size filter | + | PgUp/PgDn | Scroll detail panel | + | Esc | Back | + """; + } + + // ---- Action methods ---- + + private void startRecording() { + if (ctx.selectedPid == null || ctx.runner == null) { + return; + } + if (!loading.compareAndSet(false, true)) { + return; + } + state = State.RECORDING; + recordingStartTime = System.currentTimeMillis(); + + String pid = ctx.selectedPid; + int dur = duration; + startDaemonThread("jfr-start-" + pid, () -> { + try { + sendStartCommand(pid, dur); + } finally { + loading.set(false); + } + }); + } + + private void sendStartCommand(String pid, int dur) { + Path outputFile = ctx.getOutputFile(pid); + PathUtils.deleteFile(outputFile); + + JsonObject root = new JsonObject(); + root.put("action", "jfr-old-objects"); + root.put("command", "start"); + root.put("duration", String.valueOf(dur)); + + Path actionFile = ctx.getActionFile(pid); + PathUtils.writeTextSafely(root.toJson(), actionFile); + + JsonObject jo = pollJsonResponse(outputFile, 15000); + PathUtils.deleteFile(outputFile); + + if (jo != null && "recording".equals(jo.getString("status"))) { + if (ctx.runner != null) { + ctx.runner.runOnRenderThread(() -> { + state = State.RECORDING; + lastPid = pid; + }); + } + scheduleResultsPoll(pid, dur); + } else { + if (ctx.runner != null) { + ctx.runner.runOnRenderThread(() -> state = State.IDLE); + } + } + } + + private void stopRecording() { + if (ctx.selectedPid == null || ctx.runner == null) { + return; + } + if (!loading.compareAndSet(false, true)) { + return; + } + state = State.LOADING_RESULTS; + + String pid = ctx.selectedPid; + startDaemonThread("jfr-stop-" + pid, () -> { + try { + sendStopAndLoadResults(pid); + } finally { + loading.set(false); + } + }); + } + + private void sendStopAndLoadResults(String pid) { + Path outputFile = ctx.getOutputFile(pid); + PathUtils.deleteFile(outputFile); + + JsonObject root = new JsonObject(); + root.put("action", "jfr-old-objects"); + root.put("command", "stop"); + + Path actionFile = ctx.getActionFile(pid); + PathUtils.writeTextSafely(root.toJson(), actionFile); + + JsonObject jo = pollJsonResponse(outputFile, 30000); + PathUtils.deleteFile(outputFile); + + if (jo != null && "completed".equals(jo.getString("status"))) { + List<SampleEntry> result = parseSamples(jo); + int count = jo.getIntegerOrDefault("sampleCount", 0); + long durationMs = jo.getLongOrDefault("recordingDurationMs", 0); + long endTime = jo.getLongOrDefault("recordingEndTime", System.currentTimeMillis()); + + if (ctx.runner != null) { + ctx.runner.runOnRenderThread(() -> { + samples = result; + sampleCount = count; + recordingDurationMs = durationMs; + recordingEndTime = endTime; + state = State.HAS_RESULTS; + tableState.select(0); + lastPid = pid; + }); + } + } else { + if (ctx.runner != null) { + ctx.runner.runOnRenderThread(() -> state = State.IDLE); + } + } + } + + private void scheduleResultsPoll(String pid, int dur) { + if (ctx.runner == null) { + return; + } + startDaemonThread("jfr-poll-" + pid, () -> { + try { + Thread.sleep((dur + 3) * 1000L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + if (state != State.RECORDING) { + return; + } + if (ctx.runner != null) { + ctx.runner.runOnRenderThread(() -> state = State.LOADING_RESULTS); + } + loadQueryResults(pid); + }); + } + + private void checkStatus() { + if (ctx.selectedPid == null || ctx.runner == null) { + return; + } + if (!loading.compareAndSet(false, true)) { + return; + } + String pid = ctx.selectedPid; + startDaemonThread("jfr-status-" + pid, () -> { + try { + sendStatusCommand(pid); + } finally { + loading.set(false); + } + }); + } + + private void sendStatusCommand(String pid) { + Path outputFile = ctx.getOutputFile(pid); + PathUtils.deleteFile(outputFile); + + JsonObject root = new JsonObject(); + root.put("action", "jfr-old-objects"); + root.put("command", "status"); + + Path actionFile = ctx.getActionFile(pid); + PathUtils.writeTextSafely(root.toJson(), actionFile); + + JsonObject jo = pollJsonResponse(outputFile, 5000); + PathUtils.deleteFile(outputFile); + + if (jo == null) { + return; + } + + String status = jo.getString("status"); + if ("recording".equals(status)) { + long startTime = jo.getLongOrDefault("startTime", System.currentTimeMillis()); + if (ctx.runner != null) { + ctx.runner.runOnRenderThread(() -> { + state = State.RECORDING; + recordingStartTime = startTime; + if (jo.containsKey("durationSeconds")) { + duration = jo.getIntegerOrDefault("durationSeconds", duration); + } + }); + } + } else if ("completed".equals(status) && jo.getBooleanOrDefault("hasCachedResults", false)) { + // there are cached results, load them + loadQueryResults(pid); + } + } + + private void loadQueryResults(String pid) { + Path outputFile = ctx.getOutputFile(pid); + PathUtils.deleteFile(outputFile); + + JsonObject root = new JsonObject(); + root.put("action", "jfr-old-objects"); + root.put("command", "query"); + + Path actionFile = ctx.getActionFile(pid); + PathUtils.writeTextSafely(root.toJson(), actionFile); + + JsonObject jo = pollJsonResponse(outputFile, 10000); + PathUtils.deleteFile(outputFile); + + if (jo != null && "completed".equals(jo.getString("status"))) { + List<SampleEntry> result = parseSamples(jo); + int count = jo.getIntegerOrDefault("sampleCount", 0); + long durationMs = jo.getLongOrDefault("recordingDurationMs", 0); + long endTime = jo.getLongOrDefault("recordingEndTime", System.currentTimeMillis()); + + if (ctx.runner != null) { + ctx.runner.runOnRenderThread(() -> { + samples = result; + sampleCount = count; + recordingDurationMs = durationMs; + recordingEndTime = endTime; + state = State.HAS_RESULTS; + tableState.select(0); + lastPid = pid; + }); + } + } else { + if (ctx.runner != null) { + ctx.runner.runOnRenderThread(() -> { + if (state == State.LOADING_RESULTS) { + state = State.IDLE; + } + }); + } + } + } + + // ---- Sorting ---- + + private List<SampleEntry> sortedSamples() { + long minSize = MIN_SIZE_PRESETS[minSizeIndex]; + List<SampleEntry> result = new ArrayList<>(samples); + if (minSize > 0) { + result.removeIf(e -> e.totalSize < minSize); + } + result.sort((a, b) -> { + int cmp = switch (sort) { + case "allocationClass" -> compareStr(a.className, b.className); + case "count" -> Integer.compare(b.count, a.count); + case "objectAge" -> Long.compare(b.objectAge, a.objectAge); + default -> Long.compare(b.totalSize, a.totalSize); + }; + return sortReversed ? -cmp : cmp; + }); + return result; + } + + // ---- Parsing ---- + + private List<SampleEntry> parseSamples(JsonObject jo) { + JsonArray arr = (JsonArray) jo.get("samples"); + if (arr == null) { + return Collections.emptyList(); + } + + List<SampleEntry> result = new ArrayList<>(); + for (int i = 0; i < arr.size(); i++) { + JsonObject sj = (JsonObject) arr.get(i); + SampleEntry entry = new SampleEntry(); + entry.num = i + 1; + entry.className = sj.getStringOrDefault("allocationClass", "unknown"); + entry.count = sj.getIntegerOrDefault("count", 1); + entry.totalSize = sj.getLongOrDefault("totalSize", 0); + entry.allocationSize = sj.getLongOrDefault("allocationSize", 0); + entry.lastKnownHeapUsage = sj.getLongOrDefault("lastKnownHeapUsage", 0); + entry.objectAge = sj.getLongOrDefault("objectAge", 0); + + // reference chain + JsonArray chain = (JsonArray) sj.get("referenceChain"); + if (chain != null && !chain.isEmpty()) { + entry.referenceChain = new ArrayList<>(); + for (int j = 0; j < chain.size(); j++) { + JsonObject linkJson = (JsonObject) chain.get(j); + ChainLink link = new ChainLink(); + link.type = linkJson.getString("type"); + link.field = linkJson.getString("field"); + link.description = linkJson.getString("description"); + entry.referenceChain.add(link); + } + entry.chainSummary = buildChainSummary(entry.referenceChain); + } + + // stack trace + JsonArray st = (JsonArray) sj.get("stackTrace"); + if (st != null && !st.isEmpty()) { + entry.stackTrace = new ArrayList<>(); + for (int j = 0; j < st.size(); j++) { + JsonObject fj = (JsonObject) st.get(j); + StackEntry se = new StackEntry(); + se.method = fj.getStringOrDefault("method", "?"); + se.line = fj.getIntegerOrDefault("line", 0); + entry.stackTrace.add(se); + } + } + + result.add(entry); + } + return result; + } + + // ---- Helpers ---- + + private static String buildChainSummary(List<ChainLink> chain) { + if (chain == null || chain.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < chain.size() && i < 3; i++) { + ChainLink link = chain.get(i); + if (i > 0) { + sb.append(" → "); + } + if (link.type != null) { + sb.append(abbreviateType(link.type)); + } + if (link.field != null) { + sb.append('.').append(link.field); + } + } + if (chain.size() > 3) { + sb.append(" → ..."); + } + return sb.toString(); + } + + private static String abbreviateType(String type) { + if (type == null) { + return "?"; + } + int dot = type.lastIndexOf('.'); + return dot >= 0 ? type.substring(dot + 1) : type; + } + + static String formatBytes(long bytes) { + if (bytes < 1024) { + return bytes + " B"; + } else if (bytes < 1024 * 1024) { + return String.format(Locale.US, "%.1f KB", bytes / 1024.0); + } else if (bytes < 1024L * 1024 * 1024) { + return String.format(Locale.US, "%.1f MB", bytes / (1024.0 * 1024)); + } else { + return String.format(Locale.US, "%.1f GB", bytes / (1024.0 * 1024 * 1024)); + } + } + + static String formatDuration(long ms) { + if (ms < 1000) { + return ms + "ms"; + } else if (ms < 60000) { + return String.format(Locale.US, "%.1fs", ms / 1000.0); + } else { + long min = ms / 60000; + long sec = (ms % 60000) / 1000; + return min + "m " + sec + "s"; + } + } + + private String sortLabel(String label, String column) { + return MonitorContext.sortLabel(label, column, sort, sortReversed); + } + + private Style sortStyle(String column) { + return MonitorContext.sortStyle(column, sort); + } + + private static int compareStr(String a, String b) { + if (a == null && b == null) { + return 0; + } + if (a == null) { + return -1; + } + if (b == null) { + return 1; + } + return a.compareToIgnoreCase(b); + } + + private static boolean isJdkFrame(String method) { + return method != null + && (method.startsWith("java.") || method.startsWith("javax.") || method.startsWith("jakarta.") + || method.startsWith("jdk.") || method.startsWith("sun.") || method.startsWith("com.sun.")); + } + + private static void startDaemonThread(String name, Runnable task) { + Thread t = new Thread(task); + t.setDaemon(true); + t.setName(name); + t.start(); + } + + // ---- Data classes ---- + + static class SampleEntry { + int num; + String className; + long allocationSize; + long lastKnownHeapUsage; + long objectAge; + int count = 1; + long totalSize; + String chainSummary; + List<ChainLink> referenceChain; + List<StackEntry> stackTrace; + } + + static class ChainLink { + String type; + String field; + String description; + } + + static class StackEntry { + String method; + int line; + } +} diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/McpFacade.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/McpFacade.java index 11096df9a093..9e58b72c6e4e 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/McpFacade.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/McpFacade.java @@ -90,8 +90,8 @@ class McpFacade { static final String[] MORE_TAB_NAMES = { "Beans", "Browse", "Circuit Breaker", "Classpath", "Configuration", - "Consumers", "DataSource", "Heap Histogram", "Inflight", "Memory", "Metrics", "SQL Query", "SQL Trace", - "Spans", "Process", "Startup", "Threads" + "Consumers", "DataSource", "Heap Histogram", "JFR Old Objects", "Inflight", "Memory", "Metrics", + "SQL Query", "SQL Trace", "Spans", "Process", "Startup", "Threads" }; static final Map<String, String> TAB_DESCRIPTIONS = Map.ofEntries( @@ -113,6 +113,8 @@ class McpFacade { Map.entry("DataSource", "JDBC DataSource pool statistics (active, idle, max connections)"), Map.entry("Heap Histogram", "Class-level heap memory analysis showing instance counts, byte usage, package summary, and JAR origin per class"), + Map.entry("JFR Old Objects", + "JFR OldObjectSample recording for memory leak diagnosis — shows objects surviving multiple GC cycles with reference chains to GC roots"), Map.entry("Inflight", "Currently in-flight exchanges being processed"), Map.entry("Memory", "JVM memory usage (heap/non-heap), GC stats, and thread counts"), Map.entry("Metrics", "Micrometer metrics (counters, gauges, timers, distribution summaries)"), diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/PopupManager.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/PopupManager.java index 7687999001cc..103191c987e1 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/PopupManager.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/PopupManager.java @@ -176,7 +176,7 @@ class PopupManager { return true; } if (ke.isDown()) { - morePopupState.selectNext(17); + morePopupState.selectNext(18); return true; } int shortcutSel = morePopupShortcut(ke); @@ -239,7 +239,7 @@ class PopupManager { void renderMorePopup(Frame frame, Rect area) { int popupW = 22; - int popupH = 19; + int popupH = 20; // Position just below the "0 More▾" tab label int dividerW = CharWidth.of(" | "); int tabBarX = 0; @@ -269,6 +269,7 @@ class PopupManager { ListItem.from(Line.from(Span.raw(" Co"), Span.styled("n", keyStyle), Span.raw("sumers"))), ListItem.from(Line.from(Span.raw(" "), Span.styled("D", keyStyle), Span.raw("ataSource"))), ListItem.from(Line.from(Span.raw(" "), Span.styled("H", keyStyle), Span.raw("eap Histogram"))), + ListItem.from(Line.from(Span.raw(" "), Span.styled("J", keyStyle), Span.raw("FR Old Objects"))), ListItem.from(Line.from(Span.raw(" "), Span.styled("I", keyStyle), Span.raw("nflight"))), ListItem.from(Line.from(Span.raw(" "), Span.styled("M", keyStyle), Span.raw("emory"))), ListItem.from(Line.from(Span.raw(" M"), Span.styled("e", keyStyle), Span.raw("trics"))), @@ -401,33 +402,36 @@ class PopupManager { if (ke.isChar('h')) { return 7; } - if (ke.isChar('i')) { + if (ke.isChar('j')) { return 8; } - if (ke.isChar('m')) { + if (ke.isChar('i')) { return 9; } - if (ke.isChar('e')) { + if (ke.isChar('m')) { return 10; } - if (ke.isChar('q')) { + if (ke.isChar('e')) { return 11; } - if (ke.isChar('r')) { + if (ke.isChar('q')) { return 12; } - if (ke.isChar('o')) { + if (ke.isChar('r')) { return 13; } - if (ke.isChar('p')) { + if (ke.isChar('o')) { return 14; } - if (ke.isChar('s')) { + if (ke.isChar('p')) { return 15; } - if (ke.isChar('t')) { + if (ke.isChar('s')) { return 16; } + if (ke.isChar('t')) { + return 17; + } return -1; } diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TabRegistry.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TabRegistry.java index e0cf19d7e1f7..680eeaf3e57e 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TabRegistry.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TabRegistry.java @@ -87,6 +87,7 @@ class TabRegistry { private OverviewTab overviewTab; private DataSourceTab dataSourceTab; private HeapHistogramTab heapHistogramTab; + private JfrOldObjectSampleTab jfrOldObjectSampleTab; private SqlQueryTab sqlQueryTab; private SqlTraceTab sqlTraceTab; @@ -107,6 +108,7 @@ class TabRegistry { consumersTab = new ConsumersTab(ctx); dataSourceTab = new DataSourceTab(ctx); heapHistogramTab = new HeapHistogramTab(ctx); + jfrOldObjectSampleTab = new JfrOldObjectSampleTab(ctx); sqlQueryTab = new SqlQueryTab(ctx); sqlTraceTab = new SqlTraceTab(ctx); endpointsTab = new EndpointsTab(ctx, dataService.metrics()); @@ -131,7 +133,7 @@ class TabRegistry { resetIntegrationTabState); sqlTraceTab.setEditSqlAction(sql -> { - selectMoreTab(11); // switch to SQL Query tab + selectMoreTab(12); // switch to SQL Query tab sqlQueryTab.setInputValue("sql", sql); }); } @@ -219,15 +221,16 @@ class TabRegistry { case 5 -> consumersTab; case 6 -> dataSourceTab; case 7 -> heapHistogramTab; - case 8 -> inflightTab; - case 9 -> memoryTab; - case 10 -> metricsTab; - case 11 -> sqlQueryTab; - case 12 -> sqlTraceTab; - case 13 -> spansTab; - case 14 -> processTab; - case 15 -> startupTab; - case 16 -> threadsTab; + case 8 -> jfrOldObjectSampleTab; + case 9 -> inflightTab; + case 10 -> memoryTab; + case 11 -> metricsTab; + case 12 -> sqlQueryTab; + case 13 -> sqlTraceTab; + case 14 -> spansTab; + case 15 -> processTab; + case 16 -> startupTab; + case 17 -> threadsTab; default -> null; }; if (activeMoreTab != null) { @@ -251,6 +254,7 @@ class TabRegistry { consumersTab.onIntegrationChanged(); dataSourceTab.onIntegrationChanged(); heapHistogramTab.onIntegrationChanged(); + jfrOldObjectSampleTab.onIntegrationChanged(); sqlQueryTab.onIntegrationChanged(); sqlTraceTab.onIntegrationChanged(); circuitBreakerTab.onIntegrationChanged();
