This is an automated email from the ASF dual-hosted git repository. jonnybot pushed a commit to branch INDY-PERF-EXPLORATION in repository https://gitbox.apache.org/repos/asf/groovy.git
commit 58e6639317391e6ffe5cc874d9d9ae7efb4e1428 Author: Jonny Carter <[email protected]> AuthorDate: Fri Feb 20 16:43:05 2026 -0600 Significantly pare back JMH implementation Some of these were LLM overreach. Not all bad ideas, but we need to focus on what's good. --- .github/workflows/groovy-performance.yml | 207 +---------- subprojects/performance/README.adoc | 270 +-------------- subprojects/performance/baselines/README.md | 49 --- subprojects/performance/build.gradle | 222 ------------ .../bench/profiling/FlameGraphGenerator.java | 385 --------------------- .../bench/profiling/ProfiledBenchmarkRunner.java | 252 -------------- 6 files changed, 14 insertions(+), 1371 deletions(-) diff --git a/.github/workflows/groovy-performance.yml b/.github/workflows/groovy-performance.yml index 3273f992b1..ee9e2b64ed 100644 --- a/.github/workflows/groovy-performance.yml +++ b/.github/workflows/groovy-performance.yml @@ -48,35 +48,10 @@ permissions: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} -jobs: - # Quick smoke test for PRs - performance-smoke: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: '17' - - uses: gradle/actions/setup-gradle@v4 - - - name: Run smoke benchmarks - run: | - ./gradlew -Pindy=true -PbenchInclude=ColdCall :performance:jmh \ - -Pjmh.fork=1 -Pjmh.wi=2 -Pjmh.i=3 - timeout-minutes: 30 - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - with: - name: benchmark-smoke-results - path: subprojects/performance/build/results/jmh/ - # ============================================================================ # Full benchmark suite: build once, fan out into parallel matrix jobs # ============================================================================ - +jobs: # Step 1: Build the JMH fat jar and discover benchmark groups build-jmh-jar: if: github.event_name != 'pull_request' @@ -152,183 +127,3 @@ jobs: name: bench-${{ matrix.group }}-indy-${{ matrix.indy }} path: results.json retention-days: 5 - - # Step 4: Collect all results and generate comparison report - collect-and-compare: - needs: benchmark-matrix - if: always() && github.event_name != 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Download all benchmark results - uses: actions/download-artifact@v4 - with: - pattern: bench-* - path: results/ - merge-multiple: false - - - name: Merge results and generate comparison - run: | - python3 << 'PYEOF' - import json, os, sys - from pathlib import Path - - indy_results = {} - noindy_results = {} - - results_dir = Path("results") - for artifact_dir in sorted(results_dir.iterdir()): - if not artifact_dir.is_dir(): - continue - results_file = artifact_dir / "results.json" - if not results_file.exists(): - print(f"Warning: no results.json in {artifact_dir.name}", file=sys.stderr) - continue - - is_indy = "-indy-true" in artifact_dir.name - target = indy_results if is_indy else noindy_results - - with open(results_file) as f: - data = json.load(f) - - for bench in data: - name = bench.get("benchmark", "unknown") - score = bench.get("primaryMetric", {}).get("score", 0) - unit = bench.get("primaryMetric", {}).get("scoreUnit", "") - error = bench.get("primaryMetric", {}).get("scoreError", 0) - target[name] = {"score": score, "unit": unit, "error": error} - - # Save merged results - with open("indy-results.json", "w") as f: - json.dump(indy_results, f, indent=2) - with open("noindy-results.json", "w") as f: - json.dump(noindy_results, f, indent=2) - - # Generate comparison report - all_benchmarks = sorted(set(list(indy_results.keys()) + list(noindy_results.keys()))) - - lines = [] - lines.append("## Performance Comparison: Indy vs Non-Indy") - lines.append("") - lines.append(f"| Benchmark | Indy | Non-Indy | Diff |") - lines.append("|-----------|------|----------|------|") - - regressions = [] - improvements = [] - - for name in all_benchmarks: - short_name = name.split(".")[-1] if "." in name else name - indy = indy_results.get(name) - noindy = noindy_results.get(name) - - if indy and noindy and noindy["score"] > 0: - diff_pct = ((indy["score"] - noindy["score"]) / noindy["score"]) * 100 - diff_str = f"{diff_pct:+.1f}%" - if diff_pct > 10: - diff_str += " :arrow_up:" - improvements.append((short_name, diff_pct)) - elif diff_pct < -10: - diff_str += " :arrow_down:" - regressions.append((short_name, diff_pct)) - lines.append(f"| {short_name} | {indy['score']:.3f} {indy['unit']} | {noindy['score']:.3f} {noindy['unit']} | {diff_str} |") - elif indy: - lines.append(f"| {short_name} | {indy['score']:.3f} {indy['unit']} | N/A | - |") - elif noindy: - lines.append(f"| {short_name} | N/A | {noindy['score']:.3f} {noindy['unit']} | - |") - - lines.append("") - lines.append(f"**Total benchmarks:** {len(all_benchmarks)} | " - f"**Indy faster (>10%):** {len(improvements)} | " - f"**Non-Indy faster (>10%):** {len(regressions)}") - - report = "\n".join(lines) - - with open("comparison-report.md", "w") as f: - f.write(report) - - # Write to GitHub Step Summary - summary_path = os.environ.get("GITHUB_STEP_SUMMARY", "") - if summary_path: - with open(summary_path, "a") as f: - f.write(report + "\n") - - print(report) - PYEOF - - - name: Upload comparison report - uses: actions/upload-artifact@v4 - if: always() - with: - name: benchmark-comparison - path: | - comparison-report.md - indy-results.json - noindy-results.json - retention-days: 30 - - # Memory-focused benchmarks with GC profiler (parallel with main matrix) - performance-memory: - needs: build-jmh-jar - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: '17' - - - name: Download JMH jar - uses: actions/download-artifact@v4 - with: - name: jmh-jar - path: . - - - name: Run memory benchmarks with GC profiler - run: | - JAR=$(ls *-jmh.jar | head -1) - java \ - -Dgroovy.target.indy=true \ - -jar "$JAR" \ - ".*bench\.memory\..*" \ - -f 1 -wi 1 \ - -prof gc \ - -rf json -rff gc-profile-results.json - - - name: Upload memory profile results - uses: actions/upload-artifact@v4 - with: - name: benchmark-memory-profile - path: gc-profile-results.json - - # Threshold sweep analysis - performance-threshold-sweep: - if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: '17' - - uses: gradle/actions/setup-gradle@v4 - - - name: Run threshold sweep - run: | - ./gradlew -PbenchInclude=Warmup :performance:jmhThresholdSweep - timeout-minutes: 180 - - - name: Upload threshold sweep results - uses: actions/upload-artifact@v4 - with: - name: benchmark-threshold-sweep - path: subprojects/performance/build/results/jmh-compare/threshold-sweep/ - - - name: Display threshold summary - run: | - echo "## Threshold Sweep Analysis" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - if [ -f subprojects/performance/build/results/jmh-compare/threshold-sweep/threshold-summary.txt ]; then - echo '```' >> $GITHUB_STEP_SUMMARY - cat subprojects/performance/build/results/jmh-compare/threshold-sweep/threshold-summary.txt >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - fi diff --git a/subprojects/performance/README.adoc b/subprojects/performance/README.adoc index 4413ade130..e6e4aa15e5 100644 --- a/subprojects/performance/README.adoc +++ b/subprojects/performance/README.adoc @@ -21,25 +21,14 @@ = Performance -This subproject contains performance tests and JMH benchmarks for Apache Groovy, -with a particular focus on investigating invokedynamic performance characteristics. - -== Quick Start - -[source,bash] ----- -# Run all benchmarks -./gradlew :performance:jmh - -# Run specific benchmark suite -./gradlew -PbenchInclude=ColdCall :performance:jmh ----- +This subproject contains two sets of performance related tests, compiler tests +and benchmarks. == Compiler Performance Tests The compiler tests can be run using the following Gradle task: - ./gradlew :performance:performanceTests + ./gradlew :perf:performanceTests This will compile various source files using several past versions of Apache Groovy in addition to the current source version. @@ -48,254 +37,21 @@ Groovy in addition to the current source version. JMH Benchmarks can be run using: - ./gradlew :performance:jmh + ./gradlew :perf:jmh + +In order to run the benchmarks against InvokeDynamic generated classes use +the `indy` property: + + ./gradlew -Pindy=true :perf:jmh + +Groovy and Java sources placed in `src/test` will also be available to the +benchmarks. To run a single benchmark or a matched set of benchmarks, use the `benchInclude` property: - ./gradlew -PbenchInclude=CallsiteBench :performance:jmh + ./gradlew -PbenchInclude=CallsiteBench :perf:jmh The `benchInclude` property will perform a partial match against package names or class names. It is equivalent to `.*${benchInclude}.*`. -== InvokeDynamic Performance Benchmarks - -A comprehensive set of benchmarks designed to investigate and isolate performance -characteristics of Groovy's invokedynamic implementation. See `INDY_PERFORMANCE_PLAN.md` -for the full testing plan and background. - -=== Phase 1: Core Performance Benchmarks - -==== Cold Call Benchmarks (`ColdCallBench`) - -Measures the cost of method invocations before callsites are optimized. -Critical for web applications where objects are created per-request. - - ./gradlew -PbenchInclude=ColdCallBench :performance:jmh - -==== Warmup Behavior Benchmarks (`WarmupBehaviorBench`) - -Tests performance at different warmup levels (10, 100, 1000, 10000+ calls) -to understand how the optimization threshold affects performance. - - ./gradlew -PbenchInclude=WarmupBehavior :performance:jmh - -==== Threshold Sensitivity Benchmarks (`ThresholdSensitivityBench`) - -Tests different usage patterns (web request, batch processing, mixed) to -understand which patterns benefit from different threshold configurations. - - ./gradlew -PbenchInclude=ThresholdSensitivity :performance:jmh - -==== Cache Invalidation Benchmarks (`CacheInvalidationBench`) - -Tests polymorphic dispatch scenarios that cause inline cache invalidation, -addressing issues described in GROOVY-8298. - - ./gradlew -PbenchInclude=CacheInvalidation :performance:jmh - -==== Property Access Benchmarks (`PropertyAccessBench`) - -Tests GORM-like property access patterns common in Grails applications. - - ./gradlew -PbenchInclude=PropertyAccess :performance:jmh - -==== Memory Allocation Benchmarks (`MemoryAllocationBench`) - -Measures heap growth and allocation rates during Groovy operations. -Quantifies the memory overhead of MethodHandleWrapper AtomicLong objects -and CacheableCallSite LRU cache entries. - - ./gradlew -PbenchInclude=MemoryAllocation :performance:jmh - -For detailed allocation profiling with GC metrics: - - ./gradlew -PbenchInclude=MemoryAllocation :performance:jmh -Pjmh.profilers=gc - -==== Callsite Growth Benchmarks (`CallsiteGrowthBench`) - -Measures how memory grows as unique callsites accumulate. Tests with -parameterized callsite counts (100, 1000, 10000) to quantify the ~32 bytes -per cached method handle overhead. - - ./gradlew -PbenchInclude=CallsiteGrowth :performance:jmh - -==== Long-Running Session Benchmarks (`LongRunningSessionBench`) - -Simulates web application memory patterns over time: request cycles, -sustained load, and memory recovery after GC. - - ./gradlew -PbenchInclude=LongRunningSession :performance:jmh - -=== Phase 2: Grails-Realistic Scenario Tests - -These benchmarks simulate real-world Grails application patterns. - -==== Request Lifecycle Benchmarks (`RequestLifecycleBench`) - -Simulates full HTTP request lifecycle in a Grails-like application: - -* Controller receives request (params parsing) -* Service layer invocation (transactional context) -* Domain object manipulation (property access, validation) -* Model building for view - - ./gradlew -PbenchInclude=RequestLifecycle :performance:jmh - -==== Template Render Benchmarks (`TemplateRenderBench`) - -Simulates GSP-like template rendering with heavy property access: - -* Property access on model objects (`${person.name}`) -* Collection iteration (`g:each`) -* GString interpolation -* Spread operators (`people*.name`) - - ./gradlew -PbenchInclude=TemplateRender :performance:jmh - -==== Mini-Grails Domain Model - -The `grailslike` package contains realistic domain classes: - -* `DomainObject` - Base class with dirty tracking, validation -* `Person`, `Order`, `OrderItem` - E-commerce entities -* `User`, `Role`, `Customer`, `Product` - Additional domain classes -* `DynamicService` - Service with methodMissing for dynamic finders -* `TransactionalSimulator` - Transaction boundary simulation -* `ControllerSimulator` - Request handling patterns -* `TemplateSimulator` - GSP-like rendering - -=== Threshold Parameter Sweep - -Test multiple threshold values to find optimal settings: - -[source,bash] ----- -./gradlew :performance:jmhThresholdSweep ----- - -Tests thresholds: 0, 10, 100, 1000, 10000, 100000 - -Results saved to `build/results/jmh-threshold-sweep/threshold-summary.txt`. - -=== Profiling Integration - -==== JFR Profiling - -Run benchmarks with Java Flight Recorder: - -[source,bash] ----- -./gradlew :performance:jmhProfile ----- - -JFR output saved to `build/results/jmh-profile/benchmark.jfr`. - -Analyze with: -[source,bash] ----- -jfr print build/results/jmh-profile/benchmark.jfr -jfr summary build/results/jmh-profile/benchmark.jfr -# Or open in JDK Mission Control (jmc) ----- - -==== GC Profiling - -Run with JMH's GC profiler for detailed memory analysis: - -[source,bash] ----- -./gradlew :performance:jmhGcProfile ----- - -==== Memory Tracking Benchmarks - -Use `MemoryTrackingState` in benchmarks for detailed heap tracking: - -[source,bash] ----- -./gradlew -PbenchInclude=MemoryProfile :performance:jmh ----- - -=== Baseline Management & CI - -==== Saving Baselines - -[source,bash] ----- -# Run benchmarks -./gradlew :performance:jmh - -# Save as baseline -./gradlew :performance:jmhSaveBaseline -PbaselineName=groovy-4.0.x ----- - -==== Comparing Against Baselines - -[source,bash] ----- -# Compare against most recent baseline -./gradlew :performance:jmh -./gradlew :performance:jmhCompareBaseline - -# Compare against specific baseline -./gradlew :performance:jmhCompareBaseline -PbaselineName=groovy-4.0.x ----- - -==== CI Integration - -Performance benchmarks run automatically via GitHub Actions: - -* **On PRs**: Smoke tests for performance-related changes -* **Weekly**: Full benchmark suite -* **On demand**: Via workflow_dispatch with customizable filters - -See `.github/workflows/groovy-performance.yml` for details. - -=== Tuning Thresholds - -The invokedynamic implementation has configurable thresholds: - -* `groovy.indy.optimize.threshold` (default: 10000) - Calls before optimization -* `groovy.indy.fallback.threshold` (default: 10000) - Fallbacks before reset - -Test with different thresholds: - -[source,bash] ----- -./gradlew -PbenchInclude=ThresholdSensitivity :performance:jmh \ - --jvmArgs="-Dgroovy.indy.optimize.threshold=100" ----- - -=== Benchmark Categories by Use Case - -==== For Web Application Performance - -Focus on cold call and request lifecycle: - -[source,bash] ----- -./gradlew -PbenchInclude="ColdCall|RequestLifecycle|Template" :performance:jmh ----- - -==== For Memory Investigation - -[source,bash] ----- -./gradlew -PbenchInclude=Memory :performance:jmh -Pjmh.profilers=gc ----- - -==== For Polymorphic Dispatch Issues - -[source,bash] ----- -./gradlew -PbenchInclude="CacheInvalidation|Dispatch" :performance:jmh ----- - -==== For Full Regression Testing - -[source,bash] ----- -./gradlew :performance:jmh -./gradlew :performance:jmhCompareBaseline ----- diff --git a/subprojects/performance/baselines/README.md b/subprojects/performance/baselines/README.md deleted file mode 100644 index 76b8e0a8fc..0000000000 --- a/subprojects/performance/baselines/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Performance Baselines - -This directory contains saved JMH benchmark results for comparison purposes. - -## Creating a Baseline - -Run benchmarks and save as a baseline: - -```bash -# Run all benchmarks -./gradlew :performance:jmh - -# Save results as a named baseline -./gradlew :performance:jmhSaveBaseline -PbaselineName=my-baseline - -# Or with a specific date -./gradlew :performance:jmhSaveBaseline -PbaselineName=groovy-4.0.x-indy-20240115 -``` - -## Comparing Against a Baseline - -```bash -# Compare current results against most recent baseline -./gradlew :performance:jmh -./gradlew :performance:jmhCompareBaseline - -# Compare against a specific baseline -./gradlew :performance:jmhCompareBaseline -PbaselineName=groovy-4.0.x-indy -``` - -## Baseline Naming Convention - -Recommended naming pattern: `{version}-{mode}-{date}.txt` - -Examples: -- `groovy-3.0.x-noindy.txt` - Groovy 3.x without invokedynamic -- `groovy-4.0.x-indy.txt` - Groovy 4.x with invokedynamic (default) -- `groovy-4.0.x-noindy.txt` - Groovy 4.x without invokedynamic -- `groovy-4.0.x-threshold-100.txt` - Groovy 4.x with custom threshold - -## File Format - -Baseline files are standard JMH text output format: - -``` -Benchmark Mode Cnt Score Error Units -MemoryAllocationBench.memory_create... avgt 10 11.047 ± 0.493 us/op -MemoryAllocationBench.memory_poly... avgt 10 253.352 ± 14.6 us/op -``` diff --git a/subprojects/performance/build.gradle b/subprojects/performance/build.gradle index 01ad54aedc..cdd4d067ef 100644 --- a/subprojects/performance/build.gradle +++ b/subprojects/performance/build.gradle @@ -38,104 +38,6 @@ def jmhResultsDir = layout.buildDirectory.dir("results/jmh") // Threshold Parameter Sweep // ============================================================================ -/** - * Run benchmarks with different optimize threshold values. - * Helps determine optimal threshold for different workloads. - */ -tasks.register('jmhThresholdSweep') { - group = 'benchmark' - description = 'Run benchmarks with varying groovy.indy.optimize.threshold values' - - def thresholds = [0, 10, 100, 1000, 10000, 100000] - def sweepDir = layout.buildDirectory.dir("results/jmh-threshold-sweep") - - doFirst { - sweepDir.get().asFile.mkdirs() - } - - doLast { - def results = [:] - - thresholds.each { threshold -> - println "" - println "=" * 60 - println "Running with groovy.indy.optimize.threshold = ${threshold}" - println "=" * 60 - - def outputFile = sweepDir.get().file("threshold-${threshold}.txt").asFile - - // Run JMH with this threshold - providers.javaexec { - mainClass = 'org.openjdk.jmh.Main' - classpath = files(tasks.named('jmhJar')) - args '-rf', 'text' - args '-rff', outputFile.absolutePath - args '-wi', '3' // Reduced warmup for sweep - args '-i', '3' // Reduced iterations for sweep - args '-f', '1' // Single fork for sweep - - if (project.hasProperty('benchInclude')) { - args '.*' + project.benchInclude + '.*' - } - - jvmArgs "-Dgroovy.indy.optimize.threshold=${threshold}" - }.result.get() - - // Parse results - if (outputFile.exists()) { - outputFile.eachLine { line -> - def match = line =~ /^(\S+)\s+\S+\s+\d+\s+([\d.]+)/ - if (match) { - def name = match[0][1] - def score = match[0][2] as double - if (!results[name]) results[name] = [:] - results[name][threshold] = score - } - } - } - } - - // Generate summary report - def summaryFile = sweepDir.get().file("threshold-summary.txt").asFile - summaryFile.withWriter { writer -> - writer.writeLine "=" * 100 - writer.writeLine "THRESHOLD SWEEP SUMMARY" - writer.writeLine "Generated: ${new Date()}" - writer.writeLine "=" * 100 - writer.writeLine "" - writer.writeLine String.format("%-50s %s", "Benchmark", thresholds.collect { String.format("%10d", it) }.join("")) - writer.writeLine "-" * 100 - - results.keySet().sort().each { name -> - def scores = thresholds.collect { t -> - results[name][t] ? String.format("%10.3f", results[name][t]) : String.format("%10s", "N/A") - }.join("") - writer.writeLine String.format("%-50s %s", - name.length() > 50 ? "..." + name[-47..-1] : name, - scores) - } - - writer.writeLine "" - writer.writeLine "Optimal thresholds by benchmark:" - writer.writeLine "-" * 50 - results.keySet().sort().each { name -> - def scores = results[name] - def optimal = scores.min { it.value }?.key - if (optimal != null) { - writer.writeLine String.format("%-50s %d", - name.length() > 50 ? "..." + name[-47..-1] : name, - optimal) - } - } - } - - println "" - println "Threshold sweep summary saved to: ${summaryFile}" - println "" - println summaryFile.text - } -} - // ============================================================================ // Profiling Support // ============================================================================ @@ -205,130 +107,6 @@ tasks.register('jmhGcProfile', JavaExec) { } } -// ============================================================================ -// Baseline Management -// ============================================================================ - -def baselinesDir = file("baselines") - -/** - * Save current benchmark results as a baseline. - */ -tasks.register('jmhSaveBaseline') { - dependsOn 'jmh' - group = 'benchmark' - description = 'Save current JMH results as a baseline for future comparison' - - doLast { - def baselineName = project.hasProperty('baselineName') ? project.baselineName : "baseline-${new Date().format('yyyyMMdd-HHmmss')}" - def resultsFile = jmhResultsDir.get().file("results.txt").asFile - def baselineFile = new File(baselinesDir, "${baselineName}.txt") - - if (!resultsFile.exists()) { - println "ERROR: No JMH results found. Run jmh first." - return - } - - baselinesDir.mkdirs() - baselineFile.text = resultsFile.text - - println "Baseline saved to: ${baselineFile}" - } -} - -/** - * Compare current results against a saved baseline. - */ -tasks.register('jmhCompareBaseline') { - dependsOn 'jmh' - group = 'benchmark' - description = 'Compare current JMH results against a saved baseline' - - doLast { - def baselineName = project.hasProperty('baselineName') ? project.baselineName : null - def baselineFile = baselineName ? new File(baselinesDir, "${baselineName}.txt") : null - - // Find most recent baseline if not specified - if (!baselineFile?.exists()) { - def baselines = baselinesDir.listFiles()?.findAll { it.name.endsWith('.txt') }?.sort { -it.lastModified() } - baselineFile = baselines?.first() - } - - if (!baselineFile?.exists()) { - println "ERROR: No baseline found. Run jmhSaveBaseline first." - return - } - - def currentFile = jmhResultsDir.get().file("results.txt").asFile - if (!currentFile.exists()) { - println "ERROR: No current results found. Run jmh first." - return - } - - def parseResults = { File file -> - def results = [:] - file.eachLine { line -> - def match = line =~ /^(\S+)\s+(\S+)\s+(\d+)\s+([\d.]+)\s*±?\s*([\d.]*)\s*(\S+)/ - if (match) { - def name = match[0][1] - def score = match[0][4] as double - results[name] = score - } - } - results - } - - def baseline = parseResults(baselineFile) - def current = parseResults(currentFile) - - println "" - println "=" * 80 - println "BASELINE COMPARISON" - println "Baseline: ${baselineFile.name}" - println "Current: ${currentFile.name}" - println "=" * 80 - println "" - println String.format("%-50s %12s %12s %12s", "Benchmark", "Baseline", "Current", "Change") - println "-" * 90 - - def allBenchmarks = (baseline.keySet() + current.keySet()).unique().sort() - def regressions = [] - def improvements = [] - - allBenchmarks.each { name -> - def baseScore = baseline[name] - def currScore = current[name] - - if (baseScore && currScore) { - def change = ((currScore - baseScore) / baseScore) * 100 - def changeStr = String.format("%+.1f%%", change) - - if (change > 10) { - changeStr += " REGRESSION" - regressions << [name: name, change: change] - } else if (change < -10) { - changeStr += " IMPROVED" - improvements << [name: name, change: change] - } - - println String.format("%-50s %12.3f %12.3f %12s", - name.length() > 50 ? "..." + name[-47..-1] : name, - baseScore, currScore, changeStr) - } - } - - println "" - if (regressions) { - println "REGRESSIONS (>10% slower):" - regressions.each { println " - ${it.name}: ${String.format('%+.1f%%', it.change)}" } - } - if (improvements) { - println "IMPROVEMENTS (>10% faster):" - improvements.each { println " - ${it.name}: ${String.format('%+.1f%%', it.change)}" } - } - } -} - // ============================================================================ // Dynamic Benchmark Grouping (for CI matrix) // ============================================================================ diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/profiling/FlameGraphGenerator.java b/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/profiling/FlameGraphGenerator.java deleted file mode 100644 index 8b1df6db7d..0000000000 --- a/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/profiling/FlameGraphGenerator.java +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.groovy.bench.profiling; - -import java.io.*; -import java.nio.file.*; -import java.util.*; -import java.util.regex.*; - -/** - * Post-processes JFR output to generate flame graph compatible output. - * <p> - * This class converts JFR recording data to the collapsed stack format - * that can be consumed by flame graph tools like async-profiler's - * converter or Brendan Gregg's flamegraph.pl. - * <p> - * Usage: - * <pre> - * # First, convert JFR to text using jfr tool - * jfr print --events jdk.ExecutionSample recording.jfr > stacks.txt - * - * # Then use this tool to convert to collapsed format - * java FlameGraphGenerator stacks.txt > collapsed.txt - * - * # Finally generate SVG using flamegraph.pl - * flamegraph.pl collapsed.txt > profile.svg - * </pre> - */ -public class FlameGraphGenerator { - - private static final Pattern STACK_FRAME_PATTERN = - Pattern.compile("^\\s+(.+)$"); - - private static final Pattern EVENT_SEPARATOR_PATTERN = - Pattern.compile("^jdk\\.(ExecutionSample|NativeMethodSample|ObjectAllocationInNewTLAB|ObjectAllocationOutsideTLAB)"); - - /** - * Main entry point for command-line usage. - */ - public static void main(String[] args) throws Exception { - if (args.length < 1) { - printUsage(); - System.exit(1); - } - - String command = args[0]; - - switch (command) { - case "collapse": - if (args.length < 2) { - System.err.println("Usage: FlameGraphGenerator collapse <jfr-text-file>"); - System.exit(1); - } - collapseStacks(args[1], System.out); - break; - - case "filter": - if (args.length < 3) { - System.err.println("Usage: FlameGraphGenerator filter <collapsed-file> <pattern>"); - System.exit(1); - } - filterStacks(args[1], args[2], System.out); - break; - - case "top": - if (args.length < 2) { - System.err.println("Usage: FlameGraphGenerator top <collapsed-file> [n]"); - System.exit(1); - } - int n = args.length > 2 ? Integer.parseInt(args[2]) : 20; - topMethods(args[1], n, System.out); - break; - - case "diff": - if (args.length < 3) { - System.err.println("Usage: FlameGraphGenerator diff <baseline-file> <current-file>"); - System.exit(1); - } - diffProfiles(args[1], args[2], System.out); - break; - - default: - printUsage(); - System.exit(1); - } - } - - private static void printUsage() { - System.err.println("FlameGraphGenerator - JFR profile analysis tool for Groovy benchmarks"); - System.err.println(); - System.err.println("Commands:"); - System.err.println(" collapse <jfr-text> Convert JFR text output to collapsed stacks"); - System.err.println(" filter <collapsed> <pattern> Filter stacks matching pattern"); - System.err.println(" top <collapsed> [n] Show top n methods by sample count"); - System.err.println(" diff <baseline> <current> Show difference between profiles"); - System.err.println(); - System.err.println("Workflow:"); - System.err.println(" 1. jfr print --events jdk.ExecutionSample recording.jfr > stacks.txt"); - System.err.println(" 2. java FlameGraphGenerator collapse stacks.txt > collapsed.txt"); - System.err.println(" 3. flamegraph.pl collapsed.txt > profile.svg"); - System.err.println(); - System.err.println("Analysis:"); - System.err.println(" java FlameGraphGenerator filter collapsed.txt 'groovy' > groovy-only.txt"); - System.err.println(" java FlameGraphGenerator top collapsed.txt 30"); - System.err.println(" java FlameGraphGenerator diff baseline.txt current.txt"); - } - - /** - * Convert JFR text output to collapsed stack format. - */ - public static void collapseStacks(String inputFile, PrintStream out) throws IOException { - Map<String, Long> stackCounts = new LinkedHashMap<>(); - List<String> currentStack = new ArrayList<>(); - - try (BufferedReader reader = Files.newBufferedReader(Paths.get(inputFile))) { - String line; - while ((line = reader.readLine()) != null) { - Matcher eventMatcher = EVENT_SEPARATOR_PATTERN.matcher(line); - if (eventMatcher.find()) { - // Start of new event - save previous stack if any - if (!currentStack.isEmpty()) { - String collapsed = collapseStack(currentStack); - stackCounts.merge(collapsed, 1L, Long::sum); - currentStack.clear(); - } - continue; - } - - Matcher frameMatcher = STACK_FRAME_PATTERN.matcher(line); - if (frameMatcher.matches()) { - String frame = frameMatcher.group(1).trim(); - // Clean up frame format - frame = cleanFrame(frame); - if (!frame.isEmpty()) { - currentStack.add(frame); - } - } - } - - // Don't forget the last stack - if (!currentStack.isEmpty()) { - String collapsed = collapseStack(currentStack); - stackCounts.merge(collapsed, 1L, Long::sum); - } - } - - // Output in collapsed format - for (Map.Entry<String, Long> entry : stackCounts.entrySet()) { - out.println(entry.getKey() + " " + entry.getValue()); - } - } - - private static String collapseStack(List<String> stack) { - // Reverse because JFR shows leaf first, but flame graphs want root first - StringBuilder sb = new StringBuilder(); - for (int i = stack.size() - 1; i >= 0; i--) { - if (sb.length() > 0) { - sb.append(";"); - } - sb.append(stack.get(i)); - } - return sb.toString(); - } - - private static String cleanFrame(String frame) { - // Remove line numbers and source file info - frame = frame.replaceAll("\\(.*\\)", ""); - // Remove module prefixes like java.base/ - frame = frame.replaceAll("^[\\w.]+/", ""); - return frame.trim(); - } - - /** - * Filter stacks to only those matching a pattern. - */ - public static void filterStacks(String inputFile, String pattern, PrintStream out) throws IOException { - Pattern p = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE); - - try (BufferedReader reader = Files.newBufferedReader(Paths.get(inputFile))) { - String line; - while ((line = reader.readLine()) != null) { - if (p.matcher(line).find()) { - out.println(line); - } - } - } - } - - /** - * Show top methods by sample count. - */ - public static void topMethods(String inputFile, int n, PrintStream out) throws IOException { - Map<String, Long> methodCounts = new HashMap<>(); - - try (BufferedReader reader = Files.newBufferedReader(Paths.get(inputFile))) { - String line; - while ((line = reader.readLine()) != null) { - int lastSpace = line.lastIndexOf(' '); - if (lastSpace < 0) continue; - - String stackPart = line.substring(0, lastSpace); - long count = Long.parseLong(line.substring(lastSpace + 1).trim()); - - // Extract individual methods from stack - String[] frames = stackPart.split(";"); - for (String frame : frames) { - methodCounts.merge(frame, count, Long::sum); - } - } - } - - // Sort by count descending - List<Map.Entry<String, Long>> sorted = new ArrayList<>(methodCounts.entrySet()); - sorted.sort((a, b) -> Long.compare(b.getValue(), a.getValue())); - - out.println("Top " + n + " methods by sample count:"); - out.println("===================================="); - int shown = 0; - for (Map.Entry<String, Long> entry : sorted) { - if (shown >= n) break; - out.printf("%8d %s%n", entry.getValue(), entry.getKey()); - shown++; - } - } - - /** - * Compare two profiles and show differences. - */ - public static void diffProfiles(String baselineFile, String currentFile, PrintStream out) throws IOException { - Map<String, Long> baseline = loadCollapsed(baselineFile); - Map<String, Long> current = loadCollapsed(currentFile); - - // Find methods with biggest changes - Map<String, Double> changes = new HashMap<>(); - - Set<String> allMethods = new HashSet<>(); - allMethods.addAll(baseline.keySet()); - allMethods.addAll(current.keySet()); - - long baselineTotal = baseline.values().stream().mapToLong(Long::longValue).sum(); - long currentTotal = current.values().stream().mapToLong(Long::longValue).sum(); - - for (String method : allMethods) { - long baseCount = baseline.getOrDefault(method, 0L); - long currCount = current.getOrDefault(method, 0L); - - double basePct = baselineTotal > 0 ? (baseCount * 100.0 / baselineTotal) : 0; - double currPct = currentTotal > 0 ? (currCount * 100.0 / currentTotal) : 0; - - double diff = currPct - basePct; - if (Math.abs(diff) > 0.1) { // Only show significant changes - changes.put(method, diff); - } - } - - // Sort by absolute change - List<Map.Entry<String, Double>> sorted = new ArrayList<>(changes.entrySet()); - sorted.sort((a, b) -> Double.compare(Math.abs(b.getValue()), Math.abs(a.getValue()))); - - out.println("Profile Comparison (positive = regression, negative = improvement)"); - out.println("================================================================"); - out.printf("Baseline total samples: %d%n", baselineTotal); - out.printf("Current total samples: %d%n", currentTotal); - out.println(); - - int shown = 0; - for (Map.Entry<String, Double> entry : sorted) { - if (shown >= 30) break; - String sign = entry.getValue() > 0 ? "+" : ""; - out.printf("%s%.2f%% %s%n", sign, entry.getValue(), entry.getKey()); - shown++; - } - } - - private static Map<String, Long> loadCollapsed(String file) throws IOException { - Map<String, Long> methodCounts = new HashMap<>(); - - try (BufferedReader reader = Files.newBufferedReader(Paths.get(file))) { - String line; - while ((line = reader.readLine()) != null) { - int lastSpace = line.lastIndexOf(' '); - if (lastSpace < 0) continue; - - String stackPart = line.substring(0, lastSpace); - long count = Long.parseLong(line.substring(lastSpace + 1).trim()); - - String[] frames = stackPart.split(";"); - for (String frame : frames) { - methodCounts.merge(frame, count, Long::sum); - } - } - } - - return methodCounts; - } - - /** - * Programmatic API for generating collapsed stacks from JFR. - */ - public static Map<String, Long> collapseFromJfr(Path jfrTextFile) throws IOException { - Map<String, Long> stackCounts = new LinkedHashMap<>(); - List<String> currentStack = new ArrayList<>(); - - try (BufferedReader reader = Files.newBufferedReader(jfrTextFile)) { - String line; - while ((line = reader.readLine()) != null) { - Matcher eventMatcher = EVENT_SEPARATOR_PATTERN.matcher(line); - if (eventMatcher.find()) { - if (!currentStack.isEmpty()) { - String collapsed = collapseStack(currentStack); - stackCounts.merge(collapsed, 1L, Long::sum); - currentStack.clear(); - } - continue; - } - - Matcher frameMatcher = STACK_FRAME_PATTERN.matcher(line); - if (frameMatcher.matches()) { - String frame = cleanFrame(frameMatcher.group(1).trim()); - if (!frame.isEmpty()) { - currentStack.add(frame); - } - } - } - - if (!currentStack.isEmpty()) { - String collapsed = collapseStack(currentStack); - stackCounts.merge(collapsed, 1L, Long::sum); - } - } - - return stackCounts; - } - - /** - * Filter collapsed stacks programmatically. - */ - public static Map<String, Long> filterByPattern(Map<String, Long> stacks, String pattern) { - Pattern p = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE); - Map<String, Long> filtered = new LinkedHashMap<>(); - - for (Map.Entry<String, Long> entry : stacks.entrySet()) { - if (p.matcher(entry.getKey()).find()) { - filtered.put(entry.getKey(), entry.getValue()); - } - } - - return filtered; - } - - /** - * Get top N methods from collapsed stacks. - */ - public static List<Map.Entry<String, Long>> getTopMethods(Map<String, Long> stacks, int n) { - Map<String, Long> methodCounts = new HashMap<>(); - - for (Map.Entry<String, Long> entry : stacks.entrySet()) { - String[] frames = entry.getKey().split(";"); - for (String frame : frames) { - methodCounts.merge(frame, entry.getValue(), Long::sum); - } - } - - List<Map.Entry<String, Long>> sorted = new ArrayList<>(methodCounts.entrySet()); - sorted.sort((a, b) -> Long.compare(b.getValue(), a.getValue())); - - return sorted.subList(0, Math.min(n, sorted.size())); - } -} diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/profiling/ProfiledBenchmarkRunner.java b/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/profiling/ProfiledBenchmarkRunner.java deleted file mode 100644 index ec3a0e7883..0000000000 --- a/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/profiling/ProfiledBenchmarkRunner.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.groovy.bench.profiling; - -import org.openjdk.jmh.annotations.*; -import org.openjdk.jmh.infra.Blackhole; -import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; - -import java.lang.management.*; -import java.util.*; -import java.util.concurrent.TimeUnit; - -/** - * Profiled benchmark runner with detailed memory and performance analysis. - * <p> - * This class provides a programmatic way to run JMH benchmarks with - * additional profiling capabilities beyond what JMH provides by default. - * <p> - * Features: - * <ul> - * <li>Detailed heap memory tracking before/after benchmarks</li> - * <li>Class loading statistics</li> - * <li>Thread count monitoring</li> - * <li>GC activity reporting</li> - * <li>Groovy-specific metrics (callsite count estimation)</li> - * </ul> - * <p> - * Run with: java -cp ... ProfiledBenchmarkRunner [benchmark-pattern] - */ -public class ProfiledBenchmarkRunner { - - private static final MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); - private static final ClassLoadingMXBean classLoadingBean = ManagementFactory.getClassLoadingMXBean(); - private static final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); - private static final List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); - - /** - * Main entry point for running profiled benchmarks. - */ - public static void main(String[] args) throws RunnerException { - String pattern = args.length > 0 ? args[0] : ".*"; - - System.out.println("=" .repeat(80)); - System.out.println("PROFILED BENCHMARK RUNNER"); - System.out.println("Pattern: " + pattern); - System.out.println("=" .repeat(80)); - System.out.println(); - - // Capture initial state - SystemState initialState = captureSystemState(); - printSystemState("INITIAL STATE", initialState); - - // Build JMH options - Options opt = new OptionsBuilder() - .include(pattern) - .warmupIterations(3) - .measurementIterations(5) - .forks(1) - .build(); - - // Run benchmarks - System.out.println("\nRunning benchmarks...\n"); - new Runner(opt).run(); - - // Capture final state - SystemState finalState = captureSystemState(); - printSystemState("FINAL STATE", finalState); - - // Print delta - printStateDelta(initialState, finalState); - } - - /** - * Capture current system state. - */ - public static SystemState captureSystemState() { - // Force GC for accurate memory reading - System.gc(); - try { Thread.sleep(100); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - - SystemState state = new SystemState(); - - // Memory - MemoryUsage heap = memoryBean.getHeapMemoryUsage(); - state.heapUsed = heap.getUsed(); - state.heapCommitted = heap.getCommitted(); - state.heapMax = heap.getMax(); - - MemoryUsage nonHeap = memoryBean.getNonHeapMemoryUsage(); - state.nonHeapUsed = nonHeap.getUsed(); - state.nonHeapCommitted = nonHeap.getCommitted(); - - // Classes - state.loadedClassCount = classLoadingBean.getLoadedClassCount(); - state.totalLoadedClassCount = classLoadingBean.getTotalLoadedClassCount(); - state.unloadedClassCount = classLoadingBean.getUnloadedClassCount(); - - // Threads - state.threadCount = threadBean.getThreadCount(); - state.peakThreadCount = threadBean.getPeakThreadCount(); - - // GC - for (GarbageCollectorMXBean gc : gcBeans) { - state.gcCount += gc.getCollectionCount(); - state.gcTime += gc.getCollectionTime(); - } - - state.timestamp = System.currentTimeMillis(); - - return state; - } - - /** - * Print system state. - */ - public static void printSystemState(String label, SystemState state) { - System.out.println("-".repeat(60)); - System.out.println(label); - System.out.println("-".repeat(60)); - System.out.printf("Heap Memory: %.2f MB used / %.2f MB committed / %.2f MB max%n", - state.heapUsed / 1024.0 / 1024.0, - state.heapCommitted / 1024.0 / 1024.0, - state.heapMax / 1024.0 / 1024.0); - System.out.printf("Non-Heap Memory: %.2f MB used / %.2f MB committed%n", - state.nonHeapUsed / 1024.0 / 1024.0, - state.nonHeapCommitted / 1024.0 / 1024.0); - System.out.printf("Classes: %d loaded / %d total loaded / %d unloaded%n", - state.loadedClassCount, state.totalLoadedClassCount, state.unloadedClassCount); - System.out.printf("Threads: %d current / %d peak%n", - state.threadCount, state.peakThreadCount); - System.out.printf("GC: %d collections / %d ms total%n", - state.gcCount, state.gcTime); - System.out.println(); - } - - /** - * Print delta between two states. - */ - public static void printStateDelta(SystemState before, SystemState after) { - System.out.println("=".repeat(60)); - System.out.println("BENCHMARK IMPACT (Delta)"); - System.out.println("=".repeat(60)); - - long heapDelta = after.heapUsed - before.heapUsed; - long nonHeapDelta = after.nonHeapUsed - before.nonHeapUsed; - int classLoadDelta = after.loadedClassCount - before.loadedClassCount; - long totalClassDelta = after.totalLoadedClassCount - before.totalLoadedClassCount; - long gcCountDelta = after.gcCount - before.gcCount; - long gcTimeDelta = after.gcTime - before.gcTime; - long duration = after.timestamp - before.timestamp; - - System.out.printf("Duration: %.2f seconds%n", duration / 1000.0); - System.out.printf("Heap Growth: %+.2f MB%n", heapDelta / 1024.0 / 1024.0); - System.out.printf("Non-Heap Growth: %+.2f MB%n", nonHeapDelta / 1024.0 / 1024.0); - System.out.printf("Classes Loaded: %+d (total: %+d)%n", classLoadDelta, totalClassDelta); - System.out.printf("GC Activity: %d collections, %d ms%n", gcCountDelta, gcTimeDelta); - - // Warnings - System.out.println(); - if (heapDelta > 100 * 1024 * 1024) { - System.out.println("WARNING: Significant heap growth (>100MB) - possible memory leak"); - } - if (classLoadDelta > 1000) { - System.out.println("WARNING: Large number of classes loaded (>1000) - check dynamic class generation"); - } - if (gcTimeDelta > duration * 0.1) { - System.out.println("WARNING: High GC overhead (>10% of runtime)"); - } - - System.out.println(); - } - - /** - * System state snapshot. - */ - public static class SystemState { - public long heapUsed; - public long heapCommitted; - public long heapMax; - public long nonHeapUsed; - public long nonHeapCommitted; - public int loadedClassCount; - public long totalLoadedClassCount; - public long unloadedClassCount; - public int threadCount; - public int peakThreadCount; - public long gcCount; - public long gcTime; - public long timestamp; - } - - /** - * Estimate the number of Groovy callsites by examining loaded classes. - * This is a rough approximation based on class naming patterns. - */ - public static int estimateCallsiteCount() { - int count = 0; - // This would require reflection to inspect GroovyClassLoader or CallSite arrays - // For now, return -1 to indicate "not implemented" - return -1; - } -} - -/** - * JMH State that combines MemoryTrackingState with additional Groovy-specific tracking. - */ -@State(Scope.Benchmark) -class GroovyProfilingState { - - private ProfiledBenchmarkRunner.SystemState beforeState; - private ProfiledBenchmarkRunner.SystemState afterState; - - @Setup(Level.Trial) - public void setup() { - beforeState = ProfiledBenchmarkRunner.captureSystemState(); - ProfiledBenchmarkRunner.printSystemState("Benchmark Start", beforeState); - } - - @TearDown(Level.Trial) - public void teardown() { - afterState = ProfiledBenchmarkRunner.captureSystemState(); - ProfiledBenchmarkRunner.printSystemState("Benchmark End", afterState); - ProfiledBenchmarkRunner.printStateDelta(beforeState, afterState); - } - - public ProfiledBenchmarkRunner.SystemState getBeforeState() { - return beforeState; - } - - public ProfiledBenchmarkRunner.SystemState getAfterState() { - return afterState; - } -}
