This is an automated email from the ASF dual-hosted git repository. jamesfredley pushed a commit to branch feat/app-bench-ci in repository https://gitbox.apache.org/repos/asf/grails-core.git
commit c303782388ab52751aa5c92d9270221a98c49f0d Author: James Fredley <[email protected]> AuthorDate: Tue Aug 18 17:40:04 2026 -0400 feat: add app-level indy A/B benches to the performance workflow Land the gated HTTP harness (latency, app1, gsp-layout) and run it from benchmark.yml when a PR has the performance label. Compare -PgrailsIndy=false vs true on HEAD. Does not change the plugin default. Assisted-by: claude-code:claude-opus-5 --- .github/workflows/benchmark.yml | 62 ++++++ gradle/app-bench-config.gradle | 43 ++++ grails-benchmarks/scripts/run-app-indy-bench.ps1 | 118 +++++++++++ grails-test-examples/app1/build.gradle | 1 + .../AppBenchInterceptorDemoSpec.groovy | 49 +++++ grails-test-examples/gsp-layout/build.gradle | 4 +- .../layout/AppBenchDemoRenderTextSpec.groovy | 50 +++++ grails-test-examples/latency/build.gradle | 1 + .../groovy/latencyapp/AppBenchFastPingSpec.groovy | 50 +++++ .../testing/http/client/bench/AppHttpBench.groovy | 221 +++++++++++++++++++++ .../http/client/bench/AppHttpBenchSpec.groovy | 84 ++++++++ 11 files changed, 682 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3d1cd75675..14f1a750df 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -264,6 +264,68 @@ jobs: git worktree remove --force "$WORKTREE_ROOT/base" || true git worktree remove --force "$WORKTREE_ROOT/head" || true + app-bench: + name: "App indy benchmarks" + if: >- + contains(github.event.pull_request.labels.*.name, 'performance') && + (github.event.action != 'labeled' || github.event.label.name == 'performance') + runs-on: ubuntu-24.04 + env: + RESULT_DIR: ${{ github.workspace }}/app-bench-results + REPORT_DIR: ${{ github.workspace }}/app-bench-reports + steps: + - name: "Checkout repository" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: "Setup JDK" + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + distribution: liberica + java-version: 21 + - name: "Setup Gradle" + uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0 + with: + cache-provider: basic # 'basic' uses the MIT-licensed, open-source cache provider; the default 'enhanced' provider (v6+) is proprietary (Gradle commercial Terms of Use) + develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + - name: "Verify JMH comparison tool" + run: ./gradlew :grails-benchmarks:test --max-workers=4 + - name: "Run app benchmarks without indy" + timeout-minutes: 90 + run: | + mkdir -p "$RESULT_DIR/noindy" + ./gradlew --no-daemon :grails-test-examples-latency:integrationTest --tests "latencyapp.AppBenchFastPingSpec" -PgrailsIndy=false -PappBench=true -PappBenchWarmup=80 -PappBenchSamples=300 -PappBenchForks=2 -PappBenchOut="$RESULT_DIR/noindy/latency.json" --max-workers=4 + ./gradlew --no-daemon :grails-test-examples-app1:integrationTest --tests "functionaltests.AppBenchInterceptorDemoSpec" -PgrailsIndy=false -PappBench=true -PappBenchWarmup=80 -PappBenchSamples=300 -PappBenchForks=2 -PappBenchOut="$RESULT_DIR/noindy/app1.json" --max-workers=4 + ./gradlew --no-daemon :grails-test-examples-gsp-layout:integrationTest --tests "org.example.grails.layout.AppBenchDemoRenderTextSpec" -PgrailsIndy=false -PappBench=true -PappBenchWarmup=80 -PappBenchSamples=300 -PappBenchForks=2 -PappBenchOut="$RESULT_DIR/noindy/gsp-layout.json" --max-workers=4 + - name: "Run app benchmarks with indy" + timeout-minutes: 90 + run: | + mkdir -p "$RESULT_DIR/indy" + ./gradlew --no-daemon :grails-test-examples-latency:integrationTest --tests "latencyapp.AppBenchFastPingSpec" -PgrailsIndy=true -PappBench=true -PappBenchWarmup=80 -PappBenchSamples=300 -PappBenchForks=2 -PappBenchOut="$RESULT_DIR/indy/latency.json" --max-workers=4 + ./gradlew --no-daemon :grails-test-examples-app1:integrationTest --tests "functionaltests.AppBenchInterceptorDemoSpec" -PgrailsIndy=true -PappBench=true -PappBenchWarmup=80 -PappBenchSamples=300 -PappBenchForks=2 -PappBenchOut="$RESULT_DIR/indy/app1.json" --max-workers=4 + ./gradlew --no-daemon :grails-test-examples-gsp-layout:integrationTest --tests "org.example.grails.layout.AppBenchDemoRenderTextSpec" -PgrailsIndy=true -PappBench=true -PappBenchWarmup=80 -PappBenchSamples=300 -PappBenchForks=2 -PappBenchOut="$RESULT_DIR/indy/gsp-layout.json" --max-workers=4 + - name: "Compare indy app benchmark results" + if: always() + continue-on-error: true + run: | + mkdir -p "$REPORT_DIR" + ./gradlew -q --console=plain :grails-benchmarks:jmhCompare --args="--base $RESULT_DIR/noindy --head $RESULT_DIR/indy --output $REPORT_DIR/indy-vs-noindy.md" + - name: "Publish app benchmark report in job summary" + if: always() + run: | + if [ -f "$REPORT_DIR/indy-vs-noindy.md" ]; then + cat "$REPORT_DIR/indy-vs-noindy.md" >> "$GITHUB_STEP_SUMMARY" + else + printf '## App indy benchmark comparison\n\nNo comparison report was produced.\n' >> "$GITHUB_STEP_SUMMARY" + fi + - name: "Upload app benchmark artifacts" + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: app-bench-results + path: | + app-bench-results/ + app-bench-reports/ + if-no-files-found: warn + report: name: "Publish JMH benchmark comparison" needs: benchmark diff --git a/gradle/app-bench-config.gradle b/gradle/app-bench-config.gradle new file mode 100644 index 0000000000..edd1bbd2d5 --- /dev/null +++ b/gradle/app-bench-config.gradle @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Experimental app-level indy A/B harness wiring. +// Enable with -PappBench=true (and usually --tests '*AppBench*'). +// Optional: -PappBenchWarmup=200 -PappBenchSamples=1000 -PappBenchForks=2 -PappBenchOut=<json> + +def appBenchEnabled = providers.gradleProperty('appBench').orElse('false') +def appBenchWarmup = providers.gradleProperty('appBenchWarmup').orElse('200') +def appBenchSamples = providers.gradleProperty('appBenchSamples').orElse('1000') +def appBenchForks = providers.gradleProperty('appBenchForks').orElse('2') +def appBenchOut = providers.gradleProperty('appBenchOut').orElse('') + +tasks.withType(Test).configureEach { Test test -> + test.systemProperty('app.bench', appBenchEnabled.get()) + test.systemProperty('app.bench.warmup', appBenchWarmup.get()) + test.systemProperty('app.bench.samples', appBenchSamples.get()) + test.systemProperty('app.bench.forks', appBenchForks.get()) + if (appBenchOut.get()) { + test.systemProperty('app.bench.out', appBenchOut.get()) + test.outputs.file(appBenchOut.get()) + } + if (appBenchEnabled.get() == 'true') { + test.outputs.upToDateWhen { false } + test.outputs.cacheIf { false } + } +} diff --git a/grails-benchmarks/scripts/run-app-indy-bench.ps1 b/grails-benchmarks/scripts/run-app-indy-bench.ps1 new file mode 100644 index 0000000000..5aaa0742ba --- /dev/null +++ b/grails-benchmarks/scripts/run-app-indy-bench.ps1 @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Experimental: app-level indy vs non-indy HTTP benches against grails-test-examples. +# Run from repo root. Rebuilds framework+app with -PgrailsIndy, runs gated AppBench* specs, +# then compares JMH-shaped JSON with :grails-benchmarks:jmhCompare. +# +# Usage: +# pwsh grails-benchmarks/scripts/run-app-indy-bench.ps1 +# pwsh grails-benchmarks/scripts/run-app-indy-bench.ps1 -Samples 500 -Warmup 100 + +param( + [int]$Warmup = 200, + [int]$Samples = 1000, + [int]$Forks = 2 +) + +$ErrorActionPreference = 'Stop' +$root = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path +Set-Location -LiteralPath $root + +$outDir = Join-Path $root 'build\app-bench' +New-Item -ItemType Directory -Force -Path $outDir | Out-Null + +$apps = @( + @{ + Name = 'latency' + Project = ':grails-test-examples-latency' + Tests = 'latencyapp.AppBenchFastPingSpec' + OutName = 'latency' + }, + @{ + Name = 'app1' + Project = ':grails-test-examples-app1' + Tests = 'functionaltests.AppBenchInterceptorDemoSpec' + OutName = 'app1' + }, + @{ + Name = 'gsp-layout' + Project = ':grails-test-examples-gsp-layout' + Tests = 'org.example.grails.layout.AppBenchDemoRenderTextSpec' + OutName = 'gsp-layout' + } +) + +function Invoke-AppBenchMode { + param( + [string]$Indy, + [hashtable]$App + ) + $label = if ($Indy -eq 'true') { 'indy' } else { 'noindy' } + $outFile = Join-Path $outDir "$($App.OutName)-$label.json" + if (Test-Path -LiteralPath $outFile) { + Remove-Item -LiteralPath $outFile -Force + } + + Write-Host "=== $($App.Name) grailsIndy=$Indy -> $outFile ===" + & "$root\gradlew.bat" 'clean' "$($App.Project):integrationTest" ` + "-PgrailsIndy=$Indy" ` + '-PappBench=true' ` + "-PappBenchWarmup=$Warmup" ` + "-PappBenchSamples=$Samples" ` + "-PappBenchForks=$Forks" ` + "-PappBenchOut=$($outFile -replace '\\','/')" ` + '--tests' $App.Tests + if ($LASTEXITCODE -ne 0) { + throw "App bench failed for $($App.Name) indy=$Indy (exit $LASTEXITCODE)" + } + if (-not (Test-Path -LiteralPath $outFile)) { + throw "Missing result file: $outFile" + } +} + +foreach ($app in $apps) { + Invoke-AppBenchMode -Indy 'false' -App $app + Invoke-AppBenchMode -Indy 'true' -App $app + + $base = Join-Path $outDir "$($app.OutName)-noindy.json" + $head = Join-Path $outDir "$($app.OutName)-indy.json" + $report = Join-Path $outDir "$($app.OutName)-indy-vs-noindy.md" + Write-Host "=== compare $($app.Name) ===" + & "$root\gradlew.bat" '-q' ':grails-benchmarks:jmhCompare' ` + "--args=--base $($base -replace '\\','/') --head $($head -replace '\\','/') --output $($report -replace '\\','/')" + if ($LASTEXITCODE -ne 0) { + throw "jmhCompare failed for $($app.Name)" + } +} + +# Merged all-app compare +$mergedNo = Join-Path $outDir 'all-apps-noindy.json' +$mergedYes = Join-Path $outDir 'all-apps-indy.json' +$allNo = @() +$allYes = @() +foreach ($app in $apps) { + $allNo += (Get-Content -LiteralPath (Join-Path $outDir "$($app.OutName)-noindy.json") -Raw | ConvertFrom-Json) + $allYes += (Get-Content -LiteralPath (Join-Path $outDir "$($app.OutName)-indy.json") -Raw | ConvertFrom-Json) +} +($allNo | ConvertTo-Json -Depth 20) | Set-Content -LiteralPath $mergedNo -Encoding utf8 +($allYes | ConvertTo-Json -Depth 20) | Set-Content -LiteralPath $mergedYes -Encoding utf8 + +& "$root\gradlew.bat" '-q' ':grails-benchmarks:jmhCompare' ` + "--args=--base $($mergedNo -replace '\\','/') --head $($mergedYes -replace '\\','/') --output $((Join-Path $outDir 'all-apps-indy-vs-noindy.md') -replace '\\','/')" + +Write-Host "Done. Reports under $outDir" diff --git a/grails-test-examples/app1/build.gradle b/grails-test-examples/app1/build.gradle index a4575fb878..8184ef1180 100644 --- a/grails-test-examples/app1/build.gradle +++ b/grails-test-examples/app1/build.gradle @@ -92,4 +92,5 @@ apply { from rootProject.layout.projectDirectory.file('gradle/functional-test-config.gradle') from rootProject.layout.projectDirectory.file('gradle/test-webjar-asset-config.gradle') from rootProject.layout.projectDirectory.file('gradle/grails-extension-gradle-config.gradle') + from rootProject.layout.projectDirectory.file('gradle/app-bench-config.gradle') } diff --git a/grails-test-examples/app1/src/integration-test/groovy/functionaltests/AppBenchInterceptorDemoSpec.groovy b/grails-test-examples/app1/src/integration-test/groovy/functionaltests/AppBenchInterceptorDemoSpec.groovy new file mode 100644 index 0000000000..ed3439f578 --- /dev/null +++ b/grails-test-examples/app1/src/integration-test/groovy/functionaltests/AppBenchInterceptorDemoSpec.groovy @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package functionaltests + +import spock.lang.IgnoreIf +import spock.lang.Specification + +import grails.testing.mixin.integration.Integration +import org.apache.grails.testing.http.client.HttpClientSupport +import org.apache.grails.testing.http.client.bench.AppHttpBench + +/** + * App-level bench: interceptor URI matching + controller render on a full web app. + * Gated by {@code -PappBench=true}. Complements JMH interceptor microbenches with a real stack. + */ +@Integration +@IgnoreIf({ !AppHttpBench.enabled() }) +class AppBenchInterceptorDemoSpec extends Specification implements HttpClientSupport { + + void 'bench GET /interceptorDemo/one'() { + when: + AppHttpBench.measureAndWrite( + 'appbench.app1.InterceptorDemo.httpGet', + AppHttpBench.outputPath('app1-appbench.json') + ) { + def response = http('/interceptorDemo/one') + response.assertStatus(200) + } + + then: + noExceptionThrown() + } +} diff --git a/grails-test-examples/gsp-layout/build.gradle b/grails-test-examples/gsp-layout/build.gradle index 6f20012a7e..6e63b590da 100644 --- a/grails-test-examples/gsp-layout/build.gradle +++ b/grails-test-examples/gsp-layout/build.gradle @@ -64,6 +64,7 @@ dependencies { runtimeOnly 'org.glassfish.web:jakarta.servlet.jsp.jstl' integrationTestImplementation testFixtures('org.apache.grails:grails-geb') + integrationTestImplementation project(':grails-testing-support-http-client') testImplementation 'org.junit.jupiter:junit-jupiter-api' testImplementation 'org.springframework:spring-test' @@ -78,4 +79,5 @@ apply { from rootProject.layout.projectDirectory.file('gradle/functional-test-config.gradle') from rootProject.layout.projectDirectory.file('gradle/test-webjar-asset-config.gradle') from rootProject.layout.projectDirectory.file('gradle/grails-extension-gradle-config.gradle') -} \ No newline at end of file + from rootProject.layout.projectDirectory.file('gradle/app-bench-config.gradle') +} diff --git a/grails-test-examples/gsp-layout/src/integration-test/groovy/org/example/grails/layout/AppBenchDemoRenderTextSpec.groovy b/grails-test-examples/gsp-layout/src/integration-test/groovy/org/example/grails/layout/AppBenchDemoRenderTextSpec.groovy new file mode 100644 index 0000000000..48f3cce457 --- /dev/null +++ b/grails-test-examples/gsp-layout/src/integration-test/groovy/org/example/grails/layout/AppBenchDemoRenderTextSpec.groovy @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.example.grails.layout + +import spock.lang.IgnoreIf +import spock.lang.Specification + +import grails.testing.mixin.integration.Integration +import org.apache.grails.testing.http.client.HttpClientSupport +import org.apache.grails.testing.http.client.bench.AppHttpBench + +/** + * App-level bench: controller HTML text render through the layout app stack. + * Gated by {@code -PappBench=true}. + */ +@Integration +@IgnoreIf({ !AppHttpBench.enabled() }) +class AppBenchDemoRenderTextSpec extends Specification implements HttpClientSupport { + + void 'bench GET /demo/renderText'() { + when: + AppHttpBench.measureAndWrite( + 'appbench.gsplayout.DemoRenderText.httpGet', + AppHttpBench.outputPath('gsp-layout-appbench.json') + ) { + def response = http('/demo/renderText') + response.assertStatus(200) + response.assertContains('Hello World') + } + + then: + noExceptionThrown() + } +} diff --git a/grails-test-examples/latency/build.gradle b/grails-test-examples/latency/build.gradle index f6531a0576..64948b1a8b 100644 --- a/grails-test-examples/latency/build.gradle +++ b/grails-test-examples/latency/build.gradle @@ -45,4 +45,5 @@ dependencies { apply { from rootProject.layout.projectDirectory.file('gradle/functional-test-config.gradle') from rootProject.layout.projectDirectory.file('gradle/grails-extension-gradle-config.gradle') + from rootProject.layout.projectDirectory.file('gradle/app-bench-config.gradle') } diff --git a/grails-test-examples/latency/src/integration-test/groovy/latencyapp/AppBenchFastPingSpec.groovy b/grails-test-examples/latency/src/integration-test/groovy/latencyapp/AppBenchFastPingSpec.groovy new file mode 100644 index 0000000000..9b2ea82572 --- /dev/null +++ b/grails-test-examples/latency/src/integration-test/groovy/latencyapp/AppBenchFastPingSpec.groovy @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package latencyapp + +import spock.lang.IgnoreIf +import spock.lang.Specification + +import grails.testing.mixin.integration.Integration +import org.apache.grails.testing.http.client.HttpClientSupport +import org.apache.grails.testing.http.client.bench.AppHttpBench + +/** + * App-level bench: plain controller {@code render} on the full Spring Boot stack. + * Gated by {@code -PappBench=true}. + */ +@Integration +@IgnoreIf({ !AppHttpBench.enabled() }) +class AppBenchFastPingSpec extends Specification implements HttpClientSupport { + + void 'bench GET /fast/ping'() { + when: + AppHttpBench.measureAndWrite( + 'appbench.latency.FastPing.httpGet', + AppHttpBench.outputPath('latency-appbench.json') + ) { + def response = http('/fast/ping') + response.assertStatus(200) + response.assertContains('pong') + } + + then: + noExceptionThrown() + } +} diff --git a/grails-testing-support-http-client/src/main/groovy/org/apache/grails/testing/http/client/bench/AppHttpBench.groovy b/grails-testing-support-http-client/src/main/groovy/org/apache/grails/testing/http/client/bench/AppHttpBench.groovy new file mode 100644 index 0000000000..ec2a2061a2 --- /dev/null +++ b/grails-testing-support-http-client/src/main/groovy/org/apache/grails/testing/http/client/bench/AppHttpBench.groovy @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.testing.http.client.bench + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +import groovy.json.JsonOutput +import groovy.transform.CompileStatic + +/** + * App-level HTTP microbench helper that emits JMH-compatible JSON so results can be compared + * with {@code :grails-benchmarks:jmhCompare} using the same methodology as the framework JMH suite. + * + * <p>Measurement model (deliberately simple and reproducible on one machine): + * <ul> + * <li>warm up the full Spring Boot stack with {@code warmup} requests (discarded)</li> + * <li>collect {@code samples} timed requests as one raw series</li> + * <li>split the series into {@code forks} equal chunks to mimic JMH multi-fork rawData shape</li> + * <li>report mean ns/op with a simple standard-error-based scoreError</li> + * </ul> + * + * <p>Enable gated specs with {@code -PappBench=true}. Optional properties: + * {@code appBenchWarmup}, {@code appBenchSamples}, {@code appBenchForks}, {@code appBenchOut}. + */ +@CompileStatic +final class AppHttpBench { + + private AppHttpBench() { + } + + static boolean enabled() { + Boolean.getBoolean('app.bench') || Boolean.parseBoolean(System.getProperty('appBench', 'false')) + } + + static int warmupCount() { + Integer.getInteger('app.bench.warmup', Integer.getInteger('appBenchWarmup', 200)) + } + + static int sampleCount() { + Integer.getInteger('app.bench.samples', Integer.getInteger('appBenchSamples', 1000)) + } + + static int forkCount() { + Integer.getInteger('app.bench.forks', Integer.getInteger('appBenchForks', 2)) + } + + static Path outputPath(String defaultFileName) { + String configured = System.getProperty('app.bench.out', System.getProperty('appBenchOut', '')) + if (configured) { + return Paths.get(configured) + } + Path dir = Paths.get('build', 'app-bench') + Files.createDirectories(dir) + return dir.resolve(defaultFileName) + } + + /** + * Time a single request body. The closure must perform the HTTP call and assert success. + * + * @return elapsed nanoseconds + */ + static long timeNanos(Closure<?> request) { + long start = System.nanoTime() + request.call() + return System.nanoTime() - start + } + + /** + * Warm up, sample, and append one JMH-shaped benchmark entry to {@code out}. + * + * @param benchmark fully-qualified-style name, e.g. {@code appbench.latency.FastPing.httpGet} + * @param request closure that performs one successful request + */ + static void measureAndWrite(String benchmark, Path out, Closure<?> request) { + int warmup = warmupCount() + int samples = sampleCount() + int forks = Math.max(1, forkCount()) + + for (int i = 0; i < warmup; i++) { + request.call() + } + + double[] values = new double[samples] + for (int i = 0; i < samples; i++) { + values[i] = (double) timeNanos(request) + } + + Map<String, Object> entry = toJmhEntry(benchmark, values, forks) + appendEntry(out, entry) + } + + static Map<String, Object> toJmhEntry(String benchmark, double[] values, int forks) { + double mean = mean(values) + double stdev = stdev(values, mean) + double scoreError = stdev * 1.96d / Math.sqrt((double) values.length) + + int forkCount = Math.max(1, forks) + int perFork = Math.max(1, values.length.intdiv(forkCount)) + List<List<Double>> rawData = new ArrayList<>(forkCount) + int offset = 0 + for (int f = 0; f < forkCount; f++) { + int end = (f == forkCount - 1) ? values.length : Math.min(values.length, offset + perFork) + List<Double> chunk = new ArrayList<>(Math.max(0, end - offset)) + for (int i = offset; i < end; i++) { + chunk.add(values[i]) + } + if (chunk.isEmpty() && !rawData.isEmpty()) { + chunk.addAll(rawData.get(rawData.size() - 1)) + } + rawData.add(chunk) + offset = end + } + + Map<String, Object> percentiles = new LinkedHashMap<>() + double[] sorted = Arrays.copyOf(values, values.length) + Arrays.sort(sorted) + percentiles.put('0.0', sorted[0]) + percentiles.put('50.0', percentile(sorted, 0.50d)) + percentiles.put('90.0', percentile(sorted, 0.90d)) + percentiles.put('95.0', percentile(sorted, 0.95d)) + percentiles.put('99.0', percentile(sorted, 0.99d)) + percentiles.put('100.0', sorted[sorted.length - 1]) + + Map<String, Object> primary = new LinkedHashMap<>() + primary.put('score', mean) + primary.put('scoreError', scoreError) + primary.put('scoreConfidence', [mean - scoreError, mean + scoreError]) + primary.put('scorePercentiles', percentiles) + primary.put('scoreUnit', 'ns/op') + primary.put('rawData', rawData) + + Map<String, Object> entry = new LinkedHashMap<>() + entry.put('jmhVersion', 'app-bench-1.0') + entry.put('benchmark', benchmark) + entry.put('mode', 'avgt') + entry.put('threads', 1) + entry.put('forks', forkCount) + entry.put('jdkVersion', System.getProperty('java.version', 'unknown')) + entry.put('vmName', System.getProperty('java.vm.name', 'unknown')) + entry.put('vmVersion', System.getProperty('java.vm.version', 'unknown')) + entry.put('warmupIterations', 1) + entry.put('warmupTime', "${warmupCount()} reqs") + entry.put('measurementIterations', values.length) + entry.put('measurementTime', '1 req') + entry.put('primaryMetric', primary) + entry.put('secondaryMetrics', Collections.emptyMap()) + return entry + } + + static void appendEntry(Path out, Map<String, Object> entry) { + List<Object> entries = new ArrayList<>() + if (Files.exists(out)) { + String existing = Files.readString(out, StandardCharsets.UTF_8).trim() + if (existing.startsWith('[')) { + Object parsed = new groovy.json.JsonSlurper().parseText(existing) + if (parsed instanceof List) { + entries.addAll((List) parsed) + } + } + } + entries.add(entry) + Path parent = out.getParent() + if (parent != null) { + Files.createDirectories(parent) + } + Files.writeString(out, JsonOutput.prettyPrint(JsonOutput.toJson(entries)), StandardCharsets.UTF_8) + } + + private static double mean(double[] values) { + double sum = 0d + for (double value : values) { + sum += value + } + return sum / (double) values.length + } + + private static double stdev(double[] values, double mean) { + if (values.length < 2) { + return 0d + } + double sumSq = 0d + for (double value : values) { + double delta = value - mean + sumSq += delta * delta + } + return Math.sqrt(sumSq / (double) (values.length - 1)) + } + + private static double percentile(double[] sorted, double p) { + if (sorted.length == 1) { + return sorted[0] + } + double rank = p * (double) (sorted.length - 1) + int low = (int) Math.floor(rank) + int high = (int) Math.ceil(rank) + if (low == high) { + return sorted[low] + } + double weight = rank - (double) low + return sorted[low] * (1d - weight) + sorted[high] * weight + } +} diff --git a/grails-testing-support-http-client/src/test/groovy/org/apache/grails/testing/http/client/bench/AppHttpBenchSpec.groovy b/grails-testing-support-http-client/src/test/groovy/org/apache/grails/testing/http/client/bench/AppHttpBenchSpec.groovy new file mode 100644 index 0000000000..6f8ee631e2 --- /dev/null +++ b/grails-testing-support-http-client/src/test/groovy/org/apache/grails/testing/http/client/bench/AppHttpBenchSpec.groovy @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.testing.http.client.bench + +import java.nio.file.Files +import java.nio.file.Path + +import groovy.json.JsonSlurper +import spock.lang.Specification +import spock.lang.TempDir + +class AppHttpBenchSpec extends Specification { + + @TempDir + Path tempDir + + void cleanup() { + System.clearProperty('app.bench') + System.clearProperty('app.bench.out') + } + + void 'enabled is off by default'() { + expect: + !AppHttpBench.enabled() + } + + void 'enabled follows the app.bench system property'() { + when: + System.setProperty('app.bench', 'true') + + then: + AppHttpBench.enabled() + } + + void 'toJmhEntry reports mean ns/op and splits raw data into forks'() { + given: + double[] values = [10d, 20d, 30d, 40d] as double[] + + when: + Map<String, Object> entry = AppHttpBench.toJmhEntry('appbench.example.httpGet', values, 2) + + then: + entry.benchmark == 'appbench.example.httpGet' + entry.mode == 'avgt' + entry.forks == 2 + ((Map) entry.primaryMetric).score == 25d + ((Map) entry.primaryMetric).scoreUnit == 'ns/op' + ((List) ((Map) entry.primaryMetric).rawData).size() == 2 + } + + void 'appendEntry writes and extends a JMH JSON array'() { + given: + Path out = tempDir.resolve('bench.json') + Map<String, Object> first = AppHttpBench.toJmhEntry('one', [1d] as double[], 1) + Map<String, Object> second = AppHttpBench.toJmhEntry('two', [2d] as double[], 1) + + when: + AppHttpBench.appendEntry(out, first) + AppHttpBench.appendEntry(out, second) + List parsed = (List) new JsonSlurper().parse(out.toFile()) + + then: + parsed.size() == 2 + parsed[0].benchmark == 'one' + parsed[1].benchmark == 'two' + Files.size(out) > 0 + } +}
