This is an automated email from the ASF dual-hosted git repository. paulk-asert pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/groovy.git
commit c18dd839c712cddcaab67e4631ebe96e09b407fc Author: Paul King <[email protected]> AuthorDate: Mon May 11 16:34:33 2026 +1000 test additional user agent workarounds --- .github/workflows/groovy-build-coverage.yml | 30 ++++++++- .github/workflows/groovy-build-test.yml | 77 ++++++++++++++++++++-- .../main/groovy/org.apache.groovy-tested.gradle | 62 +++++++++++++++++ 3 files changed, 160 insertions(+), 9 deletions(-) diff --git a/.github/workflows/groovy-build-coverage.yml b/.github/workflows/groovy-build-coverage.yml index 60c746d3c2..eb045bd505 100644 --- a/.github/workflows/groovy-build-coverage.yml +++ b/.github/workflows/groovy-build-coverage.yml @@ -46,16 +46,40 @@ jobs: # # Same key prefix as `groovy-build-test.yml` so the two workflows # share their accumulated Grape cache. - - name: "π Cache @Grab artifacts (~/.groovy/grapes)" + - name: "π Cache @Grab artifacts (~/.groovy/grapes + ~/.m2/repository)" uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - path: ~/.groovy/grapes + path: | + ~/.groovy/grapes + ~/.m2/repository key: ${{ runner.os }}-grape-${{ github.run_id }} restore-keys: | ${{ runner.os }}-grape- - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - name: "π‘ Pre-warm @Grab artifacts via Maven" + shell: bash + run: | + set +e + coords=$(grep -rhEo "@Grab\(\s*(value\s*=\s*)?['\"][^'\"]+['\"]" src/test subprojects/*/src/test 2>/dev/null \ + | sed -E "s/.*@Grab\(\s*(value\s*=\s*)?['\"]([^'\"]+)['\"].*/\2/" \ + | grep -E '^[a-zA-Z0-9._-]+:[a-zA-Z0-9._-]+:[a-zA-Z0-9._+-]+$' \ + | sort -u) + [ -z "$coords" ] && { echo "No @Grab coords β skipping"; exit 0; } + n=$(printf '%s\n' "$coords" | wc -l | tr -d ' ') + echo "Pre-warming $n coords" + ok=0; fail=0 + while IFS= read -r c; do + if mvn -B -q dependency:get -Dartifact="$c" -Dtransitive=true >/dev/null 2>&1; then + ok=$((ok+1)) + else + fail=$((fail+1)); echo " β $c" + fi + done <<< "$coords" + echo "Pre-warm: $ok ok / $fail failed" + exit 0 + timeout-minutes: 15 - name: Test with Gradle - run: ./gradlew -Pcoverage=true jacocoAllReport + run: ./gradlew -Pgroovy.grape.bridge-cache=true -Pcoverage=true jacocoAllReport timeout-minutes: 60 - name: Upload coverage to Codecov uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 diff --git a/.github/workflows/groovy-build-test.yml b/.github/workflows/groovy-build-test.yml index 5cf2e3a5a0..226c2fd0f9 100644 --- a/.github/workflows/groovy-build-test.yml +++ b/.github/workflows/groovy-build-test.yml @@ -64,10 +64,16 @@ jobs: # fresh entry; the next run finds the most recent via prefix # fallback. The cache grows with new @Grab coordinates over time and # never gets invalidated by older ones being removed. - - name: "π Cache @Grab artifacts (~/.groovy/grapes)" + - name: "π Cache @Grab artifacts (~/.groovy/grapes + ~/.m2/repository)" uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - path: ~/.groovy/grapes + # ~/.groovy/grapes is what Grape/Ivy reads from at test time (via the + # bridge in org.apache.groovy-tested.gradle). ~/.m2/repository is what + # the pre-warm step below populates with mvn dependency:get β the bridge + # also copies it into the test JVM's isolated localm2 root. + path: | + ~/.groovy/grapes + ~/.m2/repository key: ${{ runner.os }}-grape-${{ github.run_id }} restore-keys: | ${{ runner.os }}-grape- @@ -75,8 +81,43 @@ jobs: uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 - name: "π Setup TestLens" uses: testlens-app/setup-testlens@v1 + # Pre-warm the @Grab artifact cache by fetching each unique Maven-shorthand + # coordinate referenced in test sources via `mvn dependency:get`. Maven uses + # Apache HttpClient (vs Ivy's java.net.URLConnection), so it currently + # passes Cloudflare's WAF in front of Maven Central while Ivy gets HTTP 404. + # Once any run succeeds, actions/cache saves ~/.m2/repository for the next + # run, and the org.apache.groovy-tested.gradle bridge copies the artifacts + # into each test task's isolated localm2 root so tests stay off the network. + # Failures here are non-fatal β Ivy will retry at test time and may succeed + # on any individual artifact even when bursts are throttled. + - name: "π‘ Pre-warm @Grab artifacts via Maven" + shell: bash + run: | + set +e + coords=$(grep -rhEo "@Grab\(\s*(value\s*=\s*)?['\"][^'\"]+['\"]" src/test subprojects/*/src/test 2>/dev/null \ + | sed -E "s/.*@Grab\(\s*(value\s*=\s*)?['\"]([^'\"]+)['\"].*/\2/" \ + | grep -E '^[a-zA-Z0-9._-]+:[a-zA-Z0-9._-]+:[a-zA-Z0-9._+-]+$' \ + | sort -u) + if [ -z "$coords" ]; then + echo "No @Grab coords discovered β skipping pre-warm" + exit 0 + fi + n=$(printf '%s\n' "$coords" | wc -l | tr -d ' ') + echo "Pre-warming $n @Grab coords via mvn dependency:get" + ok=0; fail=0 + while IFS= read -r coord; do + if mvn -B -q dependency:get -Dartifact="$coord" -Dtransitive=true >/dev/null 2>&1; then + ok=$((ok+1)) + else + fail=$((fail+1)) + echo " β $coord" + fi + done <<< "$coords" + echo "Pre-warm complete: $ok ok / $fail failed (failures retried by Ivy at test time)" + exit 0 + timeout-minutes: 15 - name: "πTest with Gradle" - run: ./gradlew test ${{ matrix.junit-network }} -Ptarget.java.home="$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" + run: ./gradlew test ${{ matrix.junit-network }} -Pgroovy.grape.bridge-cache=true -Ptarget.java.home="$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" shell: bash timeout-minutes: 60 - name: "πUpload reports" @@ -105,14 +146,38 @@ jobs: check-latest: true # See the lts job for rationale; same prefix lets both jobs share # the cache. - - name: "π Cache @Grab artifacts (~/.groovy/grapes)" + - name: "π Cache @Grab artifacts (~/.groovy/grapes + ~/.m2/repository)" uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - path: ~/.groovy/grapes + path: | + ~/.groovy/grapes + ~/.m2/repository key: ${{ runner.os }}-grape-${{ github.run_id }} restore-keys: | ${{ runner.os }}-grape- - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - name: "π‘ Pre-warm @Grab artifacts via Maven" + shell: bash + run: | + set +e + coords=$(grep -rhEo "@Grab\(\s*(value\s*=\s*)?['\"][^'\"]+['\"]" src/test subprojects/*/src/test 2>/dev/null \ + | sed -E "s/.*@Grab\(\s*(value\s*=\s*)?['\"]([^'\"]+)['\"].*/\2/" \ + | grep -E '^[a-zA-Z0-9._-]+:[a-zA-Z0-9._-]+:[a-zA-Z0-9._+-]+$' \ + | sort -u) + [ -z "$coords" ] && { echo "No @Grab coords β skipping"; exit 0; } + n=$(printf '%s\n' "$coords" | wc -l | tr -d ' ') + echo "Pre-warming $n coords" + ok=0; fail=0 + while IFS= read -r c; do + if mvn -B -q dependency:get -Dartifact="$c" -Dtransitive=true >/dev/null 2>&1; then + ok=$((ok+1)) + else + fail=$((fail+1)); echo " β $c" + fi + done <<< "$coords" + echo "Pre-warm: $ok ok / $fail failed" + exit 0 + timeout-minutes: 15 - name: "πTest with Gradle" - run: ./gradlew test -Ptarget.java.home="$JAVA_HOME_${{ matrix.java }}_X64" + run: ./gradlew test -Pgroovy.grape.bridge-cache=true -Ptarget.java.home="$JAVA_HOME_${{ matrix.java }}_X64" timeout-minutes: 60 diff --git a/build-logic/src/main/groovy/org.apache.groovy-tested.gradle b/build-logic/src/main/groovy/org.apache.groovy-tested.gradle index 1546a8f84e..e21fefdc67 100644 --- a/build-logic/src/main/groovy/org.apache.groovy-tested.gradle +++ b/build-logic/src/main/groovy/org.apache.groovy-tested.gradle @@ -46,6 +46,19 @@ dependencies { def aggregator = TestResultAggregatorService.register( objects.newInstance(TestServices).buildEventsListenerRegistry, gradle) +// Opt-in: bridge the runner's ~/.groovy/grapes (populated by actions/cache in +// .github/workflows/groovy-build-*.yml) into each Test task's per-task grape +// root at task start, and fold newly-resolved artifacts back at task end. +// Without bridging, the actions/cache restore lands at a path the test JVM +// never reads (test JVMs run with -Duser.home=<task-temp>, so $user.home/.groovy +// resolves to <task-temp>/.groovy β not the runner's real home). Enabling +// bridging lets tests reuse previously-cached artifacts instead of re-hitting +// Maven Central (mitigates HTTP 429 throttling). Disabled by default locally +// so a developer's polluted ~/.groovy/grapes doesn't leak into tests. +// Enable on CI with: ./gradlew test -Pgroovy.grape.bridge-cache=true +def grapeBridgeCache = (findProperty('groovy.grape.bridge-cache') ?: + System.properties['groovy.grape.bridge-cache']) == 'true' + tasks.withType(Test).configureEach { def fs = objects.newInstance(TestServices).fileSystemOperations def grapeDirectory = new File(temporaryDir, '.groovy') @@ -115,10 +128,59 @@ tasks.withType(Test).configureEach { // delete if it exists already to be in a clean state delete(grapeDirectory) } + if (grapeBridgeCache) { + // Copy the actions/cache-restored ~/.groovy/grapes into the test + // task's grape root so resolutions can hit the local filesystem + // resolver (`cachedGrapes` in defaultGrapeConfig.xml) instead of + // re-fetching from Maven Central. Exclude lock files and Maven's + // "lastUpdated" markers so we don't propagate orphan locks or + // stale negative-resolution state across runs. + def sharedGrapes = new File(System.getProperty('user.home'), '.groovy/grapes') + if (sharedGrapes.isDirectory()) { + fs.copy { + from(sharedGrapes) + into(new File(grapeDirectory, 'grapes')) + exclude '**/*.lck', '**/*.lastUpdated' + } + logger.lifecycle "Bridged ~/.groovy/grapes -> ${grapeDirectory}/grapes" + } + // Also bridge ~/.m2/repository so the test JVM's localm2 Ivy resolver + // (configured as file:${user.home}/.m2/repository/) finds artifacts + // that the workflow's pre-warm step populated via `mvn dependency:get`. + // Tests run with -Duser.home=<task-temp>, so without this they'd see + // an empty localm2 and fall through to Maven Central β exactly the + // path that's currently being 404-throttled by Cloudflare for Java's + // URLConnection client. + def sharedM2 = new File(System.getProperty('user.home'), '.m2/repository') + def m2Target = new File(temporaryDir, '.m2/repository') + if (sharedM2.isDirectory()) { + fs.copy { + from(sharedM2) + into(m2Target) + exclude '**/*.lastUpdated', '**/_remote.repositories' + } + logger.lifecycle "Bridged ~/.m2/repository -> ${m2Target}" + } + } logger.debug "Grape directory: ${grapeDirectory.absolutePath}" } doLast { + if (grapeBridgeCache) { + // Fold newly-resolved artifacts back into the shared cache so the + // workflow's `actions/cache` save step persists them for next run. + // The forward bridge filtered locks/lastUpdated; do the same here. + def sharedGrapes = new File(System.getProperty('user.home'), '.groovy/grapes') + def testGrapes = new File(grapeDirectory, 'grapes') + if (testGrapes.isDirectory()) { + sharedGrapes.mkdirs() + fs.copy { + from(testGrapes) + into(sharedGrapes) + exclude '**/*.lck', '**/*.lastUpdated' + } + } + } fs.delete { delete(files(".").filter { it.name.endsWith '.class' }) }
