This is an automated email from the ASF dual-hosted git repository.
adutra pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new ad77bd9d1 feat(ci): Improve Gradle cache in CI (#1928)
ad77bd9d1 is described below
commit ad77bd9d1333c93b036d98e508ea481c0f8f7ef7
Author: Alexandre Dutra <adu...@apache.org>
AuthorDate: Mon Jul 14 11:46:38 2025 +0200
feat(ci): Improve Gradle cache in CI (#1928)
---
.../actions/ci-incr-build-cache-prepare/action.yml | 82 ++++++++++++++++++++++
.../actions/ci-incr-build-cache-save/action.yml | 70 ++++++++++++++++++
.github/workflows/gradle.yml | 48 +++++++++++++
.github/workflows/regtest.yml | 10 ++-
.github/workflows/spark_client_regtests.yml | 8 +++
LICENSE | 2 +
6 files changed, 219 insertions(+), 1 deletion(-)
diff --git a/.github/actions/ci-incr-build-cache-prepare/action.yml
b/.github/actions/ci-incr-build-cache-prepare/action.yml
new file mode 100644
index 000000000..55c80c3ef
--- /dev/null
+++ b/.github/actions/ci-incr-build-cache-prepare/action.yml
@@ -0,0 +1,82 @@
+# Copyright (C) 2020 Dremio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CODE_COPIED_TO_POLARIS Copied from Project Nessie v. 0.104.2
+
+name: 'Incremental Gradle build cache prepare'
+description: 'Prepare to save incremental Gradle build cache'
+inputs:
+ cache-read-only:
+ description: 'Gradle cache read only'
+ default: 'true'
+ java-version:
+ description: 'Java version'
+ default: '21'
+ job-id:
+ description: 'Job ID to prefer'
+ default: 'polaris-gradle'
+ job-instance:
+ description: 'Job instance to prefer'
+ default: 'gradle'
+runs:
+ using: "composite"
+ steps:
+ - name: Prep env
+ shell: bash
+ run: |
+ echo "GRADLE_BUILD_ACTION_CACHE_KEY_ENVIRONMENT=java-${{
inputs.java-version }}" >> ${GITHUB_ENV}
+ echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB=${{ inputs.job-id }}" >>
${GITHUB_ENV}
+ if [[ -n "${{ inputs.no-daemon }}" ]] ; then
+ echo "G_DAEMON_FLAG=--no-daemon" >> ${GITHUB_ENV}
+ fi
+ if [[ -n "${{ inputs.job-instance }}" ]] ; then
+ echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB_INSTANCE=${{
inputs.job-instance }}" >> ${GITHUB_ENV}
+ fi
+
+ - name: Gradle / Setup
+ uses:
gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+ with:
+ cache-read-only: ${{ inputs.cache-read-only }}
+ validate-wrappers: false
+
+ - name: Gradle / Init
+ shell: bash
+ run: ./gradlew -h
+
+ - name: Download existing workflow artifacts
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
# v4
+ # Just in case, don't know the exact inner workings of Gradle's build
cache and whether
+ # the download-action complains about duplicate files.
+ continue-on-error: true
+ with:
+ path: ~/downloaded-artifacts/
+
+ - name: Extract caches
+ shell: bash
+ run: |
+ echo "::group::Gradle build cache / add incremental updates"
+ mkdir -p ~/.gradle/caches/
+
+ if [[ -d ~/downloaded-artifacts/ ]] ; then
+ find ~/downloaded-artifacts/ -type f -name "ci-gradle-caches-*-${{
inputs.java-version }}.tar" | while read arch ; do
+ echo "Adding archive content from $arch ..."
+ (cd ~/.gradle/caches/ ; tar xf $arch)
+ done
+ else
+ echo "No previous build cache artifacts downloaded."
+ fi
+
+ date +%s > ~/caches-prepared-at-epoch
+
+ echo "::endgroup::"
diff --git a/.github/actions/ci-incr-build-cache-save/action.yml
b/.github/actions/ci-incr-build-cache-save/action.yml
new file mode 100644
index 000000000..4d6085c97
--- /dev/null
+++ b/.github/actions/ci-incr-build-cache-save/action.yml
@@ -0,0 +1,70 @@
+# Copyright (C) 2020 Dremio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CODE_COPIED_TO_POLARIS Copied from Project Nessie v. 0.104.2
+
+name: 'Save incremental Gradle caches'
+description: 'Save incremental Gradle caches'
+inputs:
+ job-name:
+ description: 'job name'
+ java-version:
+ description: 'Java version'
+ default: '21'
+runs:
+ using: "composite"
+ steps:
+ - name: Prepare Gradle caches archive
+ shell: bash
+ run: |
+ if [[ -d ~/.gradle/caches/ ]] ; then
+ echo "::group::Gradle caches / identify updated cache items"
+
+ cd ~/.gradle/caches/
+
+ echo "Gradle caches/ contains $(find . -type f | wc -l) files"
+ # Identify the added and changed files in caches/.
+
+ echo "Identifying changed/added files..."
+ AGE_SECS=$(($(date +%s) - $(cat ~/caches-prepared-at-epoch)))
+ echo "Build started ~ $AGE_SECS seconds ago"
+ AGE_MINS=$(($AGE_SECS / 60 + 1))
+ echo " ... assuming that is ~ $AGE_MINS minutes"
+ # This lists all relevant files that have been created or modified
during by the Gradle
+ # runs of the current job.
+ find . -mmin -$AGE_MINS -type f '(' \
+ -path './[0-9]*/kotlin-dsl/*' -or \
+ -path './jars-*/*' -or \
+ -path './modules-*/files-*/*' -or \
+ -path './modules-*/files-*/*' -or \
+ -path './build-cache-*/*' \
+ ')' | grep -v '[.]lock$' > ~/ci-gradle-caches-diff || true
+ echo "Identified $(wc -l < ~/ci-gradle-caches-diff) changed/added
files in caches/"
+
+ # Only call 'tar', if there is some difference
+ # Note: actions/upload-artifact takes care of compressing the
artifact, no need to bug the CPU here
+ echo "Creating artifact (if necessary)..."
+ if [[ -s ~/ci-gradle-caches-diff ]] ; then
+ tar --create --ignore-failed-read --file ~/ci-gradle-caches-${{
inputs.job-name }}-${{ inputs.java-version }}.tar -T ~/ci-gradle-caches-diff
+ ls -al ~/ci-gradle-caches-${{ inputs.job-name }}-${{
inputs.java-version }}.tar
+ fi
+ echo "::endgroup::"
+ fi
+ - name: Archive code-checks incremental
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #
v4
+ with:
+ name: ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version
}}
+ path: ~/ci-gradle-caches-${{ inputs.job-name }}-${{
inputs.java-version }}.tar
+ if-no-files-found: ignore
+ retention-days: 1
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
index ae974984f..9ffc77288 100644
--- a/.github/workflows/gradle.yml
+++ b/.github/workflows/gradle.yml
@@ -50,12 +50,19 @@ jobs:
uses:
gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
with:
validate-wrappers: false
+ - name: Prepare Gradle build cache
+ uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Run unit tests
run: |
./gradlew check sourceTarball distTar distZip publishToMavenLocal \
-x :polaris-runtime-service:test \
-x :polaris-admin:test \
-x intTest --continue
+ - name: Save partial Gradle build cache
+ uses: ./.github/actions/ci-incr-build-cache-save
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+ with:
+ job-name: 'unit-tests'
- name: Archive test results
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
# v4
if: always()
@@ -80,12 +87,19 @@ jobs:
uses:
gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
with:
validate-wrappers: false
+ - name: Prepare Gradle build cache
+ uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Run Quarkus tests
run: |
./gradlew \
:polaris-runtime-service:test \
:polaris-admin:test \
--continue
+ - name: Save partial Gradle build cache
+ uses: ./.github/actions/ci-incr-build-cache-save
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+ with:
+ job-name: 'quarkus-tests'
- name: Archive test results
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
# v4
if: always()
@@ -110,8 +124,15 @@ jobs:
uses:
gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
with:
validate-wrappers: false
+ - name: Prepare Gradle build cache
+ uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Run integration tests
run: ./gradlew intTest --continue
+ - name: Save partial Gradle build cache
+ uses: ./.github/actions/ci-incr-build-cache-save
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+ with:
+ job-name: 'integration-tests'
- name: Archive test results
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
# v4
if: always()
@@ -119,3 +140,30 @@ jobs:
name: upload-integration-test-artifacts
path: |
**/build/test-results/**
+
+ store-gradle-cache:
+ name: Store Gradle Cache
+ runs-on: ubuntu-24.04
+ timeout-minutes: 30
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+ needs:
+ - unit-tests
+ - quarkus-tests
+ - integration-tests
+ steps:
+ - name: Set up JDK 21
+ uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
+ with:
+ java-version: '21'
+ distribution: 'temurin'
+ - name: Setup Gradle
+ uses:
gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+ with:
+ validate-wrappers: false
+ - name: Collect partial Gradle build caches
+ uses: ./.github/actions/ci-incr-build-cache-prepare
+ with:
+ cache-read-only: false
+ - name: Trigger Gradle home cleanup
+ run: ./gradlew --no-daemon :showVersion
+ # Note: the "Post Gradle invocation" archives the updated build cache.
diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml
index 70e433baf..e6d531b0e 100644
--- a/.github/workflows/regtest.yml
+++ b/.github/workflows/regtest.yml
@@ -40,6 +40,14 @@ jobs:
java-version: '21'
distribution: 'temurin'
+ - name: Setup Gradle
+ uses:
gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+ with:
+ validate-wrappers: false
+
+ - name: Prepare Gradle build cache
+ uses: ./.github/actions/ci-incr-build-cache-prepare
+
- name: Fix permissions
run: mkdir -p regtests/output && chmod 777 regtests/output && chmod
777 regtests/t_*/ref/*
@@ -58,4 +66,4 @@ jobs:
AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
run: |
- docker compose -f regtests/docker-compose.yml up --build
--exit-code-from regtest
\ No newline at end of file
+ docker compose -f regtests/docker-compose.yml up --build
--exit-code-from regtest
diff --git a/.github/workflows/spark_client_regtests.yml
b/.github/workflows/spark_client_regtests.yml
index c74e3f763..6cb6d9347 100644
--- a/.github/workflows/spark_client_regtests.yml
+++ b/.github/workflows/spark_client_regtests.yml
@@ -40,6 +40,14 @@ jobs:
java-version: '21'
distribution: 'temurin'
+ - name: Setup Gradle
+ uses:
gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+ with:
+ validate-wrappers: false
+
+ - name: Prepare Gradle build cache
+ uses: ./.github/actions/ci-incr-build-cache-prepare
+
- name: Fix permissions
run: mkdir -p regtests/output && chmod 777 regtests/output && chmod
777 regtests/t_*/ref/*
diff --git a/LICENSE b/LICENSE
index dff385d90..6ed4bcbc0 100644
--- a/LICENSE
+++ b/LICENSE
@@ -264,6 +264,8 @@ License: https://www.apache.org/licenses/LICENSE-2.0
This product includes code from Project Nessie.
+* .github/actions/ci-incr-build-cache-prepare/action.yml
+* .github/actions/ci-incr-build-cache-save/action.yml
* build-logic/src/main/kotlin/LicenseFileValidation.kt
* build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerPlugin.kt
* build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerExtension.kt
