This is an automated email from the ASF dual-hosted git repository. xxubai pushed a commit to branch split-ci-matrix in repository https://gitbox.apache.org/repos/asf/amoro.git
commit 8227dcdad0ed5a43771d08675d208ef929c864a9 Author: xuba <[email protected]> AuthorDate: Wed Sep 23 17:32:06 2026 +0800 [AMORO-3909][CI] Split workflows by module and compatibility matrix --- .github/workflows/ams-ci.yml | 80 ++++++++++ .github/workflows/core-hadoop2-ci.yml | 63 -------- .github/workflows/core-hadoop3-ci.yml | 69 --------- .github/workflows/dist-ci.yml | 95 ++++++++++++ .../workflows/flink-mixed-format-connector-ci.yml | 123 +++++++++++++++ .github/workflows/format-integrations-ci.yml | 117 ++++++++++++++ .github/workflows/frontend-ci.yml | 81 ++++++++++ .github/workflows/maven-static-checks.yml | 69 +++++++++ .github/workflows/optimizer-ci.yml | 171 +++++++++++++++++++++ .github/workflows/site-ci.yml | 69 +++++++++ .../workflows/spark-mixed-format-connector-ci.yml | 146 ++++++++++++++++++ .github/workflows/trino-ci.yml | 89 +++++++---- dev/ci/has-changes.sh | 32 ++++ 13 files changed, 1043 insertions(+), 161 deletions(-) diff --git a/.github/workflows/ams-ci.yml b/.github/workflows/ams-ci.yml new file mode 100644 index 000000000..b82f598cc --- /dev/null +++ b/.github/workflows/ams-ci.yml @@ -0,0 +1,80 @@ +name: AMS + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "master" + - "0.**" + pull_request: + branches: + - "master" + - "0.**" + workflow_dispatch: + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect AMS changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '^amoro-ams/|^amoro-common/|^amoro-format-iceberg/|^amoro-format-mixed/amoro-mixed-hive/|^amoro-optimizer/amoro-optimizer-(common|standalone)/|^pom\.xml$|^tools/maven/|^dev/ci/|^\.github/workflows/ams-ci\.yml$'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + + build: + needs: changes + if: needs.changes.outputs.changed == 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + jdk: [ '11', '17' ] + name: AMS / Spark 3.5 / Hadoop 3 / JDK ${{ matrix.jdk }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + - name: Verify AMS + run: >- + ./mvnw -B -ntp verify + -pl amoro-ams -am + -Pjava${{ matrix.jdk }},spark-3.5,skip-dashboard-build + - name: Upload AMS coverage + uses: codecov/codecov-action@v5 + with: + flags: ams + + gate: + name: CI / AMS + if: always() + needs: + - changes + - build + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.build.result }}" == "success" || "${{ needs.build.result }}" == "skipped" ]] diff --git a/.github/workflows/core-hadoop2-ci.yml b/.github/workflows/core-hadoop2-ci.yml deleted file mode 100644 index ad891366c..000000000 --- a/.github/workflows/core-hadoop2-ci.yml +++ /dev/null @@ -1,63 +0,0 @@ -# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: Core/hadoop2 CI with Maven - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: - - "master" - - "0.**" - - pull_request: - branches: - - "master" - - "0.**" - paths: - - ".github/workflows/**" - - "amoro-ams/**" - - "amoro-common/**" - - "amoro-metrics/**" - - "amoro-optimizer/**" - - "amoro-format-iceberg/**" - - "amoro-format-paimon/**" - - "amoro-format-hudi/**" - - "amoro-format-mixed/amoro-mixed-flink/**" - - "amoro-format-mixed/amoro-mixed-hive/**" - - "amoro-format-mixed/amoro-mixed-spark/**" - - "pom.xml" - - "tools/maven/**" - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - jdk: [ '11' ] - name: Build Amoro with JDK ${{ matrix.jdk }} - steps: - - uses: actions/checkout@v3 - - - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - cache: maven - check-latest: false - - - name: Validate checkstyle first - run: ./mvnw validate - - - name: Build all modules with Maven - run: ./mvnw clean install -Djacoco.flink.skip=true -B -ntp -Phadoop2 -Pjava${{ matrix.jdk }} - - - name: Code coverage - uses: codecov/codecov-action@v3 - with: - verbose: true - flags: core diff --git a/.github/workflows/core-hadoop3-ci.yml b/.github/workflows/core-hadoop3-ci.yml deleted file mode 100644 index aebb929f0..000000000 --- a/.github/workflows/core-hadoop3-ci.yml +++ /dev/null @@ -1,69 +0,0 @@ -# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: Core/hadoop3 CI with Maven - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: - - "master" - - "0.**" - - pull_request: - branches: - - "master" - - "0.**" - paths: - - ".github/workflows/**" - - "amoro-ams/**" - - "amoro-common/**" - - "amoro-metrics/**" - - "amoro-optimizer/**" - - "amoro-format-iceberg/**" - - "amoro-format-paimon/**" - - "amoro-format-hudi/**" - - "amoro-format-mixed/amoro-mixed-flink/**" - - "amoro-format-mixed/amoro-mixed-hive/**" - - "amoro-format-mixed/amoro-mixed-spark/**" - - "pom.xml" - - "tools/maven/**" - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - jdk: [ '11', '17' ] - spark: [ '3.3','3.4', '3.5' ] - exclude: - - jdk: '17' - spark: '3.3' - - jdk: '17' - spark: '3.4' - name: Build Amoro with JDK ${{ matrix.jdk }} Spark-${{ matrix.spark }} - steps: - - uses: actions/checkout@v3 - - - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - cache: maven - check-latest: false - - - name: Validate checkstyle first - run: ./mvnw validate - - - name: Build all modules with Maven - run: ./mvnw clean install -Djacoco.flink.skip=true -B -ntp -Pjava${{ matrix.jdk }} -Pspark-${{ matrix.spark }} - - - name: Code coverage - uses: codecov/codecov-action@v3 - with: - verbose: true - flags: core diff --git a/.github/workflows/dist-ci.yml b/.github/workflows/dist-ci.yml new file mode 100644 index 000000000..8b457b1f6 --- /dev/null +++ b/.github/workflows/dist-ci.yml @@ -0,0 +1,95 @@ +name: Dist + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "master" + - "0.**" + pull_request: + branches: + - "master" + - "0.**" + workflow_dispatch: + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect distribution changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '^dist/|^amoro-(ams|common|metrics|optimizer|web)/|^amoro-format-|^pom\.xml$|^tools/maven/|^dev/ci/|^\.github/workflows/dist-ci\.yml$'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + + build: + needs: changes + if: needs.changes.outputs.changed == 'true' + runs-on: ubuntu-latest + name: Dist / Spark 3.5 / Hadoop 3 / JDK 17 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: temurin + cache: maven + - name: Verify distribution + run: ./mvnw -B -ntp verify -pl dist -am -Pjava17,spark-3.5 + - name: Verify archive contents + shell: bash + run: | + archive=$(find dist/target -maxdepth 1 -name '*-bin.tar.gz' -print -quit) + test -n "${archive}" + extract_dir="${RUNNER_TEMP}/amoro-dist" + mkdir -p "${extract_dir}" + tar -xzf "${archive}" -C "${extract_dir}" + + test -n "$(find "${extract_dir}" -name 'amoro-ams-*.jar' -print -quit)" + test -n "$(find "${extract_dir}" -path '*/plugin/optimizer/flink/optimizer-job.jar' -print -quit)" + test -n "$(find "${extract_dir}" -path '*/plugin/optimizer/spark/optimizer-job.jar' -print -quit)" + + while IFS= read -r script; do + bash -n "${script}" + done < <(find "${extract_dir}" -path '*/bin/*.sh' -type f) + + web_jar=$(find "${extract_dir}" -name 'amoro-web-*.jar' -print -quit) + test -n "${web_jar}" + jar tf "${web_jar}" > "${RUNNER_TEMP}/amoro-web-contents.txt" + grep -q '^static/' "${RUNNER_TEMP}/amoro-web-contents.txt" + + if find "${extract_dir}" -iname '*flink-shaded-hadoop-2-uber*' -print -quit | grep -q .; then + echo "Hadoop 2 shaded runtime must not be packaged in the Hadoop 3 distribution" >&2 + exit 1 + fi + + gate: + name: CI / Dist + if: always() + needs: + - changes + - build + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.build.result }}" == "success" || "${{ needs.build.result }}" == "skipped" ]] diff --git a/.github/workflows/flink-mixed-format-connector-ci.yml b/.github/workflows/flink-mixed-format-connector-ci.yml new file mode 100644 index 000000000..ba1abfb07 --- /dev/null +++ b/.github/workflows/flink-mixed-format-connector-ci.yml @@ -0,0 +1,123 @@ +name: Flink Mixed Format Connector + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "master" + - "0.**" + pull_request: + branches: + - "master" + - "0.**" + schedule: + - cron: "20 18 * * *" + workflow_dispatch: + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect Flink connector changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '^amoro-common/|^amoro-format-iceberg/|^amoro-format-paimon/|^amoro-format-mixed/amoro-mixed-(flink|hive)/|^pom\.xml$|^tools/maven/|^dev/ci/|^\.github/workflows/flink-mixed-format-connector-ci\.yml$'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + + pr-build: + needs: changes + if: needs.changes.outputs.changed == 'true' && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - flink: '1.18' + hadoop: '2' + jdk: '11' + profiles: hadoop2,java11 + - flink: '1.19' + hadoop: '3' + jdk: '11' + profiles: java11 + - flink: '1.20' + hadoop: '3' + jdk: '17' + profiles: java17 + name: Flink ${{ matrix.flink }} / Hadoop ${{ matrix.hadoop }} / JDK ${{ matrix.jdk }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + - name: Verify Flink mixed-format runtime + run: >- + ./mvnw -B -ntp verify + -pl amoro-format-mixed/amoro-mixed-flink/v${{ matrix.flink }}/amoro-mixed-flink-runtime-${{ matrix.flink }} + -am -P${{ matrix.profiles }} + - name: Upload Flink coverage + uses: codecov/codecov-action@v5 + with: + flags: flink + + nightly-build: + needs: changes + if: needs.changes.outputs.changed == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + flink: [ '1.18', '1.19', '1.20' ] + hadoop: [ '2', '3' ] + jdk: [ '11', '17' ] + exclude: + - hadoop: '2' + jdk: '17' + name: Flink full / ${{ matrix.flink }} / Hadoop ${{ matrix.hadoop }} / JDK ${{ matrix.jdk }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + - name: Verify Flink mixed-format runtime + run: >- + ./mvnw -B -ntp verify + -pl amoro-format-mixed/amoro-mixed-flink/v${{ matrix.flink }}/amoro-mixed-flink-runtime-${{ matrix.flink }} + -am -Pjava${{ matrix.jdk }}${{ matrix.hadoop == '2' && ',hadoop2' || '' }} + + gate: + name: CI / Flink Mixed Format Connector + if: always() + needs: + - changes + - pr-build + - nightly-build + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.pr-build.result }}" == "success" || "${{ needs.pr-build.result }}" == "skipped" ]] + [[ "${{ needs.nightly-build.result }}" == "success" || "${{ needs.nightly-build.result }}" == "skipped" ]] diff --git a/.github/workflows/format-integrations-ci.yml b/.github/workflows/format-integrations-ci.yml new file mode 100644 index 000000000..3ee3c3714 --- /dev/null +++ b/.github/workflows/format-integrations-ci.yml @@ -0,0 +1,117 @@ +name: Format Integrations + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "master" + - "0.**" + pull_request: + branches: + - "master" + - "0.**" + schedule: + - cron: "20 20 * * *" + workflow_dispatch: + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect format changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '^amoro-common/|^amoro-format-(hudi|paimon|lance)/|^pom\.xml$|^tools/maven/|^dev/ci/|^\.github/workflows/format-integrations-ci\.yml$'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + + pr-build: + needs: changes + if: needs.changes.outputs.changed == 'true' && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - hadoop: '2' + jdk: '11' + profiles: hadoop2,java11 + - hadoop: '3' + jdk: '17' + profiles: java17 + name: Formats / Hadoop ${{ matrix.hadoop }} / JDK ${{ matrix.jdk }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + - name: Verify optional formats + run: >- + ./mvnw -B -ntp verify + -pl amoro-format-hudi,amoro-format-paimon,amoro-format-lance + -am -P${{ matrix.profiles }} + + nightly-build: + needs: changes + if: needs.changes.outputs.changed == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - hadoop: '2' + jdk: '11' + profiles: hadoop2,java11 + - hadoop: '3' + jdk: '11' + profiles: java11 + - hadoop: '3' + jdk: '17' + profiles: java17 + name: Formats full / Hadoop ${{ matrix.hadoop }} / JDK ${{ matrix.jdk }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + - name: Verify optional formats + run: >- + ./mvnw -B -ntp verify + -pl amoro-format-hudi,amoro-format-paimon,amoro-format-lance + -am -P${{ matrix.profiles }} + + gate: + name: CI / Format Integrations + if: always() + needs: + - changes + - pr-build + - nightly-build + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.pr-build.result }}" == "success" || "${{ needs.pr-build.result }}" == "skipped" ]] + [[ "${{ needs.nightly-build.result }}" == "success" || "${{ needs.nightly-build.result }}" == "skipped" ]] diff --git a/.github/workflows/frontend-ci.yml b/.github/workflows/frontend-ci.yml new file mode 100644 index 000000000..62b33e509 --- /dev/null +++ b/.github/workflows/frontend-ci.yml @@ -0,0 +1,81 @@ +name: Frontend + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "master" + - "0.**" + pull_request: + branches: + - "master" + - "0.**" + workflow_dispatch: + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect frontend changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '^amoro-web/|^dev/ci/|^\.github/workflows/frontend-ci\.yml$'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + + build: + needs: changes + if: needs.changes.outputs.changed == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: amoro-web + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22.23.2' + - name: Enable pnpm + run: | + corepack enable + corepack prepare [email protected] --activate + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Lint (non-blocking until the existing baseline is clean) + continue-on-error: true + run: pnpm lint + - name: Type check (non-blocking until the existing baseline is clean) + continue-on-error: true + run: pnpm typecheck + - name: Build dashboard + run: pnpm build + + gate: + name: CI / Frontend + if: always() + needs: + - changes + - build + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.build.result }}" == "success" || "${{ needs.build.result }}" == "skipped" ]] diff --git a/.github/workflows/maven-static-checks.yml b/.github/workflows/maven-static-checks.yml new file mode 100644 index 000000000..85226a9df --- /dev/null +++ b/.github/workflows/maven-static-checks.yml @@ -0,0 +1,69 @@ +name: Maven Static Checks + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "master" + - "0.**" + pull_request: + branches: + - "master" + - "0.**" + workflow_dispatch: + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect Maven source changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '(^|/)pom\.xml$|\.java$|^tools/maven/|^dev/ci/|^\.github/workflows/'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + + validate: + needs: changes + if: needs.changes.outputs.changed == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: temurin + cache: maven + - name: Validate Maven project + run: ./mvnw -B -ntp validate -Pjava11 + + gate: + name: CI / Maven Static Checks + if: always() + needs: + - changes + - validate + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.validate.result }}" == "success" || "${{ needs.validate.result }}" == "skipped" ]] diff --git a/.github/workflows/optimizer-ci.yml b/.github/workflows/optimizer-ci.yml new file mode 100644 index 000000000..18b3d3fd9 --- /dev/null +++ b/.github/workflows/optimizer-ci.yml @@ -0,0 +1,171 @@ +name: Optimizer + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "master" + - "0.**" + pull_request: + branches: + - "master" + - "0.**" + schedule: + - cron: "20 21 * * *" + workflow_dispatch: + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect optimizer changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '^amoro-optimizer/|^amoro-common/|^amoro-format-iceberg/|^amoro-format-mixed/amoro-mixed-hive/|^pom\.xml$|^tools/maven/|^dev/ci/|^\.github/workflows/optimizer-ci\.yml$'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + + pr-build: + needs: changes + if: needs.changes.outputs.changed == 'true' && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - implementation: standalone + module: amoro-optimizer/amoro-optimizer-standalone + engine: Java + jdk: '11' + profiles: java11 + - implementation: flink + module: amoro-optimizer/amoro-optimizer-flink + engine: Flink-1.18.1 + jdk: '11' + profiles: java11 + - implementation: spark + module: amoro-optimizer/amoro-optimizer-spark + engine: Spark-3.3 + jdk: '11' + profiles: java11,spark-3.3 + - implementation: spark + module: amoro-optimizer/amoro-optimizer-spark + engine: Spark-3.4 + jdk: '11' + profiles: java11,spark-3.4 + - implementation: spark + module: amoro-optimizer/amoro-optimizer-spark + engine: Spark-3.5 + jdk: '17' + profiles: java17,spark-3.5 + name: ${{ matrix.implementation }} / ${{ matrix.engine }} / JDK ${{ matrix.jdk }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + - name: Verify optimizer + run: >- + ./mvnw -B -ntp verify + -pl ${{ matrix.module }} -am + -P${{ matrix.profiles }} + - name: Upload optimizer coverage + uses: codecov/codecov-action@v5 + with: + flags: optimizer + + nightly-build: + needs: changes + if: needs.changes.outputs.changed == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - implementation: standalone + module: amoro-optimizer/amoro-optimizer-standalone + engine: Java + jdk: '11' + profiles: java11 + - implementation: standalone + module: amoro-optimizer/amoro-optimizer-standalone + engine: Java + jdk: '17' + profiles: java17 + - implementation: flink + module: amoro-optimizer/amoro-optimizer-flink + engine: Flink-1.18.1 + jdk: '11' + profiles: java11 + - implementation: flink + module: amoro-optimizer/amoro-optimizer-flink + engine: Flink-1.18.1 + jdk: '17' + profiles: java17 + - implementation: spark + module: amoro-optimizer/amoro-optimizer-spark + engine: Spark-3.3 + jdk: '11' + profiles: java11,spark-3.3 + - implementation: spark + module: amoro-optimizer/amoro-optimizer-spark + engine: Spark-3.4 + jdk: '11' + profiles: java11,spark-3.4 + - implementation: spark + module: amoro-optimizer/amoro-optimizer-spark + engine: Spark-3.5 + jdk: '11' + profiles: java11,spark-3.5 + - implementation: spark + module: amoro-optimizer/amoro-optimizer-spark + engine: Spark-3.5 + jdk: '17' + profiles: java17,spark-3.5 + name: Full / ${{ matrix.implementation }} / ${{ matrix.engine }} / JDK ${{ matrix.jdk }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + - name: Verify optimizer + run: >- + ./mvnw -B -ntp verify + -pl ${{ matrix.module }} -am + -P${{ matrix.profiles }} + + gate: + name: CI / Optimizer + if: always() + needs: + - changes + - pr-build + - nightly-build + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.pr-build.result }}" == "success" || "${{ needs.pr-build.result }}" == "skipped" ]] + [[ "${{ needs.nightly-build.result }}" == "success" || "${{ needs.nightly-build.result }}" == "skipped" ]] diff --git a/.github/workflows/site-ci.yml b/.github/workflows/site-ci.yml new file mode 100644 index 000000000..fb858b70e --- /dev/null +++ b/.github/workflows/site-ci.yml @@ -0,0 +1,69 @@ +name: Site Validation + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "master" + - "0.**" + pull_request: + branches: + - "master" + - "0.**" + workflow_dispatch: + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect site changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '^site/|^docs/|^dev/ci/|^\.github/workflows/site-ci\.yml$'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + + build: + needs: changes + if: needs.changes.outputs.changed == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: peaceiris/actions-hugo@v2 + with: + hugo-version: latest + extended: true + - name: Build project site + run: hugo --source site/amoro-site --minify + - name: Build documentation site + run: hugo --source site/amoro-docs --minify + + gate: + name: CI / Site Validation + if: always() + needs: + - changes + - build + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.build.result }}" == "success" || "${{ needs.build.result }}" == "skipped" ]] diff --git a/.github/workflows/spark-mixed-format-connector-ci.yml b/.github/workflows/spark-mixed-format-connector-ci.yml new file mode 100644 index 000000000..2ae143865 --- /dev/null +++ b/.github/workflows/spark-mixed-format-connector-ci.yml @@ -0,0 +1,146 @@ +name: Spark Mixed Format Connector + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "master" + - "0.**" + pull_request: + branches: + - "master" + - "0.**" + schedule: + - cron: "20 19 * * *" + workflow_dispatch: + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect Spark connector changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '^amoro-common/|^amoro-format-iceberg/|^amoro-format-paimon/|^amoro-format-mixed/amoro-mixed-(spark|hive)/|^pom\.xml$|^tools/maven/|^dev/ci/|^\.github/workflows/spark-mixed-format-connector-ci\.yml$'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + + pr-build: + needs: changes + if: needs.changes.outputs.changed == 'true' && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - spark: '3.3' + hadoop: '2' + jdk: '11' + profiles: hadoop2,java11,spark-3.3 + - spark: '3.4' + hadoop: '3' + jdk: '11' + profiles: java11,spark-3.4 + - spark: '3.5' + hadoop: '3' + jdk: '17' + profiles: java17,spark-3.5 + name: Spark ${{ matrix.spark }} / Hadoop ${{ matrix.hadoop }} / JDK ${{ matrix.jdk }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + - name: Verify Spark mixed-format runtime + run: >- + ./mvnw -B -ntp verify + -pl amoro-format-mixed/amoro-mixed-spark/v${{ matrix.spark }}/amoro-mixed-spark-runtime-${{ matrix.spark }} + -am -P${{ matrix.profiles }} + - name: Upload Spark coverage + uses: codecov/codecov-action@v5 + with: + flags: spark + + nightly-build: + needs: changes + if: needs.changes.outputs.changed == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - spark: '3.3' + hadoop: '2' + jdk: '11' + profiles: hadoop2,java11,spark-3.3 + - spark: '3.4' + hadoop: '2' + jdk: '11' + profiles: hadoop2,java11,spark-3.4 + - spark: '3.5' + hadoop: '2' + jdk: '11' + profiles: hadoop2,java11,spark-3.5 + - spark: '3.3' + hadoop: '3' + jdk: '11' + profiles: java11,spark-3.3 + - spark: '3.4' + hadoop: '3' + jdk: '11' + profiles: java11,spark-3.4 + - spark: '3.5' + hadoop: '3' + jdk: '11' + profiles: java11,spark-3.5 + - spark: '3.5' + hadoop: '3' + jdk: '17' + profiles: java17,spark-3.5 + name: Spark full / ${{ matrix.spark }} / Hadoop ${{ matrix.hadoop }} / JDK ${{ matrix.jdk }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + - name: Verify Spark mixed-format runtime + run: >- + ./mvnw -B -ntp verify + -pl amoro-format-mixed/amoro-mixed-spark/v${{ matrix.spark }}/amoro-mixed-spark-runtime-${{ matrix.spark }} + -am -P${{ matrix.profiles }} + + gate: + name: CI / Spark Mixed Format Connector + if: always() + needs: + - changes + - pr-build + - nightly-build + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.pr-build.result }}" == "success" || "${{ needs.pr-build.result }}" == "skipped" ]] + [[ "${{ needs.nightly-build.result }}" == "success" || "${{ needs.nightly-build.result }}" == "skipped" ]] diff --git a/.github/workflows/trino-ci.yml b/.github/workflows/trino-ci.yml index 921aaf1e9..a01f8e2d3 100644 --- a/.github/workflows/trino-ci.yml +++ b/.github/workflows/trino-ci.yml @@ -1,7 +1,4 @@ -# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: Trino CI with Maven +name: Trino Mixed Format Connector concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -12,42 +9,76 @@ on: branches: - "master" - "0.**" - pull_request: branches: - "master" - "0.**" - paths: - - ".github/workflows/**" - - "amoro-ams/**" - - "amoro-common/**" - - "amoro-format-iceberg/**" - - "amoro-format-mixed/amoro-mixed-hive/**" - - "amoro-format-mixed/amoro-mixed-trino/**" - - "pom.xml" - - "tools/maven/**" + workflow_dispatch: jobs: + changes: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.filter.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: filter + name: Detect Trino connector changes + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + elif bash dev/ci/has-changes.sh \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}" \ + '^amoro-common/|^amoro-format-iceberg/|^amoro-format-mixed/amoro-mixed-(hive|trino)/|^pom\.xml$|^tools/maven/|^dev/ci/|^\.github/workflows/trino-ci\.yml$'; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + fi + build: + needs: changes + if: needs.changes.outputs.changed == 'true' runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - hadoop: '2' + profiles: hadoop2,java17,format-mixed-format-trino,build-mixed-format-trino + - hadoop: '3' + profiles: java17,format-mixed-format-trino,build-mixed-format-trino + name: Trino 406 / Hadoop ${{ matrix.hadoop }} / JDK 17 steps: - - uses: actions/checkout@v3 - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: java-version: '17' - distribution: 'temurin' + distribution: temurin cache: maven - check-latest: false - - - name: Validate checkstyle first - run: ./mvnw validate -P format-mixed-format-trino - - - name: Build trino module with Maven - run: ./mvnw clean install -pl 'amoro-format-mixed/amoro-mixed-trino' -am -B -P hadoop2,format-mixed-format-trino,build-mixed-format-trino - - - name: Code coverage - uses: codecov/codecov-action@v3 + - name: Verify Trino mixed-format connector + run: >- + ./mvnw -B -ntp verify + -pl amoro-format-mixed/amoro-mixed-trino + -am -P${{ matrix.profiles }} + - name: Upload Trino coverage + uses: codecov/codecov-action@v5 with: - verbose: true flags: trino + + gate: + name: CI / Trino Mixed Format Connector + if: always() + needs: + - changes + - build + runs-on: ubuntu-latest + steps: + - name: Check results + shell: bash + run: | + [[ "${{ needs.changes.result }}" == "success" ]] + [[ "${{ needs.build.result }}" == "success" || "${{ needs.build.result }}" == "skipped" ]] diff --git a/dev/ci/has-changes.sh b/dev/ci/has-changes.sh new file mode 100644 index 000000000..f13924e52 --- /dev/null +++ b/dev/ci/has-changes.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 <base-revision> <head-revision> <path-regex>" >&2 + exit 2 +fi + +base_revision=$1 +head_revision=$2 +path_regex=$3 + +changed_files=$(git diff --name-only "${base_revision}" "${head_revision}") +grep -Eq "${path_regex}" <<< "${changed_files}"
