kou commented on code in PR #47600: URL: https://github.com/apache/arrow/pull/47600#discussion_r2362671608
########## .github/workflows/check_labels.yml: ########## @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name: Check Labels Reusable + +on: + workflow_call: + inputs: + label: + description: "Label to check for" + required: true + type: string + parent-workflow: + description: "The parent workflow filename (without .yml)" + required: true + type: string + outputs: + ci-extra: + description: "Whether to run the extra CI" + value: ${{ jobs.check-labels.outputs.ci-extra }} + +jobs: + check-labels: + name: Check labels + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + ci-extra: ${{ steps.check.outputs.ci-extra }} + steps: + - name: Checkout Arrow + if: github.event_name == 'pull_request' + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Check + id: check + env: + GH_TOKEN: ${{ github.token }} + run: | + set -ex + case "${GITHUB_EVENT_NAME}" in + push|schedule) + ci_extra=true Review Comment: Separated output may be easier to use: ```suggestion echo "force=true" >> "${GITHUB_OUTPUT}" ``` ########## .github/workflows/check_labels.yml: ########## @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name: Check Labels Reusable + +on: + workflow_call: + inputs: + label: + description: "Label to check for" + required: true + type: string + parent-workflow: + description: "The parent workflow filename (without .yml)" + required: true + type: string + outputs: + ci-extra: + description: "Whether to run the extra CI" + value: ${{ jobs.check-labels.outputs.ci-extra }} + +jobs: + check-labels: + name: Check labels + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + ci-extra: ${{ steps.check.outputs.ci-extra }} + steps: + - name: Checkout Arrow + if: github.event_name == 'pull_request' + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Check + id: check + env: + GH_TOKEN: ${{ github.token }} + run: | + set -ex + case "${GITHUB_EVENT_NAME}" in + push|schedule) + ci_extra=true + ;; + pull_request) + n_ci_extra_labels=$( + gh pr view ${{ github.event.number }} \ + --jq '.labels[].name | select(. == "${{ inputs.label }}")' \ + --json labels \ + --repo ${GITHUB_REPOSITORY} | wc -l) + if [ "${n_ci_extra_labels}" -eq 1 ]; then + ci_extra=true + else + git fetch origin ${GITHUB_BASE_REF} + git diff --stat origin/${GITHUB_BASE_REF}.. + if git diff --stat origin/${GITHUB_BASE_REF}.. | \ + grep \ + --fixed-strings ".github/workflows/${{ inputs.parent-workflow }}.yml" \ + --quiet; then + ci_extra=true Review Comment: ```suggestion echo "force=true" >> "${GITHUB_OUTPUT}" ``` ########## .github/workflows/linux_packaging.yml: ########## @@ -0,0 +1,435 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux Packaging Extra + +on: + push: + branches: + - '**' + - '!dependabot/**' + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + tags: + - '**' + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + version: + description: "The Arrow version" + type: string + required: true + no_rc_version: + description: "The Arrow version without RC" + type: string + required: true + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-labels: + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + label: "CI: Extra: Linux Packaging" + parent-workflow: linux_packaging + + package: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: needs.check-labels.outputs.ci-extra == 'true' + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-8 + title: AlmaLinux 8 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-8 + title: AlmaLinux 8 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-9 + title: AlmaLinux 9 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-9 + title: AlmaLinux 9 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-10 + title: AlmaLinux 10 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-10 + title: AlmaLinux 10 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: amazon-linux-2023 + title: Amazon Linux 2023 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: amazon-linux-2023 + title: Amazon Linux 2023 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-7 + title: CentOS 7 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-9-stream + title: CentOS 9 Stream AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: centos-9-stream + title: CentOS 9 Stream ARM64 + task-namespace: yum + upload-extensions: + - rpm Review Comment: We can simplify this: ```suggestion id: - almalinux-8-amd64 - almalinux-8-arm64 - almalinux-9-amd64 - almalinux-9-arm64 - almalinux-10-amd64 - almalinux-10-arm64 ... ``` ```yaml title: ${{ matrix.id }} runs-on: ${{ contains(matrix.id, 'amd64') && 'ubuntu-latest' || 'ubuntu-24.04-arm' }} ``` ```yaml name: Prepare environment variables env: ID: ${{ matrix.id }} run: | # almalinux-8-amd64 -> amd64 architecture="${ID##*-}" echo "ARCHITECTURE=${architecture}" >> "${GITHUB_ENV}" # almalinux-8-amd64 -> almalinux-8 target="${ID%-*}" case "${target}" in almalinux-*|amazon-linux-*|centos-*) echo "TASK_NAMESPACE=yum" >> "${GITHUB_ENV}" echo "YUM_TARGETS=${target}" >> "${GITHUB_ENV}" ;; *) echo "TASK_NAMESPACE=apt" >> "${GITHUB_ENV}" echo "APT_TARGETS=${target}" >> "${GITHUB_ENV}" ;; esac ``` ########## .github/workflows/linux_packaging.yml: ########## @@ -0,0 +1,435 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux Packaging Extra Review Comment: We don't need "Extra" here because there is no "Extra" version. ########## .github/workflows/cpp_extra.yml: ########## @@ -75,47 +79,11 @@ permissions: jobs: check-labels: - name: Check labels - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - ci-extra: ${{ steps.check.outputs.ci-extra }} - steps: - - name: Checkout Arrow - if: github.event_name == 'pull_request' - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Check - id: check - env: - GH_TOKEN: ${{ github.token }} - run: | - case "${GITHUB_EVENT_NAME}" in - push|schedule) - ci_extra=true - ;; - pull_request) - n_ci_extra_labels=$( - gh pr view ${{ github.event.number }} \ - --jq '.labels[].name | select(. == "CI: Extra")' \ - --json labels \ - --repo ${GITHUB_REPOSITORY} | wc -l) - if [ "${n_ci_extra_labels}" -eq 1 ]; then - ci_extra=true - else - git fetch origin ${GITHUB_BASE_REF} - if git diff --stat origin/${GITHUB_BASE_REF}.. | \ - grep \ - --fixed-strings ".github/workflows/cpp_extra.yml" \ - --quiet; then - ci_extra=true - else - ci_extra=false - fi - fi - ;; - esac - - echo "ci-extra=${ci_extra}" >> "${GITHUB_OUTPUT}" + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + label: "CI: Extra" Review Comment: I like `CI: Extra: C++`/`CI: Extra: Package: Linux` or `CI: cpp`/`CI: package: linux`/`CI: package: linux: apt`. ########## .github/workflows/check_labels.yml: ########## @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name: Check Labels Reusable Review Comment: ```suggestion name: Check Labels Reusable ``` ########## .github/workflows/check_labels.yml: ########## @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name: Check Labels Reusable + +on: + workflow_call: + inputs: + label: + description: "Label to check for" + required: true + type: string + parent-workflow: + description: "The parent workflow filename (without .yml)" + required: true + type: string + outputs: + ci-extra: + description: "Whether to run the extra CI" + value: ${{ jobs.check-labels.outputs.ci-extra }} + +jobs: + check-labels: + name: Check labels + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + ci-extra: ${{ steps.check.outputs.ci-extra }} + steps: + - name: Checkout Arrow + if: github.event_name == 'pull_request' + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Check + id: check + env: + GH_TOKEN: ${{ github.token }} + run: | + set -ex + case "${GITHUB_EVENT_NAME}" in + push|schedule) + ci_extra=true + ;; + pull_request) + n_ci_extra_labels=$( + gh pr view ${{ github.event.number }} \ + --jq '.labels[].name | select(. == "${{ inputs.label }}")' \ + --json labels \ + --repo ${GITHUB_REPOSITORY} | wc -l) Review Comment: How about detecting all `CI: Extra` labels? ```suggestion { echo "ci-extra-labels<<LABELS" gh pr view ${{ github.event.number }} \ --jq '[.labels[].name | select(startswith("CI: Extra"))]' \ --json labels \ --repo ${GITHUB_REPOSITORY} echo "LABELS" } >> "${GITHUB_OUTPUT}" ``` We can use it in parent workflow by: ```yaml if: >- needs.check-labels.outputs.force == 'true' || fromJSON(needs.check-labels.outputs.ci-extra-labels, 'CI: Extra') || fromJSON(needs.check-labels.outputs.ci-extra-labels, 'CI: Extra: C++') ``` ########## .github/workflows/linux_packaging.yml: ########## @@ -0,0 +1,435 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux Packaging Extra + +on: + push: + branches: + - '**' + - '!dependabot/**' + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + tags: + - '**' + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + version: + description: "The Arrow version" + type: string + required: true + no_rc_version: + description: "The Arrow version without RC" + type: string + required: true + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-labels: + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + label: "CI: Extra: Linux Packaging" + parent-workflow: linux_packaging + + package: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: needs.check-labels.outputs.ci-extra == 'true' + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-8 + title: AlmaLinux 8 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-8 + title: AlmaLinux 8 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-9 + title: AlmaLinux 9 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-9 + title: AlmaLinux 9 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-10 + title: AlmaLinux 10 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-10 + title: AlmaLinux 10 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: amazon-linux-2023 + title: Amazon Linux 2023 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: amazon-linux-2023 + title: Amazon Linux 2023 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-7 + title: CentOS 7 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-9-stream + title: CentOS 9 Stream AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: centos-9-stream + title: CentOS 9 Stream ARM64 + task-namespace: yum + upload-extensions: + - rpm + env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARROW_VERSION: ${{ inputs.version || ''}} + NO_RC_VERSION: ${{ inputs.no_rc_version || ''}} + steps: + - name: Checkout Arrow + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + if: runner.os == 'Linux' && runner.arch == 'X64' + shell: bash + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: .docker + key: extra-${{ matrix.target }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.target }}- Review Comment: ```suggestion key: package-linux-${{ matrix.target }}-${{ hashFiles('cpp/**') }} restore-keys: package-linux-${{ matrix.target }}- ``` ########## .github/workflows/linux_packaging.yml: ########## Review Comment: How about `package_linux.yml`? We may add `package_conan.yml`, `package_homeber.yml` and so on later. ########## .github/workflows/linux_packaging.yml: ########## @@ -0,0 +1,435 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux Packaging Extra + +on: + push: + branches: + - '**' + - '!dependabot/**' + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + tags: + - '**' + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + version: + description: "The Arrow version" + type: string + required: true + no_rc_version: + description: "The Arrow version without RC" + type: string + required: true + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-labels: + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + label: "CI: Extra: Linux Packaging" + parent-workflow: linux_packaging + + package: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: needs.check-labels.outputs.ci-extra == 'true' + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-8 + title: AlmaLinux 8 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-8 + title: AlmaLinux 8 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-9 + title: AlmaLinux 9 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-9 + title: AlmaLinux 9 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-10 + title: AlmaLinux 10 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-10 + title: AlmaLinux 10 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: amazon-linux-2023 + title: Amazon Linux 2023 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: amazon-linux-2023 + title: Amazon Linux 2023 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-7 + title: CentOS 7 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-9-stream + title: CentOS 9 Stream AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: centos-9-stream + title: CentOS 9 Stream ARM64 + task-namespace: yum + upload-extensions: + - rpm + env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARROW_VERSION: ${{ inputs.version || ''}} + NO_RC_VERSION: ${{ inputs.no_rc_version || ''}} + steps: + - name: Checkout Arrow + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + if: runner.os == 'Linux' && runner.arch == 'X64' + shell: bash + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: .docker + key: extra-${{ matrix.target }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.target }}- + - name: Set up Ruby + run: | + sudo apt update + sudo apt install -y \ + rake \ + ruby \ + ruby-dev + - name: Get Arrow Version + id: get-arrow-version + run: | + # In case of dev (no workflow_dispatch inputs) compute ARROW_VERSION and + # use the same version for NO_RC_VERSION. + if [ -z "${ARROW_VERSION}" ]; then + ARROW_VERSION=$(git describe --tags --abbrev=0 --match "apache-arrow-[0-9]*.*" | sed 's/^apache-arrow-//') + ARROW_VERSION="${ARROW_VERSION}$(date +%Y%m%d)" + echo "ARROW_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + echo "NO_RC_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + fi + - name: Build + run: | + set -e + pushd dev/tasks/linux-packages + rake version:update ARROW_RELEASE_TIME="$(date --iso-8601=seconds)" + rake docker:pull || : + rake --trace ${{ matrix.task-namespace }}:build BUILD_DIR=build + popd + env: + APT_TARGETS: ${{ matrix.target }} + # TODO: Investigate what is this REPO variable and where is coming from + REPO: ${{ secrets.REPO }} + YUM_TARGETS: ${{ matrix.target }} + - name: Login to Dockerhub + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} Review Comment: If we use `ghcr.io`, we can use `${{ github.actor }}` and `${{ secrets.GITHUB_TOKEN}}` for `username` and `password`. ########## .github/workflows/linux_packaging.yml: ########## @@ -0,0 +1,435 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux Packaging Extra + +on: + push: + branches: + - '**' + - '!dependabot/**' + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + tags: + - '**' + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + version: + description: "The Arrow version" + type: string + required: true + no_rc_version: + description: "The Arrow version without RC" + type: string + required: true + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-labels: + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + label: "CI: Extra: Linux Packaging" + parent-workflow: linux_packaging + + package: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: needs.check-labels.outputs.ci-extra == 'true' + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-8 + title: AlmaLinux 8 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-8 + title: AlmaLinux 8 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-9 + title: AlmaLinux 9 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-9 + title: AlmaLinux 9 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-10 + title: AlmaLinux 10 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-10 + title: AlmaLinux 10 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: amazon-linux-2023 + title: Amazon Linux 2023 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: amazon-linux-2023 + title: Amazon Linux 2023 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-7 + title: CentOS 7 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-9-stream + title: CentOS 9 Stream AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: centos-9-stream + title: CentOS 9 Stream ARM64 + task-namespace: yum + upload-extensions: + - rpm + env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARROW_VERSION: ${{ inputs.version || ''}} + NO_RC_VERSION: ${{ inputs.no_rc_version || ''}} + steps: + - name: Checkout Arrow + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + if: runner.os == 'Linux' && runner.arch == 'X64' + shell: bash + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: .docker + key: extra-${{ matrix.target }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.target }}- + - name: Set up Ruby + run: | + sudo apt update + sudo apt install -y \ + rake \ + ruby \ + ruby-dev + - name: Get Arrow Version + id: get-arrow-version + run: | + # In case of dev (no workflow_dispatch inputs) compute ARROW_VERSION and + # use the same version for NO_RC_VERSION. + if [ -z "${ARROW_VERSION}" ]; then + ARROW_VERSION=$(git describe --tags --abbrev=0 --match "apache-arrow-[0-9]*.*" | sed 's/^apache-arrow-//') + ARROW_VERSION="${ARROW_VERSION}$(date +%Y%m%d)" + echo "ARROW_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + echo "NO_RC_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + fi + - name: Build + run: | + set -e + pushd dev/tasks/linux-packages + rake version:update ARROW_RELEASE_TIME="$(date --iso-8601=seconds)" + rake docker:pull || : + rake --trace ${{ matrix.task-namespace }}:build BUILD_DIR=build + popd + env: + APT_TARGETS: ${{ matrix.target }} + # TODO: Investigate what is this REPO variable and where is coming from + REPO: ${{ secrets.REPO }} Review Comment: This is for the Docker image name part. We can use `ghcr.io/${{ github.repository }}-package-linux` here: ```suggestion REPO: ghcr.io/${{ github.repository }}-package-linux ``` BTW, could you improve method name? ```diff diff --git a/dev/tasks/linux-packages/helper.rb b/dev/tasks/linux-packages/helper.rb index 0354188046..ee132f12e9 100644 --- a/dev/tasks/linux-packages/helper.rb +++ b/dev/tasks/linux-packages/helper.rb @@ -72,13 +72,13 @@ module Helper raise "Failed to detect #{name} environment variable" end - def detect_repo + def docker_image_name detect_env("REPO") end def docker_image(os, architecture) architecture ||= "amd64" - "#{detect_repo}:#{architecture}-#{os}-package-#{@package}" + "#{docker_image_name}:#{architecture}-#{os}-package-#{@package}" end end end ``` ########## .github/workflows/linux_packaging.yml: ########## @@ -0,0 +1,435 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux Packaging Extra + +on: + push: + branches: + - '**' + - '!dependabot/**' + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + tags: + - '**' + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + version: + description: "The Arrow version" + type: string + required: true + no_rc_version: + description: "The Arrow version without RC" + type: string + required: true + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-labels: + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + label: "CI: Extra: Linux Packaging" + parent-workflow: linux_packaging + + package: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: needs.check-labels.outputs.ci-extra == 'true' + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-8 + title: AlmaLinux 8 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-8 + title: AlmaLinux 8 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-9 + title: AlmaLinux 9 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-9 + title: AlmaLinux 9 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-10 + title: AlmaLinux 10 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-10 + title: AlmaLinux 10 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: amazon-linux-2023 + title: Amazon Linux 2023 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: amazon-linux-2023 + title: Amazon Linux 2023 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-7 + title: CentOS 7 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-9-stream + title: CentOS 9 Stream AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: centos-9-stream + title: CentOS 9 Stream ARM64 + task-namespace: yum + upload-extensions: + - rpm + env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARROW_VERSION: ${{ inputs.version || ''}} + NO_RC_VERSION: ${{ inputs.no_rc_version || ''}} + steps: + - name: Checkout Arrow + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + if: runner.os == 'Linux' && runner.arch == 'X64' + shell: bash + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: .docker + key: extra-${{ matrix.target }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.target }}- + - name: Set up Ruby + run: | + sudo apt update + sudo apt install -y \ + rake \ + ruby \ + ruby-dev + - name: Get Arrow Version + id: get-arrow-version + run: | + # In case of dev (no workflow_dispatch inputs) compute ARROW_VERSION and + # use the same version for NO_RC_VERSION. + if [ -z "${ARROW_VERSION}" ]; then + ARROW_VERSION=$(git describe --tags --abbrev=0 --match "apache-arrow-[0-9]*.*" | sed 's/^apache-arrow-//') + ARROW_VERSION="${ARROW_VERSION}$(date +%Y%m%d)" + echo "ARROW_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + echo "NO_RC_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + fi + - name: Build + run: | + set -e + pushd dev/tasks/linux-packages + rake version:update ARROW_RELEASE_TIME="$(date --iso-8601=seconds)" + rake docker:pull || : + rake --trace ${{ matrix.task-namespace }}:build BUILD_DIR=build + popd + env: + APT_TARGETS: ${{ matrix.target }} + # TODO: Investigate what is this REPO variable and where is coming from + REPO: ${{ secrets.REPO }} + YUM_TARGETS: ${{ matrix.target }} + - name: Login to Dockerhub + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Docker Push + continue-on-error: true + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + shell: bash + run: | + pushd dev/tasks/linux-packages + rake docker:push + popd + env: + APT_TARGETS: ${{ matrix.target }} + REPO: ${{ secrets.REPO }} + YUM_TARGETS: ${{ matrix.target }} + - name: Build artifact paths + id: artifact-paths + shell: bash + run: | + paths="" + for ext in ${{ join(matrix.upload-extensions, ' ') }}; do + paths="$paths dev/tasks/linux-packages/*/*/repositories/**/*.${ext}" + done + echo $paths + echo "paths=$paths" >> $GITHUB_OUTPUT Review Comment: We can upload all files under `dev/tasks/linux-packages/*/*/repositories`: ```suggestion pushd dev/tasks/linux-packages tar cvzf ${{ matrix.id }}.tar.gz */${{ matrix.task_namespace }}/repositories mv ${{ matrix.id }}.tar.gz # We can upload only this tar.gz popd ``` ########## .github/workflows/report_ci.yml: ########## @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name: Report CI results Review Comment: ```suggestion name: Report CI results ``` ########## dev/release/04-binary-download.sh: ########## @@ -39,3 +42,11 @@ crossbow_job_prefix="release-${version_with_rc}" : ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-${CROSSBOW_JOB_NUMBER}"} archery crossbow download-artifacts --no-fetch ${CROSSBOW_JOB_ID} "$@" + +# Wait for the GitHub Workflow that creates the Linux packages +# to finish before downloading the artifacts. +. "${SOURCE_DIR}/utils-watch-gh-workflow.sh" "${release_tag}" "linux_packaging.yml" + +RUN_ID=$(get_run_id) +# Download the artifacts created by the linux_packaging.yml workflow +download_artifacts "${SOURCE_DIR}/../../packages/${CROSSBOW_JOB_ID}" Review Comment: Not flat folder is better. The current flat folder is a limitation of Crossbow. Our binary upload script reconstruct the original repository structure from a flat folder: https://github.com/apache/arrow/blob/7ef564884547e9de6d34ccd25d3bc6992111b92c/dev/release/binary-task.rb#L1666-L1675 If we can keep the repository structure, we can simplify the binary upload script. ########## .github/workflows/linux_packaging.yml: ########## @@ -0,0 +1,435 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux Packaging Extra + +on: + push: + branches: + - '**' + - '!dependabot/**' + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + tags: + - '**' + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + version: + description: "The Arrow version" + type: string + required: true + no_rc_version: + description: "The Arrow version without RC" + type: string + required: true + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-labels: + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + label: "CI: Extra: Linux Packaging" + parent-workflow: linux_packaging + + package: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: needs.check-labels.outputs.ci-extra == 'true' + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-8 + title: AlmaLinux 8 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-8 + title: AlmaLinux 8 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-9 + title: AlmaLinux 9 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-9 + title: AlmaLinux 9 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-10 + title: AlmaLinux 10 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-10 + title: AlmaLinux 10 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: amazon-linux-2023 + title: Amazon Linux 2023 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: amazon-linux-2023 + title: Amazon Linux 2023 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-7 + title: CentOS 7 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-9-stream + title: CentOS 9 Stream AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: centos-9-stream + title: CentOS 9 Stream ARM64 + task-namespace: yum + upload-extensions: + - rpm + env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARROW_VERSION: ${{ inputs.version || ''}} + NO_RC_VERSION: ${{ inputs.no_rc_version || ''}} + steps: + - name: Checkout Arrow + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + if: runner.os == 'Linux' && runner.arch == 'X64' + shell: bash + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: .docker + key: extra-${{ matrix.target }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.target }}- + - name: Set up Ruby + run: | + sudo apt update + sudo apt install -y \ + rake \ + ruby \ + ruby-dev + - name: Get Arrow Version + id: get-arrow-version + run: | + # In case of dev (no workflow_dispatch inputs) compute ARROW_VERSION and + # use the same version for NO_RC_VERSION. + if [ -z "${ARROW_VERSION}" ]; then + ARROW_VERSION=$(git describe --tags --abbrev=0 --match "apache-arrow-[0-9]*.*" | sed 's/^apache-arrow-//') + ARROW_VERSION="${ARROW_VERSION}$(date +%Y%m%d)" + echo "ARROW_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + echo "NO_RC_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + fi + - name: Build + run: | + set -e + pushd dev/tasks/linux-packages + rake version:update ARROW_RELEASE_TIME="$(date --iso-8601=seconds)" + rake docker:pull || : + rake --trace ${{ matrix.task-namespace }}:build BUILD_DIR=build + popd + env: + APT_TARGETS: ${{ matrix.target }} + # TODO: Investigate what is this REPO variable and where is coming from + REPO: ${{ secrets.REPO }} + YUM_TARGETS: ${{ matrix.target }} + - name: Login to Dockerhub + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Docker Push + continue-on-error: true + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + shell: bash + run: | + pushd dev/tasks/linux-packages + rake docker:push + popd + env: + APT_TARGETS: ${{ matrix.target }} + REPO: ${{ secrets.REPO }} + YUM_TARGETS: ${{ matrix.target }} + - name: Build artifact paths + id: artifact-paths + shell: bash + run: | + paths="" + for ext in ${{ join(matrix.upload-extensions, ' ') }}; do + paths="$paths dev/tasks/linux-packages/*/*/repositories/**/*.${ext}" + done + echo $paths + echo "paths=$paths" >> $GITHUB_OUTPUT + - name: Upload the artifacts to the job + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ matrix.target }}-${{ matrix.architecture }} + path: ${{ steps.artifact-paths.outputs.paths }} + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts-downloaded + - name: Expected uploaded yum artifacts + if: matrix.task-namespace == 'yum' + id: expected-yum-artifacts + run: | + set -ex + artifacts=" + apache-arrow-release-${NO_RC_VERSION}-1.[a-z0-9]+.noarch.rpm + apache-arrow-release-${NO_RC_VERSION}-1.[a-z0-9]+.src.rpm + arrow-${NO_RC_VERSION}-1.[a-z0-9]+.src.rpm + arrow[0-9]+-acero-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-acero-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-compute-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-compute-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-dataset-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-dataset-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-dataset-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-dataset-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-dataset-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-tools-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet[0-9]+-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet[0-9]+-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-tools-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + " + if [ "${{ matrix.target }}" != "centos-7" ]; then + artifacts="$artifacts + arrow[0-9]+-acero-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-compute-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-dataset-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-dataset-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-debugsource-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-sql-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-sql-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-sql-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-sql-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-sql-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-sql-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-sql-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-tools-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva[0-9]+-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva[0-9]+-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva[0-9]+-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva[0-9]+-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet[0-9]+-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet[0-9]+-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-tools-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm" + fi + echo $artifacts + { + echo 'artifacts<<EOF' + echo "$artifacts" + echo 'EOF' + } >> $GITHUB_OUTPUT + - name: Validate uploaded yum artifacts + if: matrix.task-namespace == 'yum' + id: validate-artifacts + shell: bash + run: | + expected_patterns="${{ steps.expected-yum-artifacts.outputs.artifacts }}" + + # Find all RPM files in the extracted directory + found_files=$(find artifacts-downloaded/${{ matrix.target }}-${{ matrix.architecture }} -type f -name '*.rpm' | sort) + + # Prepare expected patterns for matching + echo "$expected_patterns" | grep -v '^\s*$' > expected_patterns.txt + + # Check for missing artifacts + missing="" + for pattern in $(cat expected_patterns.txt); do + # Escape dots for grep + grep_pattern=$(echo "$pattern" | sed 's/\./\\./g') + match=$(echo "$found_files" | grep -E "$grep_pattern") + if [ -z "$match" ]; then + missing="$missing\n$pattern" + fi + done + + # Check for unexpected artifacts + unexpected="" + for file in $found_files; do + matched=false + for pattern in $(cat expected_patterns.txt); do + grep_pattern=$(echo "$pattern" | sed 's/\./\\./g') + if echo "$file" | grep -qE "$grep_pattern"; then + matched=true + break + fi + done + if [ "$matched" = false ]; then + unexpected="$unexpected\n$file" + fi + done + + if [ -n "$missing" ]; then + echo "Missing expected artifacts:" + echo -e "$missing" + exit 1 + fi + + if [ -n "$unexpected" ]; then + echo "Unexpected artifacts found:" + echo -e "$unexpected" + exit 1 + fi + + echo "All expected artifacts are present, and no unexpected artifacts found." + - name: Set up test + run: | + sudo apt install -y \ + apt-utils \ + cpio \ + createrepo-c \ + devscripts \ + gpg \ + rpm \ + rsync + gem install --user-install apt-dists-merge + (echo "Key-Type: RSA"; \ + echo "Key-Length: 4096"; \ + echo "Name-Real: Test"; \ + echo "Name-Email: [email protected]"; \ + echo "%no-protection") | \ + gpg --full-generate-key --batch + GPG_KEY_ID=$(gpg --list-keys --with-colon [email protected] | grep fpr | cut -d: -f10) + echo "GPG_KEY_ID=${GPG_KEY_ID}" >> ${GITHUB_ENV} + case "${{ matrix.target }}" in + almalinux-*|amazon-linux-*|centos-*) + repositories_dir=dev/tasks/linux-packages/apache-arrow-release/yum/repositories + rpm2cpio ${repositories_dir}/*/*/*/Packages/apache-arrow-release-*.rpm | \ + cpio -id + mv etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow \ + dev/tasks/linux-packages/KEYS + ;; + esac + gpg --export --armor [email protected] >> dev/tasks/linux-packages/KEYS + - name: Test + run: | + set -e + pushd dev/tasks/linux-packages + rake --trace ${{ matrix.task-namespace }}:test + rm -rf ${{ matrix.task-namespace }}/repositories Review Comment: This is needless. ```suggestion ``` ########## .github/workflows/linux_packaging.yml: ########## @@ -0,0 +1,368 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux Packaging Extra + +on: + push: + branches: + - '**' + - '!dependabot/**' + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + tags: + - '**' + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + version: + description: "The Arrow version" + type: string + required: true + no_rc_version: + description: "The Arrow version without RC" + type: string + required: true + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-labels: + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + label: "CI: Extra: Linux Packaging" + parent-workflow: linux_packaging + + package: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: needs.check-labels.outputs.ci-extra == 'true' + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-8 + title: AlmaLinux 8 AMD64 + task-namespace: yum + upload-extensions: + - rpm + env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARROW_VERSION: ${{ inputs.version || ''}} + NO_RC_VERSION: ${{ inputs.no_rc_version || ''}} + steps: + - name: Checkout Arrow + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + if: runner.os == 'Linux' && runner.arch == 'X64' + shell: bash + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: .docker + key: extra-${{ matrix.target }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.target }}- + - name: Set up Ruby + run: | + sudo apt update + sudo apt install -y \ + rake \ + ruby \ + ruby-dev + - name: Get Arrow Version + id: get-arrow-version + run: | + # In case of dev (no workflow_dispatch inputs) compute ARROW_VERSION and + # use the same version for NO_RC_VERSION. + if [ -z "${ARROW_VERSION}" ]; then + ARROW_VERSION=$(git describe --tags --abbrev=0 --match "apache-arrow-[0-9]*.*" | sed 's/^apache-arrow-//') + ARROW_VERSION="${ARROW_VERSION}$(date +%Y%m%d)" + echo "ARROW_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + echo "NO_RC_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + fi + - name: Build + run: | + set -e + pushd dev/tasks/linux-packages + rake version:update ARROW_RELEASE_TIME="$(date --iso-8601=seconds)" + rake docker:pull || : + rake --trace ${{ matrix.task-namespace }}:build BUILD_DIR=build + popd + env: + APT_TARGETS: ${{ matrix.target }} + # TODO: Investigate what is this REPO variable and where is coming from + REPO: ${{ secrets.REPO }} + YUM_TARGETS: ${{ matrix.target }} + - name: Login to Dockerhub + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Docker Push + continue-on-error: true + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + shell: bash + run: | + pushd dev/tasks/linux-packages + rake docker:push + popd + env: + APT_TARGETS: ${{ matrix.target }} + REPO: ${{ secrets.REPO }} + YUM_TARGETS: ${{ matrix.target }} + - name: Build artifact paths + id: artifact-paths + shell: bash + run: | + paths="" + for ext in ${{ join(matrix.upload-extensions, ' ') }}; do + paths="$paths dev/tasks/linux-packages/*/*/repositories/**/*.${ext}" + done + echo $paths + echo "paths=$paths" >> $GITHUB_OUTPUT + - name: Upload the artifacts to the job + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ matrix.target }}-${{ matrix.architecture }} + path: ${{ steps.artifact-paths.outputs.paths }} + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts-downloaded + - name: Expected uploaded yum artifacts Review Comment: Ah, we don't need to validate target files. Our tests will detect missing files. ########## .github/workflows/linux_packaging.yml: ########## @@ -0,0 +1,435 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux Packaging Extra + +on: + push: + branches: + - '**' + - '!dependabot/**' + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + tags: + - '**' + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/linux_packaging.yml' + - '.github/workflows/report_ci.yml' + - 'cpp/**' + - 'c_glib/**' + - 'dev/tasks/linux-packages/**' + - 'format/Flight.proto' + - 'testing' + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + version: + description: "The Arrow version" + type: string + required: true + no_rc_version: + description: "The Arrow version without RC" + type: string + required: true + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-labels: + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + label: "CI: Extra: Linux Packaging" + parent-workflow: linux_packaging + + package: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: needs.check-labels.outputs.ci-extra == 'true' + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-8 + title: AlmaLinux 8 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-8 + title: AlmaLinux 8 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-9 + title: AlmaLinux 9 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-9 + title: AlmaLinux 9 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: almalinux-10 + title: AlmaLinux 10 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: almalinux-10 + title: AlmaLinux 10 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: amazon-linux-2023 + title: Amazon Linux 2023 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: amazon-linux-2023 + title: Amazon Linux 2023 ARM64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-7 + title: CentOS 7 AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: amd64 + runs-on: ubuntu-latest + target: centos-9-stream + title: CentOS 9 Stream AMD64 + task-namespace: yum + upload-extensions: + - rpm + - architecture: arm64 + runs-on: ubuntu-24.04-arm + target: centos-9-stream + title: CentOS 9 Stream ARM64 + task-namespace: yum + upload-extensions: + - rpm + env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARROW_VERSION: ${{ inputs.version || ''}} + NO_RC_VERSION: ${{ inputs.no_rc_version || ''}} + steps: + - name: Checkout Arrow + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + if: runner.os == 'Linux' && runner.arch == 'X64' + shell: bash + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: .docker + key: extra-${{ matrix.target }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.target }}- + - name: Set up Ruby + run: | + sudo apt update + sudo apt install -y \ + rake \ + ruby \ + ruby-dev + - name: Get Arrow Version + id: get-arrow-version + run: | + # In case of dev (no workflow_dispatch inputs) compute ARROW_VERSION and + # use the same version for NO_RC_VERSION. + if [ -z "${ARROW_VERSION}" ]; then + ARROW_VERSION=$(git describe --tags --abbrev=0 --match "apache-arrow-[0-9]*.*" | sed 's/^apache-arrow-//') + ARROW_VERSION="${ARROW_VERSION}$(date +%Y%m%d)" + echo "ARROW_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + echo "NO_RC_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV + fi + - name: Build + run: | + set -e + pushd dev/tasks/linux-packages + rake version:update ARROW_RELEASE_TIME="$(date --iso-8601=seconds)" + rake docker:pull || : + rake --trace ${{ matrix.task-namespace }}:build BUILD_DIR=build + popd + env: + APT_TARGETS: ${{ matrix.target }} + # TODO: Investigate what is this REPO variable and where is coming from + REPO: ${{ secrets.REPO }} + YUM_TARGETS: ${{ matrix.target }} + - name: Login to Dockerhub + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Docker Push + continue-on-error: true + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + shell: bash + run: | + pushd dev/tasks/linux-packages + rake docker:push + popd + env: + APT_TARGETS: ${{ matrix.target }} + REPO: ${{ secrets.REPO }} + YUM_TARGETS: ${{ matrix.target }} + - name: Build artifact paths + id: artifact-paths + shell: bash + run: | + paths="" + for ext in ${{ join(matrix.upload-extensions, ' ') }}; do + paths="$paths dev/tasks/linux-packages/*/*/repositories/**/*.${ext}" + done + echo $paths + echo "paths=$paths" >> $GITHUB_OUTPUT + - name: Upload the artifacts to the job + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ matrix.target }}-${{ matrix.architecture }} + path: ${{ steps.artifact-paths.outputs.paths }} + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts-downloaded + - name: Expected uploaded yum artifacts + if: matrix.task-namespace == 'yum' + id: expected-yum-artifacts + run: | + set -ex + artifacts=" + apache-arrow-release-${NO_RC_VERSION}-1.[a-z0-9]+.noarch.rpm + apache-arrow-release-${NO_RC_VERSION}-1.[a-z0-9]+.src.rpm + arrow-${NO_RC_VERSION}-1.[a-z0-9]+.src.rpm + arrow[0-9]+-acero-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-acero-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-compute-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-compute-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-dataset-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-dataset-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-dataset-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-dataset-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-dataset-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-tools-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet[0-9]+-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet[0-9]+-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-tools-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + " + if [ "${{ matrix.target }}" != "centos-7" ]; then + artifacts="$artifacts + arrow[0-9]+-acero-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-compute-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-dataset-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-dataset-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-debugsource-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-sql-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-sql-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-flight-sql-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-sql-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-sql-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-sql-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-flight-sql-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow[0-9]+-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + arrow-tools-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva-glib-devel-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva-glib-doc-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva[0-9]+-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva[0-9]+-glib-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva[0-9]+-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + gandiva[0-9]+-libs-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet[0-9]+-glib-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet[0-9]+-libs-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm + parquet-tools-debuginfo-${NO_RC_VERSION}-1.[a-z0-9]+.[a-z0-9_]+.rpm" + fi + echo $artifacts + { + echo 'artifacts<<EOF' + echo "$artifacts" + echo 'EOF' + } >> $GITHUB_OUTPUT + - name: Validate uploaded yum artifacts + if: matrix.task-namespace == 'yum' + id: validate-artifacts + shell: bash + run: | + expected_patterns="${{ steps.expected-yum-artifacts.outputs.artifacts }}" + + # Find all RPM files in the extracted directory + found_files=$(find artifacts-downloaded/${{ matrix.target }}-${{ matrix.architecture }} -type f -name '*.rpm' | sort) + + # Prepare expected patterns for matching + echo "$expected_patterns" | grep -v '^\s*$' > expected_patterns.txt + + # Check for missing artifacts + missing="" + for pattern in $(cat expected_patterns.txt); do + # Escape dots for grep + grep_pattern=$(echo "$pattern" | sed 's/\./\\./g') + match=$(echo "$found_files" | grep -E "$grep_pattern") + if [ -z "$match" ]; then + missing="$missing\n$pattern" + fi + done + + # Check for unexpected artifacts + unexpected="" + for file in $found_files; do + matched=false + for pattern in $(cat expected_patterns.txt); do + grep_pattern=$(echo "$pattern" | sed 's/\./\\./g') + if echo "$file" | grep -qE "$grep_pattern"; then + matched=true + break + fi + done + if [ "$matched" = false ]; then + unexpected="$unexpected\n$file" + fi + done + + if [ -n "$missing" ]; then + echo "Missing expected artifacts:" + echo -e "$missing" + exit 1 + fi + + if [ -n "$unexpected" ]; then + echo "Unexpected artifacts found:" + echo -e "$unexpected" + exit 1 + fi + + echo "All expected artifacts are present, and no unexpected artifacts found." + - name: Set up test + run: | + sudo apt install -y \ Review Comment: ```suggestion pushd dev/tasks/linux-packages rm -rf */${{ matrix.task-namespace }}/repositories # Remove artifacts tar xf ${{ matrix.id }}.tar.gz # Use uploaded artifacts popd sudo apt install -y \ ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
