This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new a1c6566 feat(ci): Add suite of Docker-based release verification
script tests (#160)
a1c6566 is described below
commit a1c6566cfe0589b81853760b1b154adc559e1b4c
Author: Dewey Dunnington <[email protected]>
AuthorDate: Thu Mar 16 13:31:57 2023 -0400
feat(ci): Add suite of Docker-based release verification script tests (#160)
This PR revises the existing verification action to (1) include MacOS
and Windows and (2) use a Dockerfile/docker-compose setup for a few
extra platforms. This PR doesn't add s390x because the docker build
takes too long with qemu, but it sounds like @assignUser knows a way to
build and push those images separately (in which case the verification
will probably run in a reasonable amount of time).
---------
Co-authored-by: Jacob Wujciak-Jens <[email protected]>
---
.env | 25 ++++++
.github/workflows/docker-build.yaml | 80 +++++++++++++++++
.github/workflows/verify-source.yaml | 76 ----------------
.github/workflows/verify.yaml | 164 +++++++++++++++++++++++++++++++++++
ci/docker/alpine.dockerfile | 36 ++++++++
ci/docker/archlinux.dockerfile | 23 +++++
ci/docker/fedora.dockerfile | 23 +++++
ci/docker/ubuntu.dockerfile | 31 +++++++
docker-compose.yml | 31 +++++++
9 files changed, 413 insertions(+), 76 deletions(-)
diff --git a/.env b/.env
new file mode 100644
index 0000000..e95ee54
--- /dev/null
+++ b/.env
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# All of the following environment variables are required to set default values
+# for the parameters in docker-compose.yml.
+
+# Default repository to pull and push images from
+REPO=ghcr.io/apache/arrow-nanoarrow
+
+# Linux platform (i.e., the dockerfile to use from ci/docker)
+PLATFORM=ubuntu
diff --git a/.github/workflows/docker-build.yaml
b/.github/workflows/docker-build.yaml
new file mode 100644
index 0000000..29d63fc
--- /dev/null
+++ b/.github/workflows/docker-build.yaml
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+name: Build Docker Images
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: '5 0 * * 0'
+ pull_request:
+ paths:
+ - 'ci/docker/*'
+ - '.github/workflow/docker-build.yaml'
+ - 'docker-compose.yaml'
+
+permissions:
+ contents: read
+ packages: write
+
+jobs:
+ build-docker:
+ name: "docker-${{ matrix.config.platform }}-${{
matrix.config.docker_platform }}"
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - { platform: "ubuntu", docker_platform: "amd64" }
+ - { platform: "fedora", docker_platform: "amd64" }
+ - { platform: "archlinux", docker_platform: "amd64" }
+ - { platform: "alpine", docker_platform: "amd64" }
+ - { platform: "alpine", docker_platform: "s390x" }
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set up QEMU
+ if: matrix.config.docker_platform != 'amd64'
+ uses: docker/setup-qemu-action@v2
+
+ - name: Build
+ env:
+ PLATFORM: ${{ matrix.config.platform }}
+ DOCKER_DEFAULT_PLATFORM: "linux/${{ matrix.config.docker_platform }}"
+ run: |
+ docker compose build
+
+ - name: Push
+ if: ${{ github.event_name != 'pull_request' && github.repository ==
'apache/arrow-nanoarrow'}}
+ env:
+ PLATFORM: ${{ matrix.config.platform }}
+ DOCKER_DEFAULT_PLATFORM: "linux/${{ matrix.config.docker_platform }}"
+ run: |
+ docker compose push
diff --git a/.github/workflows/verify-source.yaml
b/.github/workflows/verify-source.yaml
deleted file mode 100644
index 2df69cf..0000000
--- a/.github/workflows/verify-source.yaml
+++ /dev/null
@@ -1,76 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: verify-rc
-
-on:
- push:
- branches:
- - main
- pull_request:
- branches:
- - main
- paths:
- - '.github/workflows/verify-source.yaml'
- - 'dev/release/verify-release-candidate.sh'
-
-jobs:
- verify-rc:
- runs-on: ubuntu-latest
- name: ${{ matrix.config.label }}
-
- steps:
- - name: Checkout repo
- uses: actions/checkout@v3
- with:
- path: src
- fetch-depth: 0
-
- - name: Install dependencies
- run: |
- sudo apt-get update
- sudo apt install -y -V ca-certificates lsb-release wget cmake
- wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id
--short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release
--codename --short).deb
- sudo apt-get install -y -V
./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
- sudo apt-get update
- sudo apt-get install -y -V libarrow-dev
- rm apache-arrow-apt-*.deb
-
- # Needed for R source verification
- - uses: r-lib/actions/setup-tinytex@v2
- - uses: r-lib/actions/setup-pandoc@v2
- - uses: r-lib/actions/setup-r@v2
- with:
- use-public-rspm: true
- - uses: r-lib/actions/setup-r-dependencies@v2
- with:
- needs: check
- working-directory: src/r
-
- - name: Run dev/release/verify-release-candidate.sh
- run: |
- mkdir nanoarrow-verify-tmp
- export NANOARROW_TMPDIR=`pwd`/nanoarrow-verify-tmp
- cd src/dev/release
- ./verify-release-candidate.sh
-
- - name: Upload temp directory
- if: failure()
- uses: actions/upload-artifact@main
- with:
- path: nanoarrow-verify-tmp
- name: nanoarrow-verify-tmp
diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml
new file mode 100644
index 0000000..ccd2a1d
--- /dev/null
+++ b/.github/workflows/verify.yaml
@@ -0,0 +1,164 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Verification
+
+on:
+ workflow_dispatch:
+ inputs:
+ version:
+ description: "Version to verify"
+ required: false
+ type: string
+ default: ""
+ rc:
+ description: "RC to verify"
+ required: false
+ type: string
+ default: ""
+ pull_request:
+ branches:
+ - main
+ paths:
+ - '.github/workflows/verify.yaml'
+ - 'dev/release/verify-release-candidate.sh'
+
+permissions:
+ contents: read
+
+jobs:
+ verify:
+ name: "${{ matrix.config.label }}"
+ runs-on: ${{ matrix.config.os }}
+
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - {os: macOS-latest, label: macos}
+ - {os: windows-latest, label: windows}
+ - {os: ubuntu-latest, label: ubuntu}
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ path: src
+ fetch-depth: 0
+ persist-credentials: false
+
+ # For R verification
+ - uses: r-lib/actions/setup-tinytex@v2
+ - uses: r-lib/actions/setup-pandoc@v2
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ use-public-rspm: true
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ needs: check
+ working-directory: src/r
+
+ - name: Cache Windows Arrow C++ Build
+ if: matrix.config.label == 'windows'
+ id: cache-arrow-build
+ uses: actions/cache@v3
+ with:
+ path: arrow
+ key: arrow-${{ runner.os }}-2
+
+ # For Arrow C++
+ - name: Install Arrow C++ (MacOS)
+ if: matrix.config.label == 'macos'
+ run: brew install apache-arrow
+
+ - name: Install Arrow C++ (Debian)
+ if: matrix.config.label == 'ubuntu'
+ shell: bash
+ run: |
+ sudo apt-get update
+ sudo apt install -y -V ca-certificates lsb-release wget cmake
+ wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id
--short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release
--codename --short).deb
+ sudo apt-get install -y -V
./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+ sudo apt-get update
+ sudo apt-get install -y -V libarrow-dev
+ rm apache-arrow-apt-*.deb
+
+ - name: Build Arrow C++ (Windows)
+ if: matrix.config.label == 'windows' &&
steps.cache-arrow-build.outputs.cache-hit != 'true'
+ shell: bash
+ run: |
+ curl
https://dlcdn.apache.org/arrow/arrow-11.0.0/apache-arrow-11.0.0.tar.gz | \
+ tar -zxf -
+ mkdir arrow-build && cd arrow-build
+ cmake ../apache-arrow-11.0.0/cpp -DCMAKE_INSTALL_PREFIX=../arrow
+ cmake --build .
+ cmake --install . --prefix=../arrow --config=Debug
+ cd ..
+
+ - name: Set CMake options (Windows)
+ if: matrix.config.label == 'windows'
+ shell: bash
+ run: |
+ echo "NANOARROW_CMAKE_OPTIONS=${NANOARROW_CMAKE_OPTIONS}
-DArrow_DIR=$(pwd -W)/arrow/lib/cmake/Arrow -Dgtest_force_shared_crt=ON" >>
$GITHUB_ENV
+
+ - name: Run dev/release/verify-release-candidate.sh
+ shell: bash
+ env:
+ VERBOSE: "1"
+ run: |
+ mkdir nanoarrow-verify-tmp
+ export NANOARROW_TMPDIR=`pwd`/nanoarrow-verify-tmp
+ cd src/dev/release
+ ./verify-release-candidate.sh ${{ inputs.version }} ${{ inputs.rc }}
+
+ - name: Upload temp directory
+ if: failure()
+ uses: actions/upload-artifact@main
+ with:
+ path: nanoarrow-verify-tmp
+ name: nanoarrow-verify-tmp
+
+ verify-docker:
+ name: "docker-${{ matrix.config.platform }}-${{
matrix.config.docker_platform }}"
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - {platform: "ubuntu", docker_platform: "amd64"}
+ - {platform: "fedora", docker_platform: "amd64"}
+ - {platform: "archlinux", docker_platform: "amd64"}
+ - {platform: "alpine", docker_platform: "amd64"}
+ # Skipping until we have pre-built images we can pull
+ # - {platform: "alpine", docker_platform: "s390x"}
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ path: src
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Set up QEMU
+ if: matrix.config.docker_platform != 'amd64'
+ uses: docker/setup-qemu-action@v2
+
+ - name: Run docker-compose verify
+ env:
+ PLATFORM: ${{ matrix.config.platform }}
+ run: |
+ cd src
+ DOCKER_DEFAULT_PLATFORM=linux/${{ matrix.config.docker_platform }}
docker-compose run verify
diff --git a/ci/docker/alpine.dockerfile b/ci/docker/alpine.dockerfile
new file mode 100644
index 0000000..235b887
--- /dev/null
+++ b/ci/docker/alpine.dockerfile
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM alpine:latest
+
+RUN apk add bash linux-headers git cmake R R-dev g++ gnupg curl
+
+# For Arrow C++
+RUN curl
https://dlcdn.apache.org/arrow/arrow-11.0.0/apache-arrow-11.0.0.tar.gz | tar
-zxf - && \
+ mkdir /arrow-build && \
+ cd /arrow-build && \
+ cmake ../apache-arrow-11.0.0/cpp \
+ -DARROW_JEMALLOC=OFF \
+ -DARROW_SIMD_LEVEL=NONE \
+ -DCMAKE_INSTALL_PREFIX=../arrow && \
+ cmake --build . && \
+ cmake --install . --prefix=../arrow
+
+# For R. Note that arrow is not installed (takes too long).
+RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org";)'
+
+ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
diff --git a/ci/docker/archlinux.dockerfile b/ci/docker/archlinux.dockerfile
new file mode 100644
index 0000000..810d00a
--- /dev/null
+++ b/ci/docker/archlinux.dockerfile
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM archlinux:latest
+
+RUN pacman -Syu --noconfirm git gcc make cmake r-base gnupg curl arrow
+
+# For R
+RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org";)'
diff --git a/ci/docker/fedora.dockerfile b/ci/docker/fedora.dockerfile
new file mode 100644
index 0000000..b2d3543
--- /dev/null
+++ b/ci/docker/fedora.dockerfile
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM fedora:latest
+
+RUN dnf install -y git cmake R gnupg curl libarrow-devel glibc-langpack-en
+
+# For R. Note that arrow is not installed (takes too long).
+RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org";)'
diff --git a/ci/docker/ubuntu.dockerfile b/ci/docker/ubuntu.dockerfile
new file mode 100644
index 0000000..cc20346
--- /dev/null
+++ b/ci/docker/ubuntu.dockerfile
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:latest
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
locales git cmake r-base gnupg curl
+RUN locale-gen en_US.UTF-8 && update-locale en_US.UTF-8
+
+# For Arrow C++
+RUN apt-get install -y -V ca-certificates lsb-release wget && \
+ wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short
| tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename
--short).deb && \
+ apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release
--codename --short).deb && \
+ apt-get update && \
+ apt-get install -y -V libarrow-dev
+
+# For R. Note that arrow is not installed (takes too long).
+RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org";)'
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..4f749e0
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: '3.5'
+
+services:
+
+ verify:
+ image: ${REPO}:${PLATFORM}
+ build:
+ context: .
+ cache_from:
+ - ${REPO}:${PLATFORM}
+ dockerfile: ci/docker/${PLATFORM}.dockerfile
+ volumes:
+ - .:/nanoarrow
+ command: "/bin/bash /nanoarrow/dev/release/verify-release-candidate.sh"
