This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new bfa1beef chore(ci): Add integration test CI docker image with all the
arrows (#557)
bfa1beef is described below
commit bfa1beef0904689851fedc7f6614f2238b0ef22e
Author: Dewey Dunnington <[email protected]>
AuthorDate: Wed Jul 17 10:18:53 2024 -0300
chore(ci): Add integration test CI docker image with all the arrows (#557)
This PR adds an integration test run for every commit by caching a
recent docker image build (weekly) containing the built versions of
integration tests from the Arrow monorepo and arrow-rs. This should
result in a CI job with a reasonable run time that provides high-level
testing and type coverage.
This is based on the arrow-rs version of the integration test job (
https://github.com/apache/arrow-rs/blob/6d4e2f2ceaf423031b0bc72f54c547dd77a0ddbb/.github/workflows/integration.yml
).
---------
Co-authored-by: Jacob Wujciak-Jens <[email protected]>
---
.github/workflows/docker-build.yaml | 35 ++++++++++++----------
.github/workflows/integration.yaml | 41 ++++++++++++++++++++++++++
ci/docker/integration.dockerfile | 59 +++++++++++++++++++++++++++++++++++++
docker-compose.yml | 18 +++++++++++
4 files changed, 138 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/docker-build.yaml
b/.github/workflows/docker-build.yaml
index 500a154c..16172536 100644
--- a/.github/workflows/docker-build.yaml
+++ b/.github/workflows/docker-build.yaml
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-name: Build Docker Images
+name: docker-build
on:
workflow_dispatch:
@@ -30,24 +30,29 @@ permissions:
contents: read
packages: write
+concurrency:
+ group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+ cancel-in-progress: true
+
jobs:
build-docker:
- name: "docker-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
+ name: "${{ matrix.config.service }}-${{ matrix.config.platform }}-${{
matrix.config.arch }}"
runs-on: ${{ matrix.config.runs_on }}
strategy:
fail-fast: false
matrix:
config:
- - { runs_on: "ubuntu-latest", platform: "ubuntu", arch: "amd64" }
- - { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64" }
- - { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64" }
- - { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64" }
+ - { runs_on: "ubuntu-latest", platform: "ubuntu", arch: "amd64",
service: "verify" }
+ - { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64",
service: "verify" }
+ - { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64",
service: "verify" }
+ - { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64",
service: "verify" }
+ - { runs_on: "ubuntu-latest", arch: "amd64", platform:
"integration", service: "integration" }
- - { runs_on: ["self-hosted", "arm"], platform: "ubuntu", arch:
"arm64" }
- - { runs_on: ["self-hosted", "arm"], platform: "fedora", arch:
"arm64" }
- - { runs_on: ["self-hosted", "arm"], platform: "alpine", arch:
"arm64" }
+ - { runs_on: ["self-hosted", "arm"], platform: "ubuntu", arch:
"arm64", service: "verify" }
+ - { runs_on: ["self-hosted", "arm"], platform: "fedora", arch:
"arm64", service: "verify" }
+ - { runs_on: ["self-hosted", "arm"], platform: "alpine", arch:
"arm64", service: "verify" }
- - { runs_on: "ubuntu-latest", platform: "alpine", arch: "s390x" }
+ - { runs_on: "ubuntu-latest", platform: "alpine", arch: "s390x",
service: "verify" }
env:
DOCKER_CLIENT_TIMEOUT: 360
@@ -64,12 +69,12 @@ jobs:
docker context create builders
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
+ uses: docker/setup-buildx-action@v3
with:
endpoint: builders
- name: Login to GitHub Container Registry
- uses: docker/login-action@v2
+ uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
@@ -77,7 +82,7 @@ jobs:
- name: Set up QEMU
if: matrix.config.arch == 's390x'
- uses: docker/setup-qemu-action@v2
+ uses: docker/setup-qemu-action@v3
- name: Build
env:
@@ -94,7 +99,7 @@ jobs:
NANOARROW_PLATFORM: ${{ matrix.config.platform }}
DOCKER_DEFAULT_PLATFORM: "linux/${{ matrix.config.arch }}"
run: |
- docker-compose push verify
+ docker-compose push ${{ matrix.config.service }}
# Build some arch-agnostic images for non-verify docker bits
build-docker-manifest:
@@ -102,7 +107,7 @@ jobs:
needs: build-docker
steps:
- name: Login to GitHub Container Registry
- uses: docker/login-action@v2
+ uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
diff --git a/.github/workflows/integration.yaml
b/.github/workflows/integration.yaml
new file mode 100644
index 00000000..8125f261
--- /dev/null
+++ b/.github/workflows/integration.yaml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: integration
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ paths:
+ - .github/workflows/integration.yaml
+ - src/nanoarrow/**
+
+permissions:
+ contents: read
+
+jobs:
+ integration:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Run integration tests
+ run: |
+ echo "::group::Docker Pull"
+ docker compose run --rm -e GITHUB_ACTIONS integration
diff --git a/ci/docker/integration.dockerfile b/ci/docker/integration.dockerfile
new file mode 100644
index 00000000..ac7c7716
--- /dev/null
+++ b/ci/docker/integration.dockerfile
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM apache/arrow-dev:amd64-conda-integration
+
+ENV ARROW_USE_CCACHE=OFF \
+ ARROW_CPP_EXE_PATH=/build/cpp/debug \
+ ARROW_NANOARROW_PATH=/build/nanoarrow \
+ ARROW_RUST_EXE_PATH=/build/rust/debug \
+ BUILD_DOCS_CPP=OFF \
+ ARROW_INTEGRATION_CPP=ON \
+ ARROW_INTEGRATION_CSHARP=ON \
+ ARROW_INTEGRATION_GO=ON \
+ ARROW_INTEGRATION_JAVA=ON \
+ ARROW_INTEGRATION_JS=ON \
+ ARCHERY_INTEGRATION_WITH_NANOARROW="1" \
+ ARCHERY_INTEGRATION_WITH_RUST="1"
+
+# These are necessary because the github runner overrides $HOME
+# https://github.com/actions/runner/issues/863
+ENV RUSTUP_HOME=/root/.rustup
+ENV CARGO_HOME=/root/.cargo
+
+ENV ARROW_USE_CCACHE=OFF
+ENV ARROW_CPP_EXE_PATH=/build/cpp/debug
+ENV ARROW_NANOARROW_PATH=/build/nanoarrow
+ENV ARROW_RUST_EXE_PATH=/build/rust/debug
+ENV BUILD_DOCS_CPP=OFF
+
+# Clone the arrow monorepo
+RUN git clone https://github.com/apache/arrow.git /arrow-integration
--recurse-submodules
+
+# Clone the arrow-rs repo
+RUN git clone https://github.com/apache/arrow-rs /arrow-integration/rust
+
+# Workaround: stable rust is not compatible with glibc provided by the
+# provided arrow docker image https://github.com/apache/arrow/issues/41637
+RUN cd /arrow-integration/rust && rustup override set 1.77
+
+# Build all the integrations except nanoarrow (since we'll do that ourselves
on each run)
+RUN ARCHERY_INTEGRATION_WITH_NANOARROW="0" \
+ conda run --no-capture-output \
+ /arrow-integration/ci/scripts/integration_arrow_build.sh \
+ /arrow-integration \
+ /build
diff --git a/docker-compose.yml b/docker-compose.yml
index 8542ebdc..96defb6e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -46,3 +46,21 @@ services:
- ./ci/scripts/build-docs.sh:/build-docs.sh
- ${NANOARROW_DOCKER_SOURCE_DIR}:/nanoarrow
command: "/bin/bash /build-docs.sh /nanoarrow"
+
+ integration:
+ # Based on an arrow-dev repo image that is amd64 only
+ image: ${REPO}:integration-amd64
+ build:
+ context: .
+ cache_from:
+ - ${REPO}:integration-amd64
+ dockerfile: ci/docker/integration.dockerfile
+ volumes:
+ - ${NANOARROW_DOCKER_SOURCE_DIR}:/arrow-integration/nanoarrow
+ command:
+ ["echo '::group::Build nanoarrow' &&
+ conda run --no-capture-output
/arrow-integration/ci/scripts/nanoarrow_build.sh /arrow-integration /build &&
+ echo '::endgroup::' &&
+ echo '::group::Run integration tests' &&
+ conda run --no-capture-output
/arrow-integration/ci/scripts/integration_arrow.sh /arrow-integration /build &&
+ echo '::endgroup::'"]
