This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new 43001a98 chore(ci): Fix failing weekly jobs (#603)
43001a98 is described below
commit 43001a98a37b602952a1e3b4254a63b70921c62b
Author: Dewey Dunnington <[email protected]>
AuthorDate: Wed Sep 11 14:35:28 2024 -0300
chore(ci): Fix failing weekly jobs (#603)
This PR:
- Updates the integration test workflow to not cache a weekly build,
which was failing with "no space left on device". It also runs more
tests than are strictly necessary at the commit level (and takes
slightly longer without the cached weekly build), so this PR makes it
part of a weekly run instead.
- Updates the alpine dockerfile to avoid a `desc` dependency, since this
fails to install on s390x
- Update two tests to account for a function that returns the value of
`errno` (that might not be the same everywhere)
It also updates the weekly jobs for consistency:
- Updates the weekly jobs to always run on the `maint-**` branches
- Update all of them to cancel previous runs of the same workflow
(because many of them have long-running jobs)
It doesn't fix IPC writing on big endian (a different kind of failure
that I'll punt to another PR).
---------
Co-authored-by: Jacob Wujciak-Jens <[email protected]>
---
.github/workflows/docker-build.yaml | 1 -
.github/workflows/integration.yaml | 11 +++++++++--
.github/workflows/python-wheels.yaml | 1 +
.github/workflows/verify.yaml | 7 +++++++
ci/docker/alpine.dockerfile | 3 +--
docker-compose.yml | 8 ++------
src/nanoarrow/ipc/reader_test.cc | 1 +
src/nanoarrow/ipc/writer_test.cc | 1 +
8 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/docker-build.yaml
b/.github/workflows/docker-build.yaml
index 4e9cea12..fb60aa91 100644
--- a/.github/workflows/docker-build.yaml
+++ b/.github/workflows/docker-build.yaml
@@ -46,7 +46,6 @@ jobs:
- { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64",
service: "verify" }
- { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64",
service: "verify" }
- { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64",
service: "verify" }
- - { runs_on: "ubuntu-latest", arch: "amd64", platform:
"integration", service: "integration" }
- { runs_on: ["self-hosted", "arm"], platform: "ubuntu", arch:
"arm64", service: "verify" }
- { runs_on: ["self-hosted", "arm"], platform: "fedora", arch:
"arm64", service: "verify" }
diff --git a/.github/workflows/integration.yaml
b/.github/workflows/integration.yaml
index 8125f261..8a2e9d63 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -21,14 +21,21 @@ on:
push:
branches:
- main
+ - 'maint-*'
pull_request:
paths:
- .github/workflows/integration.yaml
- - src/nanoarrow/**
+ - docker-compose.yml
+ schedule:
+ - cron: '5 0 * * 0'
permissions:
contents: read
+concurrency:
+ group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+ cancel-in-progress: true
+
jobs:
integration:
runs-on: ubuntu-latest
@@ -37,5 +44,5 @@ jobs:
- name: Run integration tests
run: |
- echo "::group::Docker Pull"
+ echo "::group::Docker Build"
docker compose run --rm -e GITHUB_ACTIONS integration
diff --git a/.github/workflows/python-wheels.yaml
b/.github/workflows/python-wheels.yaml
index 3e8b723a..acb9e0a6 100644
--- a/.github/workflows/python-wheels.yaml
+++ b/.github/workflows/python-wheels.yaml
@@ -31,6 +31,7 @@ on:
push:
branches:
- main
+ - 'maint-**'
workflow_dispatch:
permissions:
diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml
index 0ff5b796..b754594e 100644
--- a/.github/workflows/verify.yaml
+++ b/.github/workflows/verify.yaml
@@ -18,6 +18,9 @@
name: Verification
on:
+ push:
+ branches:
+ - 'maint-**'
workflow_dispatch:
inputs:
version:
@@ -43,6 +46,10 @@ on:
permissions:
contents: read
+concurrency:
+ group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+ cancel-in-progress: true
+
jobs:
verify:
name: "${{ matrix.config.label }}"
diff --git a/ci/docker/alpine.dockerfile b/ci/docker/alpine.dockerfile
index e4d2c58e..1c9d5137 100644
--- a/ci/docker/alpine.dockerfile
+++ b/ci/docker/alpine.dockerfile
@@ -33,9 +33,8 @@ ENV NANOARROW_PYTHON_VENV "/venv"
# For R. Note that arrow is not installed (takes too long).
RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
-RUN R -e 'install.packages("desc", repos = "https://cloud.r-project.org";)' &&
mkdir /tmp/rdeps
COPY r/DESCRIPTION /tmp/rdeps
-RUN R -e
'install.packages(setdiff(desc::desc("/tmp/rdeps")$get_deps()$package,
"arrow"), repos = "https://cloud.r-project.org";)'
+RUN R -e 'gsub("\\(.*?\\)", "", read.dcf("/tmp/rdeps")[1, "Suggests"]) |>
strsplit("[^A-Za-z0-9.]+") |> unlist(use.names = FALSE) |> setdiff("arrow") |>
install.packages(repos = "https://cloud.r-project.org";)'
RUN rm -f ~/.R/Makevars
ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
diff --git a/docker-compose.yml b/docker-compose.yml
index 96defb6e..ca629170 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -15,8 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-version: '3.5'
-
services:
verify:
image: ${REPO}:${NANOARROW_PLATFORM}-${NANOARROW_ARCH}
@@ -48,12 +46,10 @@ services:
command: "/bin/bash /build-docs.sh /nanoarrow"
integration:
- # Based on an arrow-dev repo image that is amd64 only
- image: ${REPO}:integration-amd64
+ # Don't cache here (building the image takes about the same amount of time
+ # as downloading on CI)
build:
context: .
- cache_from:
- - ${REPO}:integration-amd64
dockerfile: ci/docker/integration.dockerfile
volumes:
- ${NANOARROW_DOCKER_SOURCE_DIR}:/arrow-integration/nanoarrow
diff --git a/src/nanoarrow/ipc/reader_test.cc b/src/nanoarrow/ipc/reader_test.cc
index 4af8d096..9dda66fd 100644
--- a/src/nanoarrow/ipc/reader_test.cc
+++ b/src/nanoarrow/ipc/reader_test.cc
@@ -101,6 +101,7 @@ TEST(NanoarrowIpcReader, InputStreamBuffer) {
TEST(NanoarrowIpcReader, InputStreamFile) {
struct ArrowIpcInputStream stream;
+ errno = EINVAL;
ASSERT_EQ(ArrowIpcInputStreamInitFile(&stream, nullptr, 1), EINVAL);
uint8_t input_data[] = {0x01, 0x02, 0x03, 0x04, 0x05};
diff --git a/src/nanoarrow/ipc/writer_test.cc b/src/nanoarrow/ipc/writer_test.cc
index 5eed0809..4cee5618 100644
--- a/src/nanoarrow/ipc/writer_test.cc
+++ b/src/nanoarrow/ipc/writer_test.cc
@@ -92,6 +92,7 @@ TEST(NanoarrowIpcWriter, OutputStreamFile) {
TEST(NanoarrowIpcWriter, OutputStreamFileError) {
nanoarrow::ipc::UniqueOutputStream stream;
+ errno = EINVAL;
EXPECT_EQ(ArrowIpcOutputStreamInitFile(stream.get(), nullptr,
/*close_on_release=*/1),
EINVAL);
