This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new a4f86c9d0 ci: verify source using OS-installed dependencies (#2718)
a4f86c9d0 is described below
commit a4f86c9d0bd4b83ca9f0ca00f47f813d61c70e8c
Author: David Li <[email protected]>
AuthorDate: Thu Apr 24 11:41:19 2025 +0900
ci: verify source using OS-installed dependencies (#2718)
We've only really tested Conda so far. Validate non-Conda builds too.
This only tests Ubuntu/APT but eventually we should also test
Debian/APT, macOS/Homebrew, Fedora/RPM, and AlmaLinux/RPM.
---------
Co-authored-by: Sutou Kouhei <[email protected]>
Co-authored-by: Dewey Dunnington <[email protected]>
---
.github/workflows/nightly-verify.yml | 31 ++++++
ci/scripts/verify_ubuntu.sh | 111 +++++++++++++++++++++
compose.yaml | 9 ++
dev/release/verify-release-candidate.sh | 76 ++++++++------
.../driver/flightsql/flightsql_adbc_server_test.go | 3 +-
5 files changed, 197 insertions(+), 33 deletions(-)
diff --git a/.github/workflows/nightly-verify.yml
b/.github/workflows/nightly-verify.yml
index a689ccf7f..5caa2b26c 100644
--- a/.github/workflows/nightly-verify.yml
+++ b/.github/workflows/nightly-verify.yml
@@ -39,6 +39,10 @@ defaults:
# 'bash' will expand to -eo pipefail
shell: bash
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
jobs:
source:
# For cron: only run on the main repo, not forks
@@ -201,3 +205,30 @@ jobs:
run: |
pushd arrow-adbc
docker compose run -e PYTHON=3.12 --rm python-debug
+
+ source-verify-docker:
+ name: "Verify Source (OS)/${{ matrix.os }} ${{ matrix.version }}"
+ runs-on: ubuntu-latest
+ strategy:
+ max-parallel: 2
+ matrix:
+ include:
+ - os: ubuntu
+ version: "22.04"
+ - os: ubuntu
+ version: "24.04"
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ path: arrow-adbc
+ persist-credentials: false
+ submodules: recursive
+
+ - name: Verify
+ env:
+ OS: ${{ matrix.os }}
+ OS_VERSION: ${{ matrix.version }}
+ run: |
+ pushd arrow-adbc
+ env ${OS@U}=${OS_VERSION} docker compose run --rm verify-all-${OS}
diff --git a/ci/scripts/verify_ubuntu.sh b/ci/scripts/verify_ubuntu.sh
new file mode 100755
index 000000000..0b1eef2c5
--- /dev/null
+++ b/ci/scripts/verify_ubuntu.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Test the validation script using system dependencies instead of Conda. This
+# script is meant for automation (e.g. Docker), not a local/developer machine
+# (except via Docker).
+
+set -euo pipefail
+
+main() {
+ local -r source_dir="${1}"
+
+ # Install all the dependencies we need for all the subprojects
+
+ # When installing tzdata, don't block and wait for user input
+ export DEBIAN_FRONTEND=noninteractive
+ export TZ=Etc/UTC
+
+ apt update
+ apt install -y \
+ apt-transport-https \
+ build-essential \
+ ca-certificates \
+ cmake \
+ curl \
+ dirmngr \
+ git \
+ gobject-introspection \
+ gpg \
+ libgirepository1.0-dev \
+ libglib2.0-dev \
+ libgmock-dev \
+ libgtest-dev \
+ libpq-dev \
+ libsqlite3-dev \
+ lsb-release \
+ ninja-build \
+ pkg-config \
+ protobuf-compiler \
+ python3 \
+ python3-dev \
+ python3-pip \
+ python3-venv \
+ r-base \
+ ruby-full \
+ software-properties-common \
+ wget
+
+ # Install Java
+
+ wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | \
+ gpg --dearmor | \
+ tee /etc/apt/trusted.gpg.d/adoptium.gpg > /dev/null
+
+ echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F=
'/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | \
+ tee /etc/apt/sources.list.d/adoptium.list
+
+ # Install Arrow GLib
+ wget https://packages.apache.org/artifactory/arrow/$(lsb_release --id
--short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release
--codename --short).deb
+ apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release
--codename --short).deb
+
+ apt update
+ apt install -y \
+ libarrow-dev \
+ libarrow-glib-dev \
+ temurin-21-jdk
+
+ # Install Maven
+ wget
https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz
+ mkdir -p /opt/maven
+ tar -C /opt/maven -xzvf apache-maven-3.9.9-bin.tar.gz --strip-components=1
+ export PATH=/opt/maven/bin:$PATH
+
+ # Check if protoc is too old (Ubuntu 22.04). If so, install it from
+ # upstream instead.
+ if ! protoc --version | \
+ awk '{print $2}{print "3.15"}' | \
+ sort --version-sort | \
+ head -n1 | \
+ grep -E '^3\.15.*$' >/dev/null ; then
+ echo "protoc is too old"
+
+ wget -O protoc.zip
https://github.com/protocolbuffers/protobuf/releases/download/v30.2/protoc-30.2-linux-x86_64.zip
+ unzip -o protoc.zip -d /usr/local -x readme.txt
+ fi
+
+ # We run under Docker and this is necessary since the source dir is
+ # typically mounted as a volume
+ git config --global --add safe.directory "${source_dir}"
+
+ "${source_dir}/dev/release/verify-release-candidate.sh"
+}
+
+main "$@"
diff --git a/compose.yaml b/compose.yaml
index 3a7b07fdc..742669cc8 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -275,3 +275,12 @@ services:
retries: 5
ports:
- "5432:5432"
+
+ ################################ Verification
################################
+
+ verify-all-ubuntu:
+ image: ubuntu:${UBUNTU}
+ volumes:
+ - .:/adbc:delegated
+ command: |
+ /adbc/ci/scripts/verify_ubuntu.sh /adbc
diff --git a/dev/release/verify-release-candidate.sh
b/dev/release/verify-release-candidate.sh
index 4967493e7..fab1cb019 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -246,38 +246,42 @@ install_dotnet() {
return 0
fi
- show_info "Ensuring that .NET is installed..."
-
- if dotnet --version | grep 8\.0 > /dev/null 2>&1; then
- local csharp_bin=$(dirname $(which dotnet))
- show_info "Found C# at $(which csharp) (.NET $(dotnet --version))"
- else
- if which dotnet > /dev/null 2>&1; then
- show_info "dotnet found but it is the wrong version and will be ignored."
+ if command -v dotnet; then
+ show_info "Found $(dotnet --version) at $(which dotnet)"
+
+ if dotnet --version | grep --quiet --fixed-string 8.0; then
+ local csharp_bin=$(dirname $(which dotnet))
+ show_info "Found C# at $(which csharp) (.NET $(dotnet --version))"
+ DOTNET_ALREADY_INSTALLED=1
+ return 0
fi
- local csharp_bin=${ARROW_TMPDIR}/csharp/bin
- local dotnet_version=8.0.204
- local dotnet_platform=
- case "$(uname)" in
+ fi
+
+ show_info "dotnet found but it is the wrong version or dotnet not found"
+
+ local csharp_bin=${ARROW_TMPDIR}/csharp/bin
+ local dotnet_version=8.0.204
+ local dotnet_platform=
+ case "$(uname)" in
Linux)
- dotnet_platform=linux
- ;;
+ dotnet_platform=linux
+ ;;
Darwin)
- dotnet_platform=macos
- ;;
- esac
- local
dotnet_download_thank_you_url=https://dotnet.microsoft.com/download/thank-you/dotnet-sdk-${dotnet_version}-${dotnet_platform}-x64-binaries
- local dotnet_download_url=$( \
- curl -sL ${dotnet_download_thank_you_url} | \
- grep 'directLink' | \
- grep -E -o 'https://download[^"]+' | \
- sed -n 2p)
- mkdir -p ${csharp_bin}
- curl -sL ${dotnet_download_url} | \
+ dotnet_platform=macos
+ ;;
+ esac
+ local
dotnet_download_thank_you_url=https://dotnet.microsoft.com/download/thank-you/dotnet-sdk-${dotnet_version}-${dotnet_platform}-x64-binaries
+ show_info "Getting .NET download URL from ${dotnet_download_thank_you_url}"
+ local dotnet_download_url=$(curl -sL ${dotnet_download_thank_you_url} | \
+ grep 'directLink' | \
+ grep -E -o 'https://download[^"]+' | \
+ sed -n 2p)
+ show_info "Downloading .NET from ${dotnet_download_url}"
+ mkdir -p ${csharp_bin}
+ curl -sL ${dotnet_download_url} | \
tar xzf - -C ${csharp_bin}
- PATH=${csharp_bin}:${PATH}
- show_info "Installed C# at $(which csharp) (.NET $(dotnet --version))"
- fi
+ PATH=${csharp_bin}:${PATH}
+ show_info "Installed C# at $(which csharp) (.NET $(dotnet --version))"
DOTNET_ALREADY_INSTALLED=1
}
@@ -296,6 +300,18 @@ install_go() {
return 0
fi
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+
+ if [ -f "${prefix}/go/bin/go" ]; then
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+ show_info "Found $(go version) at ${prefix}/go/bin/go"
+ GO_ALREADY_INSTALLED=1
+ return 0
+ fi
+
local version=1.24.2
show_info "Installing go version ${version}..."
@@ -315,8 +331,6 @@ install_go() {
local archive="go${version}.${os}-${arch}.tar.gz"
curl -sLO https://go.dev/dl/$archive
- local prefix=${ARROW_TMPDIR}/go
- mkdir -p $prefix
tar -xzf $archive -C $prefix
rm -f $archive
@@ -549,7 +563,7 @@ test_python() {
show_header "Build and test Python libraries"
# Build and test Python
- maybe_setup_virtualenv cython duckdb pandas protobuf pyarrow pytest
setuptools_scm setuptools importlib_resources || exit 1
+ maybe_setup_virtualenv cython duckdb pandas polars protobuf pyarrow pytest
setuptools_scm setuptools importlib_resources || exit 1
# XXX: pin Python for now since various other packages haven't caught up
maybe_setup_conda --file "${ADBC_DIR}/ci/conda_env_python.txt" python=3.12
|| exit 1
diff --git a/go/adbc/driver/flightsql/flightsql_adbc_server_test.go
b/go/adbc/driver/flightsql/flightsql_adbc_server_test.go
index 5ec08b1b6..1261d6bec 100644
--- a/go/adbc/driver/flightsql/flightsql_adbc_server_test.go
+++ b/go/adbc/driver/flightsql/flightsql_adbc_server_test.go
@@ -1660,8 +1660,7 @@ func (ts *TimeoutTests) TestDoActionTimeout() {
var adbcErr adbc.Error
ts.ErrorAs(stmt.Prepare(context.Background()), &adbcErr)
ts.Equal(adbc.StatusTimeout, adbcErr.Code, adbcErr.Error())
- // Exact match - we don't want extra fluff in the message
- ts.Equal("[FlightSQL] context deadline exceeded (DeadlineExceeded;
Prepare)", adbcErr.Msg)
+ // It seems gRPC isn't stable about the error message, unfortunately
}
func (ts *TimeoutTests) TestDoGetTimeout() {
