This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 4ebab67 ci: set up nightly verification build (#439)
4ebab67 is described below
commit 4ebab6791a3ac8f867f100abdb6b5d0676e58767
Author: David Li <[email protected]>
AuthorDate: Wed Feb 8 16:12:22 2023 -0500
ci: set up nightly verification build (#439)
Fixes #435.
Fixes #440.
---
.gitattributes | 1 +
.github/workflows/nightly-verify.yml | 157 +++++++++++++++++++++++++++++++
.github/workflows/verify.yml | 4 -
.pre-commit-config.yaml | 2 +-
dev/release/verify-release-candidate.ps1 | 36 ++++---
dev/release/verify-release-candidate.sh | 6 +-
docs/source/development/releasing.rst | 8 ++
7 files changed, 195 insertions(+), 19 deletions(-)
diff --git a/.gitattributes b/.gitattributes
index 3681d55..d13ce8e 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -16,6 +16,7 @@
# under the License.
c/vendor/* linguist-vendored
+python/adbc_driver_flightsql/adbc_driver_flightsql/_static_version.py
export-subst
python/adbc_driver_manager/adbc_driver_manager/_static_version.py export-subst
python/adbc_driver_postgresql/adbc_driver_postgresql/_static_version.py
export-subst
python/adbc_driver_sqlite/adbc_driver_sqlite/_static_version.py export-subst
diff --git a/.github/workflows/nightly-verify.yml
b/.github/workflows/nightly-verify.yml
new file mode 100644
index 0000000..50af93f
--- /dev/null
+++ b/.github/workflows/nightly-verify.yml
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Verification (Nightly)
+
+on:
+ schedule:
+ - cron: "0 0 * * *"
+ workflow_dispatch: {}
+
+permissions:
+ contents: read
+
+jobs:
+ source:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Prepare version
+ shell: bash
+ run: |
+ VERSION=$(grep 'set(ADBC_VERSION' c/cmake_modules/AdbcVersion.cmake
| \
+ grep -E -o '[0-9]+\.[0-9]+\.[0-9]+')
+ echo "VERSION=${VERSION}" >> $GITHUB_ENV
+
+ - name: Create archive
+ shell: bash
+ run: |
+ git config --global user.name 'github-actions[bot]'
+ git config --global user.email
'github-actions[bot]@users.noreply.github.com'
+ git tag \
+ -a apache-arrow-adbc-${VERSION}-rc0 \
+ -m "ADBC Libraries ${VERSION} RC 0"
+ ci/scripts/source_build.sh \
+ apache-arrow-adbc-${VERSION} \
+ apache-arrow-adbc-${VERSION}-rc0
+
+ - name: Create fake GPG key
+ shell: bash
+ run: |
+ gpg \
+ --quick-gen-key \
+ --batch \
+ --passphrase '' \
+ user@localhost
+
+ gpg \
+ --list-sigs >> KEYS
+
+ gpg \
+ --armor \
+ --export >> KEYS
+
+ - name: Create sum/signature
+ shell: bash
+ run: |
+ gpg \
+ --armor \
+ --detach-sign \
+ --output apache-arrow-adbc-${VERSION}.tar.gz.asc \
+ apache-arrow-adbc-${VERSION}.tar.gz
+
+ shasum --algorithm 512 \
+ apache-arrow-adbc-${VERSION}.tar.gz >
apache-arrow-adbc-${VERSION}.tar.gz.sha512
+
+ - uses: actions/upload-artifact@v3
+ with:
+ name: source
+ retention-days: 7
+ path: |
+ KEYS
+ apache-arrow-adbc-${{ env.VERSION }}.tar.gz
+ apache-arrow-adbc-${{ env.VERSION }}.tar.gz.asc
+ apache-arrow-adbc-${{ env.VERSION }}.tar.gz.sha512
+
+ source-conda:
+ name: "Verify Source (Conda)/${{ matrix.os }}"
+ runs-on: ${{ matrix.os }}
+ needs:
+ - source
+ strategy:
+ fail-fast: false
+ matrix:
+ os: ["macos-latest", "ubuntu-latest", "windows-latest"]
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ path: arrow-adbc
+ persist-credentials: false
+
+ - name: Prepare version
+ shell: bash
+ run: |
+ VERSION=$(grep 'set(ADBC_VERSION'
arrow-adbc/c/cmake_modules/AdbcVersion.cmake | \
+ grep -E -o '[0-9]+\.[0-9]+\.[0-9]+')
+ echo "VERSION=${VERSION}" >> $GITHUB_ENV
+
+ - uses: actions/download-artifact@v3
+ with:
+ name: source
+ path: ${{ github.workspace }}/apache-arrow-adbc-${{ env.VERSION
}}-rc0/
+
+ - name: Setup directory structure
+ shell: bash
+ run: |
+ mv apache-arrow-adbc-${{ env.VERSION }}-rc0/KEYS .
+
+ - uses: conda-incubator/setup-miniconda@v2
+ # The Unix script will set up conda itself
+ if: matrix.os == 'windows-latest'
+ with:
+ miniforge-variant: Mambaforge
+ miniforge-version: latest
+ use-mamba: true
+
+ - name: Verify
+ if: matrix.os != 'windows-latest'
+ env:
+ REPOSITORY: ${{ github.repository }}
+ TEST_DEFAULT: "0"
+ TEST_SOURCE: "1"
+ USE_CONDA: "1"
+ VERBOSE: "1"
+ VERIFICATION_MOCK_DIST_DIR: ${{ github.workspace }}
+ run: |
+ ./arrow-adbc/dev/release/verify-release-candidate.sh $VERSION 0
+
+ - name: Verify
+ if: matrix.os == 'windows-latest'
+ shell: pwsh
+ env:
+ REPOSITORY: ${{ github.repository }}
+ TEST_DEFAULT: "0"
+ TEST_SOURCE: "1"
+ USE_CONDA: "1"
+ VERBOSE: "1"
+ VERIFICATION_MOCK_DIST_DIR: ${{ github.workspace
}}\apache-arrow-adbc-${{ env.VERSION }}-rc0
+ run: |
+ .\arrow-adbc\dev\release\verify-release-candidate.ps1 $env:VERSION 0
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
index aa1d98e..6e5d741 100644
--- a/.github/workflows/verify.yml
+++ b/.github/workflows/verify.yml
@@ -18,8 +18,6 @@
name: Verification
on:
- # schedule:
- # - cron: "1 0 * * *"
workflow_dispatch:
inputs:
version:
@@ -79,8 +77,6 @@ jobs:
miniforge-variant: Mambaforge
miniforge-version: latest
use-mamba: true
- # Required for caching
- use-only-tar-bz2: true
- name: Verify
if: matrix.os != 'windows-latest'
env:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index d6b56fd..843f55b 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -73,7 +73,7 @@ repos:
- id: flake8
types_or: [python]
- repo: https://github.com/PyCQA/isort
- rev: 5.10.1
+ rev: 5.12.0
hooks:
- id: isort
types_or: [python]
diff --git a/dev/release/verify-release-candidate.ps1
b/dev/release/verify-release-candidate.ps1
index 896a954..d451a59 100755
--- a/dev/release/verify-release-candidate.ps1
+++ b/dev/release/verify-release-candidate.ps1
@@ -51,6 +51,22 @@ Assumes Mamba is set up and available on the path.
exit 1
}
+$ArrowDistUrl = "https://dist.apache.org/repos/dist/dev/arrow";
+$DistName = "apache-arrow-adbc-$($Version)"
+
+function Download-Dist-File {
+ $DistUrl = "$($ArrowDistUrl)/$($DistName)-rc$($RcNumber)/$($args[0])"
+ $DistPath = Join-Path $ArrowTempDir $args[0]
+
+ echo "Fetching $($DistUrl)"
+ if ($env:VERIFICATION_MOCK_DIST_DIR -eq $null) {
+ Invoke-WebRequest -Uri $DistUrl -OutFile $DistPath
+ } else {
+ $SourcePath = Join-Path $env:VERIFICATION_MOCK_DIST_DIR $args[0]
+ cp $SourcePath $DistPath
+ }
+}
+
function Show-Header {
echo ""
echo "============================================================"
@@ -74,21 +90,16 @@ Show-Header "Ensure Source Directory"
if ($SourceKind -eq "local") {
$ArrowSourceDir = Join-Path $PSScriptRoot "..\.." | Resolve-Path | % {
$_.Path }
} else {
- $ArrowDistUrl = "https://dist.apache.org/repos/dist/dev/arrow";
- $DistName = "apache-arrow-adbc-$($Version)"
- $DistUrl = "$($ArrowDistUrl)/$($DistName)-rc$($RcNumber)/$DistName.tar.gz"
- $Sha512Url =
"$($ArrowDistUrl)/$($DistName)-rc$($RcNumber)/$DistName.tar.gz.sha512"
- $DistPath = Join-Path $ArrowTempDir "$($DistName).tar.gz"
- $Sha512Path = Join-Path $ArrowTempDir "$($DistName).tar.gz.sha512"
$ArrowSourceDir = Join-Path $ArrowTempDir $DistName
-
- echo "Fetching $($DistUrl)"
New-Item -ItemType Directory -Path $ArrowSourceDir -Force
- Invoke-WebRequest -Uri $DistUrl -OutFile $DistPath
- Invoke-WebRequest -Uri $Sha512Url -OutFile $Sha512Path
- $ExpectedSha512 = (Get-Content $Sha512Path).Split(" ")[0]
+ Download-Dist-File "$($DistName).tar.gz"
+ Download-Dist-File "$($DistName).tar.gz.sha512"
+ $DistPath = Join-Path $ArrowTempDir "$($DistName).tar.gz"
+ $Sha512Path = Join-Path $ArrowTempDir "$($DistName).tar.gz.sha512"
+
+ $ExpectedSha512 = (Get-Content $Sha512Path).Split(" ")[0]
if (-not ((Get-FileHash -Algorithm SHA512 $DistPath).Hash -eq
$ExpectedSha512)) {
echo "SHA512 hash mismatch"
exit 1
@@ -103,7 +114,8 @@ Show-Header "Create Conda Environment"
mamba create -c conda-forge -f -y -p $(Join-Path $ArrowTempDir conda-env) `
--file $(Join-Path $ArrowSourceDir ci\conda_env_cpp.txt) `
- --file $(Join-Path $ArrowSourceDir ci\conda_env_python.txt)
+ --file $(Join-Path $ArrowSourceDir ci\conda_env_python.txt) `
+ go=1.18
Invoke-Expression $(conda shell.powershell hook | Out-String)
conda activate $(Join-Path $ArrowTempDir conda-env)
diff --git a/dev/release/verify-release-candidate.sh
b/dev/release/verify-release-candidate.sh
index 4626b36..d3406ff 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -89,7 +89,7 @@ show_info() {
ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
download_dist_file() {
- if [[ -n "${VERIFICATION_MOCK_DIST_DIR}" && "$1" != "KEYS" ]]; then
+ if [[ -n "${VERIFICATION_MOCK_DIST_DIR}" ]]; then
cp "${VERIFICATION_MOCK_DIST_DIR}/$1" .
else
curl \
@@ -407,9 +407,11 @@ test_cpp() {
show_header "Build, install and test C++ libraries"
# Build and test C++
+ maybe_setup_go
maybe_setup_conda \
--file ci/conda_env_cpp.txt \
- compilers || exit 1
+ compilers \
+ go=1.18 || exit 1
if [ "${USE_CONDA}" -gt 0 ]; then
export
CMAKE_PREFIX_PATH="${CONDA_BACKUP_CMAKE_PREFIX_PATH}:${CMAKE_PREFIX_PATH}"
diff --git a/docs/source/development/releasing.rst
b/docs/source/development/releasing.rst
index 824daa8..ca897fb 100644
--- a/docs/source/development/releasing.rst
+++ b/docs/source/development/releasing.rst
@@ -78,6 +78,14 @@ Before creating a Release Candidate
# Activate conda environment
mamba activate adbc
+Check Nightly Verification Job
+------------------------------
+
+Ensure that the `verification job
+<https://github.com/apache/arrow-adbc/actions/workflows/nightly-verify.yml>`_
+is passing. This simulates part of the release verification workflow
+to detect issues ahead of time.
+
Creating a Release Candidate
============================
