This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new d1e203a chore: Release script improvements (#122)
d1e203a is described below
commit d1e203ad302fd296674e8e8bb05414538e865f3d
Author: Dewey Dunnington <[email protected]>
AuthorDate: Mon Feb 27 17:04:37 2023 -0400
chore: Release script improvements (#122)
---
dev/release/.env.example | 21 +++
dev/release/.gitignore | 18 +++
dev/release/01-prepare.sh | 4 +-
dev/release/02-sign.sh | 8 +-
dev/release/03-source.sh | 9 +-
dev/release/README.md | 243 ++++++++++++++++++++++++++++++++
dev/release/utils-prepare.sh | 9 +-
dev/release/verify-release-candidate.sh | 31 ++--
8 files changed, 320 insertions(+), 23 deletions(-)
diff --git a/dev/release/.env.example b/dev/release/.env.example
new file mode 100644
index 0000000..ca4778d
--- /dev/null
+++ b/dev/release/.env.example
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+GPG_KEY_ID=XXXXX
+APACHE_USERNAME=XXXXX
diff --git a/dev/release/.gitignore b/dev/release/.gitignore
new file mode 100644
index 0000000..fed460a
--- /dev/null
+++ b/dev/release/.gitignore
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+.env
diff --git a/dev/release/01-prepare.sh b/dev/release/01-prepare.sh
index ce05fa0..6dd9b49 100755
--- a/dev/release/01-prepare.sh
+++ b/dev/release/01-prepare.sh
@@ -60,7 +60,7 @@ echo "Updating changelog for $version"
(
echo ;
# Strip trailing blank line
- printf '%s\n' "$(cz ch --dry-run --unreleased-version "nanoarrow
${version}" --start-rev apache-arrow-nanoarrow-${prev_version})"
+ printf '%s\n' "$(cz ch --dry-run --unreleased-version "nanoarrow
${version}")"
) >> ${SOURCE_DIR}/../../CHANGELOG.md
git add ${SOURCE_DIR}/../../CHANGELOG.md
git commit -m "chore: update CHANGELOG.md for $version"
@@ -76,4 +76,4 @@ git commit -m "chore: update versions for ${version}"
--allow-empty
git tag -a "${release_candidate_tag}" -m "nanoarrow ${version} RC ${rc_number}"
echo "Created release candidate tag: ${release_candidate_tag}"
-echo "Push this tag before continuing!"
+echo "git push upstream ${release_candidate_tag} before continuing!"
diff --git a/dev/release/02-sign.sh b/dev/release/02-sign.sh
index b11a200..33eb0a0 100755
--- a/dev/release/02-sign.sh
+++ b/dev/release/02-sign.sh
@@ -51,7 +51,8 @@ main() {
--repo "${REPOSITORY}" \
--workflow=packaging.yaml \
--json 'databaseId,event,headBranch,status' \
- --jq ".[] | select(.event == \"push\" and .headBranch ==
\"${tag}\") | .databaseId")
+ --jq ".[] | select(.event == \"push\" and .headBranch ==
\"${tag}\") | .databaseId" | \
+ head -n 1)
sleep 1
done
@@ -147,6 +148,11 @@ upload_asset_signatures() {
"${tag}" \
"${assets[@]/%/.asc}" \
"${assets[@]/%/.sha512}"
+
+ # Clean up
+ for asset in "${assets[@]}"; do
+ rm -f "${asset}" "${asset}.asc" "${asset}.sha512"
+ done
}
main "$@"
diff --git a/dev/release/03-source.sh b/dev/release/03-source.sh
index 82c1d59..1f7bc18 100755
--- a/dev/release/03-source.sh
+++ b/dev/release/03-source.sh
@@ -63,14 +63,19 @@ main() {
# commit to svn
svn add "tmp/${tag}"
- svn ci -m "Apache Arrow nanoarrow ${version} RC${rc_number}" "tmp/${tag}"
+ svn ci --username=$APACHE_USERNAME -m "Apache Arrow nanoarrow ${version}
RC${rc_number}" "tmp/${tag}"
# clean up
rm -rf tmp
+ rm -rf "${download_dir}"
echo "Uploaded at https://dist.apache.org/repos/dist/dev/arrow/${tag}";
+}
- popd
+header() {
+ echo "============================================================"
+ echo "${1}"
+ echo "============================================================"
}
main "$@"
diff --git a/dev/release/README.md b/dev/release/README.md
new file mode 100644
index 0000000..f0486f5
--- /dev/null
+++ b/dev/release/README.md
@@ -0,0 +1,243 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Release process for Apache Arrow nanoarrow
+
+## Verifying a nanoarrow release candidate
+
+Release candidates for nanoarrow are uploaded to
https://dist.apache.org/repos/dist/dev/arrow/
+prior to a release vote being called on the
+[Apache Arrow developer mailing
list](https://lists.apache.org/[email protected]).
+A script (`verify-release-candidate.sh`) is provided to verify such a release
candidate.
+For example, to verify nanoarrow 0.1.0-rc0, one could run:
+
+```
+git clone https://github.com/apache/arrow-nanoarrow.git arrow-nanoarrow
+cd arrow-nanoarrow/dev/release
+./verify-release-candidate.sh 0.1.0 0
+```
+
+Full verification requires [CMake](https://cmake.org/download/) to build and
run the test
+suite. The test suite currently depends on an Arrow C++ installation that is
discoverable
+by CMake (e.g., using one of the methods described in the
+[Arrow installation instructions](https://arrow.apache.org/install/)). For
environments
+where binary packages are not provided, building and installing Arrow C++ from
source
+may be required. You can provide the `NANOARROW_CMAKE_OPTIONS` environment
variable to
+pass extra arguments to `cmake` (e.g.,
`-DArrow_DIR=path/to/arrow/lib/cmake/Arrow` or
+`-DCMAKE_TOOLCHAIN_FILE=[path to vcpkg]/scripts/buildsystems/vcpkg.cmake`).
+
+Verification of the R package requires an
+[R installation](https://cloud.r-project.org/) and a C/C++ compiler (e.g.,
+[RTools](https://cloud.r-project.org/bin/windows/Rtools/) on Windows or XCode
Command
+Line Tools). You can set the `R_HOME` environment variable or
+`export PATH="$PATH:/path/to/R/bin"` to point to a specific R installation.
+
+The verification script itself is written in `bash` and requires the `curl`,
`gpg`, and
+`shasum`/`sha512sum` commands. These are typically available from a packaage
+manager except on Windows (see below).
+
+To run only C library verification (requires CMake and Arrow C++ but not R):
+
+```bash
+TEST_DEFAULT=0 TEST_C=1 TEST_C_BUNDLED=1 ./verify-release-candidate.sh 0.1.0 0
+```
+
+To run only R package verification (requires R but not CMake or Arrow C++):
+
+```bash
+TEST_DEFAULT=0 TEST_R=1 ./verify-release-candidate.sh 0.1.0 0
+```
+
+### Conda
+
+Using `conda`, one can install all requirements needed for verification on
Linux,
+MacOS, or Windows.
+
+```bash
+conda create --name nanoarrow-verify-rc
+conda activate nanoarrow-verify-rc
+conda config --set channel_priority strict
+
+conda install -c conda-forge git cmake bash gnupg arrow-cpp
+```
+
+### MacOS
+
+On MacOS you can install all requirements except R using
[Homebrew](https://brew.sh):
+
+```bash
+brew install cmake gnupg apache-arrow
+```
+
+For older MacOS or MacOS without Homebrew, you will have to install the XCode
+Command Line Tools (i.e., `xcode-select --install`),
+[install GnuPG](https://gnupg.org/download/),
+[install CMake](https://cmake.org/download/), and build Arrow C++ from source.
+
+<details>
+
+```bash
+# Download + build Arrow C++
+curl https://dlcdn.apache.org/arrow/arrow-11.0.0/apache-arrow-11.0.0.tar.gz | \
+ tar -zxf -
+mkdir arrow-build && cd arrow-build
+cmake ../apache-arrow-11.0.0/cpp \
+ -DARROW_JEMALLOC=OFF -DARROW_SIMD_LEVEL=NONE \
+ # Required for Arrow on old clang
+ -DCMAKE_CXX_FLAGS="-D_LIBCPP_DISABLE_AVAILABILITY" \
+ -DCMAKE_INSTALL_PREFIX=../arrow
+cmake --build .
+cmake --install . --prefix=../arrow
+cd ..
+
+# Pass location of install to the release verification script
+export NANOARROW_CMAKE_OPTIONS="-DArrow_Dir=$(pwd)/arrow/lib/cmake/Arrow
-DCMAKE_CXX_FLAGS=-D_LIBCPP_DISABLE_AVAILABILITY"
+```
+
+</details>
+
+You can install R using the instructions provided on the
+[R Project Download page](https://cloud.r-project.org/bin/macosx/).
+
+### Windows
+
+To verify the C library on Windows, use [Conda](#conda) to supply all the
prerequisites
+for the verification script; however, you can use R and Rtools
+[installed using the official
installer](https://cloud.r-project.org/bin/windows/).
+You can verify the R package on Windows without Conda
+using Git Bash or the MSys2 Bash that ships with Rtools.
+
+```bash
+export R_HOME="/c/Program Files/R/bin"
+TEST_DEFAULT=0 TEST_R=1 ./verify-release-candidate.sh 0.1.0 0
+```
+
+### Debian/Ubuntu
+
+On Debian/Ubuntu (e.g., `docker run --rm -it ubuntu:latest`) you can install
prerequisites using `apt`.
+
+<details>
+
+```bash
+apt-get update && apt-get install -y git cmake r-base gnupg curl
+
+# For Arrow C++
+apt-get install -y -V ca-certificates lsb-release wget
+wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr
'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename
--short).deb
+apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release
--codename --short).deb
+apt-get update
+apt-get install -y -V libarrow-dev
+```
+
+</details>
+
+### Fedora
+
+On recent Fedora (e.g., `docker run --rm -it fedora:latest`), you can install
all prerequisites
+using `dnf`:
+
+```bash
+dnf install -y git cmake R gnupg curl libarrow-devel
+```
+
+### Arch Linux
+
+On Arch Linux (e.g., `docker run --rm -it archlinux:latest`, you can install
all prerequisites
+using `pacman`):
+
+```bash
+pacman -S git g++ make cmake r-base gnupg curl arrow
+```
+
+### Alpine Linux
+
+On Alpine Linux (e.g., `docker run --rm -it alpine:latest`), all prerequisites
are available
+using `apk add` except Arrow C++ which must be built and installed from source.
+
+<details>
+
+```bash
+apk add bash linux-headers git cmake R R-dev g++ gnupg curl
+
+# Build Arrow C++ from source
+curl https://dlcdn.apache.org/arrow/arrow-11.0.0/apache-arrow-11.0.0.tar.gz | \
+ tar -zxf -
+mkdir arrow-build && cd arrow-build
+cmake ../apache-arrow-11.0.0/cpp \
+ -DARROW_JEMALLOC=OFF -DARROW_SIMD_LEVEL=NONE
-DCMAKE_INSTALL_PREFIX=../arrow
+cmake --build .
+cmake --install . --prefix=../arrow
+cd ..
+
+# Pass location of Arrow to the build script
+export NANOARROW_CMAKE_OPTIONS="-DArrow_Dir=$(pwd)/arrow/lib/cmake/Arrow"
+```
+
+</details>
+
+### Big endian
+
+One can verify a nanoarrow release candidate on big endian by setting
+`DOCKER_DEFAULT_PLATFORM=linux/s390x` and following the instructions for
+[Alpine Linux](#alpine-linux).
+
+## Creating a release candidate
+
+The first step to creating a nanoarrow release is to create a `maint-VERSION`
branch
+and run
+[01-prepare.R](https://github.com/apache/arrow-nanoarrow/blob/main/dev/release/01-prepare.sh):
+
+```bash
+# from the repository root
+# 01-prepare.sh <nanoarrow-dir> <prev_veresion> <version> <next_version>
<rc-num>
+dev/release/01-prepare.sh . 0.0.0 0.1.0 0.2.0 0
+```
+
+This will update version numbers, the changelong, and create the git tag
+`apache-arrow-nanoarrow-0.1.0-rc0`. Check to make sure that the changelog
+and versions are what you expect them to be before pushing the tag (you
+may wish to do this by opening a dummy PR to run CI and look at the diff
+from the main branch). When you are satisfied that the code at this tag
+is release-candidate worthy, `git push` the tag to the `upstream` repository
+(or whatever your remote name is for the `apache/arrow-nanoarrow` repo).
+This will kick off a
+[packaging
workflow](https://github.com/apache/arrow-nanoarrow/blob/main/.github/workflows/packaging.yaml)
+that will create a GitHub release and upload assets that are required for
+later steps. This step can be done by any Arrow comitter.
+
+Next, all assets need to be signed by somebody whose GPG key is listed in the
+[Arrow developers KEYS file](https://dist.apache.org/repos/dist/dev/arrow/KEYS)
+by calling
+[02-sign.sh](https://github.com/apache/arrow-nanoarrow/blob/main/dev/release/02-sign.sh)
+The caller of the script does not need to be on any particular branch to call
+the script but *does* need the
+[dev/release/.env](https://github.com/apache/arrow-nanoarrow/blob/main/dev/release/.env.example)
+file to exist setting the appropriate `GPG_KEY_ID` environment variable.
+
+```
+# 02-sign.sh <version> <rc-num>
+dev/release/02-sign.sh . 0.1.0 0
+```
+
+Finally, run
+[03-source.sh](https://github.com/apache/arrow-nanoarrow/blob/main/dev/release/03-source.sh).
+This step can be done by any Arrow comitter. The caller of this script does
not need to
+be on any particular branch but *does* need the
+[dev/release/.env](https://github.com/apache/arrow-nanoarrow/blob/main/dev/release/.env.example)
+file to exist setting the appropriate `APACHE_USERNAME` variable set.
diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh
index c812aaf..dafab50 100755
--- a/dev/release/utils-prepare.sh
+++ b/dev/release/utils-prepare.sh
@@ -25,25 +25,26 @@ update_versions() {
case ${type} in
release)
local version=${base_version}
- local conda_version=${base_version}
local docs_version=${base_version}
+ local r_version=${base_version}
;;
snapshot)
local version=${next_version}-SNAPSHOT
- local conda_version=${next_version}
local docs_version="${next_version} (dev)"
+ local r_version="${base_version}.9000"
;;
esac
local major_version=${version%%.*}
pushd "${NANOARROW_DIR}"
sed -i.bak -E "s/set\(NANOARROW_VERSION \".+\"\)/set(NANOARROW_VERSION
\"${version}\")/g" CMakeLists.txt
- rm CMakeLists.txt
+ rm CMakeLists.txt.bak
git add CMakeLists.txt
popd
pushd "${NANOARROW_DIR}/r"
- Rscript -e "desc::desc_set(Version = '${base_version}.9000')"
+ Rscript -e "desc::desc_set(Version = '${r_version}')"
+ git add DESCRIPTION
popd
}
diff --git a/dev/release/verify-release-candidate.sh
b/dev/release/verify-release-candidate.sh
index b7cd1f8..7ea32eb 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -26,7 +26,7 @@
# - NANOARROW_CMAKE_OPTIONS (e.g., to help cmake find Arrow C++)
# - R_HOME: Path to the desired R installation. Defaults to `R` on PATH.
# - TEST_SOURCE: Set to 0 to selectively run component verification.
-# - TEST_C: Builds the C library and tests using the default CMake
+# - TEST_C: Builds the C library and tests using the default CMake
# configuration. Defaults to the value of TEST_SOURCE.
# - TEST_C_BUNDLED: Bundles and builds the nanoarrow.h/nanorrow.c distribution
# and runs tests. Defaults to the value of TEST_SOURCE.
@@ -113,10 +113,8 @@ import_gpg_keys() {
}
if type shasum >/dev/null 2>&1; then
- sha256_verify="shasum -a 256 -c"
sha512_verify="shasum -a 512 -c"
else
- sha256_verify="sha256sum -c"
sha512_verify="sha512sum -c"
fi
@@ -126,10 +124,8 @@ fetch_archive() {
local dist_name=$1
download_rc_file ${dist_name}.tar.gz
download_rc_file ${dist_name}.tar.gz.asc
- download_rc_file ${dist_name}.tar.gz.sha256
download_rc_file ${dist_name}.tar.gz.sha512
gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
- ${sha256_verify} ${dist_name}.tar.gz.sha256
${sha512_verify} ${dist_name}.tar.gz.sha512
}
@@ -168,7 +164,7 @@ setup_tempdir() {
if [ -z "${NANOARROW_TMPDIR}" ]; then
# clean up automatically if NANOARROW_TMPDIR is not defined
- NANOARROW_TMPDIR=$(mktemp -d -t "nanoarrow-${VERSION}.XXXXX")
+ NANOARROW_TMPDIR=$(mktemp -d -t "nanoarrow-${VERSION}.XXXXXX")
trap cleanup EXIT
else
# don't clean up automatically
@@ -188,7 +184,7 @@ test_and_install_c() {
cmake ${NANOARROW_SOURCE_DIR} \
-DNANOARROW_BUILD_TESTS=ON \
${NANOARROW_CMAKE_OPTIONS:-}
-
+
show_info "Build CMake Project"
cmake --build .
@@ -212,7 +208,7 @@ test_c_bundled() {
-DNANOARROW_BUILD_TESTS=ON \
-DNANOARROW_BUNDLE=ON \
${NANOARROW_CMAKE_OPTIONS:-}
-
+
show_info "Build CMake Project"
cmake --build .
@@ -232,21 +228,28 @@ test_r() {
fi
show_info "Install nanoarrow test dependencies"
- $R_BIN -e 'install.packages("pak", repos = "https://cloud.r-project.org";);
pak::local_install_dev_deps("r")'
+ # For the purposes of this script, we don't install arrow because it takes
too long
+ # (but the arrow integration tests will run if the arrow package is
installed anyway).
+ # Using a manual approach because installing pak takes a while on some
systems and
+ # beacuse the package versions don't matter much.
+ $R_BIN -e 'for (pkg in c("blob", "hms", "tibble", "rlang", "testthat",
"tibble", "vctrs", "withr")) if (!requireNamespace(pkg, quietly = TRUE))
install.packages(pkg, repos = "https://cloud.r-project.org/";)'
show_info "Build the R package source tarball"
- pushd r
- ./configure
- popd
+
+ # Running R CMD INSTALL on the R source directory is the most reliable
cross-platform
+ # method to ensure the proper version of nanoarrow is vendored into the R
package.
+ # Do this in a temporary library so not to overwrite the a user's existing
package.
+ mkdir "$NANOARROW_TMPDIR/tmplib"
+ $R_BIN CMD INSTALL r --preclean --library="$NANOARROW_TMPDIR/tmplib"
# Builds the R source tarball
pushd $NANOARROW_TMPDIR
- $R_BIN CMD build $NANOARROW_SOURCE_DIR/r
+ $R_BIN CMD build "$NANOARROW_SOURCE_DIR/r"
R_PACKAGE_TARBALL_NAME=`ls nanoarrow_*.tar.gz`
show_info "Run R CMD check"
# Runs R CMD check on the tarball
- $R_BIN CMD check $R_PACKAGE_TARBALL_NAME
+ _R_CHECK_FORCE_SUGGESTS_=false $R_BIN CMD check "$R_PACKAGE_TARBALL_NAME"
--no-manual
popd
}
