This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository
https://gitbox.apache.org/repos/asf/arrow-flight-sql-postgresql.git
The following commit(s) were added to refs/heads/main by this push:
new c626804 Add release script (#87)
c626804 is described below
commit c6268049a3c4d707893eebe114a37a3324bffc58
Author: Sutou Kouhei <[email protected]>
AuthorDate: Thu Aug 31 16:14:31 2023 +0900
Add release script (#87)
Closes GH-30
---
.github/workflows/doc.yaml | 4 +-
.github/workflows/package.yaml | 23 +--
.gitignore | 1 +
Rakefile | 333 ++++++++++++++++++++++++++++++++-
.gitignore => dev/release/.env.example | 32 ++--
src/afs.cc | 29 ++-
6 files changed, 382 insertions(+), 40 deletions(-)
diff --git a/.github/workflows/doc.yaml b/.github/workflows/doc.yaml
index a7f960c..63982af 100644
--- a/.github/workflows/doc.yaml
+++ b/.github/workflows/doc.yaml
@@ -64,9 +64,9 @@ jobs:
run: |
mkdir -p site.repository
pushd site.repository
- git init --initial-branch=asf-site
+ git init --initial-branch=asf-site --bare
popd
- git clone file://${PWD}/site.repository/.git site
+ git clone file://${PWD}/site.repository/ site
- name: Publish
run: |
git config --global user.name 'github-actions[bot]'
diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml
index 027fbd7..8ead9ce 100644
--- a/.github/workflows/package.yaml
+++ b/.github/workflows/package.yaml
@@ -40,9 +40,9 @@ jobs:
- uses: actions/checkout@v3
- name: Make RC GA
run: |
- rc_tag=$(gh release list ${GITHUB_REF_NAME} | \
+ rc_tag=$(gh release list | \
cut -d' ' -f3 | \
- grep -rc | \
+ grep -F "${GITHUB_REF_NAME}-rc" | \
head -n1)
title="$(gh release view ${rc_tag} --json name --jq .name | \
sed -E -e 's/ RC[0-9]+$//')"
@@ -52,6 +52,7 @@ jobs:
gh release create ${GITHUB_REF_NAME} \
--notes-file release-note.md \
--title "${title}" \
+ --verify-tag \
rc_artifacts/*
env:
GH_TOKEN: ${{ github.token }}
@@ -84,8 +85,8 @@ jobs:
puts(ARGF.read.split(/^## /)[1]. \
gsub(/ {.+?}/, ""). \
gsub(/\[(.+?)\]\[.+?\]/) {$1})' \
- docs/source/news.md > release-note.md
- rc=${GITHUB_REF_NAME:#*-rc}
+ doc/source/release-notes.md > release-note.md
+ rc=${GITHUB_REF_NAME#*-rc}
title="$(head -n1 release-note.md | sed -e 's/^## //') RC${rc}"
tail -n +2 release-note.md > release-note-without-version.md
gh release create ${GITHUB_REF_NAME} \
@@ -109,8 +110,8 @@ jobs:
fail-fast: false
matrix:
target:
- - "postgresql-15-pgdg-debian-bookworm-amd64"
- - "postgresql-15-pgdg-ubuntu-jammy-amd64"
+ - "debian-bookworm-amd64-postgresql-15-pgdg"
+ - "ubuntu-jammy-amd64-postgresql-15-pgdg"
steps:
- uses: actions/checkout@v3
with:
@@ -125,13 +126,13 @@ jobs:
echo "BASE_NAME=${BASE_NAME}" >> $GITHUB_ENV
echo "ARROW_SOURCE=$(pwd)/arrow" >> $GITHUB_ENV
full_target=${{ matrix.target }}
- # postgresql-15-pgdg-debian-bookworm-amd64 ->
+ # debian-bookworm-amd64-postgresql-15-pgdg ->
# postgresql-15-pgdg
- PACKAGE=${full_target%-pgdg-*}-pgdg
+ PACKAGE=postgresql-${full_target#*-postgresql-}
echo "PACKAGE=${PACKAGE}" >> $GITHUB_ENV
- # postgresql-15-pgdg-debian-bookworm-amd64 ->
+ # debian-bookworm-amd64-postgresql-15-pgdg ->
# debian-bookworm-amd64
- PACKAGE_TARGET=${full_target#*-pgdg-}
+ PACKAGE_TARGET=${full_target%-postgresql-*}
echo "APT_TARGETS=${PACKAGE_TARGET}" >> $GITHUB_ENV
echo "YUM_TARGETS=${PACKAGE_TARGET}" >> $GITHUB_ENV
case ${PACKAGE_TARGET} in
@@ -194,7 +195,7 @@ jobs:
if: |
github.ref_type == 'tag'
run: |
- gh release upload ${GITHUB_REF_NAME} \
+ gh release upload --repo ${GITHUB_REPOSITORY} ${GITHUB_REF_NAME} \
${{ matrix.target }}.tar.gz
env:
GH_TOKEN: ${{ github.token }}
diff --git a/.gitignore b/.gitignore
index b75d024..9eff26e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,7 @@
/.cache/
/Gemfile.lock
/compile_commands.json
+/dev/release/.env
/dev/release/apache-rat-0.13.jar
/dev/release/dist
/dev/release/filtered_rat.txt
diff --git a/Rakefile b/Rakefile
index 9dfa179..657eaae 100644
--- a/Rakefile
+++ b/Rakefile
@@ -17,10 +17,14 @@
# specific language governing permissions and limitations
# under the License.
+require "digest/sha2"
+require "json"
require "tmpdir"
require_relative "helper"
+project_label = "Apache Arrow Flight SQL adapter for PostgreSQL"
+
version = Helper.detect_version
archive_base_name = "apache-arrow-flight-sql-postgresql-#{version}"
archive_name = "#{archive_base_name}.tar.gz"
@@ -65,11 +69,15 @@ namespace :doc do
Dir.mktmpdir do |tmp|
is_release = (ENV["GITHUB_REF_TYPE"] == "tag")
if is_release
- new_version = ENV["GITHUB_REF_NAME"].gsub(/-rc\d+\z/, "")
+ tag = ENV["GITHUB_REF_NAME"]
+ new_version = tag.gsub(/-rc\d+\z/, "")
+ is_release_candiate = (tag != new_version)
new_doc = "#{tmp}/new"
build_doc(new_doc, for_publish: true)
- current_doc = "#{tmp}/current"
- build_doc(current_doc, release: "current", for_publish: true)
+ unless is_release_candiate
+ current_doc = "#{tmp}/current"
+ build_doc(current_doc, release: "current", for_publish: true)
+ end
else
devel_doc = "#{tmp}/devel"
build_doc(devel_doc, release: "devel", for_publish: true)
@@ -86,7 +94,7 @@ namespace :doc do
add.call(index_html, "index.html")
if is_release
add.call(new_doc, new_version)
- add.call(current_doc, "current")
+ add.call(current_doc, "current") unless is_release_candiate
else
add.call(devel_doc, "devel")
end
@@ -100,3 +108,320 @@ namespace :doc do
end
end
end
+
+def load_env
+ env_file = "dev/release/.env"
+ unless File.exist?(env_file)
+ raise "must create #{env_file} from #{env_file}.example"
+ end
+ File.readlines(env_file, chomp: true).each do |line|
+ line.strip!
+ next if line.empty?
+ next if line.start_with?("#")
+ name, value = line.split("=", 2)
+ next if value.nil?
+ name.strip!
+ value.strip!
+ ENV[name] ||= value
+ end
+end
+
+def env_value(name, default=nil)
+ value = ENV[name]
+ if value.nil? and default.nil?
+ raise "must set environment variable: #{name}"
+ end
+ value || default
+end
+
+def github_repository
+ env_value("GITHUB_REPOSITORY", "apache/arrow-flight-sql-postgresql")
+end
+
+def gpg_key_id
+ env_value("GPG_KEY_ID")
+end
+
+def arrow_source
+ env_value("ARROW_SOURCE")
+end
+
+def detect_latest_rc(version)
+ rc_tags = `git tag`.each_line(chomp: true).select do |tag|
+ tag.start_with?("#{version}-rc")
+ end
+ rcs = rc_tags.collect do |rc_tag|
+ Integer(rc_tag.delete_prefix("#{version}-rc"), 10)
+ end
+ rcs.max
+end
+
+def package_directories
+ Dir.glob("package/postgresql-*")
+end
+
+def debian_changelog_latest_version(changelog_path)
+ (File.readlines(changelog_path)[0] || "")[/\(([\d.]+)-\d+\)/, 1]
+end
+
+def validate_rc(version)
+ package_directories.each do |dir|
+ latest_version = debian_changelog_latest_version("#{dir}/debian/changelog")
+ if latest_version != version
+ raise "'rake release:rc:prepare && git push' is needed"
+ end
+ end
+
+ release_notes = File.read("doc/source/release-notes.md").split(/^## /)
+ latest_release_note = release_notes[1]
+ latest_release_note_version = latest_release_note.lines[0].strip
+ if latest_release_note_version != version
+ raise "add a release note to doc/source/release-notes.md"
+ end
+end
+
+def ensure_package_job_finished(rc_tag)
+ run_id = nil
+ while run_id.nil?
+ runs = IO.pipe do |read, write|
+ sh("gh", "run", "list",
+ "--json", "databaseId,headBranch",
+ "--repo", github_repository,
+ "--workflow", "package.yaml",
+ out: write)
+ write.close
+ read.read
+ end
+ run = JSON.parse(runs).find {|rc| rc["headBranch"] == rc_tag}
+ run_id = run&.fetch("databaseId")
+ end
+ sh("gh", "run", "watch",
+ "--repo", github_repository,
+ "--exit-status",
+ run_id.to_s)
+end
+
+namespace :release do
+ namespace :rc do
+ desc "Prepare a new RC"
+ task :prepare do
+ prepared = false
+ package_directories.each do |dir|
+ cd(dir) do
+ debian_changelog = "debian/changelog"
+ latest_version = debian_changelog_latest_version(debian_changelog)
+ next if latest_version == version
+ ruby("-S", "rake", "version:update")
+ sh("git", "add", debian_changelog)
+ prepared = true
+ end
+ end
+ sh("git", "commit", "-m", "Prepare #{version} RC #{new_rc}")
+ end
+
+ desc "Tag a new RC"
+ task :tag do
+ validate_rc(version)
+ new_rc = (detect_latest_rc(version) || 0) + 1
+ rc_tag = "#{version}-rc#{new_rc}"
+ sh("git", "tag",
+ "-a", rc_tag,
+ "-m", "#{project_label} #{version} RC #{new_rc}")
+ if env_value("TAG_PUSH", "no") == "yes"
+ sh("git", "push", "upstream", rc_tag)
+ else
+ puts("Push #{rc_tag}:")
+ puts(" git push upstream #{rc_tag}")
+ end
+ end
+
+ desc "Sign the latest RC"
+ task :sign do
+ load_env
+ rc = detect_latest_rc(version)
+ if rc.nil?
+ raise "'rake release:rc:tag && git push ...' is needed"
+ end
+ rc_tag = "#{version}-rc#{rc}"
+ ensure_package_job_finished(rc_tag)
+ Dir.mktmpdir do |tmp|
+ sh("gh", "release", "download",
+ "--dir", tmp,
+ "--pattern", archive_name,
+ "--repo", github_repository,
+ rc_tag)
+ tmp_archive_name = "#{tmp}/#{archive_name}"
+ tmp_sign_name = "#{tmp_archive_name}.asc"
+ sh("gpg",
+ "--armor",
+ "--detach-sign",
+ "--local-user", gpg_key_id,
+ "--output", tmp_sign_name,
+ "#{tmp_archive_name}")
+ tmp_checksum_name = "#{tmp_archive_name}.sha512"
+ File.open(tmp_checksum_name, "w") do |output|
+ checksum = Digest::SHA512.file(tmp_archive_name)
+ output.puts("#{checksum} #{archive_name}")
+ end
+ sh("gh", "release", "upload",
+ "--clobber",
+ "--repo", github_repository,
+ rc_tag,
+ tmp_sign_name,
+ tmp_checksum_name)
+ end
+ end
+
+ desc "Upload Linux packages"
+ task :linux do
+ load_env
+ rc = detect_latest_rc(version)
+ if rc.nil?
+ raise "'rake release:rc:tag && git push ...' is needed"
+ end
+ rc_tag = "#{version}-rc#{rc}"
+ Dir.mktmpdir do |tmp|
+ sh("gh", "release", "download",
+ "--dir", tmp,
+ "--pattern", "debian-*.tar.gz",
+ "--pattern", "ubuntu-*.tar.gz",
+ "--repo", github_repository,
+ rc_tag)
+ Dir.glob("#{tmp}/*.tar.gz") do |tar_gz|
+ sh("tar", "xf", tar_gz,
+ "--directory", tmp,
+ "--one-top-level")
+ end
+ env = {
+ "ARROW_ARTIFACTS_DIR" => tmp,
+ "DEB_PACKAGE_NAME" => "apache-arrow-flight-sql-postgresql",
+ "UPLOAD_DEFAULT" => "0",
+ "UPLOAD_DEBIAN" => "1",
+ "UPLOAD_UBUNTU" => "1",
+ }
+ sh(env,
+ "#{arrow_source}/dev/release/05-binary-upload.sh",
+ version,
+ rc.to_s)
+ end
+ end
+
+ desc "Generate a release vote e-mail"
+ task :vote do
+ load_env
+ rc = detect_latest_rc(version)
+ if rc.nil?
+ raise "'rake release:rc:tag && git push ...' is needed"
+ end
+ rc_tag = "#{version}-rc#{rc}"
+ commit = IO.pipe do |read, write|
+ sh("git", "rev-list", "-n", "1", rc_tag, write: write)
+ write.close
+ read.read.strip
+ end
+ puts(<<-MAIL)
+ local -r commit=$(git rev-list -n 1 "${tag}")
+
+ cat <<MAIL
+To: [email protected]
+Subject: [VOTE] Release Apache Arrow Flight SQL adapter for PostgreSQL
${version} - RC#{rc}
+
+Hi,
+
+I would like to propose the following release candidate (RC#{rc}) of
+Apache Arrow Flight SQL adapter for PostgreSQL version #{version}.
+
+This release candidate is based on commit: #{commit} [1]
+
+The source release rc#{rc} and changelog is hosted at [2].
+The binary artifacts are hosted at [3][4].
+
+Please download, verify checksums and signatures, build and run,
+and vote on the release. See [5] for how to validate a release
+candidate.
+
+The vote will be open for at least 24 hours because Apache Arrow
+Flight SQL adapter for PostgreSQL doesn't reach 1.0.0 yet.
+
+[ ] +1 Release this as Apache Arrow Flight SQL adapter for PostgreSQL
#{version}
+[ ] +0
+[ ] -1 Do not release this as Apache Arrow Flight SQL adapter for PostgreSQL
#{version}
+ because...
+
+[1]: https://github.com/apache/arrow-flight-sql-postgresql/commit/#{commit}
+[2]:
https://github.com/apache/arrow-flight-sql-postgresql/releases/tag/#{rc_tag}
+[3]: https://apache.jfrog.io/artifactory/arrow/debian-rc/
+[4]: https://apache.jfrog.io/artifactory/arrow/ubuntu-rc/
+[5]:
https://arrow.apache.org/flight-sql-postgresql/devel/release.html#how-to-verify-release-candidates
+ MAIL
+ end
+
+ namespace :publish do
+ desc "Publish to https://dist.apache.org/";
+ task :apache do
+ load_env
+ rc = detect_latest_rc(version)
+ rc_tag = "#{version}-rc#{rc}"
+ Dir.mktmpdir do |tmp|
+ sh("svn", "co",
+ "--depth=empty",
+ "https://dist.apache.org/repos/dist/release/arrow";,
+ tmp)
+ release_dir = "#{tmp}/apache-arrow-flight-sql-postgresql-#{verison}"
+ mkdir_p(release_dir)
+ sh("gh", "release", "download",
+ "--dir", release_dir,
+ "--pattern", "#{archive_name}*",
+ "--repo", github_repository,
+ rc_tag)
+ sh("svn", "add", release_dir)
+ sh("svn", "ci", "-m", "#{project_label} #{version}", release_dir)
+ end
+ end
+
+ desc "Register a new release to https://reporter.apache.org/";
+ task :reporter do
+ sh("open", "https://reporter.apache.org/addrelease.html?arrow";)
+ end
+
+ desc "Publish Linux packages"
+ task :linux do
+ rc = detect_latest_rc(version)
+ env = {
+ "UPLOAD_DEFAULT" => "0",
+ "UPLOAD_DEBIAN" => "1",
+ "UPLOAD_UBUNTU" => "1",
+ }
+ sh(env,
+ "#{arrow_source}/dev/release/post-02-binary.sh",
+ version,
+ rc.to_s)
+ end
+
+ desc "Tag #{version}"
+ task :tag do
+ load_env
+ rc = detect_latest_rc(version)
+ rc_tag = "#{version}-rc#{rc}"
+ sh("git", "tag",
+ "-a", version,
+ "-m", "#{project_label} #{version}",
+ "#{rc_tag}^{}")
+ if env_value("TAG_PUSH", "no") == "yes"
+ sh("git", "push", "upstream", version)
+ else
+ puts("Push #{version}:")
+ puts(" git push upstream #{version}")
+ end
+ end
+ end
+
+ desc "Publish the latest RC as a new release"
+ task :publish => [
+ "release:rc:publish:apache",
+ "release:rc:publish:reporter",
+ "release:rc:publish:linux",
+ "release:rc:publish:tag",
+ ]
+ end
+end
diff --git a/.gitignore b/dev/release/.env.example
similarity index 56%
copy from .gitignore
copy to dev/release/.env.example
index b75d024..11ff241 100644
--- a/.gitignore
+++ b/dev/release/.env.example
@@ -15,17 +15,21 @@
# specific language governing permissions and limitations
# under the License.
-/*.tar.gz
-/.cache/
-/Gemfile.lock
-/compile_commands.json
-/dev/release/apache-rat-0.13.jar
-/dev/release/dist
-/dev/release/filtered_rat.txt
-/dev/release/rat.xml
-/doc/build/
-/package/*/*.tar.gz
-/package/*/apt/build.sh
-/package/*/apt/env.sh
-/package/*/apt/repositories/
-/package/*/apt/tmp/
+# The GPG key ID to sign artifacts. The GPG key ID must be registered
+# to both of the followings:
+#
+# * https://dist.apache.org/repos/dist/dev/arrow/KEYS
+# * https://dist.apache.org/repos/dist/release/arrow/KEYS
+#
+# See these files how to import your GPG key ID to these files.
+#
+# You must set this.
+#GPG_KEY_ID=08D3564B7C6A9CAFBFF6A66791D18FCF079F8007
+
+# The Artifactory API key to upload artifacts to Artifactory.
+#
+# This can be obtained by logging in to https://apache.jfrog.io with
+# your ASF credentials and generating an API key from your user profile.
+#
+# You must set this.
+#ARTIFACTORY_API_KEY=secret
diff --git a/src/afs.cc b/src/afs.cc
index bf78472..44f8d43 100644
--- a/src/afs.cc
+++ b/src/afs.cc
@@ -46,6 +46,10 @@ extern "C"
#undef Abs
+#if PG_VERSION_NUM >= 150000
+# define PGRN_HAVE_SHMEM_REQUEST_HOOK
+#endif
+
#include <arrow/buffer.h>
#include <arrow/builder.h>
#include <arrow/flight/server_middleware.h>
@@ -134,14 +138,17 @@ afs_sigusr1(SIGNAL_ARGS)
errno = errnoSaved;
}
+#ifdef PGRN_HAVE_SHMEM_REQUEST_HOOK
static shmem_request_hook_type PreviousShmemRequestHook = nullptr;
+#endif
static const char* LWLockTrancheName = "arrow-flight-sql: lwlock tranche";
void
afs_shmem_request_hook(void)
{
+#ifdef PGRN_HAVE_SHMEM_REQUEST_HOOK
if (PreviousShmemRequestHook)
PreviousShmemRequestHook();
-
+#endif
RequestNamedLWLockTranche(LWLockTrancheName, 1);
}
@@ -2378,7 +2385,7 @@ class Proxy : public WorkerProcessor {
break;
}
}
- return std::move(reader->schema());
+ return reader->schema();
}
std::random_device randomSeed_;
@@ -2452,7 +2459,7 @@ class MainProcessor : public Processor {
BackgroundWorkerHandle* start_server()
{
- BackgroundWorker worker = {0};
+ BackgroundWorker worker = {};
snprintf(worker.bgw_name, BGW_MAXLEN, "%s: server", Tag);
snprintf(worker.bgw_type, BGW_MAXLEN, "%s: server", Tag);
worker.bgw_flags = BGWORKER_SHMEM_ACCESS;
@@ -2485,7 +2492,7 @@ class MainProcessor : public Processor {
continue;
}
- BackgroundWorker worker = {0};
+ BackgroundWorker worker = {};
snprintf(
worker.bgw_name, BGW_MAXLEN, "%s: executor: %"
PRIu64, Tag, session->id);
snprintf(
@@ -2663,7 +2670,7 @@ class FlightSQLServer : public
arrow::flight::sql::FlightSqlServerBase {
ARROW_ASSIGN_OR_RAISE(auto ticket,
arrow::flight::sql::CreateStatementQueryTicket(query));
std::vector<arrow::flight::FlightEndpoint> endpoints{
- arrow::flight::FlightEndpoint{std::move(ticket), {}}};
+
arrow::flight::FlightEndpoint{arrow::flight::Ticket{std::move(ticket)}, {}}};
ARROW_ASSIGN_OR_RAISE(
auto result,
arrow::flight::FlightInfo::Make(*schema, descriptor,
endpoints, -1, -1));
@@ -2691,7 +2698,7 @@ class FlightSQLServer : public
arrow::flight::sql::FlightSqlServerBase {
arrow::Result<arrow::flight::sql::ActionCreatePreparedStatementResult>
CreatePreparedStatement(
const arrow::flight::ServerCallContext& context,
- const arrow::flight::sql::ActionCreatePreparedStatementRequest&
request)
+ const arrow::flight::sql::ActionCreatePreparedStatementRequest&
request) override
{
ARROW_ASSIGN_OR_RAISE(auto sessionID, session_id(context));
const auto& query = request.query;
@@ -2700,7 +2707,7 @@ class FlightSQLServer : public
arrow::flight::sql::FlightSqlServerBase {
arrow::Status ClosePreparedStatement(
const arrow::flight::ServerCallContext& context,
- const arrow::flight::sql::ActionClosePreparedStatementRequest&
request)
+ const arrow::flight::sql::ActionClosePreparedStatementRequest&
request) override
{
ARROW_ASSIGN_OR_RAISE(auto sessionID, session_id(context));
const auto& handle = request.prepared_statement_handle;
@@ -2720,7 +2727,7 @@ class FlightSQLServer : public
arrow::flight::sql::FlightSqlServerBase {
ARROW_ASSIGN_OR_RAISE(auto ticket,
arrow::flight::sql::CreateStatementQueryTicket(handle));
std::vector<arrow::flight::FlightEndpoint> endpoints{
- arrow::flight::FlightEndpoint{std::move(ticket), {}}};
+
arrow::flight::FlightEndpoint{arrow::flight::Ticket{std::move(ticket)}, {}}};
ARROW_ASSIGN_OR_RAISE(
auto result,
arrow::flight::FlightInfo::Make(*schema, descriptor,
endpoints, -1, -1));
@@ -2995,10 +3002,14 @@ _PG_init(void)
NULL,
NULL);
+#ifdef PGRN_HAVE_SHMEM_REQUEST_HOOK
PreviousShmemRequestHook = shmem_request_hook;
shmem_request_hook = afs_shmem_request_hook;
+#else
+ afs_shmem_request_hook();
+#endif
- BackgroundWorker worker = {0};
+ BackgroundWorker worker = {};
snprintf(worker.bgw_name, BGW_MAXLEN, "%s: main", Tag);
snprintf(worker.bgw_type, BGW_MAXLEN, "%s: main", Tag);
worker.bgw_flags = BGWORKER_SHMEM_ACCESS;
