pingtimeout commented on code in PR #2156: URL: https://github.com/apache/polaris/pull/2156#discussion_r2225575903
########## releasey/libs/_log.sh: ########## @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Common Logging Functions +# + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color Review Comment: Ack. Fixing this. ########## releasey/libs/_log.sh: ########## @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Common Logging Functions +# + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +function print_error() { + echo -e "${RED}ERROR: $*${NC}" >&2 +} + +function print_success() { + echo -e "${GREEN}SUCCESS: $*${NC}" >&2 +} + +function print_warning() { + echo -e "${YELLOW}WARNING: $*${NC}" >&2 +} + +function print_info() { + echo "INFO: $*" >&2 +} + +function print_command() { + # This function can be used to print the bash commands that are executed by a + # script. It either prints its arguments as-is to file descriptor 3, if it + # is open, or prepends "DEBUG: " and prints them to stdout if it is unset. + # + # This allows the programmatic verification of all commands that are executed + # by a script, by running it as follows: + # + # $ ./script.sh 3>/tmp/script.log + # $ cat /tmp/script.log + # ./gradlew clean build + # rm -rf ./regtests/output Review Comment: https://github.com/apache/polaris/blob/main/regtests/README.md#prerequisites ########## releasey/libs/_nexus.sh: ########## @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Maven/Gradle Credentials Verification Script +# +# This script verifies that Maven/Gradle credentials are properly configured for Apache Polaris releases: +# 1. Checks if Apache credentials are configured in ~/.gradle/gradle.properties +# 2. Checks if Apache credentials are configured via environment variables +# 3. Validates that at least one method is configured +# 4. Attempts to verify credentials against Apache Nexus repository +# +# Returns non-zero exit code if any verification fails. +# + +set -euo pipefail + +libs_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source common logging functions and constants +source "${libs_dir}/_log.sh" +source "${libs_dir}/_constants.sh" +source "${libs_dir}/_exec.sh" + +function check_apache_credentials_are_set_in_gradle_properties() { + # Check if Apache credentials are configured in ~/.gradle/gradle.properties, return non-zero if not + print_info "Checking for Apache credentials in ${GRADLE_PROPERTIES_FILE}..." + + if [[ ! -f "${GRADLE_PROPERTIES_FILE}" ]]; then + print_warning "Gradle properties file not found: ${GRADLE_PROPERTIES_FILE}" + return 1 + fi + + if grep -q "^apacheUsername=" "${GRADLE_PROPERTIES_FILE}" 2>/dev/null && + grep -q "^apachePassword=" "${GRADLE_PROPERTIES_FILE}" 2>/dev/null; then Review Comment: That's what the release guide says. Could you open an issue to track that? ########## releasey/06-build-and-stage-docker-images.sh: ########## @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Build and Stage Docker Images Script +# +# Builds and publishes multi-platform Docker images to DockerHub for release candidates. +# This script automates the "Build and staging Docker images" step from the release guide. +# + +set -euo pipefail + +releases_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +libs_dir="${releases_dir}/libs" + +source "${libs_dir}/_log.sh" +source "${libs_dir}/_constants.sh" +source "${libs_dir}/_exec.sh" +source "${libs_dir}/_files.sh" + +function usage() { + cat << EOF +$(basename "$0") [--help | -h] + + Builds and publishes multi-platform Docker images to DockerHub for release candidates. + The version is automatically determined from the current git tag. I.e. this script must + be run from a release tag. + + Prerequisites: + - Docker with buildx support must be installed and configured + - DockerHub credentials must be configured (docker login) + - Must be run from a release tag (apache-polaris-x.y.z-incubating-rcN) + + Options: + -h --help + Print usage information. + + Examples: + $(basename "$0") + +EOF +} + +ensure_cwd_is_project_root + +while [[ $# -gt 0 ]]; do + case $1 in + --help|-h) + usage + exit 0 + ;; + *) + print_error "Unknown option/argument $1" + usage >&2 + exit 1 + ;; + esac +done + +# Determine version from current git tag +print_info "Determining version from current git tag..." + +if ! git_tag=$(git describe --tags --exact-match HEAD 2>/dev/null); then + print_error "Current HEAD is not on a release tag. Please checkout a release tag first." + print_error "Use: git checkout apache-polaris-x.y.z-incubating-rcN" + exit 1 +fi +print_info "Found git tag: ${git_tag}" + +# Extract version components from git tag in one regex match +git_tag_regex="^apache-polaris-([0-9]+)\.([0-9]+)\.([0-9]+)-incubating-rc([0-9]+)$" +if [[ ! ${git_tag} =~ ${git_tag_regex} ]]; then + print_error "Invalid git tag format: ${git_tag}" + print_error "Expected format: apache-polaris-x.y.z-incubating-rcN" + exit 1 +fi + +# Extract version components from regex match +major="${BASH_REMATCH[1]}" +minor="${BASH_REMATCH[2]}" +patch="${BASH_REMATCH[3]}" +rc_number="${BASH_REMATCH[4]}" +version="${major}.${minor}.${patch}-incubating" Review Comment: I could but it would not bring any benefit. We would have a constant for the incubating suffix, sure. Which means that as soon as Polaris graduates, that constant becomes an empty string and should be removed entirely from the scripts. This means that instead of having to remove `-incubating` everywhere, we would have to remove `${INCUBATING_SUFFIX}` everywhere. Are you sure we should do that? ########## releasey/06-build-and-stage-docker-images.sh: ########## @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Build and Stage Docker Images Script +# +# Builds and publishes multi-platform Docker images to DockerHub for release candidates. +# This script automates the "Build and staging Docker images" step from the release guide. +# + +set -euo pipefail + +releases_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +libs_dir="${releases_dir}/libs" + +source "${libs_dir}/_log.sh" +source "${libs_dir}/_constants.sh" +source "${libs_dir}/_exec.sh" +source "${libs_dir}/_files.sh" + +function usage() { + cat << EOF +$(basename "$0") [--help | -h] + + Builds and publishes multi-platform Docker images to DockerHub for release candidates. + The version is automatically determined from the current git tag. I.e. this script must + be run from a release tag. + + Prerequisites: + - Docker with buildx support must be installed and configured + - DockerHub credentials must be configured (docker login) + - Must be run from a release tag (apache-polaris-x.y.z-incubating-rcN) + + Options: + -h --help + Print usage information. + + Examples: + $(basename "$0") + +EOF +} + +ensure_cwd_is_project_root + +while [[ $# -gt 0 ]]; do + case $1 in + --help|-h) + usage + exit 0 + ;; + *) + print_error "Unknown option/argument $1" + usage >&2 + exit 1 + ;; + esac +done + +# Determine version from current git tag +print_info "Determining version from current git tag..." + +if ! git_tag=$(git describe --tags --exact-match HEAD 2>/dev/null); then + print_error "Current HEAD is not on a release tag. Please checkout a release tag first." + print_error "Use: git checkout apache-polaris-x.y.z-incubating-rcN" + exit 1 +fi +print_info "Found git tag: ${git_tag}" + +# Extract version components from git tag in one regex match +git_tag_regex="^apache-polaris-([0-9]+)\.([0-9]+)\.([0-9]+)-incubating-rc([0-9]+)$" +if [[ ! ${git_tag} =~ ${git_tag_regex} ]]; then + print_error "Invalid git tag format: ${git_tag}" + print_error "Expected format: apache-polaris-x.y.z-incubating-rcN" + exit 1 +fi + +# Extract version components from regex match +major="${BASH_REMATCH[1]}" +minor="${BASH_REMATCH[2]}" +patch="${BASH_REMATCH[3]}" +rc_number="${BASH_REMATCH[4]}" +version="${major}.${minor}.${patch}-incubating" +docker_tag="${version}-rc${rc_number}" + +print_info "Starting Docker image build and staging process..." +print_info "Version: ${version}" +print_info "RC number: ${rc_number}" +print_info "Docker tag: ${docker_tag}" +echo + +# Build and push polaris-server Docker image +print_info "Building and pushing polaris-server Docker image..." +exec_process ./gradlew :polaris-server:assemble :polaris-server:quarkusAppPartsBuild --rerun \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.push=true \ + -Dquarkus.docker.buildx.platform="linux/amd64,linux/arm64" \ + -Dquarkus.container-image.tag="${docker_tag}" + +# Build and push polaris-admin Docker image +print_info "Building and pushing polaris-admin Docker image..." +exec_process ./gradlew :polaris-admin:assemble :polaris-admin:quarkusAppPartsBuild --rerun \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.push=true \ + -Dquarkus.docker.buildx.platform="linux/amd64,linux/arm64" \ + -Dquarkus.container-image.tag="${docker_tag}" + +echo +print_success "π Docker images built and staged successfully!" +echo +print_info "Published Docker images:" +print_info "- polaris-server: ${DOCKER_REGISTRY}/apache/polaris:${docker_tag}" +print_info "- polaris-admin: ${DOCKER_REGISTRY}/apache/polaris-admin-tool:${docker_tag}" +echo +print_info "Docker Hub links:" +print_info "- ${DOCKER_HUB_URL}/r/apache/polaris/tags" +print_info "- ${DOCKER_HUB_URL}/r/apache/polaris-admin-tool/tags" +echo +print_info "Next steps:" +print_info "1. Start the vote thread" Review Comment: Wdym a proposal for the subject and body of the e-mail? I don't think we need a shell script just to run `./gradlew releaseEmailTemplate`. Are you talking about anything else? ########## releasey/libs/_constants.sh: ########## @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +libs_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# GPG constants +KEYS_URL=${KEYS_URL:-"https://downloads.apache.org/incubator/polaris/KEYS"} +KEYSERVER=${KEYSERVER:-"hkps://keyserver.ubuntu.com"} Review Comment: That is part of the release guide. > To send the key to the Ubuntu key-server, Apache Nexus needs it to validate published artifacts: > > gpg --keyserver hkps://keyserver.ubuntu.com --send-keys <YOUR KEY ID HERE> ########## releasey/libs/_log.sh: ########## @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Common Logging Functions +# + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' Review Comment: Ack. Fixing this. ########## releasey/libs/_files.sh: ########## @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Utility function for version.txt manipulation +# + +libs_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$libs_dir/_constants.sh" +source "$libs_dir/_log.sh" + +function update_version { + local version="$1" + # This function is only there for dry-run support. Because of the + # redirection, we cannot use exec_process with the exact command that will be + # executed. + if [[ ${DRY_RUN:-1} -ne 1 ]]; then + exec_process echo ${version} >$VERSION_FILE + else + exec_process "echo ${version} > $VERSION_FILE" + fi +} + +function ensure_cwd_is_project_root() { + # This function verifies that the script is executed from the project root + # directory and that the gradle wrapper script is present. + if [[ ! -f "gradlew" ]] || [[ ! -d ".git" ]]; then Review Comment: Good point, removing that. Just relying on gradlew existence is enough. ########## releasey/libs/_nexus.sh: ########## @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Maven/Gradle Credentials Verification Script +# +# This script verifies that Maven/Gradle credentials are properly configured for Apache Polaris releases: +# 1. Checks if Apache credentials are configured in ~/.gradle/gradle.properties +# 2. Checks if Apache credentials are configured via environment variables +# 3. Validates that at least one method is configured +# 4. Attempts to verify credentials against Apache Nexus repository +# +# Returns non-zero exit code if any verification fails. +# + +set -euo pipefail + +libs_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source common logging functions and constants +source "${libs_dir}/_log.sh" +source "${libs_dir}/_constants.sh" +source "${libs_dir}/_exec.sh" + +function check_apache_credentials_are_set_in_gradle_properties() { + # Check if Apache credentials are configured in ~/.gradle/gradle.properties, return non-zero if not + print_info "Checking for Apache credentials in ${GRADLE_PROPERTIES_FILE}..." + + if [[ ! -f "${GRADLE_PROPERTIES_FILE}" ]]; then + print_warning "Gradle properties file not found: ${GRADLE_PROPERTIES_FILE}" + return 1 + fi + + if grep -q "^apacheUsername=" "${GRADLE_PROPERTIES_FILE}" 2>/dev/null && + grep -q "^apachePassword=" "${GRADLE_PROPERTIES_FILE}" 2>/dev/null; then + print_success "Found Apache credentials in gradle.properties" + return 0 + else + print_warning "Apache credentials not found in ${GRADLE_PROPERTIES_FILE}" + return 1 + fi +} + +function check_apache_credentials_are_set_in_environment_variables() { + # Check if Apache credentials are configured via environment variables, return non-zero if not + print_info "Checking for Apache credentials in environment variables..." + + if [[ -n "${ORG_GRADLE_PROJECT_apacheUsername:-}" && + -n "${ORG_GRADLE_PROJECT_apachePassword:-}" ]]; then + print_success "Found Apache credentials in environment variables" + return 0 + else + print_warning "Apache credentials not found in environment variables" + return 1 + fi +} + +function ensure_credentials_are_configured() { + # Ensure that Apache credentials are configured in either gradle.properties or environment variables, return non-zero if not + print_info "Checking Apache credentials configuration..." + + if check_apache_credentials_are_set_in_gradle_properties; then + print_success "All Maven/Gradle credential checks passed! Your Apache credentials are properly configured for releases." + return 0 + fi Review Comment: Let's do this as soon as the release guide is updated. ########## releasey/libs/_constants.sh: ########## @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +libs_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# GPG constants +KEYS_URL=${KEYS_URL:-"https://downloads.apache.org/incubator/polaris/KEYS"} +KEYSERVER=${KEYSERVER:-"hkps://keyserver.ubuntu.com"} + +# Git/SVN repository constants +APACHE_REMOTE_NAME=${APACHE_REMOTE_NAME:-"apache"} Review Comment: So, on one hand what you are saying is perfectly valid and we should use `origin`. On the other hand, the release guide in its current state has been written considering that individuals may be performing releases and refers to it as `apache` > The release should be executed against https://github.com/apache/polaris.git repository (not a fork). Set it as remote with name apache for release if itβs not already set up. With the current variable, we can override it in a GH workflow to `origin`. And once we completely adopt the GH workflow based releasing process, we can switch the default value. ########## releasey/libs/_docker.sh: ########## @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Docker Setup Verification Script +# +# This script verifies that Docker is properly configured for Apache Polaris releases: +# 1. Checks that Docker is installed and available in PATH +# 2. Verifies that Docker daemon is running and accessible +# 3. Confirms that Docker buildx is available for multi-platform builds +# 4. Checks Docker Hub login status (optional) +# +# Returns non-zero exit code if any verification fails. +# + +set -euo pipefail + +libs_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source common logging functions and constants +source "${libs_dir}/_log.sh" +source "${libs_dir}/_constants.sh" +source "${libs_dir}/_exec.sh" + +function ensure_docker_is_available() { + # Check if Docker is installed, return non-zero if not + print_info "Checking if Docker is available..." + if ! command -v docker >/dev/null 2>&1; then + print_error "Docker is not installed or not in PATH" + print_error "Please install Docker to build and publish Docker images" + return 1 + fi + print_success "β Docker is installed" + return 0 +} + +function ensure_docker_daemon_is_accessible() { + # Check if Docker daemon is running and accessible, return non-zero if not + print_info "Checking if Docker daemon is accessible..." + if ! docker info >/dev/null 2>&1; then + print_error "Docker daemon is not running or not accessible" + print_error "Please start Docker daemon and ensure you have permissions to access it" + return 1 + fi + print_success "β Docker daemon is accessible" + return 0 +} + +function ensure_docker_buildx_is_available() { + # Check if Docker buildx is available, return non-zero if not + print_info "Checking if Docker buildx is available..." + if ! docker buildx version >/dev/null 2>&1; then + print_error "Docker buildx is not available" + print_error "Please install Docker with buildx support for multi-platform builds" + return 1 + fi + print_success "β Docker buildx is available" + return 0 +} + +function check_docker_hub_login_status() { + # Check if user is logged in to Docker Hub (optional check) + print_info "Checking Docker Hub login status..." + if docker info 2>/dev/null | grep -q "Username:"; then Review Comment: Do you think we should just assume docker hub login is properly configured? I wanted this to be a quick and easy check. But if it turns out to create more friction than benefits, let's drop it. ########## releasey/02-create-release-branch.sh: ########## @@ -0,0 +1,186 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Create Release Branch Script +# +# Automates the "Create release branch" section of the release guide. +# Creates a new release branch and sets the target release version. +# + +set -euo pipefail + +releases_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +libs_dir="${releases_dir}/libs" + +source "${libs_dir}/_log.sh" +source "${libs_dir}/_constants.sh" +source "${libs_dir}/_exec.sh" +source "${libs_dir}/_files.sh" + +function usage() { + cat << EOF +$(basename "$0") --version VERSION [--commit GIT_COMMIT] [--recreate] [--help | -h] + + Creates a release branch and sets the target release version. + + Options: + --version VERSION + The release version in format x.y.z-incubating + --commit GIT_COMMIT + The Git commit SHA to create the release branch from. Defaults to current HEAD. + --recreate + Drop the release branch if it already exists and recreate it to the new SHA. + -h --help + Print usage information. + + Examples: + $(basename "$0") --version 1.0.0-incubating --commit HEAD + $(basename "$0") --version 0.1.0-incubating --commit abc123def456 + $(basename "$0") --version 1.0.0-incubating --commit HEAD --recreate + +EOF +} + +ensure_cwd_is_project_root + +version="" +commit="HEAD" +recreate=false + +while [[ $# -gt 0 ]]; do + case $1 in + --version) + if [[ $# -lt 2 ]]; then + print_error "Missing argument for --version" + usage >&2 + exit 1 + fi + version="$2" + shift 2 + ;; + --commit) + if [[ $# -lt 2 ]]; then + print_error "Missing argument for --commit" + usage >&2 + exit 1 + fi + commit="$2" + shift 2 + ;; + --recreate) + recreate=true + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + print_error "Unknown option/argument $1" + usage >&2 + exit 1 + ;; + esac +done + +# Ensure the version is provided +if [[ -z ${version} ]]; then + print_error "Missing version" + usage >&2 + exit 1 +fi + +# Validate that the commit exists +if ! git rev-parse --verify "${commit}" >/dev/null 2>&1; then + print_error "Invalid Git commit: ${commit}" + usage >&2 + exit 1 +fi + +# Validate version format: x.y.z-incubating +# TODO: Remove incubating when we are a TLP +version_regex="^([0-9]+)\.([0-9]+)\.([0-9]+)-incubating$" +if [[ ! ${version} =~ ${version_regex} ]]; then Review Comment: I am not sure I understand the question. AFAICT, we are going to create a single branch per release. I.e. we won't create a release branch for different RC of the same version. ########## releasey/03-create-release-tag.sh: ########## @@ -0,0 +1,150 @@ +#!/bin/bash Review Comment: Renamed ########## releasey/03-create-release-tag.sh: ########## @@ -0,0 +1,150 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Create Release Tag Script +# +# Automates the "Create release tag" section of the release guide. Review Comment: Fixed ########## releasey/libs/_files.sh: ########## @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Utility function for version.txt manipulation +# + +libs_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$libs_dir/_constants.sh" +source "$libs_dir/_log.sh" + +function update_version { + local version="$1" + # This function is only there for dry-run support. Because of the + # redirection, we cannot use exec_process with the exact command that will be + # executed. + if [[ ${DRY_RUN:-1} -ne 1 ]]; then + exec_process echo ${version} >$VERSION_FILE + else + exec_process "echo ${version} > $VERSION_FILE" + fi +} + +function ensure_cwd_is_project_root() { Review Comment: True, with the downside that now the scripts have to `pushd/popd` in order to run any gradlew command. Let me check if I can get that changed in a not too-invasive way. If not, it's probably ok as-is given that GH Workflow will be the main execution flow for the scripts. ########## releasey/06-build-and-stage-docker-images.sh: ########## @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Build and Stage Docker Images Script +# +# Builds and publishes multi-platform Docker images to DockerHub for release candidates. +# This script automates the "Build and staging Docker images" step from the release guide. +# + +set -euo pipefail + +releases_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +libs_dir="${releases_dir}/libs" + +source "${libs_dir}/_log.sh" +source "${libs_dir}/_constants.sh" +source "${libs_dir}/_exec.sh" +source "${libs_dir}/_files.sh" + +function usage() { + cat << EOF +$(basename "$0") [--help | -h] + + Builds and publishes multi-platform Docker images to DockerHub for release candidates. + The version is automatically determined from the current git tag. I.e. this script must + be run from a release tag. + + Prerequisites: + - Docker with buildx support must be installed and configured + - DockerHub credentials must be configured (docker login) + - Must be run from a release tag (apache-polaris-x.y.z-incubating-rcN) + + Options: + -h --help + Print usage information. + + Examples: + $(basename "$0") + +EOF +} + +ensure_cwd_is_project_root + +while [[ $# -gt 0 ]]; do + case $1 in + --help|-h) + usage + exit 0 + ;; + *) + print_error "Unknown option/argument $1" + usage >&2 + exit 1 + ;; + esac +done + +# Determine version from current git tag +print_info "Determining version from current git tag..." + +if ! git_tag=$(git describe --tags --exact-match HEAD 2>/dev/null); then + print_error "Current HEAD is not on a release tag. Please checkout a release tag first." + print_error "Use: git checkout apache-polaris-x.y.z-incubating-rcN" + exit 1 +fi +print_info "Found git tag: ${git_tag}" + +# Extract version components from git tag in one regex match +git_tag_regex="^apache-polaris-([0-9]+)\.([0-9]+)\.([0-9]+)-incubating-rc([0-9]+)$" +if [[ ! ${git_tag} =~ ${git_tag_regex} ]]; then + print_error "Invalid git tag format: ${git_tag}" + print_error "Expected format: apache-polaris-x.y.z-incubating-rcN" + exit 1 +fi + +# Extract version components from regex match Review Comment: Good point, implementing this... ########## releasey/04-build-and-test.sh: ########## @@ -0,0 +1,118 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Build and Test Script +# +# Builds Polaris completely and runs regression tests to verify the release. +# Cloud-specific tests are disabled by default and require proper credentials +# to be configured in the environment. +# Review Comment: TIL the GH API endpoint. Let me look into this. It would certainly be faster to do it that way. And also it would enforce the fact that the commit must be validated by CI, as opposed to what the current `--commit` parameter allows. ########## releasey/03-create-release-tag.sh: ########## @@ -0,0 +1,150 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Create Release Tag Script +# +# Automates the "Create release tag" section of the release guide. +# + +set -euo pipefail + +releases_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +libs_dir="${releases_dir}/libs" + +source "${libs_dir}/_log.sh" +source "${libs_dir}/_constants.sh" +source "${libs_dir}/_exec.sh" +source "${libs_dir}/_files.sh" + +function usage() { + cat << EOF +$(basename "$0") --version VERSION [--help | -h] + + Creates a release tag for a release candidate. + + Options: + --version VERSION + The release version in format x.y.z-incubating-rcN where N is the RC number + -h --help + Print usage information. + + Examples: + $(basename "$0") --version 1.0.0-incubating-rc1 Review Comment: I considered pulling that part from your PR, indeed. But decided to not reuse it so that the code is overall simpler. The current approach is less smart, as it pushes the responsibility of adding `-incubating` to the release manager and enforces it via a regex validation. The alternative would be: * Have a `--version x.y.z` parameter that accepts `x.y.z` as well as `x.y.z-incubating` * Have a `--rc N` parameter * Have a logic that detects whether `-incubating` is already present and adds it, if missing Overall, I would rather have a single `--version x.y.z-incubating-rcN` parameter validated by a regex. Simpler code. The only downside, really, is that when we graduate, we will have to update multiple shell scripts to remove the `-incubating` part. I feel that it is acceptable. ########## releasey/libs/_exec.sh: ########## @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Execution wrapper function for system-modifying commands +# + +libs_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$libs_dir/_constants.sh" +source "$libs_dir/_log.sh" + +function exec_process { + if [[ ${DRY_RUN:-1} -ne 1 ]]; then + print_command "Executing '${*}'" + "$@" + elif { >&3; } 2>/dev/null; then Review Comment: I cannot seem to get something working with named FD. Ideally, test scripts should be able to use them as follows: ``` DRY_RUN=1 \ "${releases_dir}/02-create-release-branch.sh" \ --version 42.41.40-incubating \ {COMMANDS_LOG}>"$temp_file" ``` Instead of ``` DRY_RUN=1 \ "${releases_dir}/02-create-release-branch.sh" \ --version 42.41.40-incubating \ 3>"$temp_file" ``` But I cannot seem to get the underlying script to recognize `COMMANDS_LOG`. Do you have an example for a main script that can set a named fd for a subscript? ########## releasey/04-build-and-test.sh: ########## @@ -0,0 +1,118 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Build and Test Script +# +# Builds Polaris completely and runs regression tests to verify the release. +# Cloud-specific tests are disabled by default and require proper credentials +# to be configured in the environment. +# + +set -euo pipefail + +releases_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +libs_dir="${releases_dir}/libs" + +source "${libs_dir}/_log.sh" +source "${libs_dir}/_constants.sh" +source "${libs_dir}/_exec.sh" +source "${libs_dir}/_files.sh" + +function usage() { + cat << EOF +$(basename "$0") [--help | -h] + + Builds Polaris completely and runs regression tests to verify the release. + + Options: + -h --help + Print usage information. + + Examples: + $(basename "$0") + +EOF +} + +ensure_cwd_is_project_root + +while [[ $# -gt 0 ]]; do + case $1 in + --help|-h) + usage + exit 0 + ;; + *) + print_error "Unknown option/argument $1" + usage >&2 + exit 1 + ;; + esac +done + +print_info "Starting build and test process..." +echo + +# Clean and build Polaris +print_info "Building Polaris..." +exec_process ./gradlew clean build Review Comment: I agree with you on the unnecessary aspect of those commands. That being said, I think that until we develop the habit of only releasing via GH workflows, _it is possible_ that a release manager performs a release from their machine. I am going to add a check that ensures there is no uncommitted change in the repo. That way, combined with the verification in GH API that the commit has been validated by CI, we should be good. Both the "working directory is clean" and `./gradlew clean` steps will become no-op in GH workflows anyway. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@polaris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
