This is an automated email from the ASF dual-hosted git repository.
pingtimeout pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 528354e77 Fix Helm index generation during releases (#3515)
528354e77 is described below
commit 528354e77c8f2da63b80bc781b37f2a738f3ce11
Author: Pierre Laporte <[email protected]>
AuthorDate: Mon Feb 16 16:44:00 2026 +0100
Fix Helm index generation during releases (#3515)
The current release workflows force a regeneration of the Helm index
after the old (unmaintained) releases are removed from the Apache dist
release SVN server. This causes previous releases to be removed from
the Helm index (#3500).
This commit includes a standalone script that re-generates a Helm index
across
`downloads.apache.org` as well as `archive.apache.org`. The logic is as
follows:
1. All Helm charts from `downloads.a.o` and `archive.a.o` are downloaded
locally so that the index is fully rebuilt across all versions.
2. The URL of any Helm chart that is located on `archive.apache.org` but
not on `downloads.apache.org` is replaced by an absolute URL to
`archive.apache.org`. That way, previous releases are always
accessible.
3. The URL of any Helm chart that is located on `downloads.apache.org`
is replaced by a relative path `${version}/polaris-${version}.tgz`.
That way, the Helm index can be rebuild before a release vote is
started, as the relative path will work regardless of the location being
used (dist dev during RC vote, dist release and `downloads.a.o` after
vote).
The commit also includes a more permanent fix for the release workflows.
The
script is expected to only be necessary in case of a workflow error.
---
.../release-3-build-and-publish-artifacts.yml | 88 +++++++++++-
.github/workflows/release-4-publish-release.yml | 27 ++--
releasey/bin/build-helm-index.sh | 151 +++++++++++++++++++++
3 files changed, 246 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/release-3-build-and-publish-artifacts.yml
b/.github/workflows/release-3-build-and-publish-artifacts.yml
index 983d82724..84920078e 100644
--- a/.github/workflows/release-3-build-and-publish-artifacts.yml
+++ b/.github/workflows/release-3-build-and-publish-artifacts.yml
@@ -395,9 +395,6 @@ jobs:
exec_process mkdir -p
"${dist_dev_dir}/helm-chart/${version_without_rc}"
exec_process cp helm/polaris-${version_without_rc}.tgz*
"${dist_dev_dir}/helm-chart/${version_without_rc}/"
- exec_process cd "${dist_dev_dir}/helm-chart"
- exec_process helm repo index .
-
exec_process cd "${dist_dev_dir}"
exec_process svn add "helm-chart/${version_without_rc}"
@@ -405,7 +402,7 @@ jobs:
echo "## Helm Chart Summary" >> $GITHUB_STEP_SUMMARY
cat <<EOT >> $GITHUB_STEP_SUMMARY
- 🎉 Helm chart built and staged successfully:
+ 🎉 Helm chart staged successfully:
| Component | Status |
| --- | --- |
@@ -413,6 +410,89 @@ jobs:
| Apache dist dev repository | ✅ Staged |
EOT
+ - name: Regenerate Helm index with archive versions
+ env:
+ SVN_USERNAME: ${{ secrets.APACHE_USERNAME }}
+ SVN_PASSWORD: ${{ secrets.APACHE_PASSWORD }}
+ run: |
+ echo "::add-mask::$SVN_PASSWORD"
+
+ source "${LIBS_DIR}/_exec.sh"
+
+ dist_dev_dir=${RELEASEY_DIR}/polaris-dist-dev
+
+ # Base URL for Apache Polaris helm charts archive
+
ARCHIVE_URL="https://archive.apache.org/dist/incubator/polaris/helm-chart";
+
+ # Temporary directory for downloads
+ WORK_DIR=$(mktemp -d)
+ trap 'rm -rf "$WORK_DIR"' EXIT
+
+ echo "Working directory: $WORK_DIR"
+
+ echo ""
+ echo "Discovering chart versions from archive.apache.org..."
+ ARCHIVE_VERSIONS=$(curl -sL "$ARCHIVE_URL/" | \
+ grep -oE 'href="[0-9]+\.[0-9]+\.[0-9]+-incubating/"' | \
+ sed 's/href="//;s/\/"//' | \
+ sort -V | uniq || echo "")
+ echo "Found archive versions: $ARCHIVE_VERSIONS"
+
+ echo ""
+ echo "Downloading archived Helm charts..."
+ for version in $ARCHIVE_VERSIONS; do
+ echo "Processing archive version: $version"
+
+ CHART_URL="${ARCHIVE_URL}/${version}/polaris-${version}.tgz"
+ LOCAL_TGZ="$WORK_DIR/polaris-${version}.tgz"
+
+ echo " Downloading: $CHART_URL"
+ if ! curl -sLf -o "$LOCAL_TGZ" "$CHART_URL"; then
+ echo " WARNING: Failed to download $CHART_URL, skipping..."
+ continue
+ fi
+ echo " Done."
+ done
+
+ echo ""
+ echo "Adding current release candidate: ${version_without_rc}"
+ exec_process cp
"${dist_dev_dir}/helm-chart/${version_without_rc}/polaris-${version_without_rc}.tgz"
"$WORK_DIR/"
+
+ echo ""
+ echo "Generating index.yaml with helm repo index..."
+ exec_process helm repo index "$WORK_DIR"
+
+ # Post-process the index.yaml to use appropriate URLs
+ echo ""
+ echo "Updating URLs in index.yaml..."
+
+ # Replace archived versions with absolute URLs to archive.apache.org
+ for version in $ARCHIVE_VERSIONS; do
+ sed -i.bak "s|^\( *-
\)polaris-${version}.tgz$|\1${ARCHIVE_URL}/${version}/polaris-${version}.tgz|"
"$WORK_DIR/index.yaml"
+ done
+
+ # Replace current RC version with relative URL
+ sed -i.bak "s|^\( *-
\)polaris-${version_without_rc}.tgz$|\1${version_without_rc}/polaris-${version_without_rc}.tgz|"
"$WORK_DIR/index.yaml"
+
+ # Copy the updated index.yaml to the dist dev directory
+ exec_process cp "$WORK_DIR/index.yaml"
"${dist_dev_dir}/helm-chart/index.yaml"
+
+ # Commit the updated index
+ exec_process cd "${dist_dev_dir}"
+ exec_process svn commit --username "$SVN_USERNAME" --password
"$SVN_PASSWORD" --non-interactive -m "Update Helm index for
${version_without_rc} RC${rc_number}"
+
+ echo "## Helm Index Summary" >> $GITHUB_STEP_SUMMARY
+ ARCHIVE_COUNT=$(echo "$ARCHIVE_VERSIONS" | grep -v '^$' | wc -w | tr
-d ' ')
+ cat <<EOT >> $GITHUB_STEP_SUMMARY
+ 🎉 Helm index regenerated successfully:
+
+ | Component | Status |
+ | --- | --- |
+ | Charts from archive.apache.org | ✅ ${ARCHIVE_COUNT} versions
indexed |
+ | Current RC | ✅ ${version_without_rc} added with relative URL |
+ | Helm index | ✅ Committed to dist dev |
+ EOT
+
generate-release-email:
name: Generate Release Email Body
runs-on: ubuntu-latest
diff --git a/.github/workflows/release-4-publish-release.yml
b/.github/workflows/release-4-publish-release.yml
index 8b090d639..f373024d3 100644
--- a/.github/workflows/release-4-publish-release.yml
+++ b/.github/workflows/release-4-publish-release.yml
@@ -207,12 +207,7 @@ jobs:
Artifacts and Helm chart moved from dist dev to dist release
EOT
- - name: Set up Helm
- uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 #
v4.3.1
- with:
- version: 'latest'
-
- - name: Update Helm index in release space
+ - name: Transfer Helm index from dev to release
env:
SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }}
SVN_PASSWORD: ${{ secrets.POLARIS_SVN_DEV_PASSWORD }}
@@ -222,20 +217,20 @@ jobs:
source "${LIBS_DIR}/_constants.sh"
source "${LIBS_DIR}/_exec.sh"
- # Checkout the release Helm chart directory
- release_helm_dir="${RELEASEY_DIR}/polaris-dist-release-helm-chart"
-
release_helm_url="${APACHE_DIST_URL}/release/incubator/polaris/helm-chart"
+ # Define source and destination URLs for the Helm index
+
dev_helm_index_url="${APACHE_DIST_URL}/dev/incubator/polaris/helm-chart/index.yaml"
+
release_helm_index_url="${APACHE_DIST_URL}/release/incubator/polaris/helm-chart/index.yaml"
- # Retry logic for SVN checkout (Apache SVN can have transient
connectivity issues)
- exec_process_with_retries 5 60 "${release_helm_dir}" svn checkout
--username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive
"${release_helm_url}" "${release_helm_dir}"
-
- exec_process cd "${release_helm_dir}"
- exec_process helm repo index .
- exec_process svn commit --username "$SVN_USERNAME" --password
"$SVN_PASSWORD" --non-interactive -m "Update Helm index for
${version_without_rc} release"
+ # Move the index.yaml from dev to release
+ # The index was already properly generated in workflow 3 with
relative URLs for the current
+ # release and absolute URLs to archive.apache.org for previous
releases, so we just transfer it as-is
+ exec_process svn mv --username "$SVN_USERNAME" --password
"$SVN_PASSWORD" --non-interactive \
+ "${dev_helm_index_url}" "${release_helm_index_url}" \
+ -m "Transfer Helm index for ${version_without_rc} release"
cat <<EOT >> $GITHUB_STEP_SUMMARY
## Helm Index
- Helm index updated in dist release
+ Helm index transferred from dist dev to dist release
EOT
- name: Create final release tag and push to Git repository
diff --git a/releasey/bin/build-helm-index.sh b/releasey/bin/build-helm-index.sh
new file mode 100755
index 000000000..3d57b3a87
--- /dev/null
+++ b/releasey/bin/build-helm-index.sh
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# Build a Helm index.yaml file for Apache Polaris charts from both
+# https://archive.apache.org/dist/incubator/polaris/helm-chart/ and
+# https://downloads.apache.org/incubator/polaris/helm-chart/
+#
+# The current release workflows force a regeneration of the Helm index
+# after the old (unmaintained) releases are removed from the Apache dist
+# release SVN server. This causes previous releases to be removed from
+# the Helm index (#3500).
+#
+# This script is a tentative fix that re-generates a Helm index across
+# downloads.apache.org as well as archive.apache.org. The logic is as
+# follows:
+#
+# 1. All Helm charts from downloads.a.o and archive.a.o are downloaded
+# locally so that the index is fully rebuilt across all versions.
+# 2. The URL of any Helm chart that is located on archive.apache.org but
+# not on downloads.apache.org is replaced by an absolute URL to
+# archive.apache.org. That way, previous releases are always
+# accessible.
+# 3. The URL of any Helm chart that is located on downloads.apache.org is
+# replaced by a relative path `${version}/polaris-${version}.tgz`.
+#
+# That way, the Helm index can be rebuild before a release vote is
+# started, as the relative path will work regardless of the location being
+# used (dist dev during RC vote, dist release and downloads.a.o after
+# vote).
+#
+
+set -euo pipefail
+
+# Function to extract version directories from an Apache directory listing
+get_helm_chart_version_from_url() {
+ local base_url="$1"
+ curl -sL "$base_url/" |
+ grep -oE 'href="[0-9]+\.[0-9]+\.[0-9]+-incubating/"' |
+ sed 's/href="//;s/\/"//' |
+ sort -V | uniq
+}
+
+# Base URLs for Apache Polaris helm charts
+ARCHIVE_URL="https://archive.apache.org/dist/incubator/polaris/helm-chart";
+DOWNLOADS_URL="https://downloads.apache.org/incubator/polaris/helm-chart";
+
+# Temporary directory for downloads
+WORK_DIR=$(mktemp -d)
+trap 'rm -rf "$WORK_DIR"' EXIT
+
+echo "Working directory: $WORK_DIR"
+
+# Collect all unique versions from both sources
+echo "Discovering chart versions from archive.apache.org..."
+ARCHIVE_VERSIONS=$(get_helm_chart_version_from_url "$ARCHIVE_URL" || echo "")
+echo "Found archive versions: $ARCHIVE_VERSIONS"
+
+echo "Discovering chart versions from downloads.apache.org..."
+DOWNLOADS_VERSIONS=$(get_helm_chart_version_from_url "$DOWNLOADS_URL" || echo
"")
+echo "Found downloads versions: $DOWNLOADS_VERSIONS"
+
+# File to store version-to-URL mappings for archive
+URL_MAP_FILE="$WORK_DIR/url_map.txt"
+>"$URL_MAP_FILE"
+
+# Process ARCHIVE versions first with absolute URL replacement
+# Only process versions that are in archive but NOT in downloads
+echo ""
+echo "Processing versions from archive.apache.org that are not in
downloads.apache.org..."
+for version in $ARCHIVE_VERSIONS; do
+ # Skip if this version is also in downloads
+ if echo "$DOWNLOADS_VERSIONS" | grep -q "^${version}$"; then
+ echo "Skipping archive version $version (available in
downloads.apache.org)"
+ continue
+ fi
+
+ echo "Processing archive version: $version"
+
+ CHART_URL="${ARCHIVE_URL}/${version}/polaris-${version}.tgz"
+ LOCAL_TGZ="$WORK_DIR/polaris-${version}.tgz"
+
+ # Store the mapping: local filename -> remote URL for archive
+ echo "polaris-${version}.tgz|${CHART_URL}" >>"$URL_MAP_FILE"
+
+ echo " Downloading: $CHART_URL"
+ if ! curl -sLf -o "$LOCAL_TGZ" "$CHART_URL"; then
+ echo " WARNING: Failed to download $CHART_URL, skipping..."
+ continue
+ fi
+ echo " Done."
+done
+
+# Process DOWNLOADS versions without URL replacement
+echo ""
+echo "Processing versions from downloads.apache.org..."
+for version in $DOWNLOADS_VERSIONS; do
+ echo "Processing downloads version: $version"
+
+ CHART_URL="${DOWNLOADS_URL}/${version}/polaris-${version}.tgz"
+ LOCAL_TGZ="$WORK_DIR/polaris-${version}.tgz"
+
+ # Store the mapping: local filename -> relative URL for downloads
+ echo "polaris-${version}.tgz|${version}/polaris-${version}.tgz"
>>"$URL_MAP_FILE"
+
+ echo " Downloading: $CHART_URL"
+ if ! curl -sLf -o "$LOCAL_TGZ" "$CHART_URL"; then
+ echo " WARNING: Failed to download $CHART_URL, skipping..."
+ continue
+ fi
+ echo " Done."
+done
+
+# Use helm to generate the index
+echo ""
+echo "Generating index.yaml with helm repo index..."
+helm repo index "$WORK_DIR"
+
+# Post-process the index.yaml to replace local paths with remote URLs
+echo "Updating URLs to point to remote sources..."
+
+while IFS='|' read -r local_name remote_url; do
+ # Use sed to replace the local filename with the full remote URL
+ sed -i.bak "s|^\( *- \)${local_name}$|\1${remote_url}|"
"$WORK_DIR/index.yaml"
+done <"$URL_MAP_FILE"
+
+# Copy the updated index.yaml to the current directory
+cp "$WORK_DIR/index.yaml" ./index.yaml
+
+echo ""
+echo "Successfully generated index.yaml"
+ARCHIVE_COUNT=$(echo "$ARCHIVE_VERSIONS" | grep -v '^$' | wc -w | tr -d ' ')
+DOWNLOADS_COUNT=$(echo "$DOWNLOADS_VERSIONS" | grep -v '^$' | wc -w | tr -d '
')
+echo "Charts indexed from archive: $ARCHIVE_COUNT"
+echo "Charts indexed from downloads: $DOWNLOADS_COUNT"
