This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 28aeacffa57 releaseWizard: summarize dependency changes from
solr/licenses (#4966)
28aeacffa57 is described below
commit 28aeacffa57e143b273ba091f21628950e244f01
Author: David Smiley <[email protected]>
AuthorDate: Fri Oct 9 00:44:53 2026 -0400
releaseWizard: summarize dependency changes from solr/licenses (#4966)
Replaces addDepsToChanges.py, which bulk-converted solrbot commits into
changelog entries (duplicating the entries solrbot PRs already create), with
dev-tools/scripts/dependencyChanges.py.
The new script diffs the jar checksum files in solr/licenses/ between the
previous release tag and HEAD (git only; no build needed):
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.github/scripts/generate-renovate-changelog.py | 272 -----------------------
.github/workflows/renovate-changelog-prepare.yml | 123 ----------
.github/workflows/renovate-changelog-push.yml | 177 ---------------
.github/workflows/validate-changelog.yml | 2 +-
dev-docs/changelog.adoc | 7 +
dev-tools/scripts/addDepsToChanges.py | 257 ---------------------
dev-tools/scripts/dependencyChanges.py | 202 +++++++++++++++++
dev-tools/scripts/releaseWizard.yaml | 12 +-
8 files changed, 216 insertions(+), 836 deletions(-)
diff --git a/.github/scripts/generate-renovate-changelog.py
b/.github/scripts/generate-renovate-changelog.py
deleted file mode 100644
index ab37fd26288..00000000000
--- a/.github/scripts/generate-renovate-changelog.py
+++ /dev/null
@@ -1,272 +0,0 @@
-#!/usr/bin/env python3
-"""
-Generate changelog YAML entry for Renovate dependency update PRs.
-
-This script parses the PR title to extract dependency information,
-then generates a changelog YAML file in changelog/unreleased/ with the proper
-naming convention and content structure.
-
-Usage:
- python3 generate-renovate-changelog.py --pr-number 1234 --pr-title "Update
org.apache.httpcomponents to v1.2.3"
-"""
-
-import argparse
-import os
-import re
-import sys
-import yaml
-from pathlib import Path
-from typing import Optional, Tuple
-
-
-def sanitize_slug(text: str, max_length: int = 50) -> str:
- """
- Sanitize text to create a valid filename slug.
-
- - Convert to lowercase
- - Replace dots, colons, slashes with dashes
- - Replace other special chars with dashes
- - Preserve word boundaries
- - Truncate to max_length while preserving word boundaries
- """
- # Convert to lowercase
- text = text.lower()
-
- # Replace colons, slashes, dots with dashes
- text = re.sub(r'[:/.]+', '-', text)
-
- # Replace other special characters with dashes
- text = re.sub(r'[^a-z0-9\s-]', '-', text)
-
- # Replace spaces with dashes
- text = re.sub(r'\s+', '-', text)
-
- # Replace multiple dashes with single dash
- text = re.sub(r'-+', '-', text)
-
- # Remove leading/trailing dashes
- text = text.strip('-')
-
- # Truncate to max_length at word boundary
- if len(text) > max_length:
- text = text[:max_length]
- # Find last dash and truncate there
- last_dash = text.rfind('-')
- if last_dash > 0:
- text = text[:last_dash]
- text = text.rstrip('-')
-
- return text
-
-
-def parse_pr_title(title: str) -> Tuple[str, Optional[str]]:
- """
- Parse Renovate PR title to extract dependency name and version.
-
- Handles patterns like:
- - "Update dependency org.junit.jupiter:junit-jupiter to v6"
- - "Update dependency com.jayway.jsonpath:json-path to v2.10.0"
- - "Update netty to v4.2.6.Final"
- - "Update apache.kafka to v3.9.1"
- - "Update actions/checkout action to v5"
-
- Returns:
- Tuple of (title_for_changelog, dependency_slug_for_filename)
-
- Note: The slug excludes the version number so the filename remains stable
- across version updates.
- """
-
- # Pattern 1: "Update dependency {group}:{artifact} to {version}"
- match = re.match(r'Update dependency (.+?) to (.+?)(?:\s*$|\s*\(|$)',
title)
- if match:
- dep_name = match.group(1)
- version = match.group(2).strip()
- changelog_title = f"Update {dep_name} to {version}"
- # Slug contains only the dependency name, not the version
- slug = sanitize_slug(f"Update {dep_name}")
- return changelog_title, slug
-
- # Pattern 2: "Update {owner}/{action} action to {version}"
- match = re.match(r'Update ([a-z0-9-]+/[a-z0-9-]+) action to
(.+?)(?:\s*$|\s*\(|$)', title)
- if match:
- action = match.group(1)
- version = match.group(2).strip()
- changelog_title = f"Update {action} action to {version}"
- # Slug contains only the action name, not the version
- slug = sanitize_slug(f"Update {action} action")
- return changelog_title, slug
-
- # Pattern 3: "Update {package} to {version}" (short form)
- match = re.match(r'Update ([a-z0-9\-_.]+) to (.+?)(?:\s*$|\s*\(|$)', title)
- if match:
- package = match.group(1)
- version = match.group(2).strip()
- changelog_title = f"Update {package} to {version}"
- # Slug contains only the package name, not the version
- slug = sanitize_slug(f"Update {package}")
- return changelog_title, slug
-
- # Fallback: use title as-is if no pattern matches
- return title, sanitize_slug(title)
-
-
-def generate_changelog_entry(
- pr_number: int,
- changelog_title: str,
- pr_url: str = None
-) -> dict:
- """Generate the changelog YAML entry dict."""
-
- if pr_url is None:
- pr_url = f"https://github.com/apache/solr/pull/{pr_number}";
-
- return {
- 'title': changelog_title,
- 'type': 'dependency_update',
- 'authors': [
- {'name': 'solrbot'}
- ],
- 'links': [
- {
- 'name': f'PR#{pr_number}',
- 'url': pr_url
- }
- ]
- }
-
-
-def find_existing_changelog_file(pr_number: int, changelog_dir: str =
'changelog/unreleased') -> Optional[str]:
- """Find existing changelog file for this PR, returns path if exists."""
- pattern = f"PR#{pr_number}-*.yml"
- path = Path(changelog_dir)
-
- if not path.exists():
- return None
-
- for file in path.glob(f"PR#{pr_number}-*.yml"):
- return str(file)
-
- return None
-
-
-def delete_old_changelog_files(pr_number: int, changelog_dir: str =
'changelog/unreleased') -> int:
- """
- Delete all existing changelog files for this PR number.
-
- This ensures we don't accumulate orphaned files when the PR title/slug
changes.
-
- Returns:
- Number of files deleted
- """
- pattern = f"PR#{pr_number}-*.yml"
- path = Path(changelog_dir)
-
- if not path.exists():
- return 0
-
- deleted_count = 0
- for file in path.glob(pattern):
- try:
- file.unlink()
- print(f"Deleted old changelog file: {file}")
- deleted_count += 1
- except Exception as e:
- print(f"Warning: Could not delete file {file}: {e}",
file=sys.stderr)
-
- if deleted_count > 0:
- print(f"Deleted {deleted_count} old changelog file(s) for
PR#{pr_number}")
-
- return deleted_count
-
-
-def should_update_changelog(existing_file: str, new_title: str) -> bool:
- """
- Check if we need to update the changelog file.
-
- Updates if the title has changed (version was bumped).
- """
- if not existing_file or not Path(existing_file).exists():
- return False
-
- try:
- with open(existing_file, 'r') as f:
- content = yaml.safe_load(f)
-
- existing_title = content.get('title', '')
- return existing_title != new_title
- except Exception as e:
- print(f"Warning: Could not read existing file {existing_file}: {e}",
file=sys.stderr)
- return False
-
-
-def write_changelog_file(filename: str, entry: dict, changelog_dir: str =
'changelog/unreleased') -> None:
- """Write the changelog YAML file."""
- path = Path(changelog_dir)
- path.mkdir(parents=True, exist_ok=True)
-
- filepath = path / filename
-
- # Use YAML dumper that preserves order and formatting
- with open(filepath, 'w') as f:
- yaml.dump(
- entry,
- f,
- default_flow_style=False,
- sort_keys=False,
- allow_unicode=True
- )
-
- print(f"Created/updated changelog file: {filepath}")
-
-
-def main():
- parser = argparse.ArgumentParser(
- description='Generate changelog entry for Renovate PR',
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog="""
-Examples:
- python3 generate-renovate-changelog.py --pr-number 1234 --pr-title "Update
org.apache.httpcomponents to v1.2.3"
- python3 generate-renovate-changelog.py --pr-number 3751 --pr-title "Update
dependency com.microsoft.onnxruntime:onnxruntime to v1.23.1"
- """
- )
-
- parser.add_argument(
- '--pr-number',
- type=int,
- required=True,
- help='GitHub PR number'
- )
- parser.add_argument(
- '--pr-title',
- required=True,
- help='GitHub PR title (from the Renovate bot)'
- )
- parser.add_argument(
- '--changelog-dir',
- default='changelog/unreleased',
- help='Directory for changelog files (default: changelog/unreleased)'
- )
-
- args = parser.parse_args()
-
- # Delete any existing changelog files for this PR to ensure a clean slate
- # This prevents orphaned files when the PR title/slug changes
- delete_old_changelog_files(args.pr_number, args.changelog_dir)
-
- # Parse the PR title
- changelog_title, slug = parse_pr_title(args.pr_title)
-
- # Generate filename
- filename = f"PR#{args.pr_number}-{slug}.yml"
-
- # Generate the new entry
- entry = generate_changelog_entry(args.pr_number, changelog_title)
-
- # Write the changelog file
- write_changelog_file(filename, entry, args.changelog_dir)
- return 0
-
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/.github/workflows/renovate-changelog-prepare.yml
b/.github/workflows/renovate-changelog-prepare.yml
deleted file mode 100644
index 5dab69c6867..00000000000
--- a/.github/workflows/renovate-changelog-prepare.yml
+++ /dev/null
@@ -1,123 +0,0 @@
-name: Generate Renovate Changelog (Stage 1 - Prepare)
-
-# Stage 1: runs with pull_request (no secrets, no write access).
-# Checks out BASE repo code only, runs the trusted Python script,
-# and saves the generated changelog file + PR metadata as an artifact
-# for Stage 2 to pick up and push to the fork branch.
-#
-# Generation is skipped (SKIP=true in the artifact metadata) when the PR
-# carries the 'no-changelog' label, or when a user other than solrbot has
-# manually edited changelog/unreleased/ in the PR — in that case the bot
-# must not overwrite the manual changes.
-on:
- pull_request:
- types:
- - opened
- - synchronize
- branches:
- - main
- - 'branch_*'
- paths-ignore:
- - '.github/**'
-
-concurrency:
- group: renovate-changelog-prepare-${{ github.event.pull_request.number }}
- cancel-in-progress: true
-
-permissions:
- contents: read
- pull-requests: read # Needed to list PR commits for the manual-edit check
-
-jobs:
- generate:
- # Only run for Renovate bot PRs from the expected fork
- if: |
- github.event.pull_request.user.login == 'solrbot' &&
- github.event.pull_request.head.repo.full_name == 'solrbot/apache-_-solr'
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout BASE repository at base branch (NOT fork code)
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
- with:
- ref: ${{ github.event.pull_request.base.ref }}
- repository: ${{ github.repository }}
-
- - name: Check whether changelog generation should be skipped
- id: skip-check
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- REPO: ${{ github.repository }}
- HAS_NO_CHANGELOG_LABEL: ${{
contains(github.event.pull_request.labels.*.name, 'no-changelog') }}
- run: |
- set -euo pipefail
- skip=false
-
- if [ "$HAS_NO_CHANGELOG_LABEL" = "true" ]; then
- skip=true
- echo "::notice::Skipping changelog generation for PR#${PR_NUMBER}:
PR has the 'no-changelog' label"
- else
- # Skip if any PR commit from a user other than solrbot touched
- # changelog/unreleased/ — a human has taken over the changelog
- # entry and the bot must not overwrite it. Commits with an
- # unresolvable login are treated as manual edits (fail safe).
- while read -r sha login; do
- if [ "$login" != "solrbot" ]; then
- if gh api "repos/${REPO}/commits/${sha}" --jq
'.files[].filename' | grep -q '^changelog/unreleased/'; then
- skip=true
- echo "::notice::Skipping changelog generation for
PR#${PR_NUMBER}: changelog/unreleased/ was manually edited by '${login}' in
commit ${sha}"
- break
- fi
- fi
- done < <(gh api --paginate
"repos/${REPO}/pulls/${PR_NUMBER}/commits" \
- --jq '.[] | "\(.sha) \(.author.login // "unknown")"')
- fi
-
- echo "skip=${skip}" >> "$GITHUB_OUTPUT"
-
- - name: Set up Python
- if: steps.skip-check.outputs.skip != 'true'
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 #
v7.0.0
- with:
- python-version: '3.x'
-
- - name: Install dependencies
- if: steps.skip-check.outputs.skip != 'true'
- run: python3 -m pip install --quiet pyyaml
-
- - name: Generate changelog entry
- if: steps.skip-check.outputs.skip != 'true'
- env:
- PR_NUMBER: ${{ github.event.pull_request.number }}
- PR_TITLE: ${{ github.event.pull_request.title }}
- run: |
- python3 .github/scripts/generate-renovate-changelog.py \
- --pr-number "$PR_NUMBER" \
- --pr-title "$PR_TITLE"
-
- - name: Assemble artifact
- env:
- PR_NUMBER: ${{ github.event.pull_request.number }}
- HEAD_REF: ${{ github.event.pull_request.head.ref }}
- HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
- SKIP: ${{ steps.skip-check.outputs.skip }}
- run: |
- set -euo pipefail
- mkdir -p artifact
- if [ "$SKIP" != "true" ] && [ -d changelog/unreleased ]; then
- cp -r changelog/unreleased artifact/changelog-unreleased
- fi
- # Use printf + env vars to avoid shell injection from PR metadata
values
- printf 'PR_NUMBER=%s\nHEAD_REF=%s\nHEAD_REPO=%s\nSKIP=%s\n' \
- "$PR_NUMBER" "$HEAD_REF" "$HEAD_REPO" "$SKIP" \
- > artifact/pr-metadata.env
- echo "Artifact contents:"; find artifact/ -type f
-
- - name: Upload artifact
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
# v7.0.1
- with:
- name: renovate-changelog-artifact
- path: artifact/
- retention-days: 1
- if-no-files-found: error
diff --git a/.github/workflows/renovate-changelog-push.yml
b/.github/workflows/renovate-changelog-push.yml
deleted file mode 100644
index 0e44c9f434e..00000000000
--- a/.github/workflows/renovate-changelog-push.yml
+++ /dev/null
@@ -1,177 +0,0 @@
-name: Generate Renovate Changelog (Stage 2 - Push)
-
-# Stage 2: runs after Stage 1 completes, in the base-repo context with access
to
-# SOLRBOT_GITHUB_TOKEN. Downloads the pre-generated artifact (no fork code
executed here),
-# validates metadata, and pushes the changelog file to the fork branch.
-#
-# If Stage 1 flagged SKIP=true in the artifact metadata (PR has the
-# 'no-changelog' label, or a user manually edited changelog/unreleased/),
-# this workflow leaves the changelog untouched.
-on:
- workflow_run:
- workflows:
- - "Generate Renovate Changelog (Stage 1 - Prepare)"
- types:
- - completed
-
-# Each Stage 2 run corresponds to a unique Stage 1 run (unique
workflow_run.id).
-# No cancel-in-progress: Stage 1's concurrency already serializes per-PR.
-concurrency:
- group: renovate-changelog-push-${{ github.event.workflow_run.id }}
-
-permissions:
- actions: read # Required to download artifacts from another workflow run
- contents: read # Minimal; actual write access to fork uses
SOLRBOT_GITHUB_TOKEN PAT
-
-jobs:
- push-changelog:
- # Only proceed if Stage 1 succeeded for the expected fork.
- # Checking head_repository here avoids a spurious artifact-not-found
failure
- # when Stage 1 ran but skipped its generate job (e.g. non-solrbot PR).
- if: |
- github.event.workflow_run.conclusion == 'success' &&
- github.event.workflow_run.head_repository.full_name ==
'solrbot/apache-_-solr'
- runs-on: ubuntu-latest
-
- steps:
- - name: Download artifact from Stage 1
- uses:
actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- name: renovate-changelog-artifact
- run-id: ${{ github.event.workflow_run.id }}
- github-token: ${{ secrets.GITHUB_TOKEN }}
- path: downloaded-artifact/
-
- - name: Read and validate PR metadata
- id: meta
- run: |
- set -euo pipefail
- META_FILE="downloaded-artifact/pr-metadata.env"
- if [ ! -f "$META_FILE" ]; then
- echo "::error::Metadata file missing from artifact"; exit 1
- fi
-
- # Parse with grep/cut rather than source to avoid executing file
content as shell.
- # Use || true so a missing key doesn't abort under set -euo pipefail
before the
- # explicit emptiness check below can emit a meaningful error message.
- PR_NUMBER=$(grep '^PR_NUMBER=' "$META_FILE" | cut -d= -f2- || true)
- HEAD_REF=$(grep '^HEAD_REF=' "$META_FILE" | cut -d= -f2- || true)
- HEAD_REPO=$(grep '^HEAD_REPO=' "$META_FILE" | cut -d= -f2- || true)
- SKIP=$(grep '^SKIP=' "$META_FILE" | cut -d= -f2- || true)
-
- if [ -z "$PR_NUMBER" ] || [ -z "$HEAD_REF" ] || [ -z "$HEAD_REPO" ];
then
- echo "::error::Missing required metadata fields (PR_NUMBER,
HEAD_REF, or HEAD_REPO)"; exit 1
- fi
-
- # Security: verify this is the expected fork before using
SOLRBOT_GITHUB_TOKEN
- if [ "$HEAD_REPO" != "solrbot/apache-_-solr" ]; then
- echo "::error::Unexpected HEAD_REPO: '$HEAD_REPO'. Expected
'solrbot/apache-_-solr'. Aborting."; exit 1
- fi
-
- # Validate PR_NUMBER is a plain positive integer (prevents injection)
- if ! [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]]; then
- echo "::error::PR_NUMBER is not a valid positive integer:
'$PR_NUMBER'"; exit 1
- fi
-
- # Validate HEAD_REF using Git's own branch-name rules. This rejects
edge cases
- # such as '..', '@{', trailing '.lock', and ':' (dangerous in push
refspecs)
- # while still accepting valid Renovate branch names like
renovate/node@lts.
- if ! git check-ref-format --branch "$HEAD_REF" > /dev/null 2>&1; then
- echo "::error::HEAD_REF is not a valid Git branch name:
'$HEAD_REF'"; exit 1
- fi
-
- # SKIP is optional (absent in artifacts from older Stage 1 runs);
default to false
- if [ "$SKIP" = "true" ]; then
- echo "::notice::Stage 1 flagged skip for PR#${PR_NUMBER} — leaving
changelog untouched"
- else
- SKIP=false
- fi
-
- echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
- echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT"
- echo "head_repo=$HEAD_REPO" >> "$GITHUB_OUTPUT"
- echo "skip=$SKIP" >> "$GITHUB_OUTPUT"
- echo "Validated: PR#${PR_NUMBER} on ${HEAD_REPO}@${HEAD_REF}
(skip=${SKIP})"
-
- - name: Clone fork branch
- if: steps.meta.outputs.skip != 'true'
- env:
- SOLRBOT_TOKEN: ${{ secrets.SOLRBOT_GITHUB_TOKEN }}
- HEAD_REPO: ${{ steps.meta.outputs.head_repo }}
- HEAD_REF: ${{ steps.meta.outputs.head_ref }}
- run: |
- set -euo pipefail
- # Store credentials so the token never appears in the command line
or process list
- git config --global credential.helper store
- printf 'https://x-access-token:%[email protected]\n' "$SOLRBOT_TOKEN"
> ~/.git-credentials
- chmod 600 ~/.git-credentials
-
- git clone --depth=1 --branch "$HEAD_REF" \
- "https://github.com/${HEAD_REPO}.git"; \
- fork-checkout
-
- - name: Apply changelog to fork checkout
- if: steps.meta.outputs.skip != 'true'
- id: apply
- env:
- PR_NUMBER: ${{ steps.meta.outputs.pr_number }}
- run: |
- set -euo pipefail
- CHANGELOG_DIR="fork-checkout/changelog/unreleased"
- ARTIFACT_DIR="downloaded-artifact/changelog-unreleased"
-
- if [ ! -d "$CHANGELOG_DIR" ]; then
- echo "::error::changelog/unreleased not found in fork checkout";
exit 1
- fi
-
- if [ ! -d "$ARTIFACT_DIR" ]; then
- echo "::warning::No changelog-unreleased directory in artifact for
PR#${PR_NUMBER}"
- echo "has_changes=false" >> "$GITHUB_OUTPUT"
- exit 0
- fi
-
- # Remove stale PR#NNN-*.yml files (handles slug changes between
synchronize events)
- find "$CHANGELOG_DIR" -maxdepth 1 -name "PR#${PR_NUMBER}-*.yml"
-delete -print
-
- # Copy the new PR#NNN-*.yml file(s) from the artifact
- COPIED=0
- for f in "${ARTIFACT_DIR}/PR#${PR_NUMBER}-"*.yml; do
- if [ -f "$f" ]; then
- cp -v "$f" "$CHANGELOG_DIR/"
- COPIED=$((COPIED + 1))
- fi
- done
-
- if [ "$COPIED" -eq 0 ]; then
- echo "::warning::No PR#${PR_NUMBER}-*.yml file found in artifact"
- echo "has_changes=false" >> "$GITHUB_OUTPUT"
- else
- echo "Copied $COPIED changelog file(s)"
- echo "has_changes=true" >> "$GITHUB_OUTPUT"
- fi
-
- - name: Commit and push to fork branch
- if: steps.apply.outputs.has_changes == 'true'
- env:
- HEAD_REF: ${{ steps.meta.outputs.head_ref }}
- PR_NUMBER: ${{ steps.meta.outputs.pr_number }}
- run: |
- set -euo pipefail
- cd fork-checkout
-
- if [ -z "$(git status --porcelain changelog/unreleased/)" ]; then
- echo "No changelog changes (already up to date)"
- exit 0
- fi
-
- git config user.name "SolrBot"
- git config user.email "[email protected]"
-
- git add changelog/unreleased/
- git commit -m "Add changelog entry for PR#${PR_NUMBER}"
-
- # Credential store (configured in Clone step) provides authentication
- git push origin "HEAD:refs/heads/${HEAD_REF}"
-
- # Remove credentials from disk now that the push is complete
- rm -f ~/.git-credentials
diff --git a/.github/workflows/validate-changelog.yml
b/.github/workflows/validate-changelog.yml
index 66fcc1c0481..af43d05b4d5 100644
--- a/.github/workflows/validate-changelog.yml
+++ b/.github/workflows/validate-changelog.yml
@@ -20,7 +20,7 @@ concurrency:
jobs:
validate-changelog:
name: Check changelog entry
- # Skip validation for Renovate PRs (solrbot) - they get changelog entries
automatically.
+ # Skip validation for Renovate PRs (solrbot); dependency changes are
summarized at release time.
# On label events, only re-run for the 'no-changelog' label; other label
churn
# (e.g. from the labeler workflow) leaves the existing check result
untouched.
if: |
diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc
index 70536cc0f04..ac6f0504ff5 100644
--- a/dev-docs/changelog.adoc
+++ b/dev-docs/changelog.adoc
@@ -292,6 +292,13 @@ Example report output (Json or Markdown):
and the final post-vote forward-porting. It is integrated into the Release
Wizard but can also be run standalone. Run with `--help` for usage.
+=== 5.5 Dependency changes summary
+
+`dev-tools/scripts/dependencyChanges.py` summarizes the third-party dependency
changes
+since the previous release by diffing `solr/licenses/`. By default it prints
to stdout;
+with `--write`, the Release Wizard uses it to write `dependency_update`
changelog entries
+(one each for added, upgraded, and removed).
+
== 6. Further Reading
* https://github.com/logchange/logchange[Logchange web page]
diff --git a/dev-tools/scripts/addDepsToChanges.py
b/dev-tools/scripts/addDepsToChanges.py
deleted file mode 100755
index 3286d03cac4..00000000000
--- a/dev-tools/scripts/addDepsToChanges.py
+++ /dev/null
@@ -1,257 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Script to create changelog YAML entries for solrbot dependency updates
-"""
-import os
-import sys
-
-sys.path.append(os.path.dirname(__file__))
-from scriptutil import *
-
-import argparse
-import re
-import yaml
-from pathlib import Path
-
-line_re = re.compile(r"(.*?) (\(branch_\d+x\) )?\(#(\d+)\)$")
-
-
-class ChangeEntry:
- """
- Represents one dependency change entry.
- Fields:
- - pr_num: string PR number (e.g., '3605')
- - message: cleaned commit message text to be shown in CHANGES (noise
removed)
- - author: Git author (e.g., 'solrbot')
- """
-
- def __init__(self, pr_num: str, message: str, author: str):
- self.pr_num = pr_num
- self.message = message
- self.author = author
-
- def dep_key(self) -> str:
- """
- Extract a dependency key from the message after 'Update ' and before '
to', '(' or end.
- This is used for de-duplication and sorting. Case-insensitive.
- """
- m = re.search(r"(?i)update\s+(.+?)(?:\s+to\b|\s*\(|$)", self.message)
- if m:
- return m.group(1).strip()
- return self.message.strip()
-
- def __str__(self) -> str:
- # Keep trailing newline to preserve existing blank-line formatting by
update_changes
- return f"* PR#{self.pr_num}: {self.message} ({self.author})\n"
-
- def to_yaml_dict(self) -> dict:
- """
- Convert to a dictionary suitable for YAML serialization.
- Extracts JIRA IDs from the title and adds them to links.
- """
- # Extract JIRA IDs from the message
- title, jira_links = extract_jira_issues_from_title(self.message)
-
- # Build links: JIRA issues first, then PR
- links = jira_links.copy() # Start with JIRA links
- links.append({
- 'name': f'PR#{self.pr_num}',
- 'url': f'https://github.com/apache/solr/pull/{self.pr_num}'
- })
-
- return {
- 'title': title,
- 'type': 'dependency_update',
- 'authors': [
- {
- 'name': self.author
- }
- ],
- 'links': links
- }
-
- def yaml_filename(self) -> str:
- """
- Generate a filesystem-safe filename for this entry.
- Format: PR#####-slug.yaml
- Truncates slug on whitespace boundaries, allowing up to 255 chars
total.
- """
- # Clean message for slug
- slug = self.message.lower()
- # Replace whitespace with single space
- slug = re.sub(r'\s+', ' ', slug)
- # Remove non-alphanumeric except dashes
- slug = re.sub(r'[^a-z0-9-._ ]', '', slug)
-
- # Calculate available space for slug
- # Format: "PR" + pr_num + "-" + slug + ".yaml"
- # Typical PR#1234 = 8 chars + "-" = 9 chars, ".yaml" = 5 chars, total
overhead = 14 chars
- # Most filesystems limit filenames to 255 chars
- max_filename_length = 255
- overhead = len(f"PR{self.pr_num}-.yaml")
- max_slug_length = max_filename_length - overhead
-
- # Truncate to max length on word boundaries if necessary
- if len(slug) > max_slug_length:
- # Find the last space within the limit
- truncated = slug[:max_slug_length]
- last_dash = truncated.rfind(' ')
- if last_dash > max_slug_length // 2: # Keep at least half the
available space
- slug = truncated[:last_dash]
- else:
- # If no good word boundary, use hard limit and clean up
trailing spaces
- slug = truncated.rstrip(' ')
- else:
- # Remove trailing spaces
- slug = slug.rstrip(' ')
-
- return f"PR{self.pr_num}-{slug}.yaml"
-
-
-def get_prev_release_tag(ver):
- """
- Based on a given version, compute the git tag for the "previous" version
to calculate changes since.
- For a major version, we want all solrbot commits since last release, i.e.
X-1.Y.Z
- For a minor version, we want all solrbot commits since X.Y-1.0
- For a patch version, we want all solrbot commits since X.Y.Z-1
- """
- releases_arr = run('git tag |grep "releases/solr" | cut -c
15-').strip().split("\n")
- releases = list(map(lambda x: Version.parse(x), releases_arr))
- if ver.is_major_release():
- last = releases.pop()
- return "releases/solr/%s" % last.dot
- if ver.is_minor_release():
- return "releases/solr/%s.%s.0" % (ver.major, ver.minor - 1)
- if ver.is_bugfix_release():
- return "releases/solr/%s.%s.%s" % (ver.major, ver.minor, ver.bugfix -
1)
- return None
-
-
-def read_config():
- parser = argparse.ArgumentParser(description='Adds changelog entries in
changelog/ folder')
- parser.add_argument('--version', type=Version.parse, help='Solr version to
add changes to', required=True)
- parser.add_argument('--user', default='solrbot', help='Git user to get
changes for. Defaults to solrbot')
- newconf = parser.parse_args()
- return newconf
-
-
-def get_gitlog_lines(user: str, prev_tag: str):
- """
- Run git log and return a list of raw subject lines (%s), newest first.
- """
- output = run('git log --author=' + user + ' --oneline --no-merges
--pretty=format:"%s" ' + prev_tag + '..')
- return list(filter(None, output.split("\n")))
-
-
-def parse_gitlog_lines(lines, author: str):
- """
- Parse raw git log subject lines into ChangeEntry objects.
- - Extract pr_num and message
- - Strip '(branch_Nx)' noise
- - Remove optional leading 'chore(deps): '
- - Normalize 'update dependency X' to 'Update X' (case-insensitive)
- """
- entries = []
- for line in lines:
- match = line_re.search(line)
- if not match:
- print("Skipped un-parsable line: %s" % line)
- continue
- text = match.group(1)
- pr_num = match.group(3)
- # Clean message noise
- msg = text
- msg = re.sub(r"^chore\(deps\):\s*", "", msg, flags=re.IGNORECASE)
- # Normalize 'update dependency' to 'Update '
- msg = re.sub(r"(?i)^update\s+dependenc(y|ies)\s+", "Update ", msg)
- entries.append(ChangeEntry(pr_num=pr_num, message=msg, author=author))
- return entries
-
-
-def write_changelog_yaml(entries):
- """
- Write each ChangeEntry to a YAML file in changelog/unreleased/
- """
- changelog_dir = Path('changelog/unreleased')
-
- # Create directory if it doesn't exist
- changelog_dir.mkdir(parents=True, exist_ok=True)
-
- count = 0
- for entry in entries:
- filename = changelog_dir / entry.yaml_filename()
- yaml_data = entry.to_yaml_dict()
-
- # Write YAML file with proper formatting
- with open(filename, 'w') as f:
- yaml.dump(yaml_data, f, default_flow_style=False, sort_keys=False,
allow_unicode=True)
-
- print(f"Created: {filename}")
- count += 1
-
- return count
-
-
-def dedupe_entries(entries):
- """
- De-duplicate dependency update entries (newest first input) by dep_key.
- Keeps the first occurrence for each dependency key.
- """
- seen = set()
- result = []
- for e in entries:
- key = e.dep_key().lower()
- if key not in seen:
- seen.add(key)
- result.append(e)
- return result
-
-
-def sort_entries(entries):
- """Return a new list sorted alphabetically by dependency key
(case-insensitive)."""
- return sorted(entries, key=lambda e: e.dep_key().lower())
-
-
-def main():
- if not os.path.exists('CHANGELOG.md'):
- sys.exit("Tool must be run from the root of a source checkout.")
- newconf = read_config()
- prev_tag = get_prev_release_tag(newconf.version)
- print("Creating changelog YAML entries for dependency updates since git
tag %s" % prev_tag)
- try:
- gitlog_lines = get_gitlog_lines(newconf.user, prev_tag)
- entries = parse_gitlog_lines(gitlog_lines, author=newconf.user)
- if entries:
- deduped = dedupe_entries(entries)
- sorted_entries = sort_entries(deduped)
- count = write_changelog_yaml(sorted_entries)
- print(f"Successfully created {count} changelog YAML entries")
- else:
- print("No changes found for version %s" % newconf.version.dot)
- print("Done")
- except subprocess.CalledProcessError:
- print("Error running git log - check your --version")
- sys.exit(1)
-
-
-if __name__ == '__main__':
- try:
- main()
- except KeyboardInterrupt:
- print('\nReceived Ctrl-C, exiting early')
diff --git a/dev-tools/scripts/dependencyChanges.py
b/dev-tools/scripts/dependencyChanges.py
new file mode 100644
index 00000000000..c5d71a381f4
--- /dev/null
+++ b/dev-tools/scripts/dependencyChanges.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Summarizes changes to the third-party dependencies Solr ships, by diffing the
+jar checksum files in solr/licenses/ between two git refs (by default: the
+previous release tag and HEAD). Jars that moved between the same two versions
+are listed together, since they are typically from the same project.
+
+Prints the summary to stdout, or with --write, writes changelog YAML entries.
+
+This is a stopgap until Solr publishes an SBOM per release, which would be the
+better source to diff.
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from collections import defaultdict
+
+sys.path.append(os.path.dirname(__file__))
+from scriptutil import Version, find_current_version
+
+LICENSES_DIR = 'solr/licenses'
+DEFAULT_OUTPUT_DIR = 'changelog/unreleased'
+
+# A version starts at the last '-'-delimited segment that looks like a dotted
number.
+# e.g. log4j-1.2-api-2.25.3, zstd-jni-1.5.6-10, guava-33.4.8-jre
+VERSION_SEG_RE = re.compile(r'^\d+\.\d')
+# Classifiers that distinguish separate jars of the same artifact and version
+CLASSIFIER_RE =
re.compile(r'-(tests?|linux-[\w-]+|osx-[\w-]+|windows-[\w-]+)$')
+# Test artifacts sharing a LICENSE file with shipped ones (e.g.
lucene-test-framework)
+TEST_ARTIFACT_RE = re.compile(r'-(tests?|testing|test-framework)$')
+
+
+def git(*args):
+ return subprocess.run(['git'] + list(args), capture_output=True, text=True,
check=True).stdout
+
+
+def parse_jar(jar_name):
+ """Returns (artifact, version) for a jar name without the '.jar'
extension."""
+ classifier = ''
+ m = CLASSIFIER_RE.search(jar_name)
+ if m:
+ classifier = m.group(0)
+ jar_name = jar_name[:m.start()]
+ segs = jar_name.split('-')
+ idx = max((i for i, s in enumerate(segs) if i > 0 and
VERSION_SEG_RE.match(s)), default=None)
+ if idx is None:
+ idx = max((i for i, s in enumerate(segs) if i > 0 and s[:1].isdigit()),
default=len(segs))
+ return '-'.join(segs[:idx]) + classifier, '-'.join(segs[idx:])
+
+
+def read_licenses_dir(ref):
+ """Returns (artifact -> version, set of LICENSE file prefixes) at the given
ref."""
+ jars = {}
+ license_prefixes = set()
+ for name in git('ls-tree', '--name-only', f'{ref}:{LICENSES_DIR}').split():
+ if name.endswith('.jar.sha1'):
+ artifact, version = parse_jar(name[:-len('.jar.sha1')])
+ jars[artifact] = version
+ else:
+ m = re.match(r'(.+)-LICENSE-.+\.txt$', name)
+ if m:
+ license_prefixes.add(m.group(1))
+ return jars, license_prefixes
+
+
+def has_license(artifact, license_prefixes):
+ """Whether some '-'-delimited prefix of the artifact has a LICENSE file
(mirrors jar-checks.gradle)."""
+ name = CLASSIFIER_RE.sub('', artifact)
+ while True:
+ if name in license_prefixes:
+ return True
+ prefix = re.sub(r'-[^-]+$', '', name)
+ if prefix == name:
+ return False
+ name = prefix
+
+
+def find_previous_release_tag(current):
+ cur = Version.parse(current)
+ best = None
+ for tag in git('tag', '-l', 'releases/solr/*').split():
+ m = re.fullmatch(r'releases/solr/(\d+)\.(\d+)\.(\d+)', tag)
+ if not m:
+ continue
+ v = tuple(int(x) for x in m.groups())
+ if v < (cur.major, cur.minor, cur.bugfix) and (best is None or v >
best[0]):
+ best = (v, tag)
+ if best is None:
+ sys.exit(f'No release tag found before {current}; pass --from')
+ return best[1]
+
+
+def describe_artifacts(artifacts):
+ """e.g. 'asm, asm-tree', or 'jetty-* (23 jars)' when all share the first
name segment."""
+ if len(artifacts) > 3:
+ first = {a.split('-')[0] for a in artifacts}
+ if len(first) == 1:
+ return f'{first.pop()}-* ({len(artifacts)} jars)'
+ return ', '.join(artifacts)
+
+
+CATEGORIES = ('added', 'upgraded', 'removed')
+
+
+def compute_changes(from_ref, to_ref):
+ """Returns category ('added', 'upgraded', 'removed') -> sorted list of
human-readable changes."""
+ old_jars, old_license_prefixes = read_licenses_dir(from_ref)
+ new_jars, new_license_prefixes = read_licenses_dir(to_ref)
+
+ # Only jars with a LICENSE file count: *.sha1 files also cover jars we don't
ship (e.g. test
+ # dependencies), whereas LICENSE files are only required for shipped ones
(since SOLR-15465; older
+ # releases have them for non-shipped jars too, making removals from such a
release unreliable).
+ def shipped(artifact, license_prefixes):
+ return has_license(artifact, license_prefixes) and not
TEST_ARTIFACT_RE.search(artifact)
+
+ by_transition = defaultdict(list) # (old_version or None, new_version or
None) -> artifacts
+ for artifact in old_jars.keys() | new_jars.keys():
+ old, new = old_jars.get(artifact), new_jars.get(artifact)
+ if old == new:
+ continue
+ if shipped(artifact, new_license_prefixes if new else
old_license_prefixes):
+ by_transition[(old, new)].append(artifact)
+
+ changes = {c: [] for c in CATEGORIES}
+ for (old, new), artifacts in by_transition.items():
+ names = describe_artifacts(sorted(artifacts, key=str.lower))
+ if old is None:
+ changes['added'].append(f'{names} {new}')
+ elif new is None:
+ changes['removed'].append(f'{names} {old}')
+ else:
+ changes['upgraded'].append(f'{names} {old} → {new}')
+ return {c: sorted(lines, key=str.lower) for c, lines in changes.items()}
+
+
+def to_yaml(title, version):
+ def quote(s):
+ return "'" + s.replace("'", "''") + "'"
+ return (f'# Generated by dev-tools/scripts/dependencyChanges.py\n'
+ f'title: {quote(title)}\n'
+ f'type: dependency_update\n'
+ f'authors:\n'
+ f' - name: various contributors\n'
+ f'links:\n'
+ f' - name: solr/licenses\n'
+ f' url:
https://github.com/apache/solr/tree/releases/solr/{version}/solr/licenses\n')
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument('--from', dest='from_ref',
+ help='Git ref to compare from (default: the release tag
preceding the current version)')
+ parser.add_argument('--to', dest='to_ref', default='HEAD', help='Git ref to
compare to (default: HEAD)')
+ parser.add_argument('--write', nargs='?', const=DEFAULT_OUTPUT_DIR,
metavar='DIR',
+ help='Write changelog YAML entries (one per category)
instead of printing'
+ f' (default dir: {DEFAULT_OUTPUT_DIR})')
+ args = parser.parse_args()
+
+ version = find_current_version()
+ from_ref = args.from_ref or find_previous_release_tag(version)
+ changes = compute_changes(from_ref, args.to_ref)
+ since = from_ref.rsplit('/', 1)[-1]
+
+ if not args.write:
+ print(f'Dependency changes from {from_ref} to {args.to_ref}:')
+ for category in CATEGORIES:
+ print(f'{category.capitalize()}: ' + ('; '.join(changes[category]) or
'(none)'))
+ return
+
+ for i, category in enumerate(CATEGORIES, 1): # numbered so the changelog
lists them in this order
+ path = os.path.join(args.write, f'dependency-changes-{i}-{category}.yml')
+ if changes[category]:
+ title = f'Third-party dependencies {category} since {since}: ' + ';
'.join(changes[category])
+ with open(path, 'w', encoding='utf-8') as f:
+ f.write(to_yaml(title, version))
+ print(f'Wrote {len(changes[category])} {category} dependency changes to
{path}')
+ elif os.path.exists(path):
+ os.remove(path)
+ print(f'Removed {path}; no {category} dependency changes')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/dev-tools/scripts/releaseWizard.yaml
b/dev-tools/scripts/releaseWizard.yaml
index 832b73d7858..1e13c326851 100644
--- a/dev-tools/scripts/releaseWizard.yaml
+++ b/dev-tools/scripts/releaseWizard.yaml
@@ -675,21 +675,21 @@ groups:
id: dependency_updates_changes
title: Add dependency updates to changelog
description: |
- Bulk add all 'solrbot' dependency updates since last release.
- NOTE: Work in progress to let each Solrbot PR add its own changes to
changelog.
- This step will be removed once that is done.
- Until then, there may be a mix of PRs with and without changes to
changelog.
+ Add changelog entries summarizing third-party dependencies added,
upgraded, and removed since the
+ previous release, derived from the jar checksum files in
`solr/licenses/`.
+ Re-running it overwrites the entries, e.g. if dependencies changed
before a respin.
+ Run the script without `--write` to just print the summary.
depends: clean_git_checkout
commands: !Commands
root_folder: '{{ git_checkout_folder }}'
- commands_text: We call out to a helper script that compiles SolrBot
changes into `changelog/unreleased`
+ commands_text: We call out to a helper script that writes
`changelog/unreleased/dependency-changes-*.yml`
confirm_each_command: true
commands:
- !Command
cmd: git checkout {{ release_branch }}
stdout: true
- !Command
- cmd: python3 -u dev-tools/scripts/addDepsToChanges.py --user solrbot
--version {{ release_version }}
+ cmd: python3 -u dev-tools/scripts/dependencyChanges.py --write
tee: true
- !Command
cmd: git add changelog && git commit -m "Add dependency updates to
changelog for {{ release_version }}" && git push
