This is an automated email from the ASF dual-hosted git repository. rzo1 pushed a commit to branch github_issues in repository https://gitbox.apache.org/repos/asf/storm.git
commit 473f38b8bd1e2d6156c0a59bc58e7001b9bc7973 Author: Richard Zowalla <[email protected]> AuthorDate: Fri Jan 24 20:28:25 2025 +0100 Update Release Note Generation to work with GitHub Issues --- dev-tools/jira-github-join.py | 50 ------- dev-tools/jira_github/__init__.py | 296 -------------------------------------- dev-tools/release_notes.py | 157 +++++++++++--------- 3 files changed, 87 insertions(+), 416 deletions(-) diff --git a/dev-tools/jira-github-join.py b/dev-tools/jira-github-join.py deleted file mode 100755 index a6e1580b6..000000000 --- a/dev-tools/jira-github-join.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from optparse import OptionParser -from datetime import datetime -from github import GitHub -from jira_github import JiraRepo -from report.report_builder import CompleteReportBuilder - -""" -If you get certificate error when running on Mac, -(https://stackoverflow.com/questions/50236117/scraping-ssl-certificate-verify-failed-error-for-http-en-wikipedia-org) -Go to Macintosh HD - > Applications - > Python3.9 folder (or whatever version of Python you're using) - > double click on "Install Certificates.command" file. - -""" - - -def main(): - parser = OptionParser(usage="usage: %prog [options]") - parser.add_option("-g", "--github-user", dest="gituser", - type="string", help="github User, if not supplied no auth is used", metavar="USER") - - (options, args) = parser.parse_args() - - jira_repo = JiraRepo("https://issues.apache.org/jira/rest/api/2";) - github_repo = GitHub(options) - - print("=" * 100) - print("Report generated on: %s (GMT)" % (datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S"))) - print("-" * 100) - report_builder = CompleteReportBuilder(jira_repo, github_repo) - report_builder.report.print_all() - - -if __name__ == "__main__": - main() diff --git a/dev-tools/jira_github/__init__.py b/dev-tools/jira_github/__init__.py deleted file mode 100644 index 46412e6dd..000000000 --- a/dev-tools/jira_github/__init__.py +++ /dev/null @@ -1,296 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import urllib -import urllib.request -import urllib.parse -from datetime import datetime - -try: - import json -except ImportError: - import simplejson as json - - -def jiratime(obj): - if obj is None: - return None - return datetime.strptime(obj[0:19], "%Y-%m-%dT%H:%M:%S") - - -# Regex pattern definitions -github_user = re.compile(r"Git[Hh]ub user ([\w-]+)") -github_pull = re.compile(r"https://github.com/[^/\s]+/[^/\s]+/pull/[0-9]+";) -has_vote = re.compile(r"\s+([-+][01])\s*") -is_diff = re.compile("--- End diff --") - - -def search_group(reg, txt, group): - m = reg.search(txt) - if m is None: - return None - return m.group(group) - - -class JiraComment: - """A comment on a JIRA""" - - def __init__(self, data): - self.data = data - self.author = self.data['author']['name'] - self.github_author = None - self.githubPull = None - self.githubComment = (self.author == "githubbot") - body = self.get_body() - if is_diff.search(body) is not None: - self.vote = None - else: - self.vote = search_group(has_vote, body, 1) - - if self.githubComment: - self.github_author = search_group(github_user, body, 1) - self.githubPull = search_group(github_pull, body, 0) - - def get_author(self): - if self.github_author is not None: - return self.github_author - return self.author - - def get_body(self): - return self.data['body'] - - def get_pull(self): - return self.githubPull - - def has_github_pull(self): - return self.githubPull is not None - - def raw(self): - return self.data - - def has_vote(self): - return self.vote is not None - - def get_vote(self): - return self.vote - - def get_created(self): - return jiratime(self.data['created']) - - -class Jira: - """A single JIRA""" - - def __init__(self, data, parent): - self.key = data['key'] - self.fields = data['fields'] - self.parent = parent - self.notes = None - self.comments = None - - def get_id(self): - """ - Get Jira ID as a string from the string stored in self.key - :return: Jira id, example "STORM-1234" - """ - return self.key - - def get_id_num(self): - """ - Get Jira ID number as an integer from the string stored in self.key - :return: Numeric Jira Id as a number. Example "STORM-1234" and "ZKP-1234" will both return 1234 - """ - return int(self.key.split('-')[-1]) - - def get_description(self): - return self.fields['description'] - - def getReleaseNote(self): - if self.notes is None: - field = self.parent.fieldIdMap['Release Note'] - if field in self.fields: - self.notes = self.fields[field] - else: - self.notes = self.get_description() - return self.notes - - def get_status(self): - ret = "" - status = self.fields['status'] - if status is not None: - ret = status['name'] - return ret - - def get_priority(self): - ret = "" - pri = self.fields['priority'] - if pri is not None: - ret = pri['name'] - return ret - - def get_assignee_email(self): - ret = "" - mid = self.fields['assignee'] - if mid is not None: - ret = mid['emailAddress'] - return ret - - def get_assignee(self): - ret = "" - mid = self.fields['assignee'] - if mid is not None: - ret = mid['displayName'] - return ret - - def get_components(self): - return " , ".join([comp['name'] for comp in self.fields['components']]) - - def get_summary(self): - return self.fields['summary'] - - def get_trimmed_summary(self): - limit = 40 - summary = self.fields['summary'] - return summary if len(summary) < limit else summary[0:limit] + "..." - - def get_type(self): - ret = "" - mid = self.fields['issuetype'] - if mid is not None: - ret = mid['name'] - return ret - - def get_reporter(self): - ret = "" - mid = self.fields['reporter'] - if mid is not None: - ret = mid['displayName'] - return ret - - def get_project(self): - ret = "" - mid = self.fields['project'] - if mid is not None: - ret = mid['key'] - return ret - - def get_created(self): - return jiratime(self.fields['created']) - - def get_updated(self): - return jiratime(self.fields['updated']) - - def get_comments(self): - if self.comments is None: - jiraId = self.get_id() - comments = [] - at = 0 - end = 1 - count = 100 - while at < end: - params = urllib.parse.urlencode({'startAt': at, 'maxResults': count}) - resp = urllib.request.urlopen(self.parent.baseUrl + "/issue/" + jiraId + "/comment?" + params) - resp_str = resp.read().decode() - data = json.loads(resp_str) - if 'errorMessages' in data: - raise Exception(data['errorMessages']) - at = data['startAt'] + data['maxResults'] - end = data['total'] - for item in data['comments']: - j = JiraComment(item) - comments.append(j) - self.comments = comments - return self.comments - - def has_voted_comment(self): - for comment in self.get_comments(): - if comment.has_vote(): - return True - return False - - def get_trimmed_comments(self, limit=40): - comments = self.get_comments() - return comments if len(comments) < limit else comments[0:limit] + ["..."] - - def raw(self): - return self.fields - - def storm_jira_cmp(self, x, y): - xn = x.get_id().split("-")[1] - yn = y.get_id().split("-")[1] - return int(xn) - int(yn) - - -class JiraRepo: - """A Repository for JIRAs""" - - def __init__(self, baseUrl): - self.baseUrl = baseUrl - resp = urllib.request.urlopen(baseUrl + "/field") - resp_str = resp.read().decode() - data = json.loads(resp_str) - - self.fieldIdMap = {} - for part in data: - self.fieldIdMap[part['name']] = part['id'] - - def get(self, id): - resp = urllib.request.urlopen(self.baseUrl + "/issue/" + id) - resp_str = resp.read().decode() - data = json.loads(resp_str) - if 'errorMessages' in data: - raise Exception(data['errorMessages']) - j = Jira(data, self) - return j - - def query(self, query): - jiras = {} - at = 0 - end = 1 - count = 100 - while at < end: - params = urllib.parse.urlencode({'jql': query, 'startAt': at, 'maxResults': count}) - # print params - resp = urllib.request.urlopen(self.baseUrl + "/search?%s" % params) - resp_str = resp.read().decode() - data = json.loads(resp_str) - if 'errorMessages' in data: - raise Exception(data['errorMessages']) - at = data['startAt'] + data['maxResults'] - end = data['total'] - for item in data['issues']: - j = Jira(item, self) - jiras[j.get_id()] = j - return jiras - - def unresolved_jiras(self, project): - """ - :param project: The JIRA project to search for unresolved issues - :return: All JIRA issues that have the field resolution = Unresolved - """ - return self.query(f"project = {project} AND resolution = Unresolved") - - def open_jiras(self, project): - """ - :param project: The JIRA project to search for open issues - :return: All JIRA issues that have the field status = Open - """ - return self.query(f"project = {project} AND status = Open") - - def in_progress_jiras(self, project): - """ - :param project: The JIRA project to search for In Progress issues - :return: All JIRA issues that have the field status = 'In Progress' - """ - return self.query(f"project = {project} AND status = 'In Progress'") diff --git a/dev-tools/release_notes.py b/dev-tools/release_notes.py index 1c8663d2b..0c28515d3 100755 --- a/dev-tools/release_notes.py +++ b/dev-tools/release_notes.py @@ -15,9 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Usage: release_notes.py <version> > RELEASE_NOTES.html +"""Usage: release_notes.py <milestone_id_from_github> > RELEASE_NOTES.html -Depends on https://pypi.python.org/pypi/jira/, please use pip to install this module. +Depends on "requests", please use pip to install this module. Generates release notes for a Storm release by generating an HTML doc containing some introductory information about the release with links to the Storm docs followed by a list of issues resolved in the release. The script will fail if it finds @@ -26,105 +26,122 @@ Generates release notes for a Storm release by generating an HTML doc containing """ -from jira import JIRA -import itertools, sys +import requests +import sys if len(sys.argv) < 2: - print("Usage: release_notes.py <version>", file=sys.stderr) + print("Usage: release_notes.py <milestone_id>", file=sys.stderr) sys.exit(1) -version = sys.argv[1] +# GitHub configuration +GITHUB_API_BASE_URL = "https://api.github.com"; +GITHUB_TOKEN = "YOUR_PERSONAL_GITHUB_PAT" # Replace with your GitHub PAT -JIRA_BASE_URL = 'https://issues.apache.org/jira' -MAX_RESULTS = 100 # This is constrained for cloud instances so we need to fix this value +# Input arguments +owner = "apache" +repo = "storm" +milestone = sys.argv[1] # Milestone ID +print(f"Fetching issues for milestone with id= '{milestone}'...") -def get_issues(jira, query, **kwargs): +headers = { + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github.v3+json" +} + +def get_milestone_title(owner, repo, milestone_number): """ - Get all issues matching the JQL query from the JIRA instance. This handles expanding paginated results for you. - Any additional keyword arguments are forwarded to the JIRA.search_issues call. + Fetch the title of a specific milestone by its number. """ - results = [] - startAt = 0 - new_results = None - while new_results is None or len(new_results) == MAX_RESULTS: - new_results = jira.search_issues(query, startAt=startAt, maxResults=MAX_RESULTS, **kwargs) - results += new_results - startAt += len(new_results) - return results + url = f"{GITHUB_API_BASE_URL}/repos/{owner}/{repo}/milestones/{milestone_number}" + response = requests.get(url, headers=headers) + if response.status_code != 200: + print(f"Failed to fetch milestone: {response.status_code} {response.reason}", file=sys.stderr) + sys.exit(1) -def issue_link(issue): - return "%s/browse/%s" % (JIRA_BASE_URL, issue.key) + milestone = response.json() + return milestone["title"] + +def get_issues(owner, repo, milestone): + """ + Fetch all issues for a given milestone from a GitHub repository. + """ + issues_url = f"{GITHUB_API_BASE_URL}/repos/{owner}/{repo}/issues" + params = { + "milestone": milestone, + "state": "all", # Include both open and closed issues + "per_page": 100 + } + + issues = [] + while issues_url: + response = requests.get(issues_url, headers=headers, params=params) + if response.status_code != 200: + print(f"Failed to fetch issues: {response.status_code} {response.reason}", file=sys.stderr) + sys.exit(1) + + data = response.json() + issues.extend(data) + # Get next page URL from 'Link' header if available + issues_url = response.links.get("next", {}).get("url") + + return issues +def issue_link(issue): + return issue["html_url"] if __name__ == "__main__": - apache = JIRA(JIRA_BASE_URL) - issues = get_issues(apache, 'project=STORM and fixVersion=%s' % version) + issues = get_issues(owner, repo, milestone) + if not issues: - print("Didn't find any issues for the target fix version", file=sys.stderr) + print("No issues found for the specified milestone.", file=sys.stderr) sys.exit(1) - # Some resolutions, including a lack of resolution, indicate that the bug hasn't actually been addressed and we - # shouldn't even be able to create a release until they are fixed - UNRESOLVED_RESOLUTIONS = [None, - "Unresolved", - "Duplicate", - "Invalid", - "Not A Problem", - "Not A Bug", - "Won't Fix", - "Incomplete", - "Cannot Reproduce", - "Later", - "Works for Me", - "Workaround", - "Information Provided" - ] - unresolved_issues = [issue for issue in issues - if issue.fields.resolution in UNRESOLVED_RESOLUTIONS - or issue.fields.resolution.name in UNRESOLVED_RESOLUTIONS] + unresolved_issues = [issue for issue in issues if issue["state"] != "closed"] if unresolved_issues: - print("The release is not completed since unresolved issues or improperly resolved issues were found still " - "tagged with this release as the fix version:", file=sys.stderr) + print("The release is not completed since unresolved issues were found:", file=sys.stderr) for issue in unresolved_issues: - print("Unresolved issue: %15s %20s %s" % (issue.key, issue.fields.resolution, issue_link(issue)), - file=sys.stderr) - print('', file=sys.stderr) - print("Note that for some resolutions, you should simply remove the fix version as they have not been truly " - "fixed in this release.", file=sys.stderr) + print(f"Unresolved issue: {issue['number']:5d} {issue['state']:10s} {issue_link(issue)}", file=sys.stderr) sys.exit(1) - # Get list of (issue type, [issues]) sorted by the issue ID type, with each subset of issues sorted by their key so - # they are in increasing order of bug #. To get a nice ordering of the issue types we customize the key used to sort - # by issue type a bit to ensure features and improvements end up first. - def issue_type_key(issue): - if issue.fields.issuetype.name == 'New Feature': - return -2 - if issue.fields.issuetype.name == 'Improvement': - return -1 - return int(issue.fields.issuetype.id) - - by_group = [(k, sorted(g, key=lambda issue: issue.id)) - for k, g in itertools.groupby(sorted(issues, key=issue_type_key), - lambda issue: issue.fields.issuetype.name)] + # Group issues by labels + issues_by_label = {} + unlabeled_issues = [] + for issue in issues: + if issue["labels"]: # If the issue has labels + for label in issue["labels"]: + label_name = label["name"] + issues_by_label.setdefault(label_name, []).append(issue) + else: + unlabeled_issues.append(issue) # Add to the unlabeled list if no labels exist + + # Add unlabeled issues under a special "No Label" category + if unlabeled_issues: + issues_by_label["Uncategorized"] = unlabeled_issues + issues_str = "\n".join([ - f"\n\t<h2>{itype}</h2>" + + f"\n\t<h2>{label}</h2>" + f"\n\t<ul>" + - '\n\t\t'.join([f'<li>[<a href="{issue_link(issue)}">{issue.key}</a>] - {issue.fields.summary}</li>' for issue in issues]) + + "\n\t\t".join([ + f'<li>[<a href="{issue_link(issue)}">#{issue["number"]}</a>] - {issue["title"]}</li>' + for issue in issues + ]) + "\n\t</ul>" - for itype, issues in by_group]) + for label, issues in issues_by_label.items() + ]) + + version = get_milestone_title(owner, repo, milestone) print(f"""<!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8"> -<title>Storm {version} Release Notes</title> +<title>Release Notes for Apache Storm {version}</title> </head> <body> -<h1>Release Notes for Storm {version}</h1> -<p>JIRA issues addressed in the {version} release of Storm. Documentation for this - release is available at the <a href="https://storm.apache.org/";>Apache Storm project site</a>.</p> +<h1>Release Notes for Apache Storm {version}</h1> +<p>Issues addressed in {version}.</p> {issues_str} </body> </html>""")
