Repository: storm Updated Branches: refs/heads/1.x-branch e67c55a34 -> 866517c15
STORM-2665: Adapt Kafka's release note generation script for Storm Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/866517c1 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/866517c1 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/866517c1 Branch: refs/heads/1.x-branch Commit: 866517c15857f6161c5da2e6e7b6445ca92da722 Parents: e67c55a Author: Stig Rohde Døssing <s...@apache.org> Authored: Mon Jul 31 21:48:22 2017 +0200 Committer: Stig Rohde Døssing <s...@apache.org> Committed: Thu Aug 3 18:44:39 2017 +0200 ---------------------------------------------------------------------- dev-tools/jira-github-join.py | 2 +- dev-tools/jira/__init__.py | 285 --------------------------------- dev-tools/jira_github/__init__.py | 285 +++++++++++++++++++++++++++++++++ dev-tools/release_notes.py | 118 ++++++++++++++ dev-tools/report/report.py | 2 +- 5 files changed, 405 insertions(+), 287 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/866517c1/dev-tools/jira-github-join.py ---------------------------------------------------------------------- diff --git a/dev-tools/jira-github-join.py b/dev-tools/jira-github-join.py index fe0daf3..1931f19 100755 --- a/dev-tools/jira-github-join.py +++ b/dev-tools/jira-github-join.py @@ -15,7 +15,7 @@ from optparse import OptionParser from datetime import datetime from github import GitHub -from jira import JiraRepo +from jira_github import JiraRepo from report.report_builder import CompleteReportBuilder http://git-wip-us.apache.org/repos/asf/storm/blob/866517c1/dev-tools/jira/__init__.py ---------------------------------------------------------------------- diff --git a/dev-tools/jira/__init__.py b/dev-tools/jira/__init__.py deleted file mode 100755 index c98ae31..0000000 --- a/dev-tools/jira/__init__.py +++ /dev/null @@ -1,285 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import urllib -import urllib2 -from datetime import datetime - -try: - import json -except ImportError: - import simplejson as json - - -def mstr(obj): - if obj is None: - return "" - return unicode(obj) - - -def jiratime(obj): - if obj is None: - return None - return datetime.strptime(obj[0:19], "%Y-%m-%dT%H:%M:%S") - -# Regex pattern definitions -github_user = re.compile("Git[Hh]ub user ([\w-]+)") -github_pull = re.compile("https://github.com/[^/\s]+/[^/\s]+/pull/[0-9]+";) -has_vote = re.compile("\s+([-+][01])\s*") -is_diff = re.compile("--- End diff --") - - -def search_group(reg, txt, group): - m = reg.search(txt) - if m is None: - return None - return m.group(group) - - -class JiraComment: - """A comment on a JIRA""" - - def __init__(self, data): - self.data = data - self.author = mstr(self.data['author']['name']) - self.github_author = None - self.githubPull = None - self.githubComment = (self.author == "githubbot") - body = self.get_body() - if is_diff.search(body) is not None: - self.vote = None - else: - self.vote = search_group(has_vote, body, 1) - - if self.githubComment: - self.github_author = search_group(github_user, body, 1) - self.githubPull = search_group(github_pull, body, 0) - - def get_author(self): - if self.github_author is not None: - return self.github_author - return self.author - - def get_body(self): - return mstr(self.data['body']) - - def get_pull(self): - return self.githubPull - - def has_github_pull(self): - return self.githubPull is not None - - def raw(self): - return self.data - - def has_vote(self): - return self.vote is not None - - def get_vote(self): - return self.vote - - def get_created(self): - return jiratime(self.data['created']) - - -class Jira: - """A single JIRA""" - - def __init__(self, data, parent): - self.key = data['key'] - self.fields = data['fields'] - self.parent = parent - self.notes = None - self.comments = None - - def get_id(self): - return mstr(self.key) - - def get_description(self): - return mstr(self.fields['description']) - - def getReleaseNote(self): - if self.notes is None: - field = self.parent.fieldIdMap['Release Note'] - if self.fields.has_key(field): - self.notes = mstr(self.fields[field]) - else: - self.notes = self.get_description() - return self.notes - - def get_status(self): - ret = "" - status = self.fields['status'] - if status is not None: - ret = status['name'] - return mstr(ret) - - def get_priority(self): - ret = "" - pri = self.fields['priority'] - if pri is not None: - ret = pri['name'] - return mstr(ret) - - def get_assignee_email(self): - ret = "" - mid = self.fields['assignee'] - if mid is not None: - ret = mid['emailAddress'] - return mstr(ret) - - def get_assignee(self): - ret = "" - mid = self.fields['assignee'] - if mid is not None: - ret = mid['displayName'] - return mstr(ret) - - def get_components(self): - return " , ".join([comp['name'] for comp in self.fields['components']]) - - def get_summary(self): - return self.fields['summary'] - - def get_trimmed_summary(self): - limit = 40 - summary = self.fields['summary'] - return summary if len(summary) < limit else summary[0:limit] + "..." - - def get_type(self): - ret = "" - mid = self.fields['issuetype'] - if mid is not None: - ret = mid['name'] - return mstr(ret) - - def get_reporter(self): - ret = "" - mid = self.fields['reporter'] - if mid is not None: - ret = mid['displayName'] - return mstr(ret) - - def get_project(self): - ret = "" - mid = self.fields['project'] - if mid is not None: - ret = mid['key'] - return mstr(ret) - - def get_created(self): - return jiratime(self.fields['created']) - - def get_updated(self): - return jiratime(self.fields['updated']) - - def get_comments(self): - if self.comments is None: - jiraId = self.get_id() - comments = [] - at = 0 - end = 1 - count = 100 - while (at < end): - params = urllib.urlencode({'startAt': at, 'maxResults': count}) - resp = urllib2.urlopen(self.parent.baseUrl + "/issue/" + jiraId + "/comment?" + params) - data = json.loads(resp.read()) - if (data.has_key('errorMessages')): - raise Exception(data['errorMessages']) - at = data['startAt'] + data['maxResults'] - end = data['total'] - for item in data['comments']: - j = JiraComment(item) - comments.append(j) - self.comments = comments - return self.comments - - def has_voted_comment(self): - for comment in self.get_comments(): - if comment.has_vote(): - return True - return False - - def get_trimmed_comments(self, limit=40): - comments = self.get_comments() - return comments if len(comments) < limit else comments[0:limit] + "..." - - def raw(self): - return self.fields - - def storm_jira_cmp(x, y): - xn = x.get_id().split("-")[1] - yn = y.get_id().split("-")[1] - return int(xn) - int(yn) - - -class JiraRepo: - """A Repository for JIRAs""" - - def __init__(self, baseUrl): - self.baseUrl = baseUrl - resp = urllib2.urlopen(baseUrl + "/field") - data = json.loads(resp.read()) - - self.fieldIdMap = {} - for part in data: - self.fieldIdMap[part['name']] = part['id'] - - def get(self, id): - resp = urllib2.urlopen(self.baseUrl + "/issue/" + id) - data = json.loads(resp.read()) - if (data.has_key('errorMessages')): - raise Exception(data['errorMessages']) - j = Jira(data, self) - return j - - def query(self, query): - jiras = {} - at = 0 - end = 1 - count = 100 - while (at < end): - params = urllib.urlencode({'jql': query, 'startAt': at, 'maxResults': count}) - # print params - resp = urllib2.urlopen(self.baseUrl + "/search?%s" % params) - data = json.loads(resp.read()) - if (data.has_key('errorMessages')): - raise Exception(data['errorMessages']) - at = data['startAt'] + data['maxResults'] - end = data['total'] - for item in data['issues']: - j = Jira(item, self) - jiras[j.get_id()] = j - return jiras - - def unresolved_jiras(self, project): - """ - :param project: The JIRA project to search for unresolved issues - :return: All JIRA issues that have the field resolution = Unresolved - """ - return self.query("project = " + project + " AND resolution = Unresolved") - - def open_jiras(self, project): - """ - :param project: The JIRA project to search for open issues - :return: All JIRA issues that have the field status = Open - """ - return self.query("project = " + project + " AND status = Open") - - def in_progress_jiras(self, project): - """ - :param project: The JIRA project to search for In Progress issues - :return: All JIRA issues that have the field status = 'In Progress' - """ - return self.query("project = " + project + " AND status = 'In Progress'") http://git-wip-us.apache.org/repos/asf/storm/blob/866517c1/dev-tools/jira_github/__init__.py ---------------------------------------------------------------------- diff --git a/dev-tools/jira_github/__init__.py b/dev-tools/jira_github/__init__.py new file mode 100644 index 0000000..c98ae31 --- /dev/null +++ b/dev-tools/jira_github/__init__.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import urllib +import urllib2 +from datetime import datetime + +try: + import json +except ImportError: + import simplejson as json + + +def mstr(obj): + if obj is None: + return "" + return unicode(obj) + + +def jiratime(obj): + if obj is None: + return None + return datetime.strptime(obj[0:19], "%Y-%m-%dT%H:%M:%S") + +# Regex pattern definitions +github_user = re.compile("Git[Hh]ub user ([\w-]+)") +github_pull = re.compile("https://github.com/[^/\s]+/[^/\s]+/pull/[0-9]+";) +has_vote = re.compile("\s+([-+][01])\s*") +is_diff = re.compile("--- End diff --") + + +def search_group(reg, txt, group): + m = reg.search(txt) + if m is None: + return None + return m.group(group) + + +class JiraComment: + """A comment on a JIRA""" + + def __init__(self, data): + self.data = data + self.author = mstr(self.data['author']['name']) + self.github_author = None + self.githubPull = None + self.githubComment = (self.author == "githubbot") + body = self.get_body() + if is_diff.search(body) is not None: + self.vote = None + else: + self.vote = search_group(has_vote, body, 1) + + if self.githubComment: + self.github_author = search_group(github_user, body, 1) + self.githubPull = search_group(github_pull, body, 0) + + def get_author(self): + if self.github_author is not None: + return self.github_author + return self.author + + def get_body(self): + return mstr(self.data['body']) + + def get_pull(self): + return self.githubPull + + def has_github_pull(self): + return self.githubPull is not None + + def raw(self): + return self.data + + def has_vote(self): + return self.vote is not None + + def get_vote(self): + return self.vote + + def get_created(self): + return jiratime(self.data['created']) + + +class Jira: + """A single JIRA""" + + def __init__(self, data, parent): + self.key = data['key'] + self.fields = data['fields'] + self.parent = parent + self.notes = None + self.comments = None + + def get_id(self): + return mstr(self.key) + + def get_description(self): + return mstr(self.fields['description']) + + def getReleaseNote(self): + if self.notes is None: + field = self.parent.fieldIdMap['Release Note'] + if self.fields.has_key(field): + self.notes = mstr(self.fields[field]) + else: + self.notes = self.get_description() + return self.notes + + def get_status(self): + ret = "" + status = self.fields['status'] + if status is not None: + ret = status['name'] + return mstr(ret) + + def get_priority(self): + ret = "" + pri = self.fields['priority'] + if pri is not None: + ret = pri['name'] + return mstr(ret) + + def get_assignee_email(self): + ret = "" + mid = self.fields['assignee'] + if mid is not None: + ret = mid['emailAddress'] + return mstr(ret) + + def get_assignee(self): + ret = "" + mid = self.fields['assignee'] + if mid is not None: + ret = mid['displayName'] + return mstr(ret) + + def get_components(self): + return " , ".join([comp['name'] for comp in self.fields['components']]) + + def get_summary(self): + return self.fields['summary'] + + def get_trimmed_summary(self): + limit = 40 + summary = self.fields['summary'] + return summary if len(summary) < limit else summary[0:limit] + "..." + + def get_type(self): + ret = "" + mid = self.fields['issuetype'] + if mid is not None: + ret = mid['name'] + return mstr(ret) + + def get_reporter(self): + ret = "" + mid = self.fields['reporter'] + if mid is not None: + ret = mid['displayName'] + return mstr(ret) + + def get_project(self): + ret = "" + mid = self.fields['project'] + if mid is not None: + ret = mid['key'] + return mstr(ret) + + def get_created(self): + return jiratime(self.fields['created']) + + def get_updated(self): + return jiratime(self.fields['updated']) + + def get_comments(self): + if self.comments is None: + jiraId = self.get_id() + comments = [] + at = 0 + end = 1 + count = 100 + while (at < end): + params = urllib.urlencode({'startAt': at, 'maxResults': count}) + resp = urllib2.urlopen(self.parent.baseUrl + "/issue/" + jiraId + "/comment?" + params) + data = json.loads(resp.read()) + if (data.has_key('errorMessages')): + raise Exception(data['errorMessages']) + at = data['startAt'] + data['maxResults'] + end = data['total'] + for item in data['comments']: + j = JiraComment(item) + comments.append(j) + self.comments = comments + return self.comments + + def has_voted_comment(self): + for comment in self.get_comments(): + if comment.has_vote(): + return True + return False + + def get_trimmed_comments(self, limit=40): + comments = self.get_comments() + return comments if len(comments) < limit else comments[0:limit] + "..." + + def raw(self): + return self.fields + + def storm_jira_cmp(x, y): + xn = x.get_id().split("-")[1] + yn = y.get_id().split("-")[1] + return int(xn) - int(yn) + + +class JiraRepo: + """A Repository for JIRAs""" + + def __init__(self, baseUrl): + self.baseUrl = baseUrl + resp = urllib2.urlopen(baseUrl + "/field") + data = json.loads(resp.read()) + + self.fieldIdMap = {} + for part in data: + self.fieldIdMap[part['name']] = part['id'] + + def get(self, id): + resp = urllib2.urlopen(self.baseUrl + "/issue/" + id) + data = json.loads(resp.read()) + if (data.has_key('errorMessages')): + raise Exception(data['errorMessages']) + j = Jira(data, self) + return j + + def query(self, query): + jiras = {} + at = 0 + end = 1 + count = 100 + while (at < end): + params = urllib.urlencode({'jql': query, 'startAt': at, 'maxResults': count}) + # print params + resp = urllib2.urlopen(self.baseUrl + "/search?%s" % params) + data = json.loads(resp.read()) + if (data.has_key('errorMessages')): + raise Exception(data['errorMessages']) + at = data['startAt'] + data['maxResults'] + end = data['total'] + for item in data['issues']: + j = Jira(item, self) + jiras[j.get_id()] = j + return jiras + + def unresolved_jiras(self, project): + """ + :param project: The JIRA project to search for unresolved issues + :return: All JIRA issues that have the field resolution = Unresolved + """ + return self.query("project = " + project + " AND resolution = Unresolved") + + def open_jiras(self, project): + """ + :param project: The JIRA project to search for open issues + :return: All JIRA issues that have the field status = Open + """ + return self.query("project = " + project + " AND status = Open") + + def in_progress_jiras(self, project): + """ + :param project: The JIRA project to search for In Progress issues + :return: All JIRA issues that have the field status = 'In Progress' + """ + return self.query("project = " + project + " AND status = 'In Progress'") http://git-wip-us.apache.org/repos/asf/storm/blob/866517c1/dev-tools/release_notes.py ---------------------------------------------------------------------- diff --git a/dev-tools/release_notes.py b/dev-tools/release_notes.py new file mode 100644 index 0000000..c1e053a --- /dev/null +++ b/dev-tools/release_notes.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Usage: release_notes.py <version> > RELEASE_NOTES.html + +Depends on https://pypi.python.org/pypi/jira/, please use pip to install this module. + +Generates release notes for a Storm release by generating an HTML doc containing some introductory information about the + release with links to the Storm docs followed by a list of issues resolved in the release. The script will fail if it finds + any unresolved issues still marked with the target release. You should run this script after either resolving all issues or + moving outstanding issues to a later release. + +""" + +from jira import JIRA +import itertools, sys + +if len(sys.argv) < 2: + print >>sys.stderr, "Usage: release_notes.py <version>" + sys.exit(1) + +version = sys.argv[1] + +JIRA_BASE_URL = 'https://issues.apache.org/jira' +MAX_RESULTS = 100 # This is constrained for cloud instances so we need to fix this value + +def get_issues(jira, query, **kwargs): + """ + Get all issues matching the JQL query from the JIRA instance. This handles expanding paginated results for you. Any additional keyword arguments are forwarded to the JIRA.search_issues call. + """ + results = [] + startAt = 0 + new_results = None + while new_results == None or len(new_results) == MAX_RESULTS: + new_results = jira.search_issues(query, startAt=startAt, maxResults=MAX_RESULTS, **kwargs) + results += new_results + startAt += len(new_results) + return results + +def issue_link(issue): + return "%s/browse/%s" % (JIRA_BASE_URL, issue.key) + + +if __name__ == "__main__": + apache = JIRA(JIRA_BASE_URL) + issues = get_issues(apache, 'project=STORM and fixVersion=%s' % version) + if not issues: + print >>sys.stderr, "Didn't find any issues for the target fix version" + sys.exit(1) + + # Some resolutions, including a lack of resolution, indicate that the bug hasn't actually been addressed and we shouldn't even be able to create a release until they are fixed + UNRESOLVED_RESOLUTIONS = [None, + "Unresolved", + "Duplicate", + "Invalid", + "Not A Problem", + "Not A Bug", + "Won't Fix", + "Incomplete", + "Cannot Reproduce", + "Later", + "Works for Me", + "Workaround", + "Information Provided" + ] + unresolved_issues = [issue for issue in issues if issue.fields.resolution in UNRESOLVED_RESOLUTIONS or issue.fields.resolution.name in UNRESOLVED_RESOLUTIONS] + if unresolved_issues: + print >>sys.stderr, "The release is not completed since unresolved issues or improperly resolved issues were found still tagged with this release as the fix version:" + for issue in unresolved_issues: + print >>sys.stderr, "Unresolved issue: %15s %20s %s" % (issue.key, issue.fields.resolution, issue_link(issue)) + print >>sys.stderr + print >>sys.stderr, "Note that for some resolutions, you should simply remove the fix version as they have not been truly fixed in this release." + sys.exit(1) + + # Get list of (issue type, [issues]) sorted by the issue ID type, with each subset of issues sorted by their key so they + # are in increasing order of bug #. To get a nice ordering of the issue types we customize the key used to sort by issue + # type a bit to ensure features and improvements end up first. + def issue_type_key(issue): + if issue.fields.issuetype.name == 'New Feature': + return -2 + if issue.fields.issuetype.name == 'Improvement': + return -1 + return issue.fields.issuetype.id + by_group = [(k,sorted(g, key=lambda issue: issue.id)) for k,g in itertools.groupby(sorted(issues, key=issue_type_key), lambda issue: issue.fields.issuetype.name)] + + print "<!DOCTYPE html>" + print "<html lang=\"en\">" + print "<head>" + print "<meta charset=\"utf-8\">" + print "<title>Storm %(version)s Release Notes</title>" % { 'version': version } + print "</head>" + print "<body>" + print "<h1>Release Notes for Storm %s</h1>" % version + print """<p>JIRA issues addressed in the %(version)s release of Storm. Documentation for this + release is available at the <a href="http://storm.apache.org/";>Apache Storm + project site</a>.</p>""" % { 'version': version } + for itype, issues in by_group: + print "<h2>%s</h2>" % itype + print "<ul>" + for issue in issues: + print '<li>[<a href="%(link)s">%(key)s</a>] - %(summary)s</li>' % {'key': issue.key, 'link': issue_link(issue), 'summary': issue.fields.summary} + print "</ul>" + print "</body>" + print "</html>" http://git-wip-us.apache.org/repos/asf/storm/blob/866517c1/dev-tools/report/report.py ---------------------------------------------------------------------- diff --git a/dev-tools/report/report.py b/dev-tools/report/report.py index 477a498..d2df077 100644 --- a/dev-tools/report/report.py +++ b/dev-tools/report/report.py @@ -14,7 +14,7 @@ from datetime import datetime from github import mstr -from jira import Jira +from jira_github import Jira from formatter import Formatter, encode
