Repository: storm Updated Branches: refs/heads/master cddd2531c -> 413389973
STORM-931: Python Scritps to Produce Formatted JIRA and GitHub Joint Reports Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/61dceffb Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/61dceffb Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/61dceffb Branch: refs/heads/master Commit: 61dceffb622e70dd419096f0fa0ad5979d99d710 Parents: f75cf7c Author: Hugo Louro <[email protected]> Authored: Fri Jun 19 18:48:12 2015 -0700 Committer: Hugo Louro <[email protected]> Committed: Thu Jul 9 19:35:33 2015 -0700 ---------------------------------------------------------------------- dev-tools/github/__init__.py | 205 ++++++++------- dev-tools/jira-github-join.py | 77 ++---- dev-tools/jira/__init__.py | 435 ++++++++++++++++++-------------- dev-tools/report/__init__.py | 14 + dev-tools/report/formatter.py | 68 +++++ dev-tools/report/report.py | 252 ++++++++++++++++++ dev-tools/report/report_builder.py | 86 +++++++ dev-tools/storm-merge.py | 2 +- 8 files changed, 795 insertions(+), 344 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/61dceffb/dev-tools/github/__init__.py ---------------------------------------------------------------------- diff --git a/dev-tools/github/__init__.py b/dev-tools/github/__init__.py index 2c533d0..48f397b 100755 --- a/dev-tools/github/__init__.py +++ b/dev-tools/github/__init__.py @@ -13,108 +13,127 @@ import getpass import base64 -import urllib import urllib2 from datetime import datetime +import re + try: - import json + import json except ImportError: - import simplejson as json + import simplejson as json + def mstr(obj): - if (obj == None): - return "" - return unicode(obj) + if obj is None: + return "" + return unicode(obj) + + +def git_time(obj): + if obj is None: + return None + return datetime.strptime(obj[0:19], "%Y-%m-%dT%H:%M:%S") -def gittime(obj): - if (obj == None): - return None - return datetime.strptime(obj[0:19], "%Y-%m-%dT%H:%M:%S") class GitPullRequest: - """Pull Request from Git""" - def __init__(self, data, parent): - self.data = data - self.parent = parent - - def html_url(self): - return self.data["html_url"] - - def title(self): - return self.data["title"] - - def number(self): - return self.data["number"] - - #TODO def review_comments - - def user(self): - return mstr(self.data["user"]["login"]) - - def fromBranch(self): - return mstr(self.data["head"]["ref"]) - - def fromRepo(self): - return mstr(self.data["head"]["repo"]["clone_url"]) - - def merged(self): - return self.data["merged_at"] != None - - def raw(self): - return self.data - - def created_at(self): - return gittime(self.data["created_at"]) - - def updated_at(self): - return gittime(self.data["updated_at"]) - - def merged_at(self): - return gittime(self.data["merged_at"]) - - def __str__(self): - return self.html_url() - - def __repr__(self): - return self.html_url() + """Pull Request from Git""" + + storm_jira_number = re.compile("STORM-[0-9]+", re.I) + + def __init__(self, data, parent): + self.data = data + self.parent = parent + + def html_url(self): + return self.data["html_url"] + + def title(self): + return self.data["title"] + + def trimmed_title(self): + limit = 40 + title = self.data["title"] + return title if len(title) < limit else title[0:limit] + "..." + + def number(self): + return self.data["number"] + + # TODO def review_comments + + def user(self): + return mstr(self.data["user"]["login"]) + + def from_branch(self): + return mstr(self.data["head"]["ref"]) + + def from_repo(self): + return mstr(self.data["head"]["repo"]["clone_url"]) + + def merged(self): + return self.data["merged_at"] is not None + + def raw(self): + return self.data + + def created_at(self): + return git_time(self.data["created_at"]) + + def updated_at(self): + return git_time(self.data["updated_at"]) + + def merged_at(self): + return git_time(self.data["merged_at"]) + + def has_jira_id(self): + return GitPullRequest.storm_jira_number.search(self.title()) + + def jira_id(self): + return GitPullRequest.storm_jira_number.search(self.title()).group(0).upper() + + def __str__(self): + return self.html_url() + + def __repr__(self): + return self.html_url() + class GitHub: - """Github API""" - def __init__(self, options): - self.headers = {} - if options.gituser: - gitpassword = getpass.getpass("github.com user " + options.gituser+":") - authstr = base64.encodestring('%s:%s' % (options.gituser, gitpassword)).replace('\n', '') - self.headers["Authorization"] = "Basic "+authstr - - def pulls(self, user, repo, type="all"): - page=1 - ret = [] - while True: - url = "https://api.github.com/repos/"+user+"/"+repo+"/pulls?state="+type+"&page="+str(page) - - req = urllib2.Request(url,None,self.headers) - result = urllib2.urlopen(req) - contents = result.read() - if result.getcode() != 200: - raise Exception(result.getcode() + " != 200 "+ contents) - got = json.loads(contents) - for part in got: - ret.append(GitPullRequest(part, self)) - if len(got) == 0: - return ret - page = page + 1 - - def openPulls(self, user, repo): - return self.pulls(user, repo, "open") - - def pull(self, user, repo, number): - url = "https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+number - req = urllib2.Request(url,None,self.headers) - result = urllib2.urlopen(req) - contents = result.read() - if result.getcode() != 200: - raise Exception(result.getcode() + " != 200 "+ contents) - got = json.loads(contents) - return GitPullRequest(got, self) + """Github API""" + + def __init__(self, options): + self.headers = {} + if options.gituser: + gitpassword = getpass.getpass("github.com user " + options.gituser + ":") + authstr = base64.encodestring('%s:%s' % (options.gituser, gitpassword)).replace('\n', '') + self.headers["Authorization"] = "Basic " + authstr + + def pulls(self, user, repo, type="all"): + page = 1 + ret = [] + while True: + url = "https://api.github.com/repos/"; + user + "/" + repo + "/pulls?state=" + type + "&page=" + str(page) + + req = urllib2.Request(url, None, self.headers) + result = urllib2.urlopen(req) + contents = result.read() + if result.getcode() != 200: + raise Exception(result.getcode() + " != 200 " + contents) + got = json.loads(contents) + for part in got: + ret.append(GitPullRequest(part, self)) + if len(got) == 0: + return ret + page = page + 1 + + def open_pulls(self, user, repo): + return self.pulls(user, repo, "open") + def pull(self, user, repo, number): + url = "https://api.github.com/repos/"; + user + "/" + repo + "/pulls/" + number + req = urllib2.Request(url, None, self.headers) + result = urllib2.urlopen(req) + contents = result.read() + if result.getcode() != 200: + raise Exception(result.getcode() + " != 200 " + contents) + got = json.loads(contents) + return GitPullRequest(got, self) http://git-wip-us.apache.org/repos/asf/storm/blob/61dceffb/dev-tools/jira-github-join.py ---------------------------------------------------------------------- diff --git a/dev-tools/jira-github-join.py b/dev-tools/jira-github-join.py index d2526e6..fe0daf3 100755 --- a/dev-tools/jira-github-join.py +++ b/dev-tools/jira-github-join.py @@ -12,69 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jira import JiraRepo -from github import GitHub, mstr -import re from optparse import OptionParser from datetime import datetime +from github import GitHub +from jira import JiraRepo +from report.report_builder import CompleteReportBuilder -def daydiff(a, b): - return (a - b).days def main(): - parser = OptionParser(usage="usage: %prog [options]") - parser.add_option("-g", "--github-user", dest="gituser", - type="string", help="github user, if not supplied no auth is used", metavar="USER") - - (options, args) = parser.parse_args() - - jrepo = JiraRepo("https://issues.apache.org/jira/rest/api/2";) - github = GitHub(options) - - openPullRequests = github.openPulls("apache","storm") - stormJiraNumber = re.compile("STORM-[0-9]+", re.I) - openJiras = jrepo.openJiras("STORM") - - jira2Pulls = {} - pullWithoutJira = [] - pullWithBadJira = [] - - for pull in openPullRequests: - found = stormJiraNumber.search(pull.title()) - if found: - jiraNum = found.group(0).upper() - if not (jiraNum in openJiras): - pullWithBadJira.append(pull) - else: - if jira2Pulls.get(jiraNum) == None: - jira2Pulls[jiraNum] = [] - jira2Pulls[jiraNum].append(pull) - else: - pullWithoutJira.append(pull); - - now = datetime.utcnow() - print "Pull requests that need a JIRA:" - print "Pull URL\tPull Title\tPull Age\tPull Update Age" - for pull in pullWithoutJira: - print ("%s\t%s\t%s\t%s"%(pull.html_url(), pull.title(), daydiff(now, pull.created_at()), daydiff(now, pull.updated_at()))).encode("UTF-8") - - print "\nPull with bad or closed JIRA:" - print "Pull URL\tPull Title\tPull Age\tPull Update Age" - for pull in pullWithBadJira: - print ("%s\t%s\t%s\t%s"%(pull.html_url(), pull.title(), daydiff(now, pull.created_at()), daydiff(now, pull.updated_at()))).encode("UTF-8") - - print "\nOpen JIRA to Pull Requests and Possible Votes, vote detection is very approximate:" - print "JIRA\tPull Requests\tJira Summary\tJIRA Age\tPull Age\tJIRA Update Age\tPull Update Age" - print "\tComment Vote\tComment Author\tPull URL\tComment Age" - for key, value in jira2Pulls.items(): - print ("%s\t%s\t%s\t%s\t%s\t%s\t%s"%(key, mstr(value), openJiras[key].getSummary(), - daydiff(now, openJiras[key].getCreated()), daydiff(now, value[0].created_at()), - daydiff(now, openJiras[key].getUpdated()), daydiff(now, value[0].updated_at()))).encode("UTF-8") - for comment in openJiras[key].getComments(): - #print comment.raw() - if comment.hasVote(): - print (("\t%s\t%s\t%s\t%s")%(comment.getVote(), comment.getAuthor(), comment.getPull(), daydiff(now, comment.getCreated()))).encode("UTF-8") + parser = OptionParser(usage="usage: %prog [options]") + parser.add_option("-g", "--github-user", dest="gituser", + type="string", help="github User, if not supplied no auth is used", metavar="USER") -if __name__ == "__main__": - main() + (options, args) = parser.parse_args() + + jira_repo = JiraRepo("https://issues.apache.org/jira/rest/api/2";) + github_repo = GitHub(options) + print "Report generated on: %s (GMT)" % (datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S")) + + report_builder = CompleteReportBuilder(jira_repo, github_repo) + report_builder.report.print_all() + + +if __name__ == "__main__": + main() http://git-wip-us.apache.org/repos/asf/storm/blob/61dceffb/dev-tools/jira/__init__.py ---------------------------------------------------------------------- diff --git a/dev-tools/jira/__init__.py b/dev-tools/jira/__init__.py index 15380aa..c98ae31 100755 --- a/dev-tools/jira/__init__.py +++ b/dev-tools/jira/__init__.py @@ -15,218 +15,271 @@ import re import urllib import urllib2 from datetime import datetime + try: - import json + import json except ImportError: - import simplejson as json + import simplejson as json + def mstr(obj): - if (obj == None): - return "" - return unicode(obj) + if obj is None: + return "" + return unicode(obj) + def jiratime(obj): - if (obj == None): - return None - return datetime.strptime(obj[0:19], "%Y-%m-%dT%H:%M:%S") + if obj is None: + return None + return datetime.strptime(obj[0:19], "%Y-%m-%dT%H:%M:%S") + +# Regex pattern definitions +github_user = re.compile("Git[Hh]ub user ([\w-]+)") +github_pull = re.compile("https://github.com/[^/\s]+/[^/\s]+/pull/[0-9]+";) +has_vote = re.compile("\s+([-+][01])\s*") +is_diff = re.compile("--- End diff --") + -githubUser = re.compile("Git[Hh]ub user ([\w-]+)") -githubPull = re.compile("https://github.com/[^/\s]+/[^/\s]+/pull/[0-9]+";) -hasVote = re.compile("\s+([-+][01])\s*") -isDiff = re.compile("--- End diff --") +def search_group(reg, txt, group): + m = reg.search(txt) + if m is None: + return None + return m.group(group) -def searchGroup(reg, txt, group): - m = reg.search(txt) - if m == None: - return None - return m.group(group) class JiraComment: - """A comment on a JIRA""" + """A comment on a JIRA""" - def __init__(self, data): - self.data = data - self.author = mstr(self.data['author']['name']) - self.githubAuthor = None - self.githubPull = None - self.githubComment = (self.author == "githubbot") - body = self.getBody() - if isDiff.search(body) != None: - self.vote = None - else: - self.vote = searchGroup(hasVote, body, 1) + def __init__(self, data): + self.data = data + self.author = mstr(self.data['author']['name']) + self.github_author = None + self.githubPull = None + self.githubComment = (self.author == "githubbot") + body = self.get_body() + if is_diff.search(body) is not None: + self.vote = None + else: + self.vote = search_group(has_vote, body, 1) - if self.githubComment: - self.githubAuthor = searchGroup(githubUser, body, 1) - self.githubPull = searchGroup(githubPull, body, 0) - + if self.githubComment: + self.github_author = search_group(github_user, body, 1) + self.githubPull = search_group(github_pull, body, 0) - def getAuthor(self): - if self.githubAuthor != None: - return self.githubAuthor - return self.author + def get_author(self): + if self.github_author is not None: + return self.github_author + return self.author - def getBody(self): - return mstr(self.data['body']) + def get_body(self): + return mstr(self.data['body']) - def getPull(self): - return self.githubPull + def get_pull(self): + return self.githubPull - def raw(self): - return self.data + def has_github_pull(self): + return self.githubPull is not None - def hasVote(self): - return self.vote != None + def raw(self): + return self.data - def getVote(self): - return self.vote + def has_vote(self): + return self.vote is not None + + def get_vote(self): + return self.vote + + def get_created(self): + return jiratime(self.data['created']) - def getCreated(self): - return jiratime(self.data['created']) class Jira: - """A single JIRA""" - - def __init__(self, data, parent): - self.key = data['key'] - self.fields = data['fields'] - self.parent = parent - self.notes = None - self.comments = None - - def getId(self): - return mstr(self.key) - - def getDescription(self): - return mstr(self.fields['description']) - - def getReleaseNote(self): - if (self.notes == None): - field = self.parent.fieldIdMap['Release Note'] - if (self.fields.has_key(field)): - self.notes=mstr(self.fields[field]) - else: - self.notes=self.getDescription() - return self.notes - - def getPriority(self): - ret = "" - pri = self.fields['priority'] - if(pri != None): - ret = pri['name'] - return mstr(ret) - - def getAssigneeEmail(self): - ret = "" - mid = self.fields['assignee'] - if mid != None: - ret = mid['emailAddress'] - return mstr(ret) - - - def getAssignee(self): - ret = "" - mid = self.fields['assignee'] - if(mid != None): - ret = mid['displayName'] - return mstr(ret) - - def getComponents(self): - return " , ".join([ comp['name'] for comp in self.fields['components'] ]) - - def getSummary(self): - return self.fields['summary'] - - def getType(self): - ret = "" - mid = self.fields['issuetype'] - if(mid != None): - ret = mid['name'] - return mstr(ret) - - def getReporter(self): - ret = "" - mid = self.fields['reporter'] - if(mid != None): - ret = mid['displayName'] - return mstr(ret) - - def getProject(self): - ret = "" - mid = self.fields['project'] - if(mid != None): - ret = mid['key'] - return mstr(ret) - - def getCreated(self): - return jiratime(self.fields['created']) - - def getUpdated(self): - return jiratime(self.fields['updated']) - - def getComments(self): - if self.comments == None: - jiraId = self.getId() - comments = [] - at=0 - end=1 - count=100 - while (at < end): - params = urllib.urlencode({'startAt':at, 'maxResults':count}) - resp = urllib2.urlopen(self.parent.baseUrl+"/issue/"+jiraId+"/comment?"+params) - data = json.loads(resp.read()) - if (data.has_key('errorMessages')): - raise Exception(data['errorMessages']) - at = data['startAt'] + data['maxResults'] - end = data['total'] - for item in data['comments']: - j = JiraComment(item) - comments.append(j) - self.comments = comments - return self.comments - - def raw(self): - return self.fields + """A single JIRA""" + + def __init__(self, data, parent): + self.key = data['key'] + self.fields = data['fields'] + self.parent = parent + self.notes = None + self.comments = None + + def get_id(self): + return mstr(self.key) + + def get_description(self): + return mstr(self.fields['description']) + + def getReleaseNote(self): + if self.notes is None: + field = self.parent.fieldIdMap['Release Note'] + if self.fields.has_key(field): + self.notes = mstr(self.fields[field]) + else: + self.notes = self.get_description() + return self.notes + + def get_status(self): + ret = "" + status = self.fields['status'] + if status is not None: + ret = status['name'] + return mstr(ret) + + def get_priority(self): + ret = "" + pri = self.fields['priority'] + if pri is not None: + ret = pri['name'] + return mstr(ret) + + def get_assignee_email(self): + ret = "" + mid = self.fields['assignee'] + if mid is not None: + ret = mid['emailAddress'] + return mstr(ret) + + def get_assignee(self): + ret = "" + mid = self.fields['assignee'] + if mid is not None: + ret = mid['displayName'] + return mstr(ret) + + def get_components(self): + return " , ".join([comp['name'] for comp in self.fields['components']]) + + def get_summary(self): + return self.fields['summary'] + + def get_trimmed_summary(self): + limit = 40 + summary = self.fields['summary'] + return summary if len(summary) < limit else summary[0:limit] + "..." + + def get_type(self): + ret = "" + mid = self.fields['issuetype'] + if mid is not None: + ret = mid['name'] + return mstr(ret) + + def get_reporter(self): + ret = "" + mid = self.fields['reporter'] + if mid is not None: + ret = mid['displayName'] + return mstr(ret) + + def get_project(self): + ret = "" + mid = self.fields['project'] + if mid is not None: + ret = mid['key'] + return mstr(ret) + + def get_created(self): + return jiratime(self.fields['created']) + + def get_updated(self): + return jiratime(self.fields['updated']) + + def get_comments(self): + if self.comments is None: + jiraId = self.get_id() + comments = [] + at = 0 + end = 1 + count = 100 + while (at < end): + params = urllib.urlencode({'startAt': at, 'maxResults': count}) + resp = urllib2.urlopen(self.parent.baseUrl + "/issue/" + jiraId + "/comment?" + params) + data = json.loads(resp.read()) + if (data.has_key('errorMessages')): + raise Exception(data['errorMessages']) + at = data['startAt'] + data['maxResults'] + end = data['total'] + for item in data['comments']: + j = JiraComment(item) + comments.append(j) + self.comments = comments + return self.comments + + def has_voted_comment(self): + for comment in self.get_comments(): + if comment.has_vote(): + return True + return False + + def get_trimmed_comments(self, limit=40): + comments = self.get_comments() + return comments if len(comments) < limit else comments[0:limit] + "..." + + def raw(self): + return self.fields + + def storm_jira_cmp(x, y): + xn = x.get_id().split("-")[1] + yn = y.get_id().split("-")[1] + return int(xn) - int(yn) + class JiraRepo: - """A Repository for JIRAs""" - - def __init__(self, baseUrl): - self.baseUrl = baseUrl - resp = urllib2.urlopen(baseUrl+"/field") - data = json.loads(resp.read()) - - self.fieldIdMap = {} - for part in data: - self.fieldIdMap[part['name']] = part['id'] - - def get(self, id): - resp = urllib2.urlopen(self.baseUrl+"/issue/"+id) - data = json.loads(resp.read()) - if (data.has_key('errorMessages')): - raise Exception(data['errorMessages']) - j = Jira(data, self) - return j - - def query(self, query): - jiras = {} - at=0 - end=1 - count=100 - while (at < end): - params = urllib.urlencode({'jql': query, 'startAt':at, 'maxResults':count}) - #print params - resp = urllib2.urlopen(self.baseUrl+"/search?%s"%params) - data = json.loads(resp.read()) - if (data.has_key('errorMessages')): - raise Exception(data['errorMessages']) - at = data['startAt'] + data['maxResults'] - end = data['total'] - for item in data['issues']: - j = Jira(item, self) - jiras[j.getId()] = j - return jiras - - def openJiras(self, project): - return self.query("project = "+project+" AND resolution = Unresolved"); + """A Repository for JIRAs""" + + def __init__(self, baseUrl): + self.baseUrl = baseUrl + resp = urllib2.urlopen(baseUrl + "/field") + data = json.loads(resp.read()) + + self.fieldIdMap = {} + for part in data: + self.fieldIdMap[part['name']] = part['id'] + + def get(self, id): + resp = urllib2.urlopen(self.baseUrl + "/issue/" + id) + data = json.loads(resp.read()) + if (data.has_key('errorMessages')): + raise Exception(data['errorMessages']) + j = Jira(data, self) + return j + + def query(self, query): + jiras = {} + at = 0 + end = 1 + count = 100 + while (at < end): + params = urllib.urlencode({'jql': query, 'startAt': at, 'maxResults': count}) + # print params + resp = urllib2.urlopen(self.baseUrl + "/search?%s" % params) + data = json.loads(resp.read()) + if (data.has_key('errorMessages')): + raise Exception(data['errorMessages']) + at = data['startAt'] + data['maxResults'] + end = data['total'] + for item in data['issues']: + j = Jira(item, self) + jiras[j.get_id()] = j + return jiras + + def unresolved_jiras(self, project): + """ + :param project: The JIRA project to search for unresolved issues + :return: All JIRA issues that have the field resolution = Unresolved + """ + return self.query("project = " + project + " AND resolution = Unresolved") + + def open_jiras(self, project): + """ + :param project: The JIRA project to search for open issues + :return: All JIRA issues that have the field status = Open + """ + return self.query("project = " + project + " AND status = Open") + def in_progress_jiras(self, project): + """ + :param project: The JIRA project to search for In Progress issues + :return: All JIRA issues that have the field status = 'In Progress' + """ + return self.query("project = " + project + " AND status = 'In Progress'") http://git-wip-us.apache.org/repos/asf/storm/blob/61dceffb/dev-tools/report/__init__.py ---------------------------------------------------------------------- diff --git a/dev-tools/report/__init__.py b/dev-tools/report/__init__.py new file mode 100644 index 0000000..e931fe0 --- /dev/null +++ b/dev-tools/report/__init__.py @@ -0,0 +1,14 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + http://git-wip-us.apache.org/repos/asf/storm/blob/61dceffb/dev-tools/report/formatter.py ---------------------------------------------------------------------- diff --git a/dev-tools/report/formatter.py b/dev-tools/report/formatter.py new file mode 100644 index 0000000..81f574d --- /dev/null +++ b/dev-tools/report/formatter.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def encode(obj, encoding='UTF-8'): + """ + Check if the object supports encode() method, and if so, encodes it. + Encoding defaults to UTF-8. + For example objects of type 'int' do not support encode + """ + return obj.encode(encoding) if 'encode' in dir(obj) else obj + +class Formatter: + def __init__(self, fields_tuple=(), row_tuple=(), min_width_tuple=None): + # Format to pass as first argument to the print function, e.g. '%s%s%s' + self.format = "" + # data_format will be of the form ['{!s:43}'],'{!s:39}','{!s:11}','{!s:25}'] + # the widths are determined from the data in order to print output with nice format + # Each entry of the data_format list will be used by the advanced string formatter: + # "{!s:43}".format("Text") + # Advanced string formatter as detailed in here: https://www.python.org/dev/peps/pep-3101/ + self.data_format = [] + Formatter._assert(fields_tuple, row_tuple, min_width_tuple) + self._build_format_tuples(fields_tuple, row_tuple, min_width_tuple) + + @staticmethod + def _assert(o1, o2, o3): + if len(o1) != len(o2) and (o3 is not None and len(o2) != len(o3)): + raise RuntimeError("Object collections must have the same length. " + "len(o1)={0}, len(o2)={1}, len(o3)={2}" + .format(len(o1), len(o2), -1 if o3 is None else len(o3))) + + # determines the widths from the data in order to print output with nice format + @staticmethod + def _find_sizes(fields_tuple, row_tuple, min_width_tuple): + sizes = [] + padding = 3 + for i in range(0, len(row_tuple)): + max_len = max(len(encode(fields_tuple[i])), len(str(encode(row_tuple[i])))) + if min_width_tuple is not None: + max_len = max(max_len, min_width_tuple[i]) + sizes += [max_len + padding] + return sizes + + def _build_format_tuples(self, fields_tuple, row_tuple, min_width_tuple): + sizes = Formatter._find_sizes(fields_tuple, row_tuple, min_width_tuple) + + for i in range(0, len(row_tuple)): + self.format += "%s" + self.data_format += ["{!s:" + str(sizes[i]) + "}"] + + # Returns a tuple where each entry has a string that is the result of + # statements with the pattern "{!s:43}".format("Text") + def row_str_format(self, row_tuple): + format_with_values = [str(self.data_format[0].format(encode(row_tuple[0])))] + for i in range(1, len(row_tuple)): + format_with_values += [str(self.data_format[i].format(encode(row_tuple[i])))] + return tuple(format_with_values) http://git-wip-us.apache.org/repos/asf/storm/blob/61dceffb/dev-tools/report/report.py ---------------------------------------------------------------------- diff --git a/dev-tools/report/report.py b/dev-tools/report/report.py new file mode 100644 index 0000000..46ec175 --- /dev/null +++ b/dev-tools/report/report.py @@ -0,0 +1,252 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from github import mstr +from jira import Jira +from formatter import Formatter, encode + + +def daydiff(a, b): + return (a - b).days + + +class Report: + now = datetime.utcnow() + def __init__(self, header=''): + self.header = header + + # if padding starts with - it puts padding before contents, otherwise after + @staticmethod + def _build_tuple(contents, padding=''): + if padding is not '': + out = [] + for i in range(len(contents)): + out += [padding[1:] + str(contents[i])] if padding[0] is '-' else [str(contents[i]) + padding] + return tuple(out) + return contents + + # calls the native print function with the following format. Text1,Text2,... has the correct spacing + # print ("%s%s%s" % ("Text1, Text2, Text3)) + def print_(self, formatter, row_tuple): + print (formatter.format % formatter.row_str_format(row_tuple)) + +class JiraReport(Report): + def __init__(self, issues, header=''): + Report.__init__(self, header) + self.issues = issues + + def view(self, excluded): + issues_view = dict(self.issues) + for key in excluded: + issues_view.pop(key, None) + return issues_view + + def keys_view(self, excluded): + return self.view(excluded).keys().sort(Jira.storm_jira_cmp, reverse=True) + + def values_view(self, excluded=None): + temp_dic = dict(self.issues) if excluded is None else self.view(excluded) + values = temp_dic.values() + values.sort(Jira.storm_jira_cmp, reverse=True) + return values + + @staticmethod + def _row_tuple(jira): + return (jira.get_id(), jira.get_trimmed_summary(), daydiff(Report.now, jira.get_created()), + daydiff(Report.now, jira.get_updated())) + + def _min_width_tuple(self): + return -1, 43, -1, -1 + + def print_report(self): + print "%s (Count = %s) " % (self.header, len(self.issues)) + jiras = self.values_view() + fields_tuple = ('Jira Id', 'Summary', 'Created', 'Last Updated (Days)') + row_tuple = self._row_tuple(jiras[0]) + + formatter = Formatter(fields_tuple, row_tuple, self._min_width_tuple()) + + self.print_(formatter, fields_tuple) + + for jira in jiras: + row_tuple = self._row_tuple(jira) + self.print_(formatter, row_tuple) + + @staticmethod + def build_jira_url(jira_id): + BASE_URL = "https://issues.apache.org/jira/browse/"; + return BASE_URL + jira_id + + +class GitHubReport(Report): + def __init__(self, pull_requests=None, header=''): + Report.__init__(self, header) + + if pull_requests is None: + self.pull_requests = [] + self.type = '' + else: + self.pull_requests = pull_requests + self.type = type + + def _row_tuple(self, pull): + return self._build_tuple( + (pull.html_url(), pull.trimmed_title(), daydiff(Report.now, pull.created_at()), + daydiff(Report.now, pull.updated_at()), pull.user()), '') + + def _min_width_tuple(self): + return -1, 43, -1, -1, -1 + + def print_report(self): + print "%s (Count = %s) " % (self.header, len(self.pull_requests)) + + fields_tuple = self._build_tuple(('URL', 'Title', 'Created', 'Last Updated (Days)', 'User'), '') + row_tuple = self._row_tuple(self.pull_requests[0]) + + formatter = Formatter(fields_tuple, row_tuple, self._min_width_tuple()) + + self.print_(formatter, fields_tuple) + for pull in self.pull_requests: + row_tuple = self._row_tuple(pull) + self.print_(formatter, row_tuple) + + def jira_ids(self): + """ + :return: sorted list of JIRA ids present in Git pull requests + """ + jira_ids = list() + for pull in self.pull_requests: + jira_ids.append(pull.jira_id()) + return sorted(jira_ids) + +class JiraGitHubCombinedReport(Report): + def __init__(self, jira_report, github_report, header='', print_comments=False): + Report.__init__(self, header) + self.jira_report = jira_report + self.github_report = github_report + self.print_comments = print_comments + + def _jira_comments(self, jira_id): + return None if jira_id is None else self.jira_report.issues[jira_id].get_comments() + + def _idx_1st_comment_with_vote(self): + g = 0 + for pull in self.github_report.pull_requests: + c = 0 + for comment in self._jira_comments(pull.jira_id()): + if comment.has_vote(): + return(g,) + (c,) + c += 1 + g += 1 + + def _pull_request(self, pull_idx): + pull = self.github_report.pull_requests[pull_idx] + return pull + + def _jira_id(self, pull_idx): + pull = self._pull_request(pull_idx) + return encode(pull.jira_id()) + + def _jira_issue(self, jira_id): + return self.jira_report.issues[jira_id] + + def _row_tuple(self, pull_idx): + pull = self._pull_request(pull_idx) + jira_id = self._jira_id(pull_idx) + jira_issue = self._jira_issue(jira_id) + + return (jira_id, mstr(pull), jira_issue.get_trimmed_summary(), + daydiff(Report.now, jira_issue.get_created()), + daydiff(Report.now, pull.created_at()), + daydiff(Report.now, jira_issue.get_updated()), + daydiff(Report.now, pull.updated_at()), + jira_issue.get_status(), pull.user()) + + def _row_tuple_1(self, pull_idx, comment_idx): + row_tuple_1 = None + jira_id = self._jira_id(pull_idx) + jira_comments = self._jira_comments(jira_id) + comment = jira_comments[comment_idx] + if comment.has_vote(): + row_tuple_1 = (comment.get_vote(), comment.get_author(), comment.get_pull(), + daydiff(Report.now, comment.get_created())) + + return row_tuple_1 + + # variables and method names ending with _1 correspond to the comments part + def print_report(self, print_comments=False): + print "%s (Count = %s) " % (self.header, len(self.github_report.pull_requests)) + + fields_tuple = ('JIRA ID', 'Pull Request', 'Jira Summary', 'JIRA Age', + 'Pull Age', 'JIRA Update Age', 'Pull Update Age (Days)', + 'JIRA Status', 'GitHub user') + row_tuple = self._row_tuple(0) + formatter = Formatter(fields_tuple, row_tuple) + self.print_(formatter, fields_tuple) + + row_tuple_1 = () + formatter_1 = Formatter() + + if print_comments or self.print_comments: + fields_tuple_1 = self._build_tuple(('Comment Vote', 'Comment Author', 'Pull URL', 'Comment Age'), '-\t\t') + row_tuple_1 = self._build_tuple(self._row_tuple_1(*self._idx_1st_comment_with_vote()), '-\t\t') + formatter_1 = Formatter(fields_tuple_1, row_tuple_1) + self.print_(formatter_1, fields_tuple_1) + print '' + + for p in range(0, len(self.github_report.pull_requests)): + row_tuple = self._row_tuple(p) + self.print_(formatter, row_tuple) + + if print_comments or self.print_comments: + has_vote = False + comments = self._jira_comments(self._jira_id(p)) + for c in range(len(comments)): # Check cleaner way + comment = comments[c] + if comment.has_vote(): + row_tuple_1 = self._build_tuple(self._row_tuple_1(p, c), '-\t\t') + if row_tuple_1 is not None: + self.print_(formatter_1, row_tuple_1) + has_vote = True + if has_vote: + print '' + + +class CompleteReport(Report): + def __init__(self, header=''): + Report.__init__(self, header) + self.jira_reports = [] + self.github_reports = [] + self.jira_github_combined_reports = [] + + def print_all(self): + if self.header is not '': + print self.header + + self._print_github_reports() + self._print_jira_github_combined_reports() + self._print_jira_reports() + + def _print_jira_reports(self): + for jira in self.jira_reports: + jira.print_report() + + def _print_github_reports(self): + for github in self.github_reports: + github.print_report() + + def _print_jira_github_combined_reports(self): + for jira_github_combined in self.jira_github_combined_reports: + jira_github_combined.print_report() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/61dceffb/dev-tools/report/report_builder.py ---------------------------------------------------------------------- diff --git a/dev-tools/report/report_builder.py b/dev-tools/report/report_builder.py new file mode 100644 index 0000000..4b8a468 --- /dev/null +++ b/dev-tools/report/report_builder.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from report import CompleteReport, GitHubReport, JiraReport, JiraGitHubCombinedReport + + +class ReportBuilder: + def __init__(self, jira_repo=None, github_repo=None): + self.jira_repo = jira_repo + self.github_repo = github_repo + + def build(self): + pass + + +class CompleteReportBuilder(ReportBuilder): + def __init__(self, jira_repo=None, github_repo=None): + ReportBuilder.__init__(self, jira_repo, github_repo) + self.report = CompleteReport() + self.build() + + def build(self): + # all open github pull requests + github_open = GitHubReport(self.github_repo.open_pulls("apache", "storm")) + github_bad_jira = GitHubReport(None, "\nGITHUB PULL REQUESTS WITH BAD OR CLOSED JIRA ID") + github_without_jira = GitHubReport(None, "\nGITHUB PULL REQUESTS WITHOUT A JIRA ID") + github_unresolved_jira = GitHubReport(None, "\nGITHUB PULL REQUESTS WITH UNRESOLVED JIRA ID") + github_unresolved_jira_voted = GitHubReport(None, "\nGITHUB PULL REQUESTS WITH VOTES FOR UNRESOLVED JIRAS") + github_open_jira = GitHubReport(None, "\nGITHUB PULL REQUESTS WITH OPEN JIRA ID") + github_unresolved_not_open_jira = GitHubReport(None, "\nGITHUB PULL REQUESTS WITH UNRESOLVED BUT NOT OPEN JIRA ID") + + # all unresolved JIRA issues + jira_unresolved = JiraReport(self.jira_repo.unresolved_jiras("STORM")) + jira_open = JiraReport(dict((x, y) for x, y in self.jira_repo.unresolved_jiras("STORM").items() if y.get_status() == 'Open')) + jira_in_progress = JiraReport(dict((x, y) for x, y in self.jira_repo.in_progress_jiras("STORM").items() if y.get_status() == 'In Progress'), + "\nIN PROGRESS JIRA ISSUES") + + for pull in github_open.pull_requests: + if pull.has_jira_id(): + pull_jira_id = pull.jira_id() + if pull_jira_id not in jira_unresolved.issues: + github_bad_jira.pull_requests.append(pull) + else: + github_unresolved_jira.pull_requests.append(pull) + if jira_unresolved.issues[pull_jira_id].has_voted_comment(): + github_unresolved_jira_voted.pull_requests.append(pull) + if pull_jira_id in jira_open.issues: + github_open_jira.pull_requests.append(pull) + else: + github_unresolved_not_open_jira.pull_requests.append(pull) + else: + github_without_jira.pull_requests.append(pull) + + jira_github_open = JiraGitHubCombinedReport(jira_open, github_open_jira, + "\nOPEN JIRA ISSUES THAT HAVE GITHUB PULL REQUESTS") + jira_github_unresolved_not_open = JiraGitHubCombinedReport(jira_unresolved, github_unresolved_not_open_jira, + "\nIN PROGRESS OR REOPENED JIRA ISSUES THAT HAVE GITHUB PULL REQUESTS") + jira_github_unresolved_voted = JiraGitHubCombinedReport(jira_unresolved, github_unresolved_jira_voted, + "\nGITHUB PULL REQUESTS WITH VOTES FOR UNRESOLVED JIRAS", True) + # jira_github_unresolved = JiraGitHubCombinedReport(jira_unresolved, github_unresolved_jira, + # "\nUnresolved JIRA issues with GitHub pull requests") + + jira_open_no_pull = JiraReport(jira_open.view(github_open_jira.jira_ids()), + "\nOPEN JIRA ISSUES THAT DON'T HAVE GITHUB PULL REQUESTS") + + # build complete report + self.report.jira_reports.append(jira_in_progress) + self.report.jira_reports.append(jira_open_no_pull) + + self.report.github_reports.append(github_bad_jira) + self.report.github_reports.append(github_without_jira) + + self.report.jira_github_combined_reports.append(jira_github_open) + self.report.jira_github_combined_reports.append(jira_github_unresolved_voted) + self.report.jira_github_combined_reports.append(jira_github_unresolved_not_open) + # self.report.jira_github_combined_reports.append(jira_github_unresolved) http://git-wip-us.apache.org/repos/asf/storm/blob/61dceffb/dev-tools/storm-merge.py ---------------------------------------------------------------------- diff --git a/dev-tools/storm-merge.py b/dev-tools/storm-merge.py index 06ae25f..ed06216 100755 --- a/dev-tools/storm-merge.py +++ b/dev-tools/storm-merge.py @@ -24,7 +24,7 @@ def main(): for pullNumber in args: pull = github.pull("apache", "storm", pullNumber) - print "git pull "+pull.fromRepo()+" "+pull.fromBranch() + print "git pull "+pull.from_repo()+" "+pull.from_branch() if __name__ == "__main__": main()
