http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/release-doc-maker/releasedocmaker.py ---------------------------------------------------------------------- diff --git a/release-doc-maker/releasedocmaker.py b/release-doc-maker/releasedocmaker.py deleted file mode 100755 index 2b2ada4..0000000 --- a/release-doc-maker/releasedocmaker.py +++ /dev/null @@ -1,947 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -from glob import glob -from optparse import OptionParser -from time import gmtime, strftime, sleep -from distutils.version import LooseVersion -import errno -import os -import re -import shutil -import urllib -import urllib2 -import httplib -import json -sys.dont_write_bytecode = True -# pylint: disable=wrong-import-position -from utils import get_jira, to_unicode, sanitize_text, processrelnote, Outputs -# pylint: enable=wrong-import-position - -try: - import dateutil.parser -except ImportError: - print "This script requires python-dateutil module to be installed. " \ - "You can install it using:\n\t pip install python-dateutil" - sys.exit(1) - -RELEASE_VERSION = {} - -JIRA_BASE_URL = "https://issues.apache.org/jira"; -SORTTYPE = 'resolutiondate' -SORTORDER = 'older' -NUM_RETRIES = 5 - -# label to be used to mark an issue as Incompatible change. -BACKWARD_INCOMPATIBLE_LABEL = 'backward-incompatible' - -ASF_LICENSE = ''' -<!--- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---> -''' - -def buildindex(title, asf_license): - """Write an index file for later conversion using mvn site""" - versions = glob("[0-9]*.[0-9]*") - versions.sort(key=LooseVersion, reverse=True) - with open("index.md", "w") as indexfile: - if asf_license is True: - indexfile.write(ASF_LICENSE) - for version in versions: - indexfile.write("* %s v%s\n" % (title, version)) - for k in ("Changelog", "Release Notes"): - indexfile.write(" * [%s](%s/%s.%s.html)\n" % - (k, version, k.upper().replace(" ", ""), - version)) - - -def buildreadme(title, asf_license): - """Write an index file for Github using README.md""" - versions = glob("[0-9]*.[0-9]*") - versions.sort(key=LooseVersion, reverse=True) - with open("README.md", "w") as indexfile: - if asf_license is True: - indexfile.write(ASF_LICENSE) - for version in versions: - indexfile.write("* %s v%s\n" % (title, version)) - for k in ("Changelog", "Release Notes"): - indexfile.write(" * [%s](%s/%s.%s.md)\n" % - (k, version, k.upper().replace(" ", ""), - version)) - - -class GetVersions(object): - """ List of version strings """ - - def __init__(self, versions, projects): - versions = versions - projects = projects - self.newversions = [] - versions.sort(key=LooseVersion) - print "Looking for %s through %s" % (versions[0], versions[-1]) - newversions = set() - for project in projects: - url = JIRA_BASE_URL + \ - "/rest/api/2/project/%s/versions" % project.upper() - try: - resp = get_jira(url) - except (urllib2.HTTPError, urllib2.URLError, httplib.BadStatusLine): - sys.exit(1) - - datum = json.loads(resp.read()) - for data in datum: - newversions.add(data['name']) - newlist = list(newversions.copy()) - newlist.append(versions[0]) - newlist.append(versions[-1]) - newlist.sort(key=LooseVersion) - start_index = newlist.index(versions[0]) - end_index = len(newlist) - 1 - newlist[::-1].index(versions[-1]) - for newversion in newlist[start_index + 1:end_index]: - if newversion in newversions: - print "Adding %s to the list" % newversion - self.newversions.append(newversion) - - def getlist(self): - return self.newversions - - -class Version(object): - """Represents a version number""" - - def __init__(self, data): - self.mod = False - self.data = data - found = re.match(r'^((\d+)(\.\d+)*).*$', data) - if found: - self.parts = [int(p) for p in found.group(1).split('.')] - else: - self.parts = [] - # backfill version with zeros if missing parts - self.parts.extend((0,) * (3 - len(self.parts))) - - def __str__(self): - if self.mod: - return '.'.join([str(p) for p in self.parts]) - return self.data - - def __cmp__(self, other): - return cmp(self.parts, other.parts) - - -class Jira(object): - """A single JIRA""" - - def __init__(self, data, parent): - self.key = data['key'] - self.fields = data['fields'] - self.parent = parent - self.notes = None - self.incompat = None - self.reviewed = None - self.important = None - - def get_id(self): - return to_unicode(self.key) - - def get_description(self): - return to_unicode(self.fields['description']) - - def get_release_note(self): - if self.notes is None: - field = self.parent.field_id_map['Release Note'] - if field in self.fields: - self.notes = to_unicode(self.fields[field]) - elif self.get_incompatible_change() or self.get_important(): - self.notes = self.get_description() - else: - self.notes = "" - return self.notes - - def get_priority(self): - ret = "" - pri = self.fields['priority'] - if pri is not None: - ret = pri['name'] - return to_unicode(ret) - - def get_assignee(self): - ret = "" - mid = self.fields['assignee'] - if mid is not None: - ret = mid['displayName'] - return to_unicode(ret) - - def get_components(self): - if self.fields['components']: - return ", ".join([comp['name'] for comp in self.fields['components'] - ]) - return "" - - def get_summary(self): - return self.fields['summary'] - - def get_type(self): - ret = "" - mid = self.fields['issuetype'] - if mid is not None: - ret = mid['name'] - return to_unicode(ret) - - def get_reporter(self): - ret = "" - mid = self.fields['reporter'] - if mid is not None: - ret = mid['displayName'] - return to_unicode(ret) - - def get_project(self): - ret = "" - mid = self.fields['project'] - if mid is not None: - ret = mid['key'] - return to_unicode(ret) - - def __cmp__(self, other): - result = 0 - - if SORTTYPE == 'issueid': - # compare by issue name-number - selfsplit = self.get_id().split('-') - othersplit = other.get_id().split('-') - result = cmp(selfsplit[0], othersplit[0]) - if result == 0: - result = cmp(int(selfsplit[1]), int(othersplit[1])) - # dec is supported for backward compatibility - if SORTORDER in ['dec', 'desc']: - result *= -1 - - elif SORTTYPE == 'resolutiondate': - dts = dateutil.parser.parse(self.fields['resolutiondate']) - dto = dateutil.parser.parse(other.fields['resolutiondate']) - result = cmp(dts, dto) - if SORTORDER == 'newer': - result *= -1 - - return result - - def get_incompatible_change(self): - if self.incompat is None: - field = self.parent.field_id_map['Hadoop Flags'] - self.reviewed = False - self.incompat = False - if field in self.fields: - if self.fields[field]: - for flag in self.fields[field]: - if flag['value'] == "Incompatible change": - self.incompat = True - if flag['value'] == "Reviewed": - self.reviewed = True - else: - # Custom field 'Hadoop Flags' is not defined, - # search for 'backward-incompatible' label - field = self.parent.field_id_map['Labels'] - if field in self.fields and self.fields[field]: - if BACKWARD_INCOMPATIBLE_LABEL in self.fields[field]: - self.incompat = True - self.reviewed = True - return self.incompat - - def get_important(self): - if self.important is None: - field = self.parent.field_id_map['Flags'] - self.important = False - if field in self.fields: - if self.fields[field]: - for flag in self.fields[field]: - if flag['value'] == "Important": - self.important = True - return self.important - - -class JiraIter(object): - """An Iterator of JIRAs""" - - @staticmethod - def collect_fields(): - """send a query to JIRA and collect field-id map""" - try: - resp = get_jira(JIRA_BASE_URL + "/rest/api/2/field") - data = json.loads(resp.read()) - except (urllib2.HTTPError, urllib2.URLError, httplib.BadStatusLine, ValueError): - sys.exit(1) - field_id_map = {} - for part in data: - field_id_map[part['name']] = part['id'] - return field_id_map - - @staticmethod - def query_jira(ver, projects, pos): - """send a query to JIRA and collect - a certain number of issue information""" - count = 100 - pjs = "','".join(projects) - jql = "project in ('%s') and \ - fixVersion in ('%s') and \ - resolution = Fixed" % (pjs, ver) - params = urllib.urlencode({'jql': jql, - 'startAt': pos, - 'maxResults': count}) - return JiraIter.load_jira(params, 0) - - @staticmethod - def load_jira(params, fail_count): - """send query to JIRA and collect with retries""" - try: - resp = get_jira(JIRA_BASE_URL + "/rest/api/2/search?%s" % params) - except (urllib2.URLError, httplib.BadStatusLine) as err: - return JiraIter.retry_load(err, params, fail_count) - - try: - data = json.loads(resp.read()) - except httplib.IncompleteRead as err: - return JiraIter.retry_load(err, params, fail_count) - return data - - @staticmethod - def retry_load(err, params, fail_count): - """Retry connection up to NUM_RETRIES times.""" - print(err) - fail_count += 1 - if fail_count <= NUM_RETRIES: - print "Connection failed %d times. Retrying." % (fail_count) - sleep(1) - return JiraIter.load_jira(params, fail_count) - else: - print "Connection failed %d times. Aborting." % (fail_count) - sys.exit(1) - - @staticmethod - def collect_jiras(ver, projects): - """send queries to JIRA and collect all issues - that belongs to given version and projects""" - jiras = [] - pos = 0 - end = 1 - while pos < end: - data = JiraIter.query_jira(ver, projects, pos) - if 'error_messages' in data: - print "JIRA returns error message: %s" % data['error_messages'] - sys.exit(1) - pos = data['startAt'] + data['maxResults'] - end = data['total'] - jiras.extend(data['issues']) - - if ver not in RELEASE_VERSION: - for issue in data['issues']: - for fix_version in issue['fields']['fixVersions']: - if 'releaseDate' in fix_version: - RELEASE_VERSION[fix_version['name']] = fix_version[ - 'releaseDate'] - return jiras - - def __init__(self, version, projects): - self.version = version - self.projects = projects - self.field_id_map = JiraIter.collect_fields() - ver = str(version).replace("-SNAPSHOT", "") - self.jiras = JiraIter.collect_jiras(ver, projects) - self.iter = self.jiras.__iter__() - - def __iter__(self): - return self - - def next(self): - data = self.iter.next() - j = Jira(data, self) - return j - - -class Linter(object): - """Encapsulates lint-related functionality. - Maintains running lint statistics about JIRAs.""" - - _valid_filters = ["incompatible", "important", "version", "component", - "assignee"] - - def __init__(self, version, options): - self._warning_count = 0 - self._error_count = 0 - self._lint_message = "" - self._version = version - - self._filters = dict(zip(self._valid_filters, [False] * len( - self._valid_filters))) - - self.enabled = False - self._parse_options(options) - - @staticmethod - def add_parser_options(parser): - """Add Linter options to passed optparse parser.""" - filter_string = ", ".join("'" + f + "'" for f in Linter._valid_filters) - parser.add_option( - "-n", - "--lint", - dest="lint", - action="append", - type="string", - help="Specify lint filters. Valid filters are " + filter_string + - ". " + "'all' enables all lint filters. " + - "Multiple filters can be specified comma-delimited and " + - "filters can be negated, e.g. 'all,-component'.") - - def _parse_options(self, options): - """Parse options from optparse.""" - - if options.lint is None or not options.lint: - return - self.enabled = True - - # Valid filter specifications are - # self._valid_filters, negations, and "all" - valid_list = self._valid_filters - valid_list += ["-" + v for v in valid_list] - valid_list += ["all"] - valid = set(valid_list) - - enabled = [] - disabled = [] - - for o in options.lint: - for token in o.split(","): - if token not in valid: - print "Unknown lint filter '%s', valid options are: %s" % \ - (token, ", ".join(v for v in sorted(valid))) - sys.exit(1) - if token.startswith("-"): - disabled.append(token[1:]) - else: - enabled.append(token) - - for e in enabled: - if e == "all": - for f in self._valid_filters: - self._filters[f] = True - else: - self._filters[e] = True - for d in disabled: - self._filters[d] = False - - def had_errors(self): - """Returns True if a lint error was encountered, else False.""" - return self._error_count > 0 - - def message(self): - """Return summary lint message suitable for printing to stdout.""" - if not self.enabled: - return None - return self._lint_message + \ - "\n=======================================" + \ - "\n%s: Error:%d, Warning:%d \n" % \ - (self._version, self._error_count, self._warning_count) - - def _check_missing_component(self, jira): - """Return if JIRA has a 'missing component' lint error.""" - if not self._filters["component"]: - return False - - if jira.fields['components']: - return False - return True - - def _check_missing_assignee(self, jira): - """Return if JIRA has a 'missing assignee' lint error.""" - if not self._filters["assignee"]: - return False - - if jira.fields['assignee'] is not None: - return False - return True - - def _check_version_string(self, jira): - """Return if JIRA has a version string lint error.""" - if not self._filters["version"]: - return False - - field = jira.parent.field_id_map['Fix Version/s'] - for ver in jira.fields[field]: - found = re.match(r'^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', ver['name']) - if not found: - return True - return False - - def lint(self, jira): - """Run lint check on a JIRA.""" - if not self.enabled: - return - if not jira.get_release_note(): - if self._filters["incompatible"] and jira.get_incompatible_change(): - self._warning_count += 1 - self._lint_message += "\nWARNING: incompatible change %s lacks release notes." % \ - (sanitize_text(jira.get_id())) - if self._filters["important"] and jira.get_important(): - self._warning_count += 1 - self._lint_message += "\nWARNING: important issue %s lacks release notes." % \ - (sanitize_text(jira.get_id())) - - if self._check_version_string(jira): - self._warning_count += 1 - self._lint_message += "\nWARNING: Version string problem for %s " % jira.get_id( - ) - - if self._check_missing_component(jira) or self._check_missing_assignee( - jira): - self._error_count += 1 - error_message = [] - if self._check_missing_component(jira): - error_message.append("component") - if self._check_missing_assignee(jira): - error_message.append("assignee") - self._lint_message += "\nERROR: missing %s for %s " \ - % (" and ".join(error_message), jira.get_id()) - - -def parse_args(): - """Parse command-line arguments with optparse.""" - usage = "usage: %prog [OPTIONS] " + \ - "--project PROJECT [--project PROJECT] " + \ - "--version VERSION [--version VERSION2 ...]" - parser = OptionParser( - usage=usage, - epilog= - "Markdown-formatted CHANGELOG and RELEASENOTES files will be stored" - " in a directory named after the highest version provided.") - parser.add_option("--dirversions", - dest="versiondirs", - action="store_true", - default=False, - help="Put files in versioned directories") - parser.add_option("--fileversions", - dest="versionfiles", - action="store_true", - default=False, - help="Write files with embedded versions") - parser.add_option("-i", - "--index", - dest="index", - action="store_true", - default=False, - help="build an index file") - parser.add_option("-l", - "--license", - dest="license", - action="store_true", - default=False, - help="Add an ASF license") - parser.add_option("-p", - "--project", - dest="projects", - action="append", - type="string", - help="projects in JIRA to include in releasenotes", - metavar="PROJECT") - parser.add_option("-r", - "--range", - dest="range", - action="store_true", - default=False, - help="Given versions are a range") - parser.add_option( - "--sortorder", - dest="sortorder", - metavar="TYPE", - default=SORTORDER, - # dec is supported for backward compatibility - choices=["asc", "dec", "desc", "newer", "older"], - help="Sorting order for sort type (default: %s)" % SORTORDER) - parser.add_option("--sorttype", - dest="sorttype", - metavar="TYPE", - default=SORTTYPE, - choices=["resolutiondate", "issueid"], - help="Sorting type for issues (default: %s)" % SORTTYPE) - parser.add_option( - "-t", - "--projecttitle", - dest="title", - type="string", - help="Title to use for the project (default is Apache PROJECT)") - parser.add_option("-u", - "--usetoday", - dest="usetoday", - action="store_true", - default=False, - help="use current date for unreleased versions") - parser.add_option("-v", - "--version", - dest="versions", - action="append", - type="string", - help="versions in JIRA to include in releasenotes", - metavar="VERSION") - parser.add_option( - "-V", - dest="release_version", - action="store_true", - default=False, - help="display version information for releasedocmaker and exit.") - parser.add_option("-O", - "--outputdir", - dest="output_directory", - action="append", - type="string", - help="specify output directory to put release docs to.") - parser.add_option("-B", - "--baseurl", - dest="base_url", - action="append", - type="string", - help="specify base URL of the JIRA instance.") - parser.add_option( - "--retries", - dest="retries", - action="append", - type="int", - help="Specify how many times to retry connection for each URL.") - parser.add_option( - "--skip-credits", - dest="skip_credits", - action="store_true", - default=False, - help="While creating release notes skip the 'reporter' and 'contributor' columns") - parser.add_option("-X", - "--incompatiblelabel", - dest="incompatible_label", - default="backward-incompatible", - type="string", - help="Specify the label to indicate backward incompatibility.") - - Linter.add_parser_options(parser) - - if len(sys.argv) <= 1: - parser.print_help() - sys.exit(1) - - (options, _) = parser.parse_args() - - # Handle the version string right away and exit - if options.release_version: - with open( - os.path.join( - os.path.dirname(__file__), "../VERSION"), 'r') as ver_file: - print ver_file.read() - sys.exit(0) - - # Validate options - if not options.release_version: - if options.versions is None: - parser.error("At least one version needs to be supplied") - if options.projects is None: - parser.error("At least one project needs to be supplied") - if options.base_url is not None: - if len(options.base_url) > 1: - parser.error("Only one base URL should be given") - else: - options.base_url = options.base_url[0] - if options.output_directory is not None: - if len(options.output_directory) > 1: - parser.error("Only one output directory should be given") - else: - options.output_directory = options.output_directory[0] - - if options.range or len(options.versions) > 1: - if not options.versiondirs and not options.versionfiles: - parser.error("Multiple versions require either --fileversions or --dirversions") - - return options - - -def main(): - options = parse_args() - - if options.output_directory is not None: - # Create the output directory if it does not exist. - try: - os.makedirs(options.output_directory) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir( - options.output_directory): - pass - else: - print "Unable to create output directory %s: %s" % \ - (options.output_directory, exc.message) - sys.exit(1) - os.chdir(options.output_directory) - - if options.base_url is not None: - global JIRA_BASE_URL - JIRA_BASE_URL = options.base_url - - if options.incompatible_label is not None: - global BACKWARD_INCOMPATIBLE_LABEL - BACKWARD_INCOMPATIBLE_LABEL = options.incompatible_label - - - projects = options.projects - - if options.range is True: - versions = [Version(v) - for v in GetVersions(options.versions, projects).getlist()] - else: - versions = [Version(v) for v in options.versions] - versions.sort() - - global SORTTYPE - SORTTYPE = options.sorttype - global SORTORDER - SORTORDER = options.sortorder - - if options.title is None: - title = projects[0] - else: - title = options.title - - if options.retries is not None: - global NUM_RETRIES - NUM_RETRIES = options.retries[0] - - haderrors = False - - for version in versions: - vstr = str(version) - linter = Linter(vstr, options) - jlist = sorted(JiraIter(vstr, projects)) - if not jlist: - print "There is no issue which has the specified version: %s" % version - continue - - if vstr in RELEASE_VERSION: - reldate = RELEASE_VERSION[vstr] - elif options.usetoday: - reldate = strftime("%Y-%m-%d", gmtime()) - else: - reldate = "Unreleased (as of %s)" % strftime("%Y-%m-%d", gmtime()) - - if not os.path.exists(vstr) and options.versiondirs: - os.mkdir(vstr) - - if options.versionfiles and options.versiondirs: - reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", - "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - choutputs = Outputs("%(ver)s/CHANGELOG.%(ver)s.md", - "%(ver)s/CHANGELOG.%(key)s.%(ver)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - elif options.versiondirs: - reloutputs = Outputs("%(ver)s/RELEASENOTES.md", - "%(ver)s/RELEASENOTES.%(key)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - choutputs = Outputs("%(ver)s/CHANGELOG.md", - "%(ver)s/CHANGELOG.%(key)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - elif options.versionfiles: - reloutputs = Outputs("RELEASENOTES.%(ver)s.md", - "RELEASENOTES.%(key)s.%(ver)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - choutputs = Outputs("CHANGELOG.%(ver)s.md", - "CHANGELOG.%(key)s.%(ver)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - else: - reloutputs = Outputs("RELEASENOTES.md", - "RELEASENOTES.%(key)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - choutputs = Outputs("CHANGELOG.md", - "CHANGELOG.%(key)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - - if options.license is True: - reloutputs.write_all(ASF_LICENSE) - choutputs.write_all(ASF_LICENSE) - - relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \ - 'These release notes cover new developer and user-facing ' \ - 'incompatibilities, important issues, features, and major improvements.\n\n' - chhead = '# %(title)s Changelog\n\n' \ - '## Release %(ver)s - %(date)s\n'\ - '\n' - - reloutputs.write_all(relhead) - choutputs.write_all(chhead) - - incompatlist = [] - importantlist = [] - buglist = [] - improvementlist = [] - newfeaturelist = [] - subtasklist = [] - tasklist = [] - testlist = [] - otherlist = [] - - for jira in jlist: - if jira.get_incompatible_change(): - incompatlist.append(jira) - elif jira.get_important(): - importantlist.append(jira) - elif jira.get_type() == "Bug": - buglist.append(jira) - elif jira.get_type() == "Improvement": - improvementlist.append(jira) - elif jira.get_type() == "New Feature": - newfeaturelist.append(jira) - elif jira.get_type() == "Sub-task": - subtasklist.append(jira) - elif jira.get_type() == "Task": - tasklist.append(jira) - elif jira.get_type() == "Test": - testlist.append(jira) - else: - otherlist.append(jira) - - line = '* [%s](' % (sanitize_text(jira.get_id())) + JIRA_BASE_URL + \ - '/browse/%s) | *%s* | **%s**\n' \ - % (sanitize_text(jira.get_id()), - sanitize_text(jira.get_priority()), sanitize_text(jira.get_summary())) - - if jira.get_release_note() or \ - jira.get_incompatible_change() or jira.get_important(): - reloutputs.write_key_raw(jira.get_project(), "\n---\n\n") - reloutputs.write_key_raw(jira.get_project(), line) - if not jira.get_release_note(): - line = '\n**WARNING: No release note provided for this change.**\n\n' - else: - line = '\n%s\n\n' % ( - processrelnote(jira.get_release_note())) - reloutputs.write_key_raw(jira.get_project(), line) - - linter.lint(jira) - - if linter.enabled: - print linter.message() - if linter.had_errors(): - haderrors = True - shutil.rmtree(vstr) - continue - - reloutputs.write_all("\n\n") - reloutputs.close() - - if options.skip_credits: - CHANGEHDR1 = "| JIRA | Summary | Priority | " + \ - "Component |\n" - CHANGEHDR2 = "|:---- |:---- | :--- |:---- |\n" - else: - CHANGEHDR1 = "| JIRA | Summary | Priority | " + \ - "Component | Reporter | Contributor |\n" - CHANGEHDR2 = "|:---- |:---- | :--- |:---- |:---- |:---- |\n" - - if incompatlist: - choutputs.write_all("### INCOMPATIBLE CHANGES:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) - choutputs.write_list(incompatlist, options.skip_credits, JIRA_BASE_URL) - - if importantlist: - choutputs.write_all("\n\n### IMPORTANT ISSUES:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) - choutputs.write_list(importantlist, options.skip_credits, JIRA_BASE_URL) - - if newfeaturelist: - choutputs.write_all("\n\n### NEW FEATURES:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) - choutputs.write_list(newfeaturelist, options.skip_credits, JIRA_BASE_URL) - - if improvementlist: - choutputs.write_all("\n\n### IMPROVEMENTS:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) - choutputs.write_list(improvementlist, options.skip_credits, JIRA_BASE_URL) - - if buglist: - choutputs.write_all("\n\n### BUG FIXES:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) - choutputs.write_list(buglist, options.skip_credits, JIRA_BASE_URL) - - if testlist: - choutputs.write_all("\n\n### TESTS:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) - choutputs.write_list(testlist, options.skip_credits, JIRA_BASE_URL) - - if subtasklist: - choutputs.write_all("\n\n### SUB-TASKS:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) - choutputs.write_list(subtasklist, options.skip_credits, JIRA_BASE_URL) - - if tasklist or otherlist: - choutputs.write_all("\n\n### OTHER:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) - choutputs.write_list(otherlist, options.skip_credits, JIRA_BASE_URL) - choutputs.write_list(tasklist, options.skip_credits, JIRA_BASE_URL) - - choutputs.write_all("\n\n") - choutputs.close() - - if options.index: - buildindex(title, options.license) - buildreadme(title, options.license) - - if haderrors is True: - sys.exit(1) - - -if __name__ == "__main__": - main()
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/release-doc-maker/utils.py ---------------------------------------------------------------------- diff --git a/release-doc-maker/utils.py b/release-doc-maker/utils.py deleted file mode 100644 index db957d3..0000000 --- a/release-doc-maker/utils.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python2 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 -import os -import re -import urllib2 -import sys -import json -import httplib -sys.dont_write_bytecode = True - -NAME_PATTERN = re.compile(r' \([0-9]+\)') - -def clean(input_string): - return sanitize_markdown(re.sub(NAME_PATTERN, "", input_string)) - - -def get_jira(jira_url): - """ Provide standard method for fetching content from apache jira and - handling of potential errors. Returns urllib2 response or - raises one of several exceptions.""" - - username = os.environ.get('RDM_JIRA_USERNAME') - password = os.environ.get('RDM_JIRA_PASSWORD') - - req = urllib2.Request(jira_url) - if username and password: - basicauth = base64.encodestring("%s:%s" % (username, password)).replace('\n', '') - req.add_header('Authorization', 'Basic %s' % basicauth) - - try: - response = urllib2.urlopen(req) - except urllib2.HTTPError as http_err: - code = http_err.code - print "JIRA returns HTTP error %d: %s. Aborting." % \ - (code, http_err.msg) - error_response = http_err.read() - try: - error_response = json.loads(error_response) - print "- Please ensure that specified authentication, projects,"\ - " fixVersions etc. are correct." - for message in error_response['errorMessages']: - print "-", message - except ValueError: - print "FATAL: Could not parse json response from server." - sys.exit(1) - except urllib2.URLError as url_err: - print "Error contacting JIRA: %s\n" % jira_url - print "Reason: %s" % url_err.reason - raise url_err - except httplib.BadStatusLine as err: - raise err - return response - - -def format_components(input_string): - input_string = re.sub(NAME_PATTERN, '', input_string).replace("'", "") - if input_string != "": - ret = input_string - else: - # some markdown parsers don't like empty tables - ret = "." - return clean(ret) - - -# Return the string encoded as UTF-8. -# -# This is necessary for handling markdown in Python. -def encode_utf8(input_string): - return input_string.encode('utf-8') - - -# Sanitize Markdown input so it can be handled by Python. -# -# The expectation is that the input is already valid Markdown, -# so no additional escaping is required. -def sanitize_markdown(input_string): - input_string = encode_utf8(input_string) - input_string = input_string.replace("\r", "") - input_string = input_string.rstrip() - return input_string - - -# Sanitize arbitrary text so it can be embedded in MultiMarkdown output. -# -# Note that MultiMarkdown is not Markdown, and cannot be parsed as such. -# For instance, when using pandoc, invoke it as `pandoc -f markdown_mmd`. -# -# Calls sanitize_markdown at the end as a final pass. -def sanitize_text(input_string): - escapes = dict() - # See: https://daringfireball.net/projects/markdown/syntax#backslash - # We only escape a subset of special characters. We ignore characters - # that only have significance at the start of a line. - slash_escapes = "_<>*|" - slash_escapes += "`" - slash_escapes += "\\" - all_chars = set() - # Construct a set of escapes - for c in slash_escapes: - all_chars.add(c) - for c in all_chars: - escapes[c] = "\\" + c - - # Build the output string character by character to prevent double escaping - output_string = "" - for c in input_string: - o = c - if c in escapes: - o = escapes[c] - output_string += o - - return sanitize_markdown(output_string.rstrip()) - - -# if release notes have a special marker, -# we'll treat them as already in markdown format -def processrelnote(input_string): - relnote_pattern = re.compile('^\<\!\-\- ([a-z]+) \-\-\>') - fmt = relnote_pattern.match(input_string) - if fmt is None: - return sanitize_text(input_string) - return { - 'markdown': sanitize_markdown(input_string), - }.get(fmt.group(1), sanitize_text(input_string)) - - -def to_unicode(obj): - if obj is None: - return "" - return unicode(obj) - - -class Outputs(object): - """Several different files to output to at the same time""" - - def __init__(self, base_file_name, file_name_pattern, keys, params=None): - if params is None: - params = {} - self.params = params - self.base = open(base_file_name % params, 'w') - self.others = {} - for key in keys: - both = dict(params) - both['key'] = key - self.others[key] = open(file_name_pattern % both, 'w') - - def write_all(self, pattern): - both = dict(self.params) - both['key'] = '' - self.base.write(pattern % both) - for key in self.others: - both = dict(self.params) - both['key'] = key - self.others[key].write(pattern % both) - - def write_key_raw(self, key, input_string): - self.base.write(input_string) - if key in self.others: - self.others[key].write(input_string) - - def close(self): - self.base.close() - for value in self.others.values(): - value.close() - - def write_list(self, mylist, skip_credits, base_url): - """ Take a Jira object and write out the relevants parts in a multimarkdown table line""" - for jira in sorted(mylist): - if skip_credits: - line = '| [{id}]({base_url}/browse/{id}) | {summary} | ' \ - '{priority} | {component} |\n' - else: - line = '| [{id}]({base_url}/browse/{id}) | {summary} | ' \ - '{priority} | {component} | {reporter} | {assignee} |\n' - args = {'id': encode_utf8(jira.get_id()), - 'base_url': base_url, - 'summary': sanitize_text(jira.get_summary()), - 'priority': sanitize_text(jira.get_priority()), - 'component': format_components(jira.get_components()), - 'reporter': sanitize_text(jira.get_reporter()), - 'assignee': sanitize_text(jira.get_assignee()) - } - line = line.format(**args) - self.write_key_raw(jira.get_project(), line) http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/pom.xml ---------------------------------------------------------------------- diff --git a/releasedocmaker/pom.xml b/releasedocmaker/pom.xml new file mode 100644 index 0000000..75d541d --- /dev/null +++ b/releasedocmaker/pom.xml @@ -0,0 +1,149 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.yetus</groupId> + <artifactId>yetus-project</artifactId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + <artifactId>releasedocmaker</artifactId> + <description>JIRA-based Changelog and Release Notes Generator</description> + <name>Apache Yetus - releasedocmaker</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.scijava</groupId> + <artifactId>jython-shaded</artifactId> + <version>${jython-shaded.version}</version> + </dependency> + </dependencies> + + <build> + <resources> + <resource> + <directory>src/main/python</directory> + </resource> + </resources> + + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.yetus</groupId> + <artifactId>yetus-assemblies</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>pre-dist</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>dist/apache-yetus-${project.version}</finalName> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <descriptors> + <descriptor>src/main/assemblies/${project.artifactId}.xml</descriptor> + </descriptors> + </configuration> + </execution> + + <execution> + <id>module-dist</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <attach>true</attach> + <descriptorRefs> + <descriptorRef>module-dist</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + + <execution> + <id>make-jar-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <archive> + <manifest> + <mainClass>org.apache.yetus.releasedocmaker.ReleaseDocMaker</mainClass> + </manifest> + </archive> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + + </executions> + </plugin> + + <plugin> + <artifactId>maven-deploy-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + + <plugin> + <groupId>net.sf.mavenjython</groupId> + <artifactId>jython-compile-maven-plugin</artifactId> + <version>${jython-compile-maven-plugin.version}</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>jython</goal> + </goals> + <configuration> + <libraries> + <param>--index-url=https://pypi.python.org/simple/</param> + <param>python-dateutil</param> + </libraries> + </configuration> + </execution> + </executions> + </plugin> + + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/src/main/assemblies/releasedocmaker.xml ---------------------------------------------------------------------- diff --git a/releasedocmaker/src/main/assemblies/releasedocmaker.xml b/releasedocmaker/src/main/assemblies/releasedocmaker.xml new file mode 100644 index 0000000..af5eedc --- /dev/null +++ b/releasedocmaker/src/main/assemblies/releasedocmaker.xml @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd";> + <id>releasedocmaker</id> + <formats> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${basedir}/src/main/python</directory> + <outputDirectory>lib/releasedocmaker</outputDirectory> + <includes> + <include>**</include> + </includes> + <fileMode>0755</fileMode> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/shell</directory> + <outputDirectory>bin</outputDirectory> + <includes> + <include>*</include> + </includes> + <fileMode>0755</fileMode> + </fileSet> + </fileSets> +</assembly> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/src/main/java/org/apache/yetus/releasedocmaker/ReleaseDocMaker.java ---------------------------------------------------------------------- diff --git a/releasedocmaker/src/main/java/org/apache/yetus/releasedocmaker/ReleaseDocMaker.java b/releasedocmaker/src/main/java/org/apache/yetus/releasedocmaker/ReleaseDocMaker.java new file mode 100644 index 0000000..5633180 --- /dev/null +++ b/releasedocmaker/src/main/java/org/apache/yetus/releasedocmaker/ReleaseDocMaker.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.yetus.releasedocmaker; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import org.python.core.Py; +import org.python.core.PyException; +import org.python.core.PySystemState; +import org.python.util.PythonInterpreter; + +public class ReleaseDocMaker { + public static void main(final String[] args) throws PyException { + List<String> list = new LinkedList<String>(Arrays.asList(args)); + list.add(0,"releasedocmaker"); + String[] newargs = list.toArray(new String[list.size()]); + PythonInterpreter.initialize(System.getProperties(), System.getProperties(), newargs); + PySystemState systemState = Py.getSystemState(); + PythonInterpreter interpreter = new PythonInterpreter(); + systemState.__setattr__("_jy_interpreter", Py.java2py(interpreter)); + String command = "try:\n " + + " import releasedocmaker\n " + + " releasedocmaker.main()\n" + + "except " + + " SystemExit: pass"; + interpreter.exec(command); + } +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/src/main/python/releasedocmaker.py ---------------------------------------------------------------------- diff --git a/releasedocmaker/src/main/python/releasedocmaker.py b/releasedocmaker/src/main/python/releasedocmaker.py new file mode 100755 index 0000000..63c2e00 --- /dev/null +++ b/releasedocmaker/src/main/python/releasedocmaker.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python2 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.dont_write_bytecode = True +# pylint: disable=wrong-import-position,import-self +import releasedocmaker +#pylint: disable=no-member +releasedocmaker.main()
