http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/src/main/python/releasedocmaker/__init__.py ---------------------------------------------------------------------- diff --git a/releasedocmaker/src/main/python/releasedocmaker/__init__.py b/releasedocmaker/src/main/python/releasedocmaker/__init__.py new file mode 100755 index 0000000..63476bf --- /dev/null +++ b/releasedocmaker/src/main/python/releasedocmaker/__init__.py @@ -0,0 +1,946 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from glob import glob +from optparse import OptionParser +from time import gmtime, strftime, sleep +from distutils.version import LooseVersion +import errno +import os +import re +import shutil +import urllib +import urllib2 +import httplib +import json +sys.dont_write_bytecode = True +# pylint: disable=wrong-import-position,relative-import +from utils import get_jira, to_unicode, sanitize_text, processrelnote, Outputs +# pylint: enable=wrong-import-position + +try: + import dateutil.parser +except ImportError: + print "This script requires python-dateutil module to be installed. " \ + "You can install it using:\n\t pip install python-dateutil" + sys.exit(1) + +RELEASE_VERSION = {} + +JIRA_BASE_URL = "https://issues.apache.org/jira"; +SORTTYPE = 'resolutiondate' +SORTORDER = 'older' +NUM_RETRIES = 5 + +# label to be used to mark an issue as Incompatible change. +BACKWARD_INCOMPATIBLE_LABEL = 'backward-incompatible' + +ASF_LICENSE = ''' +<!--- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--> +''' + +def buildindex(title, asf_license): + """Write an index file for later conversion using mvn site""" + versions = glob("[0-9]*.[0-9]*") + versions.sort(key=LooseVersion, reverse=True) + with open("index.md", "w") as indexfile: + if asf_license is True: + indexfile.write(ASF_LICENSE) + for version in versions: + indexfile.write("* %s v%s\n" % (title, version)) + for k in ("Changelog", "Release Notes"): + indexfile.write(" * [%s](%s/%s.%s.html)\n" % + (k, version, k.upper().replace(" ", ""), + version)) + + +def buildreadme(title, asf_license): + """Write an index file for Github using README.md""" + versions = glob("[0-9]*.[0-9]*") + versions.sort(key=LooseVersion, reverse=True) + with open("README.md", "w") as indexfile: + if asf_license is True: + indexfile.write(ASF_LICENSE) + for version in versions: + indexfile.write("* %s v%s\n" % (title, version)) + for k in ("Changelog", "Release Notes"): + indexfile.write(" * [%s](%s/%s.%s.md)\n" % + (k, version, k.upper().replace(" ", ""), + version)) + + +class GetVersions(object): + """ List of version strings """ + + def __init__(self, versions, projects): + versions = versions + projects = projects + self.newversions = [] + versions.sort(key=LooseVersion) + print "Looking for %s through %s" % (versions[0], versions[-1]) + newversions = set() + for project in projects: + url = JIRA_BASE_URL + \ + "/rest/api/2/project/%s/versions" % project.upper() + try: + resp = get_jira(url) + except (urllib2.HTTPError, urllib2.URLError, httplib.BadStatusLine): + sys.exit(1) + + datum = json.loads(resp.read()) + for data in datum: + newversions.add(data['name']) + newlist = list(newversions.copy()) + newlist.append(versions[0]) + newlist.append(versions[-1]) + newlist.sort(key=LooseVersion) + start_index = newlist.index(versions[0]) + end_index = len(newlist) - 1 - newlist[::-1].index(versions[-1]) + for newversion in newlist[start_index + 1:end_index]: + if newversion in newversions: + print "Adding %s to the list" % newversion + self.newversions.append(newversion) + + def getlist(self): + return self.newversions + + +class Version(object): + """Represents a version number""" + + def __init__(self, data): + self.mod = False + self.data = data + found = re.match(r'^((\d+)(\.\d+)*).*$', data) + if found: + self.parts = [int(p) for p in found.group(1).split('.')] + else: + self.parts = [] + # backfill version with zeros if missing parts + self.parts.extend((0,) * (3 - len(self.parts))) + + def __str__(self): + if self.mod: + return '.'.join([str(p) for p in self.parts]) + return self.data + + def __cmp__(self, other): + return cmp(self.parts, other.parts) + + +class Jira(object): + """A single JIRA""" + + def __init__(self, data, parent): + self.key = data['key'] + self.fields = data['fields'] + self.parent = parent + self.notes = None + self.incompat = None + self.reviewed = None + self.important = None + + def get_id(self): + return to_unicode(self.key) + + def get_description(self): + return to_unicode(self.fields['description']) + + def get_release_note(self): + if self.notes is None: + field = self.parent.field_id_map['Release Note'] + if field in self.fields: + self.notes = to_unicode(self.fields[field]) + elif self.get_incompatible_change() or self.get_important(): + self.notes = self.get_description() + else: + self.notes = "" + return self.notes + + def get_priority(self): + ret = "" + pri = self.fields['priority'] + if pri is not None: + ret = pri['name'] + return to_unicode(ret) + + def get_assignee(self): + ret = "" + mid = self.fields['assignee'] + if mid is not None: + ret = mid['displayName'] + return to_unicode(ret) + + def get_components(self): + if self.fields['components']: + return ", ".join([comp['name'] for comp in self.fields['components'] + ]) + return "" + + def get_summary(self): + return self.fields['summary'] + + def get_type(self): + ret = "" + mid = self.fields['issuetype'] + if mid is not None: + ret = mid['name'] + return to_unicode(ret) + + def get_reporter(self): + ret = "" + mid = self.fields['reporter'] + if mid is not None: + ret = mid['displayName'] + return to_unicode(ret) + + def get_project(self): + ret = "" + mid = self.fields['project'] + if mid is not None: + ret = mid['key'] + return to_unicode(ret) + + def __cmp__(self, other): + result = 0 + + if SORTTYPE == 'issueid': + # compare by issue name-number + selfsplit = self.get_id().split('-') + othersplit = other.get_id().split('-') + result = cmp(selfsplit[0], othersplit[0]) + if result == 0: + result = cmp(int(selfsplit[1]), int(othersplit[1])) + # dec is supported for backward compatibility + if SORTORDER in ['dec', 'desc']: + result *= -1 + + elif SORTTYPE == 'resolutiondate': + dts = dateutil.parser.parse(self.fields['resolutiondate']) + dto = dateutil.parser.parse(other.fields['resolutiondate']) + result = cmp(dts, dto) + if SORTORDER == 'newer': + result *= -1 + + return result + + def get_incompatible_change(self): + if self.incompat is None: + field = self.parent.field_id_map['Hadoop Flags'] + self.reviewed = False + self.incompat = False + if field in self.fields: + if self.fields[field]: + for flag in self.fields[field]: + if flag['value'] == "Incompatible change": + self.incompat = True + if flag['value'] == "Reviewed": + self.reviewed = True + else: + # Custom field 'Hadoop Flags' is not defined, + # search for 'backward-incompatible' label + field = self.parent.field_id_map['Labels'] + if field in self.fields and self.fields[field]: + if BACKWARD_INCOMPATIBLE_LABEL in self.fields[field]: + self.incompat = True + self.reviewed = True + return self.incompat + + def get_important(self): + if self.important is None: + field = self.parent.field_id_map['Flags'] + self.important = False + if field in self.fields: + if self.fields[field]: + for flag in self.fields[field]: + if flag['value'] == "Important": + self.important = True + return self.important + + +class JiraIter(object): + """An Iterator of JIRAs""" + + @staticmethod + def collect_fields(): + """send a query to JIRA and collect field-id map""" + try: + resp = get_jira(JIRA_BASE_URL + "/rest/api/2/field") + data = json.loads(resp.read()) + except (urllib2.HTTPError, urllib2.URLError, httplib.BadStatusLine, ValueError): + sys.exit(1) + field_id_map = {} + for part in data: + field_id_map[part['name']] = part['id'] + return field_id_map + + @staticmethod + def query_jira(ver, projects, pos): + """send a query to JIRA and collect + a certain number of issue information""" + count = 100 + pjs = "','".join(projects) + jql = "project in ('%s') and \ + fixVersion in ('%s') and \ + resolution = Fixed" % (pjs, ver) + params = urllib.urlencode({'jql': jql, + 'startAt': pos, + 'maxResults': count}) + return JiraIter.load_jira(params, 0) + + @staticmethod + def load_jira(params, fail_count): + """send query to JIRA and collect with retries""" + try: + resp = get_jira(JIRA_BASE_URL + "/rest/api/2/search?%s" % params) + except (urllib2.URLError, httplib.BadStatusLine) as err: + return JiraIter.retry_load(err, params, fail_count) + + try: + data = json.loads(resp.read()) + except httplib.IncompleteRead as err: + return JiraIter.retry_load(err, params, fail_count) + return data + + @staticmethod + def retry_load(err, params, fail_count): + """Retry connection up to NUM_RETRIES times.""" + print(err) + fail_count += 1 + if fail_count <= NUM_RETRIES: + print "Connection failed %d times. Retrying." % (fail_count) + sleep(1) + return JiraIter.load_jira(params, fail_count) + else: + print "Connection failed %d times. Aborting." % (fail_count) + sys.exit(1) + + @staticmethod + def collect_jiras(ver, projects): + """send queries to JIRA and collect all issues + that belongs to given version and projects""" + jiras = [] + pos = 0 + end = 1 + while pos < end: + data = JiraIter.query_jira(ver, projects, pos) + if 'error_messages' in data: + print "JIRA returns error message: %s" % data['error_messages'] + sys.exit(1) + pos = data['startAt'] + data['maxResults'] + end = data['total'] + jiras.extend(data['issues']) + + if ver not in RELEASE_VERSION: + for issue in data['issues']: + for fix_version in issue['fields']['fixVersions']: + if 'releaseDate' in fix_version: + RELEASE_VERSION[fix_version['name']] = fix_version[ + 'releaseDate'] + return jiras + + def __init__(self, version, projects): + self.version = version + self.projects = projects + self.field_id_map = JiraIter.collect_fields() + ver = str(version).replace("-SNAPSHOT", "") + self.jiras = JiraIter.collect_jiras(ver, projects) + self.iter = self.jiras.__iter__() + + def __iter__(self): + return self + + def next(self): + data = self.iter.next() + j = Jira(data, self) + return j + + +class Linter(object): + """Encapsulates lint-related functionality. + Maintains running lint statistics about JIRAs.""" + + _valid_filters = ["incompatible", "important", "version", "component", + "assignee"] + + def __init__(self, version, options): + self._warning_count = 0 + self._error_count = 0 + self._lint_message = "" + self._version = version + + self._filters = dict(zip(self._valid_filters, [False] * len( + self._valid_filters))) + + self.enabled = False + self._parse_options(options) + + @staticmethod + def add_parser_options(parser): + """Add Linter options to passed optparse parser.""" + filter_string = ", ".join("'" + f + "'" for f in Linter._valid_filters) + parser.add_option( + "-n", + "--lint", + dest="lint", + action="append", + type="string", + help="Specify lint filters. Valid filters are " + filter_string + + ". " + "'all' enables all lint filters. " + + "Multiple filters can be specified comma-delimited and " + + "filters can be negated, e.g. 'all,-component'.") + + def _parse_options(self, options): + """Parse options from optparse.""" + + if options.lint is None or not options.lint: + return + self.enabled = True + + # Valid filter specifications are + # self._valid_filters, negations, and "all" + valid_list = self._valid_filters + valid_list += ["-" + v for v in valid_list] + valid_list += ["all"] + valid = set(valid_list) + + enabled = [] + disabled = [] + + for o in options.lint: + for token in o.split(","): + if token not in valid: + print "Unknown lint filter '%s', valid options are: %s" % \ + (token, ", ".join(v for v in sorted(valid))) + sys.exit(1) + if token.startswith("-"): + disabled.append(token[1:]) + else: + enabled.append(token) + + for e in enabled: + if e == "all": + for f in self._valid_filters: + self._filters[f] = True + else: + self._filters[e] = True + for d in disabled: + self._filters[d] = False + + def had_errors(self): + """Returns True if a lint error was encountered, else False.""" + return self._error_count > 0 + + def message(self): + """Return summary lint message suitable for printing to stdout.""" + if not self.enabled: + return None + return self._lint_message + \ + "\n=======================================" + \ + "\n%s: Error:%d, Warning:%d \n" % \ + (self._version, self._error_count, self._warning_count) + + def _check_missing_component(self, jira): + """Return if JIRA has a 'missing component' lint error.""" + if not self._filters["component"]: + return False + + if jira.fields['components']: + return False + return True + + def _check_missing_assignee(self, jira): + """Return if JIRA has a 'missing assignee' lint error.""" + if not self._filters["assignee"]: + return False + + if jira.fields['assignee'] is not None: + return False + return True + + def _check_version_string(self, jira): + """Return if JIRA has a version string lint error.""" + if not self._filters["version"]: + return False + + field = jira.parent.field_id_map['Fix Version/s'] + for ver in jira.fields[field]: + found = re.match(r'^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', ver['name']) + if not found: + return True + return False + + def lint(self, jira): + """Run lint check on a JIRA.""" + if not self.enabled: + return + if not jira.get_release_note(): + if self._filters["incompatible"] and jira.get_incompatible_change(): + self._warning_count += 1 + self._lint_message += "\nWARNING: incompatible change %s lacks release notes." % \ + (sanitize_text(jira.get_id())) + if self._filters["important"] and jira.get_important(): + self._warning_count += 1 + self._lint_message += "\nWARNING: important issue %s lacks release notes." % \ + (sanitize_text(jira.get_id())) + + if self._check_version_string(jira): + self._warning_count += 1 + self._lint_message += "\nWARNING: Version string problem for %s " % jira.get_id( + ) + + if self._check_missing_component(jira) or self._check_missing_assignee( + jira): + self._error_count += 1 + error_message = [] + if self._check_missing_component(jira): + error_message.append("component") + if self._check_missing_assignee(jira): + error_message.append("assignee") + self._lint_message += "\nERROR: missing %s for %s " \ + % (" and ".join(error_message), jira.get_id()) + + +def parse_args(): + """Parse command-line arguments with optparse.""" + usage = "usage: %prog [OPTIONS] " + \ + "--project PROJECT [--project PROJECT] " + \ + "--version VERSION [--version VERSION2 ...]" + parser = OptionParser( + usage=usage, + epilog= + "Markdown-formatted CHANGELOG and RELEASENOTES files will be stored" + " in a directory named after the highest version provided.") + parser.add_option("--dirversions", + dest="versiondirs", + action="store_true", + default=False, + help="Put files in versioned directories") + parser.add_option("--fileversions", + dest="versionfiles", + action="store_true", + default=False, + help="Write files with embedded versions") + parser.add_option("-i", + "--index", + dest="index", + action="store_true", + default=False, + help="build an index file") + parser.add_option("-l", + "--license", + dest="license", + action="store_true", + default=False, + help="Add an ASF license") + parser.add_option("-p", + "--project", + dest="projects", + action="append", + type="string", + help="projects in JIRA to include in releasenotes", + metavar="PROJECT") + parser.add_option("-r", + "--range", + dest="range", + action="store_true", + default=False, + help="Given versions are a range") + parser.add_option( + "--sortorder", + dest="sortorder", + metavar="TYPE", + default=SORTORDER, + # dec is supported for backward compatibility + choices=["asc", "dec", "desc", "newer", "older"], + help="Sorting order for sort type (default: %s)" % SORTORDER) + parser.add_option("--sorttype", + dest="sorttype", + metavar="TYPE", + default=SORTTYPE, + choices=["resolutiondate", "issueid"], + help="Sorting type for issues (default: %s)" % SORTTYPE) + parser.add_option( + "-t", + "--projecttitle", + dest="title", + type="string", + help="Title to use for the project (default is Apache PROJECT)") + parser.add_option("-u", + "--usetoday", + dest="usetoday", + action="store_true", + default=False, + help="use current date for unreleased versions") + parser.add_option("-v", + "--version", + dest="versions", + action="append", + type="string", + help="versions in JIRA to include in releasenotes", + metavar="VERSION") + parser.add_option( + "-V", + dest="release_version", + action="store_true", + default=False, + help="display version information for releasedocmaker and exit.") + parser.add_option("-O", + "--outputdir", + dest="output_directory", + action="append", + type="string", + help="specify output directory to put release docs to.") + parser.add_option("-B", + "--baseurl", + dest="base_url", + action="append", + type="string", + help="specify base URL of the JIRA instance.") + parser.add_option( + "--retries", + dest="retries", + action="append", + type="int", + help="Specify how many times to retry connection for each URL.") + parser.add_option( + "--skip-credits", + dest="skip_credits", + action="store_true", + default=False, + help="While creating release notes skip the 'reporter' and 'contributor' columns") + parser.add_option("-X", + "--incompatiblelabel", + dest="incompatible_label", + default="backward-incompatible", + type="string", + help="Specify the label to indicate backward incompatibility.") + + Linter.add_parser_options(parser) + + if len(sys.argv) <= 1: + parser.print_help() + sys.exit(1) + + (options, _) = parser.parse_args() + + # Handle the version string right away and exit + if options.release_version: + with open( + os.path.join( + os.path.dirname(__file__), "../VERSION"), 'r') as ver_file: + print ver_file.read() + sys.exit(0) + + # Validate options + if not options.release_version: + if options.versions is None: + parser.error("At least one version needs to be supplied") + if options.projects is None: + parser.error("At least one project needs to be supplied") + if options.base_url is not None: + if len(options.base_url) > 1: + parser.error("Only one base URL should be given") + else: + options.base_url = options.base_url[0] + if options.output_directory is not None: + if len(options.output_directory) > 1: + parser.error("Only one output directory should be given") + else: + options.output_directory = options.output_directory[0] + + if options.range or len(options.versions) > 1: + if not options.versiondirs and not options.versionfiles: + parser.error("Multiple versions require either --fileversions or --dirversions") + + return options + + +def main(): + options = parse_args() + + if options.output_directory is not None: + # Create the output directory if it does not exist. + try: + os.makedirs(options.output_directory) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir( + options.output_directory): + pass + else: + print "Unable to create output directory %s: %s" % \ + (options.output_directory, exc.message) + sys.exit(1) + os.chdir(options.output_directory) + + if options.base_url is not None: + global JIRA_BASE_URL + JIRA_BASE_URL = options.base_url + + if options.incompatible_label is not None: + global BACKWARD_INCOMPATIBLE_LABEL + BACKWARD_INCOMPATIBLE_LABEL = options.incompatible_label + + + projects = options.projects + + if options.range is True: + versions = [Version(v) + for v in GetVersions(options.versions, projects).getlist()] + else: + versions = [Version(v) for v in options.versions] + versions.sort() + + global SORTTYPE + SORTTYPE = options.sorttype + global SORTORDER + SORTORDER = options.sortorder + + if options.title is None: + title = projects[0] + else: + title = options.title + + if options.retries is not None: + global NUM_RETRIES + NUM_RETRIES = options.retries[0] + + haderrors = False + + for version in versions: + vstr = str(version) + linter = Linter(vstr, options) + jlist = sorted(JiraIter(vstr, projects)) + if not jlist: + print "There is no issue which has the specified version: %s" % version + continue + + if vstr in RELEASE_VERSION: + reldate = RELEASE_VERSION[vstr] + elif options.usetoday: + reldate = strftime("%Y-%m-%d", gmtime()) + else: + reldate = "Unreleased (as of %s)" % strftime("%Y-%m-%d", gmtime()) + + if not os.path.exists(vstr) and options.versiondirs: + os.mkdir(vstr) + + if options.versionfiles and options.versiondirs: + reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", + "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + choutputs = Outputs("%(ver)s/CHANGELOG.%(ver)s.md", + "%(ver)s/CHANGELOG.%(key)s.%(ver)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + elif options.versiondirs: + reloutputs = Outputs("%(ver)s/RELEASENOTES.md", + "%(ver)s/RELEASENOTES.%(key)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + choutputs = Outputs("%(ver)s/CHANGELOG.md", + "%(ver)s/CHANGELOG.%(key)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + elif options.versionfiles: + reloutputs = Outputs("RELEASENOTES.%(ver)s.md", + "RELEASENOTES.%(key)s.%(ver)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + choutputs = Outputs("CHANGELOG.%(ver)s.md", + "CHANGELOG.%(key)s.%(ver)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + else: + reloutputs = Outputs("RELEASENOTES.md", + "RELEASENOTES.%(key)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + choutputs = Outputs("CHANGELOG.md", + "CHANGELOG.%(key)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + + if options.license is True: + reloutputs.write_all(ASF_LICENSE) + choutputs.write_all(ASF_LICENSE) + + relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \ + 'These release notes cover new developer and user-facing ' \ + 'incompatibilities, important issues, features, and major improvements.\n\n' + chhead = '# %(title)s Changelog\n\n' \ + '## Release %(ver)s - %(date)s\n'\ + '\n' + + reloutputs.write_all(relhead) + choutputs.write_all(chhead) + + incompatlist = [] + importantlist = [] + buglist = [] + improvementlist = [] + newfeaturelist = [] + subtasklist = [] + tasklist = [] + testlist = [] + otherlist = [] + + for jira in jlist: + if jira.get_incompatible_change(): + incompatlist.append(jira) + elif jira.get_important(): + importantlist.append(jira) + elif jira.get_type() == "Bug": + buglist.append(jira) + elif jira.get_type() == "Improvement": + improvementlist.append(jira) + elif jira.get_type() == "New Feature": + newfeaturelist.append(jira) + elif jira.get_type() == "Sub-task": + subtasklist.append(jira) + elif jira.get_type() == "Task": + tasklist.append(jira) + elif jira.get_type() == "Test": + testlist.append(jira) + else: + otherlist.append(jira) + + line = '* [%s](' % (sanitize_text(jira.get_id())) + JIRA_BASE_URL + \ + '/browse/%s) | *%s* | **%s**\n' \ + % (sanitize_text(jira.get_id()), + sanitize_text(jira.get_priority()), sanitize_text(jira.get_summary())) + + if jira.get_release_note() or \ + jira.get_incompatible_change() or jira.get_important(): + reloutputs.write_key_raw(jira.get_project(), "\n---\n\n") + reloutputs.write_key_raw(jira.get_project(), line) + if not jira.get_release_note(): + line = '\n**WARNING: No release note provided for this change.**\n\n' + else: + line = '\n%s\n\n' % ( + processrelnote(jira.get_release_note())) + reloutputs.write_key_raw(jira.get_project(), line) + + linter.lint(jira) + + if linter.enabled: + print linter.message() + if linter.had_errors(): + haderrors = True + shutil.rmtree(vstr) + continue + + reloutputs.write_all("\n\n") + reloutputs.close() + + if options.skip_credits: + CHANGEHDR1 = "| JIRA | Summary | Priority | " + \ + "Component |\n" + CHANGEHDR2 = "|:---- |:---- | :--- |:---- |\n" + else: + CHANGEHDR1 = "| JIRA | Summary | Priority | " + \ + "Component | Reporter | Contributor |\n" + CHANGEHDR2 = "|:---- |:---- | :--- |:---- |:---- |:---- |\n" + + if incompatlist: + choutputs.write_all("### INCOMPATIBLE CHANGES:\n\n") + choutputs.write_all(CHANGEHDR1) + choutputs.write_all(CHANGEHDR2) + choutputs.write_list(incompatlist, options.skip_credits, JIRA_BASE_URL) + + if importantlist: + choutputs.write_all("\n\n### IMPORTANT ISSUES:\n\n") + choutputs.write_all(CHANGEHDR1) + choutputs.write_all(CHANGEHDR2) + choutputs.write_list(importantlist, options.skip_credits, JIRA_BASE_URL) + + if newfeaturelist: + choutputs.write_all("\n\n### NEW FEATURES:\n\n") + choutputs.write_all(CHANGEHDR1) + choutputs.write_all(CHANGEHDR2) + choutputs.write_list(newfeaturelist, options.skip_credits, JIRA_BASE_URL) + + if improvementlist: + choutputs.write_all("\n\n### IMPROVEMENTS:\n\n") + choutputs.write_all(CHANGEHDR1) + choutputs.write_all(CHANGEHDR2) + choutputs.write_list(improvementlist, options.skip_credits, JIRA_BASE_URL) + + if buglist: + choutputs.write_all("\n\n### BUG FIXES:\n\n") + choutputs.write_all(CHANGEHDR1) + choutputs.write_all(CHANGEHDR2) + choutputs.write_list(buglist, options.skip_credits, JIRA_BASE_URL) + + if testlist: + choutputs.write_all("\n\n### TESTS:\n\n") + choutputs.write_all(CHANGEHDR1) + choutputs.write_all(CHANGEHDR2) + choutputs.write_list(testlist, options.skip_credits, JIRA_BASE_URL) + + if subtasklist: + choutputs.write_all("\n\n### SUB-TASKS:\n\n") + choutputs.write_all(CHANGEHDR1) + choutputs.write_all(CHANGEHDR2) + choutputs.write_list(subtasklist, options.skip_credits, JIRA_BASE_URL) + + if tasklist or otherlist: + choutputs.write_all("\n\n### OTHER:\n\n") + choutputs.write_all(CHANGEHDR1) + choutputs.write_all(CHANGEHDR2) + choutputs.write_list(otherlist, options.skip_credits, JIRA_BASE_URL) + choutputs.write_list(tasklist, options.skip_credits, JIRA_BASE_URL) + + choutputs.write_all("\n\n") + choutputs.close() + + if options.index: + buildindex(title, options.license) + buildreadme(title, options.license) + + if haderrors is True: + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/src/main/python/releasedocmaker/utils.py ---------------------------------------------------------------------- diff --git a/releasedocmaker/src/main/python/releasedocmaker/utils.py b/releasedocmaker/src/main/python/releasedocmaker/utils.py new file mode 100644 index 0000000..db957d3 --- /dev/null +++ b/releasedocmaker/src/main/python/releasedocmaker/utils.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python2 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import re +import urllib2 +import sys +import json +import httplib +sys.dont_write_bytecode = True + +NAME_PATTERN = re.compile(r' \([0-9]+\)') + +def clean(input_string): + return sanitize_markdown(re.sub(NAME_PATTERN, "", input_string)) + + +def get_jira(jira_url): + """ Provide standard method for fetching content from apache jira and + handling of potential errors. Returns urllib2 response or + raises one of several exceptions.""" + + username = os.environ.get('RDM_JIRA_USERNAME') + password = os.environ.get('RDM_JIRA_PASSWORD') + + req = urllib2.Request(jira_url) + if username and password: + basicauth = base64.encodestring("%s:%s" % (username, password)).replace('\n', '') + req.add_header('Authorization', 'Basic %s' % basicauth) + + try: + response = urllib2.urlopen(req) + except urllib2.HTTPError as http_err: + code = http_err.code + print "JIRA returns HTTP error %d: %s. Aborting." % \ + (code, http_err.msg) + error_response = http_err.read() + try: + error_response = json.loads(error_response) + print "- Please ensure that specified authentication, projects,"\ + " fixVersions etc. are correct." + for message in error_response['errorMessages']: + print "-", message + except ValueError: + print "FATAL: Could not parse json response from server." + sys.exit(1) + except urllib2.URLError as url_err: + print "Error contacting JIRA: %s\n" % jira_url + print "Reason: %s" % url_err.reason + raise url_err + except httplib.BadStatusLine as err: + raise err + return response + + +def format_components(input_string): + input_string = re.sub(NAME_PATTERN, '', input_string).replace("'", "") + if input_string != "": + ret = input_string + else: + # some markdown parsers don't like empty tables + ret = "." + return clean(ret) + + +# Return the string encoded as UTF-8. +# +# This is necessary for handling markdown in Python. +def encode_utf8(input_string): + return input_string.encode('utf-8') + + +# Sanitize Markdown input so it can be handled by Python. +# +# The expectation is that the input is already valid Markdown, +# so no additional escaping is required. +def sanitize_markdown(input_string): + input_string = encode_utf8(input_string) + input_string = input_string.replace("\r", "") + input_string = input_string.rstrip() + return input_string + + +# Sanitize arbitrary text so it can be embedded in MultiMarkdown output. +# +# Note that MultiMarkdown is not Markdown, and cannot be parsed as such. +# For instance, when using pandoc, invoke it as `pandoc -f markdown_mmd`. +# +# Calls sanitize_markdown at the end as a final pass. +def sanitize_text(input_string): + escapes = dict() + # See: https://daringfireball.net/projects/markdown/syntax#backslash + # We only escape a subset of special characters. We ignore characters + # that only have significance at the start of a line. + slash_escapes = "_<>*|" + slash_escapes += "`" + slash_escapes += "\\" + all_chars = set() + # Construct a set of escapes + for c in slash_escapes: + all_chars.add(c) + for c in all_chars: + escapes[c] = "\\" + c + + # Build the output string character by character to prevent double escaping + output_string = "" + for c in input_string: + o = c + if c in escapes: + o = escapes[c] + output_string += o + + return sanitize_markdown(output_string.rstrip()) + + +# if release notes have a special marker, +# we'll treat them as already in markdown format +def processrelnote(input_string): + relnote_pattern = re.compile('^\<\!\-\- ([a-z]+) \-\-\>') + fmt = relnote_pattern.match(input_string) + if fmt is None: + return sanitize_text(input_string) + return { + 'markdown': sanitize_markdown(input_string), + }.get(fmt.group(1), sanitize_text(input_string)) + + +def to_unicode(obj): + if obj is None: + return "" + return unicode(obj) + + +class Outputs(object): + """Several different files to output to at the same time""" + + def __init__(self, base_file_name, file_name_pattern, keys, params=None): + if params is None: + params = {} + self.params = params + self.base = open(base_file_name % params, 'w') + self.others = {} + for key in keys: + both = dict(params) + both['key'] = key + self.others[key] = open(file_name_pattern % both, 'w') + + def write_all(self, pattern): + both = dict(self.params) + both['key'] = '' + self.base.write(pattern % both) + for key in self.others: + both = dict(self.params) + both['key'] = key + self.others[key].write(pattern % both) + + def write_key_raw(self, key, input_string): + self.base.write(input_string) + if key in self.others: + self.others[key].write(input_string) + + def close(self): + self.base.close() + for value in self.others.values(): + value.close() + + def write_list(self, mylist, skip_credits, base_url): + """ Take a Jira object and write out the relevants parts in a multimarkdown table line""" + for jira in sorted(mylist): + if skip_credits: + line = '| [{id}]({base_url}/browse/{id}) | {summary} | ' \ + '{priority} | {component} |\n' + else: + line = '| [{id}]({base_url}/browse/{id}) | {summary} | ' \ + '{priority} | {component} | {reporter} | {assignee} |\n' + args = {'id': encode_utf8(jira.get_id()), + 'base_url': base_url, + 'summary': sanitize_text(jira.get_summary()), + 'priority': sanitize_text(jira.get_priority()), + 'component': format_components(jira.get_components()), + 'reporter': sanitize_text(jira.get_reporter()), + 'assignee': sanitize_text(jira.get_assignee()) + } + line = line.format(**args) + self.write_key_raw(jira.get_project(), line) http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/src/main/shell/releasedocmaker ---------------------------------------------------------------------- diff --git a/releasedocmaker/src/main/shell/releasedocmaker b/releasedocmaker/src/main/shell/releasedocmaker new file mode 100755 index 0000000..30b13b1 --- /dev/null +++ b/releasedocmaker/src/main/shell/releasedocmaker @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Make a special version of the shell wrapper for releasedocmaker +# that maintains the ability to have '--lint' mean '--lint=all' + +args=() +for arg in "${@}"; do + if [ "${arg}" = "-n" ] || [ "${arg}" = "--lint" ]; then + args=("${args[@]}" "--lint=all") + else + args=("${args[@]}" "${arg}") + fi +done + +exec "$(dirname -- "${BASH_SOURCE-0}")/../lib/releasedocmaker/releasedocmaker.py" "${args[@]}" http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/pom.xml ---------------------------------------------------------------------- diff --git a/shelldocs/pom.xml b/shelldocs/pom.xml new file mode 100644 index 0000000..c5c3ec3 --- /dev/null +++ b/shelldocs/pom.xml @@ -0,0 +1,161 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.yetus</groupId> + <artifactId>yetus-project</artifactId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + <artifactId>shelldocs</artifactId> + <description>API Documentation for Shell Scripts</description> + <name>Apache Yetus - shelldocs</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.scijava</groupId> + <artifactId>jython-shaded</artifactId> + <version>${jython-shaded.version}</version> + </dependency> + </dependencies> + + <build> + + <resources> + <resource> + <directory>src/main/python</directory> + </resource> + </resources> + + <plugins> + + <plugin> + <groupId>net.sf.mavenjython</groupId> + <artifactId>jython-compile-maven-plugin</artifactId> + <version>${jython-compile-maven-plugin.version}</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>jython</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.yetus</groupId> + <artifactId>yetus-assemblies</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + + <executions> + + <execution> + <id>make-jar-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <archive> + <manifest> + <mainClass>org.apache.yetus.shelldocs.ShellDocs</mainClass> + </manifest> + </archive> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + + <execution> + <id>build</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>dist/apache-yetus-${project.version}</finalName> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <descriptorRefs> + <descriptorRef>script-bundle</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + + <execution> + <id>module-dist</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <attach>true</attach> + <descriptorRefs> + <descriptorRef>module-dist</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + + </executions> + </plugin> + + <plugin> + <groupId>org.apache.yetus</groupId> + <artifactId>yetus-minimaven-plugin</artifactId> + <version>${project.version}</version> + <executions> + <execution> + <id>bins4libs</id> + <phase>prepare-package</phase> + <goals> + <goal>bin4libs</goal> + </goals> + <configuration> + <libdir>lib/shelldocs</libdir> + <basedir>${project.build.directory}/dist/apache-yetus-${project.version}</basedir> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-deploy-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/shelldocs.py ---------------------------------------------------------------------- diff --git a/shelldocs/shelldocs.py b/shelldocs/shelldocs.py deleted file mode 100755 index a096cbe..0000000 --- a/shelldocs/shelldocs.py +++ /dev/null @@ -1,426 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Do this immediately to prevent compiled forms -import sys -import os -import re -from optparse import OptionParser - -sys.dont_write_bytecode = True - -ASFLICENSE = ''' -<!--- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---> -''' - - -def docstrip(key, dstr): - '''remove extra spaces from shelldoc phrase''' - dstr = re.sub("^## @%s " % key, "", dstr) - dstr = dstr.lstrip() - dstr = dstr.rstrip() - return dstr - - -def toc(tlist): - '''build a table of contents''' - tocout = [] - header = () - for i in tlist: - if header != i.getinter(): - header = i.getinter() - line = " * %s\n" % (i.headerbuild()) - tocout.append(line) - line = " * [%s](#%s)\n" % (i.getname().replace("_", r"\_"), - i.getname()) - tocout.append(line) - return tocout - - -class ShellFunction(object): - """a shell function""" - - def __init__(self, filename): - '''Initializer''' - self.name = None - self.audience = None - self.stability = None - self.replaceb = None - self.returnt = None - self.desc = None - self.params = None - self.filename = filename - self.linenum = 0 - - def __cmp__(self, other): - '''comparison''' - if self.audience == other.audience: - if self.stability == other.stability: - if self.replaceb == other.replaceb: - return cmp(self.name, other.name) - else: - if self.replaceb == "Yes": - return -1 - else: - if self.stability == "Stable": - return -1 - else: - if self.audience == "Public": - return -1 - return 1 - - def reset(self): - '''empties current function''' - self.name = None - self.audience = None - self.stability = None - self.replaceb = None - self.returnt = None - self.desc = None - self.params = None - self.linenum = 0 - self.filename = None - - def getfilename(self): - '''get the name of the function''' - if self.filename is None: - return "undefined" - return self.filename - - def setname(self, text): - '''set the name of the function''' - definition = text.split() - self.name = definition[1] - - def getname(self): - '''get the name of the function''' - if self.name is None: - return "None" - return self.name - - def setlinenum(self, linenum): - '''set the line number of the function''' - self.linenum = linenum - - def getlinenum(self): - '''get the line number of the function''' - return self.linenum - - def setaudience(self, text): - '''set the audience of the function''' - self.audience = docstrip("audience", text) - self.audience = self.audience.capitalize() - - def getaudience(self): - '''get the audience of the function''' - if self.audience is None: - return "None" - return self.audience - - def setstability(self, text): - '''set the stability of the function''' - self.stability = docstrip("stability", text) - self.stability = self.stability.capitalize() - - def getstability(self): - '''get the stability of the function''' - if self.stability is None: - return "None" - return self.stability - - def setreplace(self, text): - '''set the replacement state''' - self.replaceb = docstrip("replaceable", text) - self.replaceb = self.replaceb.capitalize() - - def getreplace(self): - '''get the replacement state''' - if self.replaceb == "Yes": - return self.replaceb - return "No" - - def getinter(self): - '''get the function state''' - return self.getaudience(), self.getstability(), self.getreplace() - - def addreturn(self, text): - '''add a return state''' - if self.returnt is None: - self.returnt = [] - self.returnt.append(docstrip("return", text)) - - def getreturn(self): - '''get the complete return state''' - if self.returnt is None: - return "Nothing" - return "\n\n".join(self.returnt) - - def adddesc(self, text): - '''add to the description''' - if self.desc is None: - self.desc = [] - self.desc.append(docstrip("description", text)) - - def getdesc(self): - '''get the description''' - if self.desc is None: - return "None" - return " ".join(self.desc) - - def addparam(self, text): - '''add a parameter''' - if self.params is None: - self.params = [] - self.params.append(docstrip("param", text)) - - def getparams(self): - '''get all of the parameters''' - if self.params is None: - return "" - return " ".join(self.params) - - def getusage(self): - '''get the usage string''' - line = "%s %s" % (self.name, self.getparams()) - return line.rstrip() - - def headerbuild(self): - '''get the header for this function''' - if self.getreplace() == "Yes": - replacetext = "Replaceable" - else: - replacetext = "Not Replaceable" - line = "%s/%s/%s" % (self.getaudience(), self.getstability(), - replacetext) - return line - - def getdocpage(self): - '''get the built document page for this function''' - line = "### `%s`\n\n"\ - "* Synopsis\n\n"\ - "```\n%s\n"\ - "```\n\n" \ - "* Description\n\n" \ - "%s\n\n" \ - "* Returns\n\n" \ - "%s\n\n" \ - "| Classification | Level |\n" \ - "| :--- | :--- |\n" \ - "| Audience | %s |\n" \ - "| Stability | %s |\n" \ - "| Replaceable | %s |\n\n" \ - % (self.getname(), - self.getusage(), - self.getdesc(), - self.getreturn(), - self.getaudience(), - self.getstability(), - self.getreplace()) - return line - - def lint(self): - '''Lint this function''' - getfuncs = { - "audience": self.getaudience, - "stability": self.getstability, - "replaceable": self.getreplace, - } - validvalues = { - "audience": ("Public", "Private"), - "stability": ("Stable", "Evolving"), - "replaceable": ("Yes", "No"), - } - messages = [] - for attr in ("audience", "stability", "replaceable"): - value = getfuncs[attr]() - if value == "None": - messages.append("%s:%u: ERROR: function %s has no @%s" % - (self.getfilename(), self.getlinenum(), - self.getname(), attr.lower())) - elif value not in validvalues[attr]: - validvalue = "|".join(v.lower() for v in validvalues[attr]) - messages.append( - "%s:%u: ERROR: function %s has invalid value (%s) for @%s (%s)" - % (self.getfilename(), self.getlinenum(), self.getname(), - value.lower(), attr.lower(), validvalue)) - return "\n".join(messages) - - def __str__(self): - '''Generate a string for this function''' - line = "{%s %s %s %s}" \ - % (self.getname(), - self.getaudience(), - self.getstability(), - self.getreplace()) - return line - - -def marked_as_ignored(file_path): - """Checks for the presence of the marker(SHELLDOC-IGNORE) to ignore the file. - - Marker needs to be in a line of its own and can not - be an inline comment. - - A leading '#' and white-spaces(leading or trailing) - are trimmed before checking equality. - - Comparison is case sensitive and the comment must be in - UPPERCASE. - """ - with open(file_path) as input_file: - for line_num, line in enumerate(input_file, 1): - if line.startswith("#") and line[1:].strip() == "SHELLDOC-IGNORE": - print >> sys.stderr, "Yo! Got an ignore directive in",\ - "file:{} on line number:{}".format(file_path, line_num) - return True - return False - - -def main(): - '''main entry point''' - parser = OptionParser( - usage="usage: %prog [--skipprnorep] " + "[--output OUTFILE|--lint] " + - "--input INFILE " + "[--input INFILE ...]", - epilog= - "You can mark a file to be ignored by shelldocs by adding" - " 'SHELLDOC-IGNORE' as comment in its own line." - ) - parser.add_option("-o", - "--output", - dest="outfile", - action="store", - type="string", - help="file to create", - metavar="OUTFILE") - parser.add_option("-i", - "--input", - dest="infile", - action="append", - type="string", - help="file to read", - metavar="INFILE") - parser.add_option("--skipprnorep", - dest="skipprnorep", - action="store_true", - help="Skip Private & Not Replaceable") - parser.add_option("--lint", - dest="lint", - action="store_true", - help="Enable lint mode") - parser.add_option( - "-V", - "--version", - dest="release_version", - action="store_true", - default=False, - help="display version information for shelldocs and exit.") - - (options, dummy_args) = parser.parse_args() - - if options.release_version: - with open( - os.path.join( - os.path.dirname(__file__), "../VERSION"), 'r') as ver_file: - print ver_file.read() - sys.exit(0) - - if options.infile is None: - parser.error("At least one input file needs to be supplied") - elif options.outfile is None and options.lint is None: - parser.error( - "At least one of output file and lint mode needs to be specified") - - allfuncs = [] - try: - for filename in options.infile: - with open(filename, "r") as shellcode: - # if the file contains a comment containing - # only "SHELLDOC-IGNORE" then skip that file - if marked_as_ignored(filename): - continue - funcdef = ShellFunction(filename) - linenum = 0 - for line in shellcode: - linenum = linenum + 1 - if line.startswith('## @description'): - funcdef.adddesc(line) - elif line.startswith('## @audience'): - funcdef.setaudience(line) - elif line.startswith('## @stability'): - funcdef.setstability(line) - elif line.startswith('## @replaceable'): - funcdef.setreplace(line) - elif line.startswith('## @param'): - funcdef.addparam(line) - elif line.startswith('## @return'): - funcdef.addreturn(line) - elif line.startswith('function'): - funcdef.setname(line) - funcdef.setlinenum(linenum) - if options.skipprnorep and \ - funcdef.getaudience() == "Private" and \ - funcdef.getreplace() == "No": - pass - else: - allfuncs.append(funcdef) - funcdef = ShellFunction(filename) - except IOError, err: - print >> sys.stderr, "ERROR: Failed to read from file: %s. Aborting." % err.filename - sys.exit(1) - - allfuncs = sorted(allfuncs) - - if options.lint: - for funcs in allfuncs: - message = funcs.lint() - if message: - print message - - if options.outfile is not None: - with open(options.outfile, "w") as outfile: - outfile.write(ASFLICENSE) - for line in toc(allfuncs): - outfile.write(line) - outfile.write("\n------\n\n") - - header = [] - for funcs in allfuncs: - if header != funcs.getinter(): - header = funcs.getinter() - line = "## %s\n" % (funcs.headerbuild()) - outfile.write(line) - outfile.write(funcs.getdocpage()) - - -if __name__ == "__main__": - main() http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/src/main/java/org/apache/yetus/shelldocs/ShellDocs.java ---------------------------------------------------------------------- diff --git a/shelldocs/src/main/java/org/apache/yetus/shelldocs/ShellDocs.java b/shelldocs/src/main/java/org/apache/yetus/shelldocs/ShellDocs.java new file mode 100644 index 0000000..facbc48 --- /dev/null +++ b/shelldocs/src/main/java/org/apache/yetus/shelldocs/ShellDocs.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.yetus.shelldocs; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import org.python.core.Py; +import org.python.core.PyException; +import org.python.core.PySystemState; +import org.python.util.PythonInterpreter; + +public class ShellDocs { + public static void main(final String[] args) throws PyException { + List<String> list = new LinkedList<String>(Arrays.asList(args)); + list.add(0,"shelldocs"); + String[] newargs = list.toArray(new String[list.size()]); + PythonInterpreter.initialize(System.getProperties(), System.getProperties(), newargs); + PySystemState systemState = Py.getSystemState(); + PythonInterpreter interpreter = new PythonInterpreter(); + systemState.__setattr__("_jy_interpreter", Py.java2py(interpreter)); + String command = "try:\n" + + " import shelldocs\n" + + " shelldocs.main()\n" + + "except" + + " SystemExit: pass"; + interpreter.exec(command); + } +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/src/main/python/shelldocs.py ---------------------------------------------------------------------- diff --git a/shelldocs/src/main/python/shelldocs.py b/shelldocs/src/main/python/shelldocs.py new file mode 100755 index 0000000..57f912f --- /dev/null +++ b/shelldocs/src/main/python/shelldocs.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python2 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.dont_write_bytecode = True +# pylint: disable=wrong-import-position,import-self +import shelldocs +# pylint: enable=wrong-import-position +# pylint: disable=no-member +shelldocs.main() http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/src/main/python/shelldocs/__init__.py ---------------------------------------------------------------------- diff --git a/shelldocs/src/main/python/shelldocs/__init__.py b/shelldocs/src/main/python/shelldocs/__init__.py new file mode 100755 index 0000000..a096cbe --- /dev/null +++ b/shelldocs/src/main/python/shelldocs/__init__.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Do this immediately to prevent compiled forms +import sys +import os +import re +from optparse import OptionParser + +sys.dont_write_bytecode = True + +ASFLICENSE = ''' +<!--- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--> +''' + + +def docstrip(key, dstr): + '''remove extra spaces from shelldoc phrase''' + dstr = re.sub("^## @%s " % key, "", dstr) + dstr = dstr.lstrip() + dstr = dstr.rstrip() + return dstr + + +def toc(tlist): + '''build a table of contents''' + tocout = [] + header = () + for i in tlist: + if header != i.getinter(): + header = i.getinter() + line = " * %s\n" % (i.headerbuild()) + tocout.append(line) + line = " * [%s](#%s)\n" % (i.getname().replace("_", r"\_"), + i.getname()) + tocout.append(line) + return tocout + + +class ShellFunction(object): + """a shell function""" + + def __init__(self, filename): + '''Initializer''' + self.name = None + self.audience = None + self.stability = None + self.replaceb = None + self.returnt = None + self.desc = None + self.params = None + self.filename = filename + self.linenum = 0 + + def __cmp__(self, other): + '''comparison''' + if self.audience == other.audience: + if self.stability == other.stability: + if self.replaceb == other.replaceb: + return cmp(self.name, other.name) + else: + if self.replaceb == "Yes": + return -1 + else: + if self.stability == "Stable": + return -1 + else: + if self.audience == "Public": + return -1 + return 1 + + def reset(self): + '''empties current function''' + self.name = None + self.audience = None + self.stability = None + self.replaceb = None + self.returnt = None + self.desc = None + self.params = None + self.linenum = 0 + self.filename = None + + def getfilename(self): + '''get the name of the function''' + if self.filename is None: + return "undefined" + return self.filename + + def setname(self, text): + '''set the name of the function''' + definition = text.split() + self.name = definition[1] + + def getname(self): + '''get the name of the function''' + if self.name is None: + return "None" + return self.name + + def setlinenum(self, linenum): + '''set the line number of the function''' + self.linenum = linenum + + def getlinenum(self): + '''get the line number of the function''' + return self.linenum + + def setaudience(self, text): + '''set the audience of the function''' + self.audience = docstrip("audience", text) + self.audience = self.audience.capitalize() + + def getaudience(self): + '''get the audience of the function''' + if self.audience is None: + return "None" + return self.audience + + def setstability(self, text): + '''set the stability of the function''' + self.stability = docstrip("stability", text) + self.stability = self.stability.capitalize() + + def getstability(self): + '''get the stability of the function''' + if self.stability is None: + return "None" + return self.stability + + def setreplace(self, text): + '''set the replacement state''' + self.replaceb = docstrip("replaceable", text) + self.replaceb = self.replaceb.capitalize() + + def getreplace(self): + '''get the replacement state''' + if self.replaceb == "Yes": + return self.replaceb + return "No" + + def getinter(self): + '''get the function state''' + return self.getaudience(), self.getstability(), self.getreplace() + + def addreturn(self, text): + '''add a return state''' + if self.returnt is None: + self.returnt = [] + self.returnt.append(docstrip("return", text)) + + def getreturn(self): + '''get the complete return state''' + if self.returnt is None: + return "Nothing" + return "\n\n".join(self.returnt) + + def adddesc(self, text): + '''add to the description''' + if self.desc is None: + self.desc = [] + self.desc.append(docstrip("description", text)) + + def getdesc(self): + '''get the description''' + if self.desc is None: + return "None" + return " ".join(self.desc) + + def addparam(self, text): + '''add a parameter''' + if self.params is None: + self.params = [] + self.params.append(docstrip("param", text)) + + def getparams(self): + '''get all of the parameters''' + if self.params is None: + return "" + return " ".join(self.params) + + def getusage(self): + '''get the usage string''' + line = "%s %s" % (self.name, self.getparams()) + return line.rstrip() + + def headerbuild(self): + '''get the header for this function''' + if self.getreplace() == "Yes": + replacetext = "Replaceable" + else: + replacetext = "Not Replaceable" + line = "%s/%s/%s" % (self.getaudience(), self.getstability(), + replacetext) + return line + + def getdocpage(self): + '''get the built document page for this function''' + line = "### `%s`\n\n"\ + "* Synopsis\n\n"\ + "```\n%s\n"\ + "```\n\n" \ + "* Description\n\n" \ + "%s\n\n" \ + "* Returns\n\n" \ + "%s\n\n" \ + "| Classification | Level |\n" \ + "| :--- | :--- |\n" \ + "| Audience | %s |\n" \ + "| Stability | %s |\n" \ + "| Replaceable | %s |\n\n" \ + % (self.getname(), + self.getusage(), + self.getdesc(), + self.getreturn(), + self.getaudience(), + self.getstability(), + self.getreplace()) + return line + + def lint(self): + '''Lint this function''' + getfuncs = { + "audience": self.getaudience, + "stability": self.getstability, + "replaceable": self.getreplace, + } + validvalues = { + "audience": ("Public", "Private"), + "stability": ("Stable", "Evolving"), + "replaceable": ("Yes", "No"), + } + messages = [] + for attr in ("audience", "stability", "replaceable"): + value = getfuncs[attr]() + if value == "None": + messages.append("%s:%u: ERROR: function %s has no @%s" % + (self.getfilename(), self.getlinenum(), + self.getname(), attr.lower())) + elif value not in validvalues[attr]: + validvalue = "|".join(v.lower() for v in validvalues[attr]) + messages.append( + "%s:%u: ERROR: function %s has invalid value (%s) for @%s (%s)" + % (self.getfilename(), self.getlinenum(), self.getname(), + value.lower(), attr.lower(), validvalue)) + return "\n".join(messages) + + def __str__(self): + '''Generate a string for this function''' + line = "{%s %s %s %s}" \ + % (self.getname(), + self.getaudience(), + self.getstability(), + self.getreplace()) + return line + + +def marked_as_ignored(file_path): + """Checks for the presence of the marker(SHELLDOC-IGNORE) to ignore the file. + + Marker needs to be in a line of its own and can not + be an inline comment. + + A leading '#' and white-spaces(leading or trailing) + are trimmed before checking equality. + + Comparison is case sensitive and the comment must be in + UPPERCASE. + """ + with open(file_path) as input_file: + for line_num, line in enumerate(input_file, 1): + if line.startswith("#") and line[1:].strip() == "SHELLDOC-IGNORE": + print >> sys.stderr, "Yo! Got an ignore directive in",\ + "file:{} on line number:{}".format(file_path, line_num) + return True + return False + + +def main(): + '''main entry point''' + parser = OptionParser( + usage="usage: %prog [--skipprnorep] " + "[--output OUTFILE|--lint] " + + "--input INFILE " + "[--input INFILE ...]", + epilog= + "You can mark a file to be ignored by shelldocs by adding" + " 'SHELLDOC-IGNORE' as comment in its own line." + ) + parser.add_option("-o", + "--output", + dest="outfile", + action="store", + type="string", + help="file to create", + metavar="OUTFILE") + parser.add_option("-i", + "--input", + dest="infile", + action="append", + type="string", + help="file to read", + metavar="INFILE") + parser.add_option("--skipprnorep", + dest="skipprnorep", + action="store_true", + help="Skip Private & Not Replaceable") + parser.add_option("--lint", + dest="lint", + action="store_true", + help="Enable lint mode") + parser.add_option( + "-V", + "--version", + dest="release_version", + action="store_true", + default=False, + help="display version information for shelldocs and exit.") + + (options, dummy_args) = parser.parse_args() + + if options.release_version: + with open( + os.path.join( + os.path.dirname(__file__), "../VERSION"), 'r') as ver_file: + print ver_file.read() + sys.exit(0) + + if options.infile is None: + parser.error("At least one input file needs to be supplied") + elif options.outfile is None and options.lint is None: + parser.error( + "At least one of output file and lint mode needs to be specified") + + allfuncs = [] + try: + for filename in options.infile: + with open(filename, "r") as shellcode: + # if the file contains a comment containing + # only "SHELLDOC-IGNORE" then skip that file + if marked_as_ignored(filename): + continue + funcdef = ShellFunction(filename) + linenum = 0 + for line in shellcode: + linenum = linenum + 1 + if line.startswith('## @description'): + funcdef.adddesc(line) + elif line.startswith('## @audience'): + funcdef.setaudience(line) + elif line.startswith('## @stability'): + funcdef.setstability(line) + elif line.startswith('## @replaceable'): + funcdef.setreplace(line) + elif line.startswith('## @param'): + funcdef.addparam(line) + elif line.startswith('## @return'): + funcdef.addreturn(line) + elif line.startswith('function'): + funcdef.setname(line) + funcdef.setlinenum(linenum) + if options.skipprnorep and \ + funcdef.getaudience() == "Private" and \ + funcdef.getreplace() == "No": + pass + else: + allfuncs.append(funcdef) + funcdef = ShellFunction(filename) + except IOError, err: + print >> sys.stderr, "ERROR: Failed to read from file: %s. Aborting." % err.filename + sys.exit(1) + + allfuncs = sorted(allfuncs) + + if options.lint: + for funcs in allfuncs: + message = funcs.lint() + if message: + print message + + if options.outfile is not None: + with open(options.outfile, "w") as outfile: + outfile.write(ASFLICENSE) + for line in toc(allfuncs): + outfile.write(line) + outfile.write("\n------\n\n") + + header = [] + for funcs in allfuncs: + if header != funcs.getinter(): + header = funcs.getinter() + line = "## %s\n" % (funcs.headerbuild()) + outfile.write(line) + outfile.write(funcs.getdocpage()) + + +if __name__ == "__main__": + main() http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/start-build-env.sh ---------------------------------------------------------------------- diff --git a/start-build-env.sh b/start-build-env.sh new file mode 100755 index 0000000..e773e4a --- /dev/null +++ b/start-build-env.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e # exit on error +ROOTDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")" >/dev/null && pwd -P) + +# moving to the path of the Dockerfile reduces the context +cd "${ROOTDIR}/precommit/src/main/shell/test-patch-docker" + +BRANCH=$(git branch | grep '\*' | cut -d ' ' -f2) +if [[ "${BRANCH}" =~ HEAD ]]; then + BRANCH=$(git branch | grep '\*' | awk '{print $NF}' | sed -e s,rel/,,g -e s,\),,g ) +fi + +echo "Attempting a pull of apache/yetus-base:${BRANCH} and apache/yetus-base:latest to save time" +echo "Errors here will be ignored!" +docker pull "apache/yetus-base:${BRANCH}" || docker pull "apache/yetus-base:latest" || true + +docker build -t "apache/yetus-build:${BRANCH}" . + +USER_NAME=${SUDO_USER:=$USER} +USER_ID=$(id -u "${USER_NAME}") +GROUP_ID=$(id -g "${USER_NAME}") + +# When using SELinux, mounted directories may not be accessible +# to the container. To work around this, with Docker prior to 1.7 +# one needs to run the "chcon -Rt svirt_sandbox_file_t" command on +# the directories. With Docker 1.7 and later the z mount option +# does this automatically. +if command -v selinuxenabled >/dev/null && selinuxenabled; then + DCKR_VER=$(docker -v| + awk '$1 == "Docker" && $2 == "version" {split($3,ver,".");print ver[1]"."ver[2]}') + DCKR_MAJ=${DCKR_VER%.*} + DCKR_MIN=${DCKR_VER#*.} + if [[ "${DCKR_MAJ}" -eq 1 ]] && [[ "${DCKR_MIN}" -ge 7 ]] || + [[ "${DCKR_MAJ}" -gt 1 ]]; then + V_OPTS=:z + else + for d in "${PWD}" "${HOME}/.m2"; do + ctx=$(stat --printf='%C' "$d"|cut -d':' -f3) + if [ "$ctx" != svirt_sandbox_file_t ] && [ "$ctx" != container_file_t ]; then + printf 'INFO: SELinux is enabled.\n' + printf '\tMounted %s may not be accessible to the container.\n' "$d" + printf 'INFO: If so, on the host, run the following command:\n' + printf '\t# chcon -Rt svirt_sandbox_file_t %s\n' "$d" + fi + done + fi +fi + +cd "${ROOTDIR}/asf-site-src" +docker build \ + -t "apache/yetus-build-${USER_ID}:${BRANCH}" \ + --build-arg GROUP_ID="${GROUP_ID}" \ + --build-arg USER_ID="${USER_ID}" \ + --build-arg USER_NAME="${USER_NAME}" \ + --build-arg DOCKER_TAG="${BRANCH}" \ + . + +# now cd back +cd "${ROOTDIR}" +# By mapping the .m2 directory you can do an mvn install from +# within the container and use the result on your normal +# system. And this also is a significant speedup in subsequent +# builds because the dependencies are downloaded only once. +# Additionally, we mount GPG and SSH directories so that +# release managers can use the container to do releases +docker run --rm=true -i -t \ + -v "${PWD}:/home/${USER_NAME}/yetus${V_OPTS:-}" \ + -w "/home/${USER_NAME}/yetus" \ + -v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}" \ + -v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg" \ + -v "${HOME}/.ssh:/home/${USER_NAME}/.ssh" \ + -u "${USER_NAME}" \ + "apache/yetus-build-${USER_ID}:${BRANCH}" "$@"
