http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/releasedocmaker.py ---------------------------------------------------------------------- diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py deleted file mode 100755 index ae3ce45..0000000 --- a/dev-support/releasedocmaker.py +++ /dev/null @@ -1,642 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from glob import glob -from optparse import OptionParser -from time import gmtime, strftime -from distutils.version import LooseVersion -import os -import re -import sys -import urllib -import urllib2 -try: - import json -except ImportError: - import simplejson as json - -RELEASE_VERSION = {} -NAME_PATTERN = re.compile(r' \([0-9]+\)') -RELNOTE_PATTERN = re.compile('^\<\!\-\- ([a-z]+) \-\-\>') - -ASF_LICENSE = ''' -<!--- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---> -''' - -def clean(_str): - return tableclean(re.sub(NAME_PATTERN, "", _str)) - -def format_components(_str): - _str = re.sub(NAME_PATTERN, '', _str).replace("'", "") - if _str != "": - ret = _str - else: - # some markdown parsers don't like empty tables - ret = "." - return clean(ret) - -# convert to utf-8 -# protect some known md metachars -# or chars that screw up doxia -def tableclean(_str): - _str = _str.encode('utf-8') - _str = _str.replace("_", r"\_") - _str = _str.replace("\r", "") - _str = _str.rstrip() - return _str - -# same thing as tableclean, -# except table metachars are also -# escaped as well as more -# things we don't want doxia to -# screw up -def notableclean(_str): - _str = tableclean(_str) - _str = _str.replace("|", r"\|") - _str = _str.replace("<", r"\<") - _str = _str.replace(">", r"\>") - _str = _str.replace("*", r"\*") - _str = _str.rstrip() - return _str - -# if release notes have a special marker, -# we'll treat them as already in markdown format -def processrelnote(_str): - fmt = RELNOTE_PATTERN.match(_str) - if fmt is None: - return notableclean(_str) - else: - return { - 'markdown' : tableclean(_str), - }.get(fmt.group(1),notableclean(_str)) - -# clean output dir -def clean_output_dir(directory): - files = os.listdir(directory) - for name in files: - os.remove(os.path.join(directory, name)) - os.rmdir(directory) - -def mstr(obj): - if obj is None: - return "" - return unicode(obj) - -def buildindex(title, asf_license): - """Write an index file for later conversion using mvn site""" - versions = glob("[0-9]*.[0-9]*.[0-9]*") - versions.sort(key=LooseVersion, reverse=True) - with open("index.md", "w") as indexfile: - if asf_license is True: - indexfile.write(ASF_LICENSE) - for version in versions: - indexfile.write("* %s v%s\n" % (title, version)) - for k in ("Changes", "Release Notes"): - indexfile.write(" * [%s](%s/%s.%s.html)\n" \ - % (k, version, k.upper().replace(" ", ""), version)) - -def buildreadme(title, asf_license): - """Write an index file for Github using README.md""" - versions = glob("[0-9]*.[0-9]*.[0-9]*") - versions.sort(key=LooseVersion, reverse=True) - with open("README.md", "w") as indexfile: - if asf_license is True: - indexfile.write(ASF_LICENSE) - for version in versions: - indexfile.write("* %s v%s\n" % (title, version)) - for k in ("Changes", "Release Notes"): - indexfile.write(" * [%s](%s/%s.%s.md)\n" \ - % (k, version, k.upper().replace(" ", ""), version)) - -class GetVersions(object): - """ List of version strings """ - def __init__(self, versions, projects): - versions = versions - projects = projects - self.newversions = [] - versions.sort(key=LooseVersion) - print "Looking for %s through %s"%(versions[0], versions[-1]) - for project in projects: - url = "https://issues.apache.org/jira/rest/api/2/project/%s/versions"; % project - resp = urllib2.urlopen(url) - datum = json.loads(resp.read()) - for data in datum: - name = data['name'] - if name[0].isdigit and versions[0] <= name and name <= versions[-1]: - print "Adding %s to the list" % name - self.newversions.append(name) - newlist = list(set(self.newversions)) - self.newversions = newlist - - def getlist(self): - return self.newversions - -class Version(object): - """Represents a version number""" - def __init__(self, data): - self.mod = False - self.data = data - found = re.match(r'^((\d+)(\.\d+)*).*$', data) - if found: - self.parts = [int(p) for p in found.group(1).split('.')] - else: - self.parts = [] - # backfill version with zeroes if missing parts - self.parts.extend((0,) * (3 - len(self.parts))) - - def __str__(self): - if self.mod: - return '.'.join([str(p) for p in self.parts]) - return self.data - - def __cmp__(self, other): - return cmp(self.parts, other.parts) - -class Jira(object): - """A single JIRA""" - - def __init__(self, data, parent): - self.key = data['key'] - self.fields = data['fields'] - self.parent = parent - self.notes = None - self.incompat = None - self.reviewed = None - self.important = None - - def get_id(self): - return mstr(self.key) - - def get_description(self): - return mstr(self.fields['description']) - - def get_release_note(self): - if self.notes is None: - field = self.parent.field_id_map['Release Note'] - if self.fields.has_key(field): - self.notes = mstr(self.fields[field]) - elif self.get_incompatible_change() or self.get_important(): - self.notes = self.get_description() - else: - self.notes = "" - return self.notes - - def get_priority(self): - ret = "" - pri = self.fields['priority'] - if pri is not None: - ret = pri['name'] - return mstr(ret) - - def get_assignee(self): - ret = "" - mid = self.fields['assignee'] - if mid is not None: - ret = mid['displayName'] - return mstr(ret) - - def get_components(self): - if len(self.fields['components']) > 0: - return ", ".join([comp['name'] for comp in self.fields['components']]) - else: - return "" - - def get_summary(self): - return self.fields['summary'] - - def get_type(self): - ret = "" - mid = self.fields['issuetype'] - if mid is not None: - ret = mid['name'] - return mstr(ret) - - def get_reporter(self): - ret = "" - mid = self.fields['reporter'] - if mid is not None: - ret = mid['displayName'] - return mstr(ret) - - def get_project(self): - ret = "" - mid = self.fields['project'] - if mid is not None: - ret = mid['key'] - return mstr(ret) - - def __cmp__(self, other): - selfsplit = self.get_id().split('-') - othersplit = other.get_id().split('-') - result = cmp(selfsplit[0], othersplit[0]) - if result != 0: - return result - else: - if selfsplit[1] < othersplit[1]: - return True - elif selfsplit[1] > othersplit[1]: - return False - return False - - def get_incompatible_change(self): - if self.incompat is None: - field = self.parent.field_id_map['Hadoop Flags'] - self.reviewed = False - self.incompat = False - if self.fields.has_key(field): - if self.fields[field]: - for flag in self.fields[field]: - if flag['value'] == "Incompatible change": - self.incompat = True - if flag['value'] == "Reviewed": - self.reviewed = True - return self.incompat - - def get_important(self): - if self.important is None: - field = self.parent.field_id_map['Flags'] - self.important = False - if self.fields.has_key(field): - if self.fields[field]: - for flag in self.fields[field]: - if flag['value'] == "Important": - self.important = True - return self.important - - def check_missing_component(self): - if len(self.fields['components']) > 0: - return False - return True - - def check_missing_assignee(self): - if self.fields['assignee'] is not None: - return False - return True - - def check_version_string(self): - field = self.parent.field_id_map['Fix Version/s'] - for ver in self.fields[field]: - found = re.match(r'^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', ver['name']) - if not found: - return True - return False - - def get_release_date(self, version): - fix_versions = self.fields['fixVersions'] - for j in range(len(fix_versions)): - if fix_versions[j] == version: - return fix_versions[j]['releaseDate'] - return None - -class JiraIter(object): - """An Iterator of JIRAs""" - - def __init__(self, version, projects): - self.version = version - self.projects = projects - ver = str(version).replace("-SNAPSHOT", "") - - resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field";) - data = json.loads(resp.read()) - - self.field_id_map = {} - for part in data: - self.field_id_map[part['name']] = part['id'] - - self.jiras = [] - pos = 0 - end = 1 - count = 100 - while pos < end: - pjs = "','".join(projects) - jql = "project in ('%s') and fixVersion in ('%s') and resolution = Fixed" % (pjs, ver) - params = urllib.urlencode({'jql': jql, 'startAt':pos, 'maxResults':count}) - resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"; % params) - data = json.loads(resp.read()) - if data.has_key('error_messages'): - raise Exception(data['error_messages']) - pos = data['startAt'] + data['maxResults'] - end = data['total'] - self.jiras.extend(data['issues']) - - needaversion = False - if ver not in RELEASE_VERSION: - needaversion = True - - if needaversion is True: - issues = data['issues'] - for i in range(len(issues)): - fix_versions = issues[i]['fields']['fixVersions'] - for j in range(len(fix_versions)): - fields = fix_versions[j] - if 'releaseDate' in fields: - RELEASE_VERSION[fields['name']] = fields['releaseDate'] - - self.iter = self.jiras.__iter__() - - def __iter__(self): - return self - - def next(self): - data = self.iter.next() - j = Jira(data, self) - return j - -class Outputs(object): - """Several different files to output to at the same time""" - - def __init__(self, base_file_name, file_name_pattern, keys, params=None): - if params is None: - params = {} - self.params = params - self.base = open(base_file_name%params, 'w') - self.others = {} - for key in keys: - both = dict(params) - both['key'] = key - self.others[key] = open(file_name_pattern%both, 'w') - - def write_all(self, pattern): - both = dict(self.params) - both['key'] = '' - self.base.write(pattern%both) - for key in self.others.keys(): - both = dict(self.params) - both['key'] = key - self.others[key].write(pattern%both) - - def write_key_raw(self, key, _str): - self.base.write(_str) - if self.others.has_key(key): - self.others[key].write(_str) - - def close(self): - self.base.close() - for value in self.others.values(): - value.close() - - def write_list(self, mylist): - for jira in sorted(mylist): - line = '| [%s](https://issues.apache.org/jira/browse/%s) | %s | %s | %s | %s | %s |\n' - line = line % (notableclean(jira.get_id()), - notableclean(jira.get_id()), - notableclean(jira.get_summary()), - notableclean(jira.get_priority()), - format_components(jira.get_components()), - notableclean(jira.get_reporter()), - notableclean(jira.get_assignee())) - self.write_key_raw(jira.get_project(), line) - -def main(): - usage = "usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]" - parser = OptionParser(usage=usage, - epilog="Markdown-formatted CHANGES and RELEASENOTES files will be stored" - "in a directory named after the highest version provided.") - parser.add_option("-i", "--index", dest="index", action="store_true", - default=False, help="build an index file") - parser.add_option("-l", "--license", dest="license", action="store_false", - default=True, help="Add an ASF license") - parser.add_option("-n", "--lint", dest="lint", action="store_true", - help="use lint flag to exit on failures") - parser.add_option("-p", "--project", dest="projects", - action="append", type="string", - help="projects in JIRA to include in releasenotes", metavar="PROJECT") - parser.add_option("-r", "--range", dest="range", action="store_true", - default=False, help="Given versions are a range") - parser.add_option("-t", "--projecttitle", dest="title", type="string", - help="Title to use for the project (default is Apache PROJECT)") - parser.add_option("-u", "--usetoday", dest="usetoday", action="store_true", - default=False, help="use current date for unreleased versions") - parser.add_option("-v", "--version", dest="versions", action="append", type="string", - help="versions in JIRA to include in releasenotes", metavar="VERSION") - (options, _) = parser.parse_args() - - if options.versions is None: - parser.error("At least one version needs to be supplied") - - proxy = urllib2.ProxyHandler() - opener = urllib2.build_opener(proxy) - urllib2.install_opener(opener) - - projects = options.projects - if projects is None: - parser.error("At least one project needs to be supplied") - - if options.range is True: - versions = [Version(v) for v in GetVersions(options.versions, projects).getlist()] - else: - versions = [Version(v) for v in options.versions] - versions.sort() - - if options.title is None: - title = projects[0] - else: - title = options.title - - haderrors = False - - for version in versions: - vstr = str(version) - jlist = JiraIter(vstr, projects) - - if vstr in RELEASE_VERSION: - reldate = RELEASE_VERSION[vstr] - elif options.usetoday: - reldate = strftime("%Y-%m-%d", gmtime()) - else: - reldate = "Unreleased (as of %s)" % strftime("%Y-%m-%d", gmtime()) - - if not os.path.exists(vstr): - os.mkdir(vstr) - - reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", - "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", - [], {"ver":version, "date":reldate, "title":title}) - choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md", - "%(ver)s/CHANGES.%(key)s.%(ver)s.md", - [], {"ver":version, "date":reldate, "title":title}) - - if options.license is True: - reloutputs.write_all(ASF_LICENSE) - choutputs.write_all(ASF_LICENSE) - - relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \ - 'These release notes cover new developer and user-facing ' \ - 'incompatibilities, important issues, features, and major improvements.\n\n' - chhead = '# %(title)s Changelog\n\n' \ - '## Release %(ver)s - %(date)s\n'\ - '\n' - - reloutputs.write_all(relhead) - choutputs.write_all(chhead) - error_count = 0 - warning_count = 0 - lint_message = "" - incompatlist = [] - importantlist = [] - buglist = [] - improvementlist = [] - newfeaturelist = [] - subtasklist = [] - tasklist = [] - testlist = [] - otherlist = [] - - for jira in sorted(jlist): - if jira.get_incompatible_change(): - incompatlist.append(jira) - elif jira.get_important(): - importantlist.append(jira) - elif jira.get_type() == "Bug": - buglist.append(jira) - elif jira.get_type() == "Improvement": - improvementlist.append(jira) - elif jira.get_type() == "New Feature": - newfeaturelist.append(jira) - elif jira.get_type() == "Sub-task": - subtasklist.append(jira) - elif jira.get_type() == "Task": - tasklist.append(jira) - elif jira.get_type() == "Test": - testlist.append(jira) - else: - otherlist.append(jira) - - line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \ - % (notableclean(jira.get_id()), notableclean(jira.get_id()), - notableclean(jira.get_priority()), notableclean(jira.get_summary())) - - if jira.get_incompatible_change() and len(jira.get_release_note()) == 0: - warning_count += 1 - reloutputs.write_key_raw(jira.get_project(), "\n---\n\n") - reloutputs.write_key_raw(jira.get_project(), line) - line = '\n**WARNING: No release note provided for this incompatible change.**\n\n' - lint_message += "\nWARNING: incompatible change %s lacks release notes." % \ - (notableclean(jira.get_id())) - reloutputs.write_key_raw(jira.get_project(), line) - - if jira.get_important() and len(jira.get_release_note()) == 0: - warning_count += 1 - reloutputs.write_key_raw(jira.get_project(), "\n---\n\n") - reloutputs.write_key_raw(jira.get_project(), line) - line = '\n**WARNING: No release note provided for this important issue.**\n\n' - lint_message += "\nWARNING: important issue %s lacks release notes." % \ - (notableclean(jira.get_id())) - reloutputs.write_key_raw(jira.get_project(), line) - - if jira.check_version_string(): - warning_count += 1 - lint_message += "\nWARNING: Version string problem for %s " % jira.get_id() - - if jira.check_missing_component() or jira.check_missing_assignee(): - error_count += 1 - error_message = [] - if jira.check_missing_component(): - error_message.append("component") - if jira.check_missing_assignee(): - error_message.append("assignee") - lint_message += "\nERROR: missing %s for %s " \ - % (" and ".join(error_message), jira.get_id()) - - if len(jira.get_release_note()) > 0: - reloutputs.write_key_raw(jira.get_project(), "\n---\n\n") - reloutputs.write_key_raw(jira.get_project(), line) - line = '\n%s\n\n' % (processrelnote(jira.get_release_note())) - reloutputs.write_key_raw(jira.get_project(), line) - - if options.lint is True: - print lint_message - print "=======================================" - print "%s: Error:%d, Warning:%d \n" % (vstr, error_count, warning_count) - if error_count > 0: - haderrors = True - clean_output_dir(vstr) - continue - - reloutputs.write_all("\n\n") - reloutputs.close() - - choutputs.write_all("### INCOMPATIBLE CHANGES:\n\n") - choutputs.write_all("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.write_all("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.write_list(incompatlist) - - choutputs.write_all("\n\n### IMPORTANT ISSUES:\n\n") - choutputs.write_all("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.write_all("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.write_list(importantlist) - - choutputs.write_all("\n\n### NEW FEATURES:\n\n") - choutputs.write_all("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.write_all("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.write_list(newfeaturelist) - - choutputs.write_all("\n\n### IMPROVEMENTS:\n\n") - choutputs.write_all("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.write_all("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.write_list(improvementlist) - - choutputs.write_all("\n\n### BUG FIXES:\n\n") - choutputs.write_all("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.write_all("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.write_list(buglist) - - choutputs.write_all("\n\n### TESTS:\n\n") - choutputs.write_all("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.write_all("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.write_list(testlist) - - choutputs.write_all("\n\n### SUB-TASKS:\n\n") - choutputs.write_all("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.write_all("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.write_list(subtasklist) - - choutputs.write_all("\n\n### OTHER:\n\n") - choutputs.write_all("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") - choutputs.write_all("|:---- |:---- | :--- |:---- |:---- |:---- |\n") - choutputs.write_list(otherlist) - choutputs.write_list(tasklist) - - choutputs.write_all("\n\n") - choutputs.close() - - if options.index: - buildindex(title, options.license) - buildreadme(title, options.license) - - if haderrors is True: - sys.exit(1) - -if __name__ == "__main__": - main()
http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/shelldocs.py ---------------------------------------------------------------------- diff --git a/dev-support/shelldocs.py b/dev-support/shelldocs.py deleted file mode 100755 index 7a1fe21..0000000 --- a/dev-support/shelldocs.py +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import sys -import string -from optparse import OptionParser - -asflicense=''' -<!--- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---> -''' - -def docstrip(key,string): - string=re.sub("^## @%s " % key ,"",string) - string=string.lstrip() - string=string.rstrip() - return string - -def toc(list): - tocout=[] - header=() - for i in list: - if header != i.getinter(): - header=i.getinter() - line=" * %s\n" % (i.headerbuild()) - tocout.append(line) - line=" * [%s](#%s)\n" % (i.getname().replace("_","\_"),i.getname()) - tocout.append(line) - return tocout - -class ShellFunction: - def __init__(self): - self.reset() - - def __cmp__(self,other): - if (self.audience == other.audience): - if (self.stability == other.stability): - if (self.replaceb == other.replaceb): - return(cmp(self.name,other.name)) - else: - if (self.replaceb == "Yes"): - return -1 - else: - return 1 - else: - if (self.stability == "Stable"): - return -1 - else: - return 1 - else: - if (self.audience == "Public"): - return -1 - else: - return 1 - - def reset(self): - self.name=None - self.audience=None - self.stability=None - self.replaceb=None - self.returnt=None - self.desc=None - self.params=None - - def setname(self,text): - definition=text.split(); - self.name=definition[1] - - def getname(self): - if (self.name is None): - return "None" - else: - return self.name - - def setaudience(self,text): - self.audience=docstrip("audience",text) - self.audience=self.audience.capitalize() - - def getaudience(self): - if (self.audience is None): - return "None" - else: - return self.audience - - def setstability(self,text): - self.stability=docstrip("stability",text) - self.stability=self.stability.capitalize() - - def getstability(self): - if (self.stability is None): - return "None" - else: - return self.stability - - def setreplace(self,text): - self.replaceb=docstrip("replaceable",text) - self.replaceb=self.replaceb.capitalize() - - def getreplace(self): - if (self.replaceb is None): - return "None" - else: - return self.replaceb - - def getinter(self): - return( (self.getaudience(), self.getstability(), self.getreplace())) - - def addreturn(self,text): - if (self.returnt is None): - self.returnt = [] - self.returnt.append(docstrip("return",text)) - - def getreturn(self): - if (self.returnt is None): - return "Nothing" - else: - return "\n\n".join(self.returnt) - - def adddesc(self,text): - if (self.desc is None): - self.desc = [] - self.desc.append(docstrip("description",text)) - - def getdesc(self): - if (self.desc is None): - return "None" - else: - return " ".join(self.desc) - - def addparam(self,text): - if (self.params is None): - self.params = [] - self.params.append(docstrip("param",text)) - - def getparams(self): - if (self.params is None): - return "" - else: - return " ".join(self.params) - - def getusage(self): - line="%s %s" % (self.name, self.getparams()) - return line - - def headerbuild(self): - if self.getreplace() == "Yes": - replacetext="Replaceable" - else: - replacetext="Not Replaceable" - line="%s/%s/%s" % (self.getaudience(), self.getstability(), replacetext) - return(line) - - def getdocpage(self): - line="### `%s`\n\n"\ - "* Synopsis\n\n"\ - "```\n%s\n"\ - "```\n\n" \ - "* Description\n\n" \ - "%s\n\n" \ - "* Returns\n\n" \ - "%s\n\n" \ - "| Classification | Level |\n" \ - "| :--- | :--- |\n" \ - "| Audience | %s |\n" \ - "| Stability | %s |\n" \ - "| Replaceable | %s |\n\n" \ - % (self.getname(), - self.getusage(), - self.getdesc(), - self.getreturn(), - self.getaudience(), - self.getstability(), - self.getreplace()) - return line - - def __str__(self): - line="{%s %s %s %s}" \ - % (self.getname(), - self.getaudience(), - self.getstability(), - self.getreplace()) - return line - -def main(): - parser=OptionParser(usage="usage: %prog --skipprnorep --output OUTFILE --input INFILE [--input INFILE ...]") - parser.add_option("-o","--output", dest="outfile", - action="store", type="string", - help="file to create", metavar="OUTFILE") - parser.add_option("-i","--input", dest="infile", - action="append", type="string", - help="file to read", metavar="INFILE") - parser.add_option("--skipprnorep", dest="skipprnorep", - action="store_true", help="Skip Private & Not Replaceable") - - (options, args)=parser.parse_args() - - allfuncs=[] - for filename in options.infile: - with open(filename,"r") as shellcode: - funcdef=ShellFunction() - for line in shellcode: - if line.startswith('## @description'): - funcdef.adddesc(line) - elif line.startswith('## @audience'): - funcdef.setaudience(line) - elif line.startswith('## @stability'): - funcdef.setstability(line) - elif line.startswith('## @replaceable'): - funcdef.setreplace(line) - elif line.startswith('## @param'): - funcdef.addparam(line) - elif line.startswith('## @return'): - funcdef.addreturn(line) - elif line.startswith('function'): - funcdef.setname(line) - if options.skipprnorep and \ - funcdef.getaudience() == "Private" and \ - funcdef.getreplace() == "No": - pass - else: - allfuncs.append(funcdef) - funcdef=ShellFunction() - - allfuncs=sorted(allfuncs) - - outfile=open(options.outfile, "w") - outfile.write(asflicense) - for line in toc(allfuncs): - outfile.write(line) - - outfile.write("\n------\n\n") - - header=[] - for funcs in allfuncs: - if header != funcs.getinter(): - header=funcs.getinter() - line="## %s\n" % (funcs.headerbuild()) - outfile.write(line) - outfile.write(funcs.getdocpage()) - outfile.close() - -if __name__ == "__main__": - main() - http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/smart-apply-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh deleted file mode 100755 index ee06030..0000000 --- a/dev-support/smart-apply-patch.sh +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Make sure that bash version meets the pre-requisite - -if [[ -z "${BASH_VERSINFO}" ]] \ - || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ - || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then - echo "bash v3.2+ is required. Sorry." - exit 1 -fi - -this="${BASH_SOURCE-$0}" -BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) -#shellcheck disable=SC2034 -QATESTMODE=false - -# dummy functions -function add_vote_table -{ - true -} - -function add_footer_table -{ - true -} - -function big_console_header -{ - true -} - -function add_test -{ - true -} - -## @description Clean the filesystem as appropriate and then exit -## @audience private -## @stability evolving -## @replaceable no -## @param runresult -function cleanup_and_exit -{ - local result=$1 - - if [[ ${PATCH_DIR} =~ ^/tmp/yetus - && -d ${PATCH_DIR} ]]; then - rm -rf "${PATCH_DIR}" - fi - - # shellcheck disable=SC2086 - exit ${result} -} - -## @description Setup the default global variables -## @audience public -## @stability stable -## @replaceable no -function setup_defaults -{ - common_defaults -} - -## @description Print the usage information -## @audience public -## @stability stable -## @replaceable no -function yetus_usage -{ - echo "Usage: smart-apply-patch.sh [options] patch" - echo - echo "--committer Apply patches like a boss." - echo "--debug If set, then output some extra stuff to stderr" - echo "--dry-run Check for patch viability without applying" - echo "--list-plugins List all installed plug-ins and then exit" - echo "--modulelist=<list> Specify additional modules to test (comma delimited)" - echo "--offline Avoid connecting to the Internet" - echo "--patch-dir=<dir> The directory for working and output files (default '/tmp/yetus-(random))" - echo "--personality=<file> The personality file to load" - echo "--plugins=<list> Specify which plug-ins to add/delete (comma delimited; use 'all' for all found)" - echo "--project=<name> The short name for project currently using test-patch (default 'yetus')" - echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d" - echo "--user-plugins=<dir> A directory of user provided plugins. see test-patch.d for examples (default empty)" - echo "" - echo "Shell binary overrides:" - echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')" - echo "--curl-cmd=<cmd> The 'curl' command to use (default 'curl')" - echo "--diff-cmd=<cmd> The GNU-compatible 'diff' command to use (default 'diff')" - echo "--file-cmd=<cmd> The 'file' command to use (default 'file')" - echo "--git-cmd=<cmd> The 'git' command to use (default 'git')" - echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')" - echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')" - echo "--sed-cmd=<cmd> The 'sed' command to use (default 'sed')" - - importplugins - - unset TESTFORMATS - unset TESTTYPES - unset BUILDTOOLS - - for plugin in ${BUGSYSTEMS}; do - if declare -f ${plugin}_usage >/dev/null 2>&1; then - echo - "${plugin}_usage" - fi - done -} - -## @description Interpret the command line parameters -## @audience private -## @stability stable -## @replaceable no -## @params $@ -## @return May exit on failure -function parse_args -{ - local i - - common_args "$@" - - for i in "$@"; do - case ${i} in - --committer) - COMMITMODE=true - ;; - --dry-run) - PATCH_DRYRUNMODE=true - ;; - --*) - ## PATCH_OR_ISSUE can't be a --. So this is probably - ## a plugin thing. - continue - ;; - *) - PATCH_OR_ISSUE=${i#*=} - ;; - esac - done - - if [[ ! -d ${PATCH_DIR} ]]; then - mkdir -p "${PATCH_DIR}" - if [[ $? != 0 ]] ; then - yetus_error "ERROR: Unable to create ${PATCH_DIR}" - cleanup_and_exit 1 - fi - fi -} - -## @description git am dryrun -## @replaceable no -## @audience private -## @stability evolving -function gitam_dryrun -{ - - # there is no dryrun method for git-am, so just - # use apply instead. - gitapply_dryrun "$@" - - if [[ ${PATCH_METHOD} = "gitapply" ]]; then - PATCH_METHOD="gitam" - fi -} - -## @description git am signoff -## @replaceable no -## @audience private -## @stability evolving -function gitam_apply -{ - declare patchfile=$1 - - echo "Applying the patch:" - yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-am.log" \ - "${GIT}" am --signoff --whitespace=fix "-p${PATCH_LEVEL}" "${patchfile}" - ${GREP} -v "^Checking" "${PATCH_DIR}/apply-patch-git-am.log" -} - -## @description import core library routines -## @audience private -## @stability evolving -function import_core -{ - declare filename - - for filename in "${BINDIR}/core.d"/*; do - # shellcheck disable=SC1091 - # shellcheck source=core.d/01-common.sh - . "${filename}" - done -} - -trap "cleanup_and_exit 1" HUP INT QUIT TERM - -import_core - -setup_defaults - -parse_args "$@" - -importplugins -yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list" -unset BUILDTOOLS -unset TESTTYPES -unset TESTFORMATS - -parse_args_plugins "$@" - -plugins_initialize - -locate_patch - -if [[ ${COMMITMODE} = true ]]; then - PATCH_METHODS=("gitam" "${PATCH_METHODS[@]}") -fi - -patchfile_dryrun_driver "${PATCH_DIR}/patch" -RESULT=$? - -if [[ ${RESULT} -gt 0 ]]; then - yetus_error "ERROR: Aborting! ${PATCH_OR_ISSUE} cannot be verified." - cleanup_and_exit ${RESULT} -fi - -if [[ ${PATCH_DRYRUNMODE} == false ]]; then - patchfile_apply_driver "${PATCH_DIR}/patch" - RESULT=$? -fi - -if [[ ${COMMITMODE} = true - && ${PATCH_METHOD} != "gitam" ]]; then - yetus_debug "Running git add -A" - git add -A -fi - -cleanup_and_exit ${RESULT} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/test-patch-docker/Dockerfile-endstub ---------------------------------------------------------------------- diff --git a/dev-support/test-patch-docker/Dockerfile-endstub b/dev-support/test-patch-docker/Dockerfile-endstub deleted file mode 100644 index aa0463e..0000000 --- a/dev-support/test-patch-docker/Dockerfile-endstub +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ADD launch-test-patch.sh /testptch/launch-test-patch.sh -RUN chmod a+rx /testptch/launch-test-patch.sh -CMD /testptch/launch-test-patch.sh \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/test-patch-docker/Dockerfile-startstub ---------------------------------------------------------------------- diff --git a/dev-support/test-patch-docker/Dockerfile-startstub b/dev-support/test-patch-docker/Dockerfile-startstub deleted file mode 100644 index d191285..0000000 --- a/dev-support/test-patch-docker/Dockerfile-startstub +++ /dev/null @@ -1,91 +0,0 @@ - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ubuntu:trusty - -WORKDIR /root - -###### -# Install common dependencies from packages -###### -RUN apt-get update && apt-get install --no-install-recommends -y \ - git curl ant make maven \ - cmake gcc g++ protobuf-compiler \ - build-essential libtool \ - zlib1g-dev pkg-config libssl-dev \ - snappy libsnappy-dev \ - bzip2 libbz2-dev \ - libjansson-dev \ - fuse libfuse-dev \ - libcurl4-openssl-dev \ - python python2.7 pylint \ - ruby \ - openjdk-7-jdk \ - libperl-critic-perl - -# Fixing the Apache commons / Maven dependency problem under Ubuntu: -# See http://wiki.apache.org/commons/VfsProblems -RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar . - -####### -# Oracle Java -####### - -RUN apt-get install -y software-properties-common -RUN add-apt-repository -y ppa:webupd8team/java -RUN apt-get update - - -# Auto-accept the Oracle JDK license -RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -RUN apt-get install -y oracle-java7-installer - -# Auto-accept the Oracle JDK license -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -RUN apt-get install -y oracle-java8-installer - -###### -# Install findbugs -###### -RUN mkdir -p /opt/findbugs && \ - curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ - -o /opt/findbugs.tar.gz && \ - tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs -ENV FINDBUGS_HOME /opt/findbugs - -#### -# Install shellcheck -#### -RUN apt-get install -y cabal-install -RUN cabal update && cabal install shellcheck --global - -#### -# Install rubocop -### -RUN gem install rubocop - -#### -# Install ruby-lint -### -RUN gem install ruby-lint - -#### -# Install bats -#### -RUN add-apt-repository -y ppa:duggan/bats -RUN apt-get update -RUN apt-get install -y bats http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/test-patch-docker/launch-test-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch-docker/launch-test-patch.sh b/dev-support/test-patch-docker/launch-test-patch.sh deleted file mode 100755 index ac976ad..0000000 --- a/dev-support/test-patch-docker/launch-test-patch.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cd "${BASEDIR}" - -if [[ -n ${JAVA_HOME} - && ! -d ${JAVA_HOME} ]]; then - echo "JAVA_HOME: ${JAVA_HOME} does not exist. Dockermode: attempting to switch to another." 1>&2 - JAVA_HOME="" -fi - -if [[ -z ${JAVA_HOME} ]]; then - JAVA_HOME=$(find /usr/lib/jvm/ -name "java-*" -type d | tail -1) - export JAVA_HOME -fi - -# Avoid out of memory errors in builds -MAVEN_OPTS=${MAVEN_OPTS:-"-Xms256m -Xmx1g"} -export MAVEN_OPTS - -# strip out --docker param to prevent re-exec again -TESTPATCHMODE=${TESTPATCHMODE/--docker } - - -cd "${BASEDIR}" -PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P) - -cd "${PATCH_DIR}/precommit/" -#shellcheck disable=SC2086 -"${PATCH_DIR}/precommit/test-patch.sh" \ - --reexec \ - --dockermode ${TESTPATCHMODE} \ - --basedir="${BASEDIR}" \ - --patch-dir="${PATCH_DIR}" \ - --java-home="${JAVA_HOME}" \ - --personality="${PATCH_DIR}/precommit/personality/provided.sh" \ - --user-plugins="${PATCH_DIR}/precommit/user-plugins" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/test-patch-docker/test-patch-docker.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch-docker/test-patch-docker.sh b/dev-support/test-patch-docker/test-patch-docker.sh deleted file mode 100755 index 8b19502..0000000 --- a/dev-support/test-patch-docker/test-patch-docker.sh +++ /dev/null @@ -1,388 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -DID=${RANDOM} - -## @description Print a message to stderr if --debug is turned on -## @audience private -## @stability stable -## @replaceable no -## @param string -function yetus_debug -{ - if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then - echo "[$(date) DEBUG]: $*" 1>&2 - fi -} - -## @description Run docker with some arguments, and -## @description optionally send to debug -## @audience private -## @stability evolving -## @replaceable no -## @param args -function dockercmd -{ - yetus_debug "docker $*" - docker "$@" -} - -## @description Handle command line arguments -## @audience private -## @stability evolving -## @replaceable no -## @param args -function parse_args -{ - local i - - for i in "$@"; do - case ${i} in - --debug) - YETUS_SHELL_SCRIPT_DEBUG=true - ;; - --dockerversion=*) - DOCKER_VERSION=${i#*=} - ;; - --help|-help|-h|help|--h|--\?|-\?|\?) - yetus_usage - exit 0 - ;; - --java-home=*) - JAVA_HOME=${i#*=} - ;; - --patch-dir=*) - PATCH_DIR=${i#*=} - ;; - --project=*) - PROJECT_NAME=${i#*=} - ;; - *) - ;; - esac - done -} - -## @description Stop and delete all defunct containers -## @audience private -## @stability evolving -## @replaceable no -## @param args -function stop_exited_containers -{ - local line - local id - local value - local size - - echo "Docker containers in exit state:" - - dockercmd ps -a | grep Exited - - # stop *all* containers that are in exit state for - # more than > 8 hours - while read line; do - id=$(echo "${line}" | cut -f1 -d' ') - value=$(echo "${line}" | cut -f2 -d' ') - size=$(echo "${line}" | cut -f3 -d' ') - - if [[ ${size} =~ day - || ${size} =~ week - || ${size} =~ month - || ${size} =~ year ]]; then - echo "Removing docker ${id}" - dockercmd rm "${id}" - fi - - if [[ ${size} =~ hours - && ${value} -gt 8 ]]; then - echo "Removing docker ${id}" - dockercmd rm "${id}" - fi - done < <( - dockercmd ps -a \ - | grep Exited \ - | sed -e 's,ago,,g' \ - | awk '{print $1" "$(NF - 2)" "$(NF - 1)}') -} - -## @description Remove all containers that are not -## @description are not running + older than 1 day -## @audience private -## @stability evolving -## @replaceable no -## @param args -function rm_old_containers -{ - local line - local id - local value - local size - - while read line; do - id=$(echo "${line}" | cut -f1 -d, ) - state=$(echo "${line}" | cut -f2 -d, ) - stoptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. ) - - # believe it or not, date is not even close to standardized... - if [[ $(uname -s) == Linux ]]; then - - # GNU date - stoptime=$(date -d "${stoptime}" "+%s") - else - - # BSD date - stoptime=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${stoptime}" "+%s") - fi - - if [[ ${state} == false ]]; then - curtime=$(date "+%s") - ((difftime = curtime - stoptime)) - if [[ ${difftime} -gt 86400 ]]; then - echo "Removing docker ${id}" - dockercmd rm "${id}" - fi - fi - done < <( - # see https://github.com/koalaman/shellcheck/issues/375 - # shellcheck disable=SC2046 - dockercmd inspect \ - -f '{{.Id}},{{.State.Running}},{{.State.FinishedAt}}' \ - $(dockercmd ps -qa) 2>/dev/null) -} - -## @description Remove untagged/unused images -## @audience private -## @stability evolving -## @replaceable no -## @param args -function remove_untagged_images -{ - # this way is a bit more compatible with older docker versions - dockercmd images | tail -n +2 | awk '$1 == "<none>" {print $3}' | \ - xargs --no-run-if-empty docker rmi -} - -## @description Remove defunct tagged images -## @audience private -## @stability evolving -## @replaceable no -## @param args -function remove_old_tagged_images -{ - local line - local id - local created - - while read line; do - id=$(echo "${line}" | awk '{print $1}') - created=$(echo "${line}" | awk '{print $5}') - - if [[ ${created} =~ week - || ${created} =~ month - || ${created} =~ year ]]; then - echo "Removing docker image ${id}" - dockercmd rmi "${id}" - fi - - if [[ ${id} =~ test-patch-base-${PROJECT_NAME}-date ]]; then - if [[ ${created} =~ day - || ${created} =~ hours ]]; then - echo "Removing docker image ${id}" - dockercmd rmi "${id}" - fi - fi - done < <(dockercmd images) - -} - -## @description Performance docker maintenance on Jenkins -## @audience private -## @stability evolving -## @replaceable no -## @param args -function cleanup_apache_jenkins_docker -{ - echo "==========================" - echo "Docker Images:" - dockercmd images - echo "==========================" - echo "Docker Containers:" - dockercmd ps -a - echo "==========================" - - stop_exited_containers - - rm_old_containers - - remove_untagged_images - - remove_old_tagged_images -} - -## @description Clean up our old images used for patch testing -## @audience private -## @stability evolving -## @replaceable no -## @param args -function cleanup_test_patch_images -{ - local images - local imagecount - local rmimage - local rmi - - # we always want to leave at least one of our images - # so that the whole thing doesn't have to be rebuilt. - # This also let's us purge any old images so that - # we can get fresh stuff sometimes - images=$(dockercmd images | grep --color=none "test-patch-tp-${PROJECT_NAME}" | awk '{print $1}') 2>&1 - - # shellcheck disable=SC2086 - imagecount=$(echo ${images} | tr ' ' '\n' | wc -l) - ((imagecount = imagecount - 1 )) - - # shellcheck disable=SC2086 - rmimage=$(echo ${images} | tr ' ' '\n' | tail -${imagecount}) - for rmi in ${rmimage} - do - echo "Removing image ${rmi}" - dockercmd rmi "${rmi}" - done -} - -## @description Perform pre-run maintenance to free up -## @description resources. With --jenkins, it is a lot -## @description more destructive. -## @audience private -## @stability evolving -## @replaceable no -## @param args -function cleanup -{ - if [[ ${TESTPATCHMODE} =~ jenkins ]]; then - cleanup_apache_jenkins_docker - fi - - cleanup_test_patch_images -} - -## @description Deterine the user name and user id of the user -## @description that the docker container should use -## @audience private -## @stability evolving -## @replaceable no -## @param args -function determine_user -{ - # On the Apache Jenkins hosts, $USER is pretty much untrustable beacuse some - # ... person ... sets it to an account that doesn't actually exist. - # so instead, we need to try and override it with something that's - # probably close to reality. - if [[ ${TESTPATCHMODE} =~ jenkins ]]; then - USER=$(id | cut -f2 -d\( | cut -f1 -d\)) - fi - - if [[ "$(uname -s)" == "Linux" ]]; then - USER_NAME=${SUDO_USER:=$USER} - USER_ID=$(id -u "${USER_NAME}") - GROUP_ID=$(id -g "${USER_NAME}") - else # boot2docker uid and gid - USER_NAME=${USER} - USER_ID=1000 - GROUP_ID=50 - fi -} - -## @description Determine the revision of a dockerfile -## @audience private -## @stability evolving -## @replaceable no -## @param args -function getdockerfilerev -{ - grep 'TEST_PATCH_PRIVATE: gitrev=' \ - "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \ - | cut -f2 -d= -} - -## @description Start a test patch docker container -## @audience private -## @stability evolving -## @replaceable no -## @param args -function run_image -{ - local dockerfilerev - local baseimagename - - dockerfilerev=$(getdockerfilerev) - - baseimagename="test-patch-base-${PROJECT_NAME}-${dockerfilerev}" - - # make a base image, if it isn't available - dockercmd build -t "${baseimagename}" "${PATCH_DIR}/precommit/test-patch-docker" - - # using the base image, make one that is patch specific - dockercmd build -t "test-patch-tp-${PROJECT_NAME}-${DID}" - <<PatchSpecificDocker -FROM ${baseimagename} -RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} -RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} -RUN chown -R ${USER_NAME} /home/${USER_NAME} -ENV HOME /home/${USER_NAME} -USER ${USER_NAME} -PatchSpecificDocker - - if [[ -f "${PATCH_DIR}/buildtool-docker-params.txt" ]]; then - extraargs=$(cat "${PATCH_DIR}/buildtool-docker-params.txt") - else - extraargs="" - fi - - if [[ ${PATCH_DIR} =~ ^/ ]]; then - dockercmd run --rm=true -i \ - ${extraargs} \ - -v "${PWD}:/testptch/${PROJECT_NAME}" \ - -v "${PATCH_DIR}:/testptch/patchprocess" \ - -u "${USER_NAME}" \ - -w "/testptch/${PROJECT_NAME}" \ - --env=BASEDIR="/testptch/${PROJECT_NAME}" \ - --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ - --env=JAVA_HOME="${JAVA_HOME}" \ - --env=PATCH_DIR=/testptch/patchprocess \ - --env=PROJECT_NAME="${PROJECT_NAME}" \ - --env=TESTPATCHMODE="${TESTPATCHMODE}" \ - "test-patch-tp-${PROJECT_NAME}-${DID}" - else - dockercmd run --rm=true -i \ - ${extraargs} \ - -v "${PWD}:/testptch/${PROJECT_NAME}" \ - -u "${USER_NAME}" \ - -w "/testptch/${PROJECT_NAME}" \ - --env=BASEDIR="/testptch/${PROJECT_NAME}" \ - --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ - --env=JAVA_HOME="${JAVA_HOME}" \ - --env=PATCH_DIR="${PATCH_DIR}" \ - --env=PROJECT_NAME="${PROJECT_NAME}" \ - --env=TESTPATCHMODE="${TESTPATCHMODE}" \ - "test-patch-tp-${PROJECT_NAME}-${DID}" - fi -} - -parse_args "$@" -cleanup -determine_user -run_image http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/test-patch.d/ant.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/ant.sh b/dev-support/test-patch.d/ant.sh deleted file mode 100755 index 9d931a5..0000000 --- a/dev-support/test-patch.d/ant.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -z "${ANT_HOME:-}" ]]; then - ANT=ant -else - ANT=${ANT_HOME}/bin/ant -fi - -add_build_tool ant - -declare -a ANT_ARGS=("-noinput") - -function ant_usage -{ - echo "ant specific:" - echo "--ant-cmd=<cmd> The 'ant' command to use (default \${ANT_HOME}/bin/ant, or 'ant')" -} - -function ant_parse_args -{ - local i - - for i in "$@"; do - case ${i} in - --ant-cmd=*) - ANT=${i#*=} - ;; - esac - done - - # if we requested offline, pass that to ant - if [[ ${OFFLINE} == "true" ]]; then - ANT_ARGS=("${ANT_ARGS[@]}" -Doffline=) - fi -} - -function ant_initialize -{ - # we need to do this before docker kicks in - if [[ -e "${HOME}/.ivy2" - && ! -d "${HOME}/.ivy2" ]]; then - yetus_error "ERROR: ${HOME}/.ivy2 is not a directory." - return 1 - elif [[ ! -e "${HOME}/.ivy2" ]]; then - yetus_debug "Creating ${HOME}/.ivy2" - mkdir -p "${HOME}/.ivy2" - fi -} - -function ant_buildfile -{ - echo "build.xml" -} - -function ant_executor -{ - echo "${ANT}" "${ANT_ARGS[@]}" -} - -function ant_modules_worker -{ - declare repostatus=$1 - declare tst=$2 - shift 2 - - # shellcheck disable=SC2034 - UNSUPPORTED_TEST=false - - case ${tst} in - findbugs) - modules_workers "${repostatus}" findbugs findbugs - ;; - compile) - modules_workers "${repostatus}" compile - ;; - distclean) - modules_workers "${repostatus}" distclean clean - ;; - javadoc) - modules_workers "${repostatus}" javadoc clean javadoc - ;; - unit) - modules_workers "${repostatus}" unit - ;; - *) - # shellcheck disable=SC2034 - UNSUPPORTED_TEST=true - if [[ ${repostatus} = patch ]]; then - add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" - fi - yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" - return 1 - ;; - esac -} - -function ant_javac_count_probs -{ - declare warningfile=$1 - declare val1 - declare val2 - - #shellcheck disable=SC2016 - val1=$(${GREP} -E "\[javac\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') - #shellcheck disable=SC2016 - val2=$(${GREP} -E "\[javac\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') - echo $((val1+val2)) -} - -## @description Helper for check_patch_javadoc -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success -## @return 1 on failure -function ant_javadoc_count_probs -{ - local warningfile=$1 - local val1 - local val2 - - #shellcheck disable=SC2016 - val1=$(${GREP} -E "\[javadoc\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') - #shellcheck disable=SC2016 - val2=$(${GREP} -E "\[javadoc\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') - echo $((val1+val2)) -} - -function ant_builtin_personality_modules -{ - local repostatus=$1 - local testtype=$2 - - local module - - yetus_debug "Using builtin personality_modules" - yetus_debug "Personality: ${repostatus} ${testtype}" - - clear_personality_queue - - for module in ${CHANGED_MODULES}; do - # shellcheck disable=SC2086 - personality_enqueue_module ${module} - done -} - -function ant_builtin_personality_file_tests -{ - local filename=$1 - - yetus_debug "Using builtin ant personality_file_tests" - - if [[ ${filename} =~ \.sh - || ${filename} =~ \.cmd - ]]; then - yetus_debug "tests/shell: ${filename}" - elif [[ ${filename} =~ \.c$ - || ${filename} =~ \.cc$ - || ${filename} =~ \.h$ - || ${filename} =~ \.hh$ - || ${filename} =~ \.proto$ - || ${filename} =~ src/test - || ${filename} =~ \.cmake$ - || ${filename} =~ CMakeLists.txt - ]]; then - yetus_debug "tests/units: ${filename}" - add_test javac - add_test unit - elif [[ ${filename} =~ build.xml - || ${filename} =~ ivy.xml - || ${filename} =~ \.java$ - ]]; then - yetus_debug "tests/javadoc+units: ${filename}" - add_test javac - add_test javadoc - add_test unit - fi - - if [[ ${filename} =~ \.java$ ]]; then - add_test findbugs - fi -} - -function ant_docker_support -{ - echo "-v ${HOME}/.ivy2:${HOME}/.ivy2" > "${PATCH_DIR}/buildtool-docker-params.txt" -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/test-patch.d/asflicense.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/asflicense.sh b/dev-support/test-patch.d/asflicense.sh deleted file mode 100755 index befe7a1..0000000 --- a/dev-support/test-patch.d/asflicense.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -add_test_type asflicense -add_test asflicense - -function asflicense_parse_args -{ - declare i - - for i in "$@"; do - case ${i} in - --asflicense-rat-excludes=*) - ASFLICENSE_RAT_EXCLUDES=${i#*=} - ;; - --asflicense-rat-jar=*) - ASFLICENSE_RAT_JAR=${i#*=} - ;; - esac - done - - case ${BUILDTOOL} in - ant|gradle|maven) - add_test asflicense - ;; - *) - if [[ -f "${ASFLICENSE_RAT_JAR}" ]]; then - add_test asflicense - fi - ;; - esac -} - -## @description Verify all files have an Apache License -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success -## @return 1 on failure -function asflicense_tests -{ - local numpatch - local btfails=true - local asfex - - big_console_header "Determining number of patched ASF License errors" - - start_clock - - personality_modules patch asflicense - case ${BUILDTOOL} in - ant) - modules_workers patch asflicense releaseaudit - ;; - gradle) - btfails=false - modules_workers patch asflicense rat - ;; - maven) - modules_workers patch asflicense apache-rat:check - ;; - *) - if [[ -z "${ASFLICENSE_RAT_JAR}" ]]; then - return 0 - fi - - if [[ -f ${ASFLICENSE_RAT_EXCLUDES} ]]; then - asfex="-E ${ASFLICENSE_RAT_EXCLUDES} -d ${BASEDIR}" - else - asfex="${BASEDIR}" - fi - - asflicense_writexsl "${PATCH_DIR}/asf.xsl" - echo_and_redirect "${PATCH_DIR}/patch-asflicense.txt" \ - "${JAVA_HOME}/bin/java" \ - -jar "${ASFLICENSE_RAT_JAR}" \ - -s "${PATCH_DIR}/asf.xsl" \ - "${asfex}" - ;; - esac - - # RAT fails the build if there are license problems. - # so let's take advantage of that a bit. - if [[ $? == 0 && ${btfails} = true ]]; then - add_vote_table 1 asflicense "Patch does not generate ASF License warnings." - return 0 - fi - - if [[ ! -f "${PATCH_DIR}/patch-asflicense.txt" ]]; then - #shellcheck disable=SC2038 - find "${BASEDIR}" -name rat.txt \ - -o -name releaseaudit_report.txt \ - -o -name rat-report.txt \ - | xargs cat > "${PATCH_DIR}/patch-asflicense.txt" - fi - - if [[ ! -s "${PATCH_DIR}/patch-asflicense.txt" ]]; then - if [[ ${btfails} = true ]]; then - # if we're here, then build actually failed - modules_messages patch asflicense true - else - add_vote_table 0 asflicense "ASF License check generated no output?" - return 0 - fi - fi - - numpatch=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patch-asflicense.txt") - echo "" - echo "" - echo "There appear to be ${numpatch} ASF License warnings after applying the patch." - if [[ -n ${numpatch} - && ${numpatch} -gt 0 ]] ; then - add_vote_table -1 asflicense "Patch generated ${numpatch} ASF License warnings." - - echo "Lines that start with ????? in the ASF License "\ - "report indicate files that do not have an Apache license header:" \ - > "${PATCH_DIR}/patch-asflicense-problems.txt" - - ${GREP} '\!?????' "${PATCH_DIR}/patch-asflicense.txt" \ - >> "${PATCH_DIR}/patch-asflicense-problems.txt" - - add_footer_table asflicense "@@BASE@@/patch-asflicense-problems.txt" - return 1 - fi - add_vote_table 1 asflicense "Patch does not generate ASF License warnings." - return 0 -} - -function asflicense_writexsl -{ -cat > "${1}" << EOF -<?xml version='1.0' ?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one * - or more contributor license agreements. See the NOTICE file * - distributed with this work for additional information * - regarding copyright ownership. The ASF licenses this file * - to you under the Apache License, Version 2.0 (the * - "License"); you may not use this file except in compliance * - with the License. You may obtain a copy of the License at * - * - http://www.apache.org/licenses/LICENSE-2.0 * - * - Unless required by applicable law or agreed to in writing, * - software distributed under the License is distributed on an * - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - KIND, either express or implied. See the License for the * - specific language governing permissions and limitations * - under the License. * ---> -<xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform";> -<xsl:output method='text'/> -<xsl:template match='/'> - Files with Apache License headers will be marked AL - Binary files (which do not require any license headers) will be marked B - Compressed archives will be marked A - Notices, licenses etc. will be marked N - - <xsl:for-each select='descendant::resource'> - <xsl:choose> - <xsl:when test='license-approval/@name="false"'>!</xsl:when> - <xsl:otherwise><xsl:text> </xsl:text></xsl:otherwise> - </xsl:choose> - <xsl:choose> - <xsl:when test='type/@name="notice"'>N </xsl:when> - <xsl:when test='type/@name="archive"'>A </xsl:when> - <xsl:when test='type/@name="binary"'>B </xsl:when> - <xsl:when test='type/@name="standard"'><xsl:value-of select='header-type/@name'/></xsl:when> - <xsl:otherwise>!!!!!</xsl:otherwise> - </xsl:choose> - <xsl:text> </xsl:text> - <xsl:value-of select='@name'/> - <xsl:text> - </xsl:text> - </xsl:for-each> -</xsl:template> -</xsl:stylesheet> -EOF -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/test-patch.d/author.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/author.sh b/dev-support/test-patch.d/author.sh deleted file mode 100755 index d456bbb..0000000 --- a/dev-support/test-patch.d/author.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_test_type author - -## @description Check the current directory for @author tags -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success -## @return 1 on failure -function author_patchfile -{ - declare patchfile=$1 - declare authorTags - # shellcheck disable=SC2155 - declare -r appname=$(basename "${BASH_SOURCE-$0}") - - big_console_header "Checking there are no @author tags in the patch." - - start_clock - - if [[ ${CHANGED_FILES} =~ ${appname} ]]; then - echo "Skipping @author checks as ${appname} has been patched." - add_vote_table 0 @author "Skipping @author checks as ${appname} has been patched." - return 0 - fi - - authorTags=$("${GREP}" -c -i '^[^-].*@author' "${patchfile}") - echo "There appear to be ${authorTags} @author tags in the patch." - if [[ ${authorTags} != 0 ]] ; then - add_vote_table -1 @author \ - "The patch appears to contain ${authorTags} @author tags which the" \ - " community has agreed to not allow in code contributions." - return 1 - fi - add_vote_table +1 @author "The patch does not contain any @author tags." - return 0 -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/test-patch.d/cc.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/cc.sh b/dev-support/test-patch.d/cc.sh deleted file mode 100755 index cae91d3..0000000 --- a/dev-support/test-patch.d/cc.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_test_type cc - -function cc_filefilter -{ - declare filename=$1 - - if [[ ${filename} =~ \.c$ - || ${filename} =~ \.cc$ - || ${filename} =~ \.cpp$ - || ${filename} =~ \.cxx$ - || ${filename} =~ \.h$ - || ${filename} =~ \.hh$ - ]]; then - yetus_debug "tests/cc: ${filename}" - add_test cc - add_test compile - fi -} - -## @description check for C/C++ compiler errors -## @audience private -## @stability stable -## @replaceable no -## @return 0 on success -## @return 1 on failure -function cc_compile -{ - declare codebase=$1 - declare multijdkmode=$2 - - verify_needed_test cc - if [[ $? = 0 ]]; then - return 0 - fi - - if [[ ${codebase} = patch ]]; then - generic_postlog_compare compile cc "${multijdkmode}" - fi -} - -function cc_count_probs -{ - declare warningfile=$1 - - #shellcheck disable=SC2016,SC2046 - ${GREP} -E '^.*\.(c|cc|h|hh)\:[[:digit:]]*\:' "${warningfile}" | ${AWK} '{sum+=1} END {print sum}' -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/dev-support/test-patch.d/checkstyle.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh deleted file mode 100755 index 24b9ffc..0000000 --- a/dev-support/test-patch.d/checkstyle.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_test_type checkstyle - -CHECKSTYLE_TIMER=0 - -function checkstyle_filefilter -{ - local filename=$1 - - if [[ ${BUILDTOOL} == maven - || ${BUILDTOOL} == ant ]]; then - if [[ ${filename} =~ \.java$ ]]; then - add_test checkstyle - fi - fi -} - -function checkstyle_runner -{ - local repostatus=$1 - local tmp=${PATCH_DIR}/$$.${RANDOM} - local j - local i=0 - local fn - local savestart=${TIMER} - local savestop - local output - local logfile - local repo - local modulesuffix - local cmd - - modules_reset - - if [[ ${repostatus} == branch ]]; then - repo=${PATCH_BRANCH} - else - repo="the patch" - fi - - #shellcheck disable=SC2153 - until [[ $i -eq ${#MODULE[@]} ]]; do - start_clock - fn=$(module_file_fragment "${MODULE[${i}]}") - modulesuffix=$(basename "${MODULE[${i}]}") - output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt" - logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt" - - if [[ ${BUILDTOOLCWD} == true ]]; then - pushd "${BASEDIR}/${MODULE[${i}]}" >/dev/null - fi - - case ${BUILDTOOL} in - ant) - cmd="${ANT} \ - -Dcheckstyle.consoleOutput=true \ - ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} \ - ${ANT_ARGS[*]} checkstyle" - ;; - maven) - cmd="${MAVEN} ${MAVEN_ARGS[*]} \ - checkstyle:checkstyle \ - -Dcheckstyle.consoleOutput=true \ - ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch" - ;; - *) - UNSUPPORTED_TEST=true - return 0 - ;; - esac - - #shellcheck disable=SC2086 - echo ${cmd} "> ${logfile}" - #shellcheck disable=SC2086 - ${cmd} 2>&1 \ - | tee "${logfile}" \ - | ${GREP} ^/ \ - | ${SED} -e "s,${BASEDIR},.,g" \ - > "${tmp}" - - if [[ $? == 0 ]] ; then - module_status ${i} +1 "${logfile}" "${modulesuffix} in ${repo} passed checkstyle" - else - module_status ${i} -1 "${logfile}" "${modulesuffix} in ${repo} failed checkstyle" - ((result = result + 1)) - fi - savestop=$(stop_clock) - #shellcheck disable=SC2034 - MODULE_STATUS_TIMER[${i}]=${savestop} - - for j in ${CHANGED_FILES}; do - ${GREP} "${j}" "${tmp}" >> "${output}" - done - - rm "${tmp}" 2>/dev/null - - if [[ ${BUILDTOOLCWD} == true ]]; then - popd >/dev/null - fi - ((i=i+1)) - done - - TIMER=${savestart} - - if [[ ${result} -gt 0 ]]; then - return 1 - fi - return 0 -} - -function checkstyle_postcompile -{ - declare repostatus=$1 - - if [[ "${repostatus}" = branch ]]; then - checkstyle_preapply - else - checkstyle_postapply - fi -} - -function checkstyle_preapply -{ - local result - - verify_needed_test checkstyle - if [[ $? == 0 ]]; then - return 0 - fi - - big_console_header "${PATCH_BRANCH} checkstyle" - - start_clock - - personality_modules branch checkstyle - checkstyle_runner branch - result=$? - modules_messages branch checkstyle true - - # keep track of how much as elapsed for us already - CHECKSTYLE_TIMER=$(stop_clock) - if [[ ${result} != 0 ]]; then - return 1 - fi - return 0 -} - -function checkstyle_postapply -{ - local result - local module - local mod - local fn - local i=0 - local numprepatch=0 - local numpostpatch=0 - local diffpostpatch=0 - - verify_needed_test checkstyle - if [[ $? == 0 ]]; then - return 0 - fi - - big_console_header "Patch checkstyle plugin" - - start_clock - - personality_modules patch checkstyle - checkstyle_runner patch - result=$? - - if [[ ${UNSUPPORTED_TEST} = true ]]; then - return 0 - fi - - # add our previous elapsed to our new timer - # by setting the clock back - offset_clock "${CHECKSTYLE_TIMER}" - - until [[ $i -eq ${#MODULE[@]} ]]; do - if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then - ((result=result+1)) - ((i=i+1)) - continue - fi - module=${MODULE[$i]} - fn=$(module_file_fragment "${module}") - - if [[ ! -f "${PATCH_DIR}/branch-checkstyle-${fn}.txt" ]]; then - touch "${PATCH_DIR}/branch-checkstyle-${fn}.txt" - fi - - calcdiffs "${PATCH_DIR}/branch-checkstyle-${fn}.txt" "${PATCH_DIR}/patch-checkstyle-${fn}.txt" > "${PATCH_DIR}/diff-checkstyle-${fn}.txt" - #shellcheck disable=SC2016 - diffpostpatch=$(wc -l "${PATCH_DIR}/diff-checkstyle-${fn}.txt" | ${AWK} '{print $1}') - - if [[ ${diffpostpatch} -gt 0 ]] ; then - ((result = result + 1)) - - # shellcheck disable=SC2016 - numprepatch=$(wc -l "${PATCH_DIR}/branch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') - # shellcheck disable=SC2016 - numpostpatch=$(wc -l "${PATCH_DIR}/patch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') - - mod=${module} - if [[ ${mod} == . ]]; then - mod=root - fi - module_status ${i} -1 "diff-checkstyle-${fn}.txt" "Patch generated "\ - "${diffpostpatch} new checkstyle issues in "\ - "${mod} (total was ${numprepatch}, now ${numpostpatch})." - fi - ((i=i+1)) - done - - modules_messages patch checkstyle true - - if [[ ${result} != 0 ]]; then - return 1 - fi - return 0 -}
