This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/master by this push:
new c78ec24 YETUS-842. remove pylintrc
c78ec24 is described below
commit c78ec24c74d6443ee8de768b9e5c855d7001c812
Author: Allen Wittenauer <a...@apache.org>
AuthorDate: Thu Apr 4 10:06:31 2019 -0700
YETUS-842. remove pylintrc
Signed-off-by: Allen Wittenauer <a...@apache.org>
---
.pylintrc | 305 ---------------------
pom.xml | 1 -
precommit/src/main/python/jenkins-admin.py | 167 +++++------
releasedocmaker/src/main/python/releasedocmaker.py | 2 +
.../src/main/python/releasedocmaker/__init__.py | 217 ++++++++-------
.../src/main/python/releasedocmaker/utils.py | 81 +++---
shelldocs/src/main/python/shelldocs.py | 2 +
shelldocs/src/main/python/shelldocs/__init__.py | 12 +-
8 files changed, 257 insertions(+), 530 deletions(-)
diff --git a/.pylintrc b/.pylintrc
deleted file mode 100644
index b4cc79a..0000000
--- a/.pylintrc
+++ /dev/null
@@ -1,305 +0,0 @@
-[MASTER]
-
-# Specify a configuration file.
-#rcfile=
-
-# Python code to execute, usually for sys.path manipulation such as
-# pygtk.require().
-#init-hook=
-
-# Profiled execution.
-#profile=no
-
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=CVS
-
-# Pickle collected data for later comparisons.
-persistent=yes
-
-# List of plugins (as comma separated values of python modules names) to load,
-# usually to register additional checkers.
-load-plugins=
-
-
-[MESSAGES CONTROL]
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time.
-#enable=
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifier separated by comma (,) or put this option
-# multiple time (only on the command line, not in the configuration file where
-# it should appear only once).
-# CHANGED:
-# C0103: Invalid name ""
-# C0111: Missing docstring
-# C0302: Too many lines in module (N)
-# I0010: Unable to consider inline option ''
-# I0011: Locally disabling WNNNN
-#
-# R0801: Similar lines in N files
-# R0901: Too many ancestors (8/7)
-# R0902: Too many instance attributes (N/7)
-# R0903: Too few public methods (N/2)
-# R0904: Too many public methods (N/20)
-# R0911: Too many return statements (N/6)
-# R0912: Too many branches (N/12)
-# R0913: Too many arguments (N/5)
-# R0914: Too many local variables (N/15)
-# R0915: Too many statements (N/50)
-# R0921: Abstract class not referenced
-# R0922: Abstract class is only referenced 1 times
-# W0122: Use of the exec statement
-# W0141: Used builtin function ''
-# W0142: Used * or ** magic
-# W0402: Uses of a deprecated module 'string'
-# W0404: 41: Reimport 'XX' (imported line NN)
-# W0511: TODO
-# W0603: Using the global statement
-# W0703: Catch "Exception"
-# W1201: Specify string format arguments as logging function parameters
-#
-# These should get enabled, but the codebase has too many violations currently.
-# bad-continuation
-# anomalous-backslash-in-string
-# bad-context-manager
-# bad-indentation
-# bad-str-strip-call
-# bad-whitespace
-# cell-var-from-loop
-# deprecated-lambda
-# eval-used
-# function-redefined
-# import-error
-# locally-enabled
-# missing-final-newline
-# no-init
-# no-name-in-module
-# no-self-use
-# not-callable
-# old-style-class
-# protected-access
-# superfluous-parens
-# super-on-old-class
-# too-many-function-args
-# trailing-whitespace
-# unnecessary-semicolon
-# unpacking-non-sequence
-# unused-import
-# useless-else-on-loop
-disable=C0103,C0111,C0302,I0010,I0011,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0921,R0922,W0122,W0141,W0142,W0402,W0404,W0511,W0603,W0703,W1201,bad-continuation,anomalous-backslash-in-string,bad-context-manager,bad-indentation,bad-str-strip-call,bad-whitespace,cell-var-from-loop,deprecated-lambda,eval-used,function-redefined,import-error,locally-enabled,missing-final-newline,no-init,no-name-in-module,no-self-use,not-callable,old-style-class,protected-access,superfluou
[...]
-
-
-[REPORTS]
-
-# Set the output format. Available formats are text, parseable, colorized, msvs
-# (visual studio) and html
-output-format=text
-
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
-# Tells whether to display a full report or only the messages
-# CHANGED:
-reports=no
-
-# Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables errors warning, statement which
-# respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) /
statement) * 10)
-
-# Add a comment according to your evaluation note. This is used by the global
-# evaluation report (RP0004).
-#comment=no
-
-
-[VARIABLES]
-
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
-
-# A regular expression matching the beginning of the name of dummy variables
-# (i.e. not used).
-dummy-variables-rgx=_|dummy
-
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid to define new builtins when possible.
-additional-builtins=
-
-
-[TYPECHECK]
-
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
-
-# List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamically set).
-ignored-classes=SQLObject,twisted.internet.reactor,hashlib,google.appengine.api.memcache
-
-# When zope mode is activated, add a predefined set of Zope acquired attributes
-# to generated-members.
-#zope=no
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E0201 when accessed. Python regular
-# expressions are accepted.
-generated-members=REQUEST,acl_users,aq_parent,multiprocessing.managers.SyncManager
-
-
-[MISCELLANEOUS]
-
-# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
-
-
-[SIMILARITIES]
-
-# Minimum lines number of a similarity.
-min-similarity-lines=4
-
-# Ignore comments when computing similarities.
-ignore-comments=yes
-
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
-
-
-[FORMAT]
-
-# Maximum number of characters on a single line.
-max-line-length=100
-
-# Maximum number of lines in a module
-max-module-lines=1000
-
-# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
-# tab).
-# CHANGED:
-indent-string=' '
-
-
-[BASIC]
-
-# Required attributes for module, separated by a comma
-#required-attributes=
-
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input
-
-# Regular expression which should only match correct module names
-module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
-
-# Regular expression which should only match correct module level names
-const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
-
-# Regular expression which should only match correct class names
-class-rgx=[A-Z_][a-zA-Z0-9]+$
-
-# Regular expression which should only match correct function names
-function-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct method names
-method-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct instance attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct list comprehension /
-# generator expression variable names
-inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
-
-# Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_
-
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,bar,baz,toto,tutu,tata
-
-# Regular expression which should only match functions or classes name which do
-# not require a docstring
-no-docstring-rgx=__.*__
-
-
-[DESIGN]
-
-# Maximum number of arguments for function / method
-max-args=5
-
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore
-ignored-argument-names=_.*
-
-# Maximum number of locals for function / method body
-max-locals=15
-
-# Maximum number of return / yield for function / method body
-max-returns=6
-
-# Maximum number of branch for function / method body
-max-branchs=12
-
-# Maximum number of statements in function / method body
-max-statements=50
-
-# Maximum number of parents for a class (see R0901).
-max-parents=7
-
-# Maximum number of attributes for a class (see R0902).
-max-attributes=7
-
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
-
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=20
-
-
-[CLASSES]
-
-# List of interface methods to ignore, separated by a comma. This is used for
-# instance to not check methods defines in Zope's Interface base class.
-#ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
-
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,__new__,setUp
-
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
-
-
-[IMPORTS]
-
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
-
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled)
-import-graph=
-
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled)
-ext-import-graph=
-
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled)
-int-import-graph=
-
-
-[EXCEPTIONS]
-
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception
diff --git a/pom.xml b/pom.xml
index 19f262f..9a6568f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -216,7 +216,6 @@
<configuration>
<excludeSubprojects>false</excludeSubprojects>
<excludes>
- <exclude>.pylintrc</exclude>
<exclude>.rubocop.yml</exclude>
<exclude>Formula/.rubocop.yml</exclude>
</excludes>
diff --git a/precommit/src/main/python/jenkins-admin.py
b/precommit/src/main/python/jenkins-admin.py
index f6ffdfe..0786cf4 100755
--- a/precommit/src/main/python/jenkins-admin.py
+++ b/precommit/src/main/python/jenkins-admin.py
@@ -1,4 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
+#pylint: disable=invalid-name
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,6 +19,8 @@
# under the License.
#
+""" Process patch file attachments from JIRA using a query """
+
from optparse import OptionParser
from tempfile import NamedTemporaryFile
from xml.etree import ElementTree
@@ -28,69 +31,68 @@ import re
import sys
import urllib2
-def httpGet(resource, ignoreError=False, username=None, password=None):
+def http_get(resource, ignore_error=False, username=None, password=None):
+ """ get the contents of a URL """
request = urllib2.Request(resource)
if username and password:
- base64string = base64.b64encode('%s:%s' % (username, password))
- request.add_header("Authorization", "Basic %s" % base64string)
+ base64string = base64.b64encode('%s:%s' % (username, password))
+ request.add_header("Authorization", "Basic %s" % base64string)
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError, http_err:
code = http_err.code
print '%s returns HTTP error %d: %s' \
% (resource, code, http_err.reason)
- if ignoreError:
+ if ignore_error:
return ''
else:
print 'Aborting.'
sys.exit(1)
except urllib2.URLError, url_err:
print 'Error contacting %s: %s' % (resource, url_err.reason)
- if ignoreError:
+ if ignore_error:
return ''
else:
raise url_err
except httplib.BadStatusLine, err:
- if ignoreError:
+ if ignore_error:
return ''
else:
raise err
return response.read()
-
-# returns a map of (project, issue) => attachment id
-
-def parseJiraData(fileName):
- tree = ElementTree.parse(fileName)
+def parse_jira_data(filename):
+ """ returns a map of (project, issue) => attachment id """
+ tree = ElementTree.parse(filename)
root = tree.getroot()
- jiraPattern = re.compile('([A-Z]+)\-([0-9]+)')
+ jirapattern = re.compile(r'([A-Z]+)\-([0-9]+)')
result = {}
for item in root.findall('./channel/item'):
jirakey = item.find('key')
if jirakey is None:
continue
jiraissue = jirakey.text
- matcher = jiraPattern.match(jiraissue)
+ matcher = jirapattern.match(jiraissue)
if not matcher:
continue
jiraissue = (matcher.group(1), matcher.group(2))
- attachmentIds = []
+ attachmentids = []
for jiraattachment in item.findall('./attachments/attachment'):
- attachmentId = jiraattachment.get('id')
+ attachmentid = jiraattachment.get('id')
try:
- attachmentIds.append(int(attachmentId))
+ attachmentids.append(int(attachmentid))
except ValueError:
pass
- if attachmentIds:
- attachmentIds.sort()
- result[jiraissue] = attachmentIds[-1]
+ if attachmentids:
+ attachmentids.sort()
+ result[jiraissue] = attachmentids[-1]
return result
-
-if __name__ == '__main__':
- parser = OptionParser(prog = 'jenkins-admin')
+def main(): #pylint: disable=too-many-branches, too-many-statements,
too-many-locals
+ """ main program """
+ parser = OptionParser(prog='jenkins-admin')
if os.getenv('JENKINS_URL'):
- parser.set_defaults(jenkinsUrl=os.getenv('JENKINS_URL'))
+ parser.set_defaults(jenkinsurl=os.getenv('JENKINS_URL'))
if os.getenv('JOB_NAME'):
parser.set_defaults(jenkinsJobName=os.getenv('JOB_NAME'))
else:
@@ -110,12 +112,12 @@ if __name__ == '__main__':
parser.add_option('--jenkins-token', type='string',
dest='jenkinsToken', help='Jenkins Token',
metavar='TOKEN')
- parser.add_option('--jenkins-url', type='string', dest='jenkinsUrl'
+ parser.add_option('--jenkins-url', type='string', dest='jenkinsurl'
, help='Jenkins base URL', metavar='URL')
parser.add_option(
'--jenkins-url-override',
type='string',
- dest='jenkinsUrlOverrides',
+ dest='jenkinsurloverrides',
action='append',
help='Project specific Jenkins base URL',
metavar='PROJECT=URL',
@@ -138,108 +140,111 @@ if __name__ == '__main__':
default=False,
help="display version information for jenkins-admin and exit.")
- (options, args) = parser.parse_args()
+ (options, args) = parser.parse_args() #pylint: disable=unused-variable
# Handle the version string right away and exit
if options.release_version:
with open(
- os.path.join(
- os.path.dirname(__file__), "../../VERSION"), 'r') as
ver_file:
+ os.path.join(
+ os.path.dirname(__file__), "../../VERSION"), 'r') as ver_file:
print ver_file.read()
sys.exit(0)
- tokenFrag = ''
+ token_frag = ''
if options.jenkinsToken:
- tokenFrag = 'token=%s' % options.jenkinsToken
+ token_frag = 'token=%s' % options.jenkinsToken
else:
- tokenFrag = 'token={project}-token'
+ token_frag = 'token={project}-token'
if not options.jiraFilter:
parser.error('ERROR: --jira-filter is a required argument.')
- if not options.jenkinsUrl:
+ if not options.jenkinsurl:
parser.error('ERROR: --jenkins-url or the JENKINS_URL environment
variable is required.'
- )
+ )
if options.history < 0:
parser.error('ERROR: --max-history must be 0 or a positive integer.'
- )
- jenkinsUrlOverrides = {}
- if options.jenkinsUrlOverrides:
- for override in options.jenkinsUrlOverrides:
+ )
+ jenkinsurloverrides = {}
+ if options.jenkinsurloverrides:
+ for override in options.jenkinsurloverrides:
if '=' not in override:
parser.error('Invalid Jenkins Url Override: '
+ override)
(project, url) = override.split('=', 1)
- jenkinsUrlOverrides[project.upper()] = url
- tempFile = NamedTemporaryFile(delete=False)
+ jenkinsurloverrides[project.upper()] = url
+ tempfile = NamedTemporaryFile(delete=False)
try:
- jobLogHistory = None
+ jobloghistory = None
if not options.jenkinsInit:
- jobLogHistory = httpGet(options.jenkinsUrl
- +
'/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt'
+ jobloghistory = http_get(options.jenkinsurl
+ +
'/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt'
% options.jenkinsJobName, True)
# if we don't have a successful build available try the last build
- if not jobLogHistory:
- jobLogHistory = httpGet(options.jenkinsUrl
- +
'/job/%s/lastCompletedBuild/artifact/patch_tested.txt'
- % options.jenkinsJobName)
- jobLogHistory = jobLogHistory.strip().split('\n')
- if 'TESTED ISSUES' not in jobLogHistory[0]:
+ if not jobloghistory:
+ jobloghistory = http_get(options.jenkinsurl
+ +
'/job/%s/lastCompletedBuild/artifact/patch_tested.txt'
+ % options.jenkinsJobName)
+ jobloghistory = jobloghistory.strip().split('\n')
+ if 'TESTED ISSUES' not in jobloghistory[0]:
print 'Downloaded patch_tested.txt control file may be
corrupted. Failing.'
sys.exit(1)
# we are either going to write a new one or rewrite the old one
- jobLog = open('patch_tested.txt', 'w+')
+ joblog = open('patch_tested.txt', 'w+')
- if jobLogHistory:
- if len(jobLogHistory) > options.history:
- jobLogHistory = [jobLogHistory[0]] \
- + jobLogHistory[len(jobLogHistory)
- - options.history:]
- for jobHistoryRecord in jobLogHistory:
- jobLog.write(jobHistoryRecord + '\n')
+ if jobloghistory:
+ if len(jobloghistory) > options.history:
+ jobloghistory = [jobloghistory[0]] \
+ + jobloghistory[len(jobloghistory) \
+ - options.history:]
+ for jobhistoryrecord in jobloghistory:
+ joblog.write(jobhistoryrecord + '\n')
else:
- jobLog.write('TESTED ISSUES\n')
- jobLog.flush()
- rssData =
httpGet(options.jiraFilter,False,options.jiraUser,options.jiraPassword)
- tempFile.write(rssData)
- tempFile.flush()
- for (key, attachment) in parseJiraData(tempFile.name).items():
+ joblog.write('TESTED ISSUES\n')
+ joblog.flush()
+ rssdata = http_get(options.jiraFilter, False, options.jiraUser,
options.jiraPassword)
+ tempfile.write(rssdata)
+ tempfile.flush()
+ for (key, attachment) in parse_jira_data(tempfile.name).items():
(project, issue) = key
- if jenkinsUrlOverrides.has_key(project):
- url = jenkinsUrlOverrides[project]
+ if jenkinsurloverrides.has_key(project):
+ url = jenkinsurloverrides[project]
else:
- url = options.jenkinsUrl
+ url = options.jenkinsurl
- jenkinsUrlTemplate = url + '/job/' \
+ jenkinsurltemplate = url + '/job/' \
+ options.jenkinsJobTemplate \
- + '/buildWithParameters?' + tokenFrag \
+ + '/buildWithParameters?' + token_frag \
+ '&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}'
- urlArgs = {
+ url_args = {
'project': project,
'issue': issue,
'attachment': attachment,
}
- jenkinsUrl = jenkinsUrlTemplate.format(**urlArgs)
+ jenkinsurl = jenkinsurltemplate.format(**url_args)
# submit job
- jobName = '%s-%s,%s' % (project, issue, attachment)
- if not jobLogHistory or jobName not in jobLogHistory:
- print jobName + ' has not been processed, submitting'
- jobLog.write(jobName + '\n')
- jobLog.flush()
+ jobname = '%s-%s,%s' % (project, issue, attachment)
+ if not jobloghistory or jobname not in jobloghistory:
+ print jobname + ' has not been processed, submitting'
+ joblog.write(jobname + '\n')
+ joblog.flush()
if options.live:
- httpGet(jenkinsUrl, True)
+ http_get(jenkinsurl, True)
else:
- print 'GET ' + jenkinsUrl
+ print 'GET ' + jenkinsurl
else:
- print jobName + ' has been processed, ignoring'
- jobLog.close()
+ print jobname + ' has been processed, ignoring'
+ joblog.close()
finally:
if options.live:
- os.remove(tempFile.name)
+ os.remove(tempfile.name)
else:
- print 'JIRA Data is located: ' + tempFile.name
+ print 'JIRA Data is located: ' + tempfile.name
+
+if __name__ == '__main__':
+ main()
diff --git a/releasedocmaker/src/main/python/releasedocmaker.py
b/releasedocmaker/src/main/python/releasedocmaker.py
index 63c2e00..f4527c7 100755
--- a/releasedocmaker/src/main/python/releasedocmaker.py
+++ b/releasedocmaker/src/main/python/releasedocmaker.py
@@ -16,6 +16,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+""" wrapper to launch releasedocmaker from the CLI """
+
import sys
sys.dont_write_bytecode = True
# pylint: disable=wrong-import-position,import-self
diff --git a/releasedocmaker/src/main/python/releasedocmaker/__init__.py
b/releasedocmaker/src/main/python/releasedocmaker/__init__.py
index 900f497..47f104c 100755
--- a/releasedocmaker/src/main/python/releasedocmaker/__init__.py
+++ b/releasedocmaker/src/main/python/releasedocmaker/__init__.py
@@ -1,4 +1,4 @@
-#
+#!/usr/bin/env python2
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,6 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+""" Generate releasenotes based upon JIRA """
+
+from __future__ import print_function
import sys
from glob import glob
from optparse import OptionParser
@@ -36,8 +39,8 @@ from utils import get_jira, to_unicode, sanitize_text,
processrelnote, Outputs
try:
import dateutil.parser
except ImportError:
- print "This script requires python-dateutil module to be installed. " \
- "You can install it using:\n\t pip install python-dateutil"
+ print("This script requires python-dateutil module to be installed. " \
+ "You can install it using:\n\t pip install python-dateutil")
sys.exit(1)
RELEASE_VERSION = {}
@@ -100,7 +103,7 @@ def buildreadme(title, asf_license):
version))
-class GetVersions(object):
+class GetVersions(object): # pylint: disable=too-few-public-methods
""" List of version strings """
def __init__(self, versions, projects):
@@ -108,7 +111,7 @@ class GetVersions(object):
projects = projects
self.newversions = []
versions.sort(key=LooseVersion)
- print "Looking for %s through %s" % (versions[0], versions[-1])
+ print("Looking for %s through %s" % (versions[0], versions[-1]))
newversions = set()
for project in projects:
url = JIRA_BASE_URL + \
@@ -129,10 +132,11 @@ class GetVersions(object):
end_index = len(newlist) - 1 - newlist[::-1].index(versions[-1])
for newversion in newlist[start_index + 1:end_index]:
if newversion in newversions:
- print "Adding %s to the list" % newversion
+ print("Adding %s to the list" % newversion)
self.newversions.append(newversion)
def getlist(self):
+ """ Get the list of versions """
return self.newversions
@@ -172,12 +176,15 @@ class Jira(object):
self.important = None
def get_id(self):
+ """ get the Issue ID """
return to_unicode(self.key)
def get_description(self):
+ """ get the description """
return to_unicode(self.fields['description'])
def get_release_note(self):
+ """ get the release note field """
if self.notes is None:
field = self.parent.field_id_map['Release Note']
if field in self.fields:
@@ -189,6 +196,7 @@ class Jira(object):
return self.notes
def get_priority(self):
+ """ Get the priority """
ret = ""
pri = self.fields['priority']
if pri is not None:
@@ -196,6 +204,7 @@ class Jira(object):
return to_unicode(ret)
def get_assignee(self):
+ """ Get the assignee """
ret = ""
mid = self.fields['assignee']
if mid is not None:
@@ -203,15 +212,18 @@ class Jira(object):
return to_unicode(ret)
def get_components(self):
+ """ Get the component(s) """
if self.fields['components']:
return ", ".join([comp['name'] for comp in
self.fields['components']
])
return ""
def get_summary(self):
+ """ Get the summary """
return self.fields['summary']
def get_type(self):
+ """ Get the Issue type """
ret = ""
mid = self.fields['issuetype']
if mid is not None:
@@ -219,6 +231,7 @@ class Jira(object):
return to_unicode(ret)
def get_reporter(self):
+ """ Get the issue reporter """
ret = ""
mid = self.fields['reporter']
if mid is not None:
@@ -226,6 +239,7 @@ class Jira(object):
return to_unicode(ret)
def get_project(self):
+ """ get the project """
ret = ""
mid = self.fields['project']
if mid is not None:
@@ -244,18 +258,19 @@ class Jira(object):
result = cmp(int(selfsplit[1]), int(othersplit[1]))
# dec is supported for backward compatibility
if SORTORDER in ['dec', 'desc']:
- result *= -1
+ result *= -1
elif SORTTYPE == 'resolutiondate':
dts = dateutil.parser.parse(self.fields['resolutiondate'])
dto = dateutil.parser.parse(other.fields['resolutiondate'])
result = cmp(dts, dto)
if SORTORDER == 'newer':
- result *= -1
+ result *= -1
return result
def get_incompatible_change(self):
+ """ get incompatible flag """
if self.incompat is None:
field = self.parent.field_id_map['Hadoop Flags']
self.reviewed = False
@@ -278,6 +293,7 @@ class Jira(object):
return self.incompat
def get_important(self):
+ """ get importat flag """
if self.important is None:
field = self.parent.field_id_map['Flags']
self.important = False
@@ -339,11 +355,11 @@ class JiraIter(object):
print(err)
fail_count += 1
if fail_count <= NUM_RETRIES:
- print "Connection failed %d times. Retrying." % (fail_count)
+ print("Connection failed %d times. Retrying." % (fail_count))
sleep(1)
return JiraIter.load_jira(params, fail_count)
else:
- print "Connection failed %d times. Aborting." % (fail_count)
+ print("Connection failed %d times. Aborting." % (fail_count))
sys.exit(1)
@staticmethod
@@ -356,7 +372,7 @@ class JiraIter(object):
while pos < end:
data = JiraIter.query_jira(ver, projects, pos)
if 'error_messages' in data:
- print "JIRA returns error message: %s" % data['error_messages']
+ print("JIRA returns error message: %s" %
data['error_messages'])
sys.exit(1)
pos = data['startAt'] + data['maxResults']
end = data['total']
@@ -382,6 +398,7 @@ class JiraIter(object):
return self
def next(self):
+ """ get next """
data = self.iter.next()
j = Jira(data, self)
return j
@@ -438,25 +455,25 @@ class Linter(object):
enabled = []
disabled = []
- for o in options.lint:
- for token in o.split(","):
+ for opt in options.lint:
+ for token in opt.split(","):
if token not in valid:
- print "Unknown lint filter '%s', valid options are: %s" % \
- (token, ", ".join(v for v in sorted(valid)))
+ print("Unknown lint filter '%s', valid options are: %s" % \
+ (token, ", ".join(v for v in sorted(valid))))
sys.exit(1)
if token.startswith("-"):
disabled.append(token[1:])
else:
enabled.append(token)
- for e in enabled:
- if e == "all":
- for f in self._valid_filters:
- self._filters[f] = True
+ for eopt in enabled:
+ if eopt == "all":
+ for filt in self._valid_filters:
+ self._filters[filt] = True
else:
- self._filters[e] = True
- for d in disabled:
- self._filters[d] = False
+ self._filters[eopt] = True
+ for disopt in disabled:
+ self._filters[disopt] = False
def had_errors(self):
"""Returns True if a lint error was encountered, else False."""
@@ -532,7 +549,7 @@ class Linter(object):
% (" and ".join(error_message), jira.get_id())
-def parse_args():
+def parse_args(): # pylint: disable=too-many-branches
"""Parse command-line arguments with optparse."""
usage = "usage: %prog [OPTIONS] " + \
"--project PROJECT [--project PROJECT] " + \
@@ -663,9 +680,9 @@ def parse_args():
# Handle the version string right away and exit
if options.release_version:
with open(
- os.path.join(
- os.path.dirname(__file__), "../VERSION"), 'r') as ver_file:
- print ver_file.read()
+ os.path.join(
+ os.path.dirname(__file__), "../VERSION"), 'r') as ver_file:
+ print(ver_file.read())
sys.exit(0)
# Validate options
@@ -686,13 +703,20 @@ def parse_args():
options.output_directory = options.output_directory[0]
if options.range or len(options.versions) > 1:
- if not options.versiondirs and not options.versionfiles:
- parser.error("Multiple versions require either --fileversions or
--dirversions")
+ if not options.versiondirs and not options.versionfiles:
+ parser.error("Multiple versions require either --fileversions or
--dirversions")
return options
-def main():
+def main(): # pylint: disable=too-many-statements, too-many-branches,
too-many-locals
+ """ hey, it's main """
+ global JIRA_BASE_URL #pylint: disable=global-statement
+ global BACKWARD_INCOMPATIBLE_LABEL #pylint: disable=global-statement
+ global SORTTYPE #pylint: disable=global-statement
+ global SORTORDER #pylint: disable=global-statement
+ global NUM_RETRIES #pylint: disable=global-statement
+
options = parse_args()
if options.output_directory is not None:
@@ -705,17 +729,15 @@ def main():
options.output_directory):
pass
else:
- print "Unable to create output directory %s: %u, %s" % \
- (options.output_directory, exc.errno, exc.message)
+ print("Unable to create output directory %s: %u, %s" % \
+ (options.output_directory, exc.errno, exc.message))
sys.exit(1)
os.chdir(options.output_directory)
if options.base_url is not None:
- global JIRA_BASE_URL
JIRA_BASE_URL = options.base_url
if options.incompatible_label is not None:
- global BACKWARD_INCOMPATIBLE_LABEL
BACKWARD_INCOMPATIBLE_LABEL = options.incompatible_label
@@ -728,9 +750,7 @@ def main():
versions = [Version(v) for v in options.versions]
versions.sort()
- global SORTTYPE
SORTTYPE = options.sorttype
- global SORTORDER
SORTORDER = options.sortorder
if options.title is None:
@@ -739,7 +759,6 @@ def main():
title = options.title
if options.retries is not None:
- global NUM_RETRIES
NUM_RETRIES = options.retries[0]
haderrors = False
@@ -749,7 +768,7 @@ def main():
linter = Linter(vstr, options)
jlist = sorted(JiraIter(vstr, projects))
if not jlist and not options.empty:
- print "There is no issue which has the specified version: %s" %
version
+ print("There is no issue which has the specified version: %s" %
version)
continue
if vstr in RELEASE_VERSION:
@@ -763,49 +782,49 @@ def main():
os.mkdir(vstr)
if options.versionfiles and options.versiondirs:
- reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
- "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", [],
- {"ver": version,
- "date": reldate,
- "title": title})
- choutputs = Outputs("%(ver)s/CHANGELOG.%(ver)s.md",
- "%(ver)s/CHANGELOG.%(key)s.%(ver)s.md", [],
- {"ver": version,
- "date": reldate,
- "title": title})
+ reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
+ "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", [],
+ {"ver": version,
+ "date": reldate,
+ "title": title})
+ choutputs = Outputs("%(ver)s/CHANGELOG.%(ver)s.md",
+ "%(ver)s/CHANGELOG.%(key)s.%(ver)s.md", [],
+ {"ver": version,
+ "date": reldate,
+ "title": title})
elif options.versiondirs:
- reloutputs = Outputs("%(ver)s/RELEASENOTES.md",
- "%(ver)s/RELEASENOTES.%(key)s.md", [],
- {"ver": version,
- "date": reldate,
- "title": title})
- choutputs = Outputs("%(ver)s/CHANGELOG.md",
- "%(ver)s/CHANGELOG.%(key)s.md", [],
- {"ver": version,
- "date": reldate,
- "title": title})
+ reloutputs = Outputs("%(ver)s/RELEASENOTES.md",
+ "%(ver)s/RELEASENOTES.%(key)s.md", [],
+ {"ver": version,
+ "date": reldate,
+ "title": title})
+ choutputs = Outputs("%(ver)s/CHANGELOG.md",
+ "%(ver)s/CHANGELOG.%(key)s.md", [],
+ {"ver": version,
+ "date": reldate,
+ "title": title})
elif options.versionfiles:
- reloutputs = Outputs("RELEASENOTES.%(ver)s.md",
- "RELEASENOTES.%(key)s.%(ver)s.md", [],
- {"ver": version,
- "date": reldate,
- "title": title})
- choutputs = Outputs("CHANGELOG.%(ver)s.md",
- "CHANGELOG.%(key)s.%(ver)s.md", [],
- {"ver": version,
- "date": reldate,
- "title": title})
+ reloutputs = Outputs("RELEASENOTES.%(ver)s.md",
+ "RELEASENOTES.%(key)s.%(ver)s.md", [],
+ {"ver": version,
+ "date": reldate,
+ "title": title})
+ choutputs = Outputs("CHANGELOG.%(ver)s.md",
+ "CHANGELOG.%(key)s.%(ver)s.md", [],
+ {"ver": version,
+ "date": reldate,
+ "title": title})
else:
- reloutputs = Outputs("RELEASENOTES.md",
- "RELEASENOTES.%(key)s.md", [],
- {"ver": version,
- "date": reldate,
- "title": title})
- choutputs = Outputs("CHANGELOG.md",
- "CHANGELOG.%(key)s.md", [],
- {"ver": version,
- "date": reldate,
- "title": title})
+ reloutputs = Outputs("RELEASENOTES.md",
+ "RELEASENOTES.%(key)s.md", [],
+ {"ver": version,
+ "date": reldate,
+ "title": title})
+ choutputs = Outputs("CHANGELOG.md",
+ "CHANGELOG.%(key)s.md", [],
+ {"ver": version,
+ "date": reldate,
+ "title": title})
if options.license is True:
reloutputs.write_all(ASF_LICENSE)
@@ -870,7 +889,7 @@ def main():
linter.lint(jira)
if linter.enabled:
- print linter.message()
+ print(linter.message())
if linter.had_errors():
haderrors = True
shutil.rmtree(vstr)
@@ -880,60 +899,60 @@ def main():
reloutputs.close()
if options.skip_credits:
- CHANGEHDR1 = "| JIRA | Summary | Priority | " + \
+ change_header21 = "| JIRA | Summary | Priority | " + \
"Component |\n"
- CHANGEHDR2 = "|:---- |:---- | :--- |:---- |\n"
+ change_header22 = "|:---- |:---- | :--- |:---- |\n"
else:
- CHANGEHDR1 = "| JIRA | Summary | Priority | " + \
+ change_header21 = "| JIRA | Summary | Priority | " + \
"Component | Reporter | Contributor |\n"
- CHANGEHDR2 = "|:---- |:---- | :--- |:---- |:---- |:---- |\n"
+ change_header22 = "|:---- |:---- | :--- |:---- |:---- |:---- |\n"
if incompatlist:
choutputs.write_all("### INCOMPATIBLE CHANGES:\n\n")
- choutputs.write_all(CHANGEHDR1)
- choutputs.write_all(CHANGEHDR2)
+ choutputs.write_all(change_header21)
+ choutputs.write_all(change_header22)
choutputs.write_list(incompatlist, options.skip_credits,
JIRA_BASE_URL)
if importantlist:
choutputs.write_all("\n\n### IMPORTANT ISSUES:\n\n")
- choutputs.write_all(CHANGEHDR1)
- choutputs.write_all(CHANGEHDR2)
+ choutputs.write_all(change_header21)
+ choutputs.write_all(change_header22)
choutputs.write_list(importantlist, options.skip_credits,
JIRA_BASE_URL)
if newfeaturelist:
choutputs.write_all("\n\n### NEW FEATURES:\n\n")
- choutputs.write_all(CHANGEHDR1)
- choutputs.write_all(CHANGEHDR2)
+ choutputs.write_all(change_header21)
+ choutputs.write_all(change_header22)
choutputs.write_list(newfeaturelist, options.skip_credits,
JIRA_BASE_URL)
if improvementlist:
choutputs.write_all("\n\n### IMPROVEMENTS:\n\n")
- choutputs.write_all(CHANGEHDR1)
- choutputs.write_all(CHANGEHDR2)
+ choutputs.write_all(change_header21)
+ choutputs.write_all(change_header22)
choutputs.write_list(improvementlist, options.skip_credits,
JIRA_BASE_URL)
if buglist:
choutputs.write_all("\n\n### BUG FIXES:\n\n")
- choutputs.write_all(CHANGEHDR1)
- choutputs.write_all(CHANGEHDR2)
+ choutputs.write_all(change_header21)
+ choutputs.write_all(change_header22)
choutputs.write_list(buglist, options.skip_credits, JIRA_BASE_URL)
if testlist:
choutputs.write_all("\n\n### TESTS:\n\n")
- choutputs.write_all(CHANGEHDR1)
- choutputs.write_all(CHANGEHDR2)
+ choutputs.write_all(change_header21)
+ choutputs.write_all(change_header22)
choutputs.write_list(testlist, options.skip_credits, JIRA_BASE_URL)
if subtasklist:
choutputs.write_all("\n\n### SUB-TASKS:\n\n")
- choutputs.write_all(CHANGEHDR1)
- choutputs.write_all(CHANGEHDR2)
+ choutputs.write_all(change_header21)
+ choutputs.write_all(change_header22)
choutputs.write_list(subtasklist, options.skip_credits,
JIRA_BASE_URL)
if tasklist or otherlist:
choutputs.write_all("\n\n### OTHER:\n\n")
- choutputs.write_all(CHANGEHDR1)
- choutputs.write_all(CHANGEHDR2)
+ choutputs.write_all(change_header21)
+ choutputs.write_all(change_header22)
choutputs.write_list(otherlist, options.skip_credits,
JIRA_BASE_URL)
choutputs.write_list(tasklist, options.skip_credits, JIRA_BASE_URL)
@@ -949,4 +968,4 @@ def main():
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/releasedocmaker/src/main/python/releasedocmaker/utils.py
b/releasedocmaker/src/main/python/releasedocmaker/utils.py
index db957d3..ae57fd1 100644
--- a/releasedocmaker/src/main/python/releasedocmaker/utils.py
+++ b/releasedocmaker/src/main/python/releasedocmaker/utils.py
@@ -16,6 +16,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+""" Utility methods used by releasedocmaker """
+
+from __future__ import print_function
import base64
import os
import re
@@ -27,10 +30,6 @@ sys.dont_write_bytecode = True
NAME_PATTERN = re.compile(r' \([0-9]+\)')
-def clean(input_string):
- return sanitize_markdown(re.sub(NAME_PATTERN, "", input_string))
-
-
def get_jira(jira_url):
""" Provide standard method for fetching content from apache jira and
handling of potential errors. Returns urllib2 response or
@@ -48,21 +47,21 @@ def get_jira(jira_url):
response = urllib2.urlopen(req)
except urllib2.HTTPError as http_err:
code = http_err.code
- print "JIRA returns HTTP error %d: %s. Aborting." % \
- (code, http_err.msg)
+ print("JIRA returns HTTP error %d: %s. Aborting." % \
+ (code, http_err.msg))
error_response = http_err.read()
try:
error_response = json.loads(error_response)
- print "- Please ensure that specified authentication, projects,"\
- " fixVersions etc. are correct."
+ print("- Please ensure that specified authentication, projects,"\
+ " fixVersions etc. are correct.")
for message in error_response['errorMessages']:
- print "-", message
+ print("-", message)
except ValueError:
- print "FATAL: Could not parse json response from server."
+ print("FATAL: Could not parse json response from server.")
sys.exit(1)
except urllib2.URLError as url_err:
- print "Error contacting JIRA: %s\n" % jira_url
- print "Reason: %s" % url_err.reason
+ print("Error contacting JIRA: %s\n" % jira_url)
+ print("Reason: %s" % url_err.reason)
raise url_err
except httplib.BadStatusLine as err:
raise err
@@ -70,40 +69,41 @@ def get_jira(jira_url):
def format_components(input_string):
+ """ format the string """
input_string = re.sub(NAME_PATTERN, '', input_string).replace("'", "")
if input_string != "":
ret = input_string
else:
# some markdown parsers don't like empty tables
ret = "."
- return clean(ret)
+ return sanitize_markdown(re.sub(NAME_PATTERN, "", ret))
-
-# Return the string encoded as UTF-8.
-#
-# This is necessary for handling markdown in Python.
def encode_utf8(input_string):
+ """ Return the string encoded as UTF-8.
+ This is necessary for handling markdown in Python. """
return input_string.encode('utf-8')
-# Sanitize Markdown input so it can be handled by Python.
-#
-# The expectation is that the input is already valid Markdown,
-# so no additional escaping is required.
def sanitize_markdown(input_string):
+ """ Sanitize Markdown input so it can be handled by Python.
+
+ The expectation is that the input is already valid Markdown,
+ so no additional escaping is required. """
input_string = encode_utf8(input_string)
input_string = input_string.replace("\r", "")
input_string = input_string.rstrip()
return input_string
-# Sanitize arbitrary text so it can be embedded in MultiMarkdown output.
-#
-# Note that MultiMarkdown is not Markdown, and cannot be parsed as such.
-# For instance, when using pandoc, invoke it as `pandoc -f markdown_mmd`.
-#
-# Calls sanitize_markdown at the end as a final pass.
+
def sanitize_text(input_string):
+ """ Sanitize arbitrary text so it can be embedded in MultiMarkdown output.
+
+ Note that MultiMarkdown is not Markdown, and cannot be parsed as such.
+ For instance, when using pandoc, invoke it as `pandoc -f markdown_mmd`.
+
+ Calls sanitize_markdown at the end as a final pass.
+ """
escapes = dict()
# See: https://daringfireball.net/projects/markdown/syntax#backslash
# We only escape a subset of special characters. We ignore characters
@@ -113,26 +113,25 @@ def sanitize_text(input_string):
slash_escapes += "\\"
all_chars = set()
# Construct a set of escapes
- for c in slash_escapes:
- all_chars.add(c)
- for c in all_chars:
- escapes[c] = "\\" + c
+ for char in slash_escapes:
+ all_chars.add(char)
+ for char in all_chars:
+ escapes[char] = "\\" + char
# Build the output string character by character to prevent double escaping
output_string = ""
- for c in input_string:
- o = c
- if c in escapes:
- o = escapes[c]
- output_string += o
+ for char in input_string:
+ out = char
+ if char in escapes:
+ out = escapes[char]
+ output_string += out
return sanitize_markdown(output_string.rstrip())
-# if release notes have a special marker,
-# we'll treat them as already in markdown format
def processrelnote(input_string):
- relnote_pattern = re.compile('^\<\!\-\- ([a-z]+) \-\-\>')
+ """ if release notes have a special marker, we'll treat them as already in
markdown format """
+ relnote_pattern = re.compile(r'^\<\!\-\- ([a-z]+) \-\-\>')
fmt = relnote_pattern.match(input_string)
if fmt is None:
return sanitize_text(input_string)
@@ -142,6 +141,7 @@ def processrelnote(input_string):
def to_unicode(obj):
+ """ convert string to unicode """
if obj is None:
return ""
return unicode(obj)
@@ -162,6 +162,7 @@ class Outputs(object):
self.others[key] = open(file_name_pattern % both, 'w')
def write_all(self, pattern):
+ """ write everything given a pattern """
both = dict(self.params)
both['key'] = ''
self.base.write(pattern % both)
@@ -171,11 +172,13 @@ class Outputs(object):
self.others[key].write(pattern % both)
def write_key_raw(self, key, input_string):
+ """ write everything withotu changes """
self.base.write(input_string)
if key in self.others:
self.others[key].write(input_string)
def close(self):
+ """ close all the outputs """
self.base.close()
for value in self.others.values():
value.close()
diff --git a/shelldocs/src/main/python/shelldocs.py
b/shelldocs/src/main/python/shelldocs.py
index 57f912f..f1c28fa 100755
--- a/shelldocs/src/main/python/shelldocs.py
+++ b/shelldocs/src/main/python/shelldocs.py
@@ -16,6 +16,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+""" wrapper to run shelldocs from the CLI """
+
import sys
sys.dont_write_bytecode = True
# pylint: disable=wrong-import-position,import-self
diff --git a/shelldocs/src/main/python/shelldocs/__init__.py
b/shelldocs/src/main/python/shelldocs/__init__.py
index 108f3b3..e6c16c2 100755
--- a/shelldocs/src/main/python/shelldocs/__init__.py
+++ b/shelldocs/src/main/python/shelldocs/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+""" process bash scripts and generate documentation from them """
+
# Do this immediately to prevent compiled forms
import sys
import os
@@ -67,7 +69,7 @@ def toc(tlist):
return tocout
-class ShellFunction(object):
+class ShellFunction(object): # pylint: disable=too-many-public-methods,
too-many-instance-attributes
"""a shell function"""
def __init__(self, filename):
@@ -304,7 +306,7 @@ def marked_as_ignored(file_path):
return False
-def main():
+def main(): # pylint: disable=too-many-statements, too-many-branches
'''main entry point'''
parser = OptionParser(
usage="usage: %prog [--skipprnorep] " + "[--output OUTFILE|--lint] " +
@@ -347,8 +349,8 @@ def main():
if options.release_version:
with open(
- os.path.join(
- os.path.dirname(__file__), "../VERSION"), 'r') as ver_file:
+ os.path.join(
+ os.path.dirname(__file__), "../VERSION"), 'r') as ver_file:
print ver_file.read()
sys.exit(0)
