Repository: yetus Updated Branches: refs/heads/master ef2b90a80 -> 34b1ff77c
YETUS-288. move precommit-admin from Hadoop This is a copy of the jenkinsPrecommitAdmin.py written by Brock Noland <[email protected]> as series of commits in the svn://svn.apache.org/repos/asf/hadoop/nightly source tree. The commits correspond with HADOOP-9765, HADOOP-10199, and HADOOP-10200. Signed-off-by: Brock Nelson <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/34b1ff77 Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/34b1ff77 Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/34b1ff77 Branch: refs/heads/master Commit: 34b1ff77cf13389ca21c41a2ecc9f335e6da7aea Parents: ef2b90a Author: Allen Wittenauer <[email protected]> Authored: Fri Dec 8 11:13:24 2017 -0800 Committer: Allen Wittenauer <[email protected]> Committed: Sat Dec 9 16:14:39 2017 -0800 ---------------------------------------------------------------------- precommit/jenkins/jenkins-admin.py | 143 ++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/34b1ff77/precommit/jenkins/jenkins-admin.py ---------------------------------------------------------------------- diff --git a/precommit/jenkins/jenkins-admin.py b/precommit/jenkins/jenkins-admin.py new file mode 100755 index 0000000..0fe34b0 --- /dev/null +++ b/precommit/jenkins/jenkins-admin.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Script uses a JIRA search filter to submit test Jenkins +# jobs for patch avilable issues. For more information see: +# http://wiki.apache.org/general/PreCommitBuilds +# https://builds.apache.org/job/PreCommit-Admin/ + +from optparse import OptionParser +from tempfile import NamedTemporaryFile +from xml.etree import ElementTree +import os, re, sys, urllib2 + +# max number of entries to keep in the patch_tested.txt file +MAX_HISTORY = 5000 + +def httpGet(resource, ignoreError=False): + if ignoreError: + try: + return urllib2.urlopen(resource).read() + except urllib2.URLError, e: + print "ERROR retrieving resource %s: %s" % (resource, e) + return "" + return urllib2.urlopen(resource).read() + + +# returns a map of (project, issue) => attachment id +def parseJiraData(fileName): + tree = ElementTree.parse(fileName) + root = tree.getroot() + jiraPattern = re.compile('([A-Z]+)\-([0-9]+)') + result = {} + for item in root.findall("./channel/item"): + key = item.find('key') + if key == None: continue + issue = key.text + matcher = jiraPattern.match(issue) + if not matcher: continue + issue = matcher.group(1), matcher.group(2) + attachmentIds = [] + for attachment in item.findall('./attachments/attachment'): + attachmentId = attachment.get('id') + try: + attachmentIds.append(int(attachmentId)) + except ValueError: + pass + if len(attachmentIds) > 0: + attachmentIds.sort() + result[issue] = attachmentIds[-1] + return result + +if __name__ == '__main__': + parser = OptionParser() + parser.add_option("--jenkins-url", dest="jenkinsUrl", + help="Jenkins base URL", metavar="URL") + parser.add_option("--jenkins-token", dest="jenkinsToken", + help="Jenkins Token", metavar="TOKEN") + parser.add_option("--jira-filter", dest="jiraFilter", + help="JIRA filter URL", metavar="URL") + parser.add_option("--jenkins-url-override", dest="jenkinsUrlOverrides", action="append", + help="Project specific Jenkins base URL", metavar="PROJECT=URL") + parser.add_option("--live", dest="live", action="store_true", + help="Submit Job to jenkins") + (options, args) = parser.parse_args() + if not options.jiraFilter: + parser.error("JIRA Filter is a required argument") + if not options.jenkinsUrl: + parser.error("Jenkins URL is a required argument") + if options.live and not options.jenkinsToken: + parser.error("Jenkins Token is required when in live mode") + jenkinsUrlOverrides = {} + if options.jenkinsUrlOverrides: + for override in options.jenkinsUrlOverrides: + if "=" not in override: + parser.error("Invalid Jenkins Url Override: " + override) + (project, url) = override.split("=", 1) + jenkinsUrlOverrides[project.upper()] = url + tempFile = NamedTemporaryFile(delete=False) + try: + jobLogHistory = httpGet(options.jenkinsUrl + \ + "/job/PreCommit-Admin/lastSuccessfulBuild/artifact/patch_tested.txt", True) + # if we don't have a successful build available try the last build + if not jobLogHistory: + jobLogHistory = httpGet(options.jenkinsUrl + \ + "/job/PreCommit-Admin/lastCompletedBuild/artifact/patch_tested.txt") + jobLogHistory = jobLogHistory.strip().split("\n") + if "TESTED ISSUES" not in jobLogHistory[0]: + print "Downloaded patch_tested.txt control file may be corrupted. Failing." + sys.exit(1) + jobLog = open('patch_tested.txt', 'w+') + if len(jobLogHistory) > MAX_HISTORY: + jobLogHistory = [ jobLogHistory[0] ] + jobLogHistory[len(jobLogHistory) - MAX_HISTORY:] + for jobHistoryRecord in jobLogHistory: + jobLog.write(jobHistoryRecord + "\n") + jobLog.flush() + rssData = httpGet(options.jiraFilter) + tempFile.write(rssData) + tempFile.flush() + for key, attachment in parseJiraData(tempFile.name).items(): + (project, issue) = key + if jenkinsUrlOverrides.has_key(project): + url = jenkinsUrlOverrides[project] + else: + url = options.jenkinsUrl + jenkinsUrlTemplate = url + "/job/PreCommit-{project}-Build/buildWithParameters" + \ + "?token={token}&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}" + urlArgs = {'token': options.jenkinsToken, 'project': project, 'issue': issue, 'attachment': attachment } + jenkinsUrl = jenkinsUrlTemplate.format(**urlArgs) + # submit job + jobName = "%s-%s,%s" % (project, issue, attachment) + if jobName not in jobLogHistory: + print jobName + " has not been processed, submitting" + jobLog.write(jobName + "\n") + jobLog.flush() + if options.live: + httpGet(jenkinsUrl, True) + else: + print "GET " + jenkinsUrl + else: + print jobName + " has been processed, ignoring" + jobLog.close() + finally: + if options.live: + os.remove(tempFile.name) + else: + print "JIRA Data is located: " + tempFile.name
