http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/tajo.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/tajo.sh b/precommit/personality/tajo.sh deleted file mode 100755 index fb86ff7..0000000 --- a/precommit/personality/tajo.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - # shellcheck disable=SC2034 - BUILDTOOL=maven - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=master - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^TAJO-[0-9]+$' - #shellcheck disable=SC2034 - GITHUB_REPO="apache/tajo" - #shellcheck disable=SC2034 - PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/TAJO/How+to+Contribute+to+Tajo"; -}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/tez.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/tez.sh b/precommit/personality/tez.sh deleted file mode 100755 index 32d9ffc..0000000 --- a/precommit/personality/tez.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - # shellcheck disable=SC2034 - BUILDTOOL=maven - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=master - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^TEZ-[0-9]+$' - #shellcheck disable=SC2034 - GITHUB_REPO="apache/tez" - #shellcheck disable=SC2034 - PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/TEZ/How+to+Contribute+to+Tez"; -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/pom.xml ---------------------------------------------------------------------- diff --git a/precommit/pom.xml b/precommit/pom.xml new file mode 100644 index 0000000..1c11a8b --- /dev/null +++ b/precommit/pom.xml @@ -0,0 +1,154 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.yetus</groupId> + <artifactId>yetus-project</artifactId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + <artifactId>precommit</artifactId> + <description>Build and Patch Management and Testing</description> + <name>Apache Yetus - Precommit</name> + <packaging>pom</packaging> + + <build> + <plugins> + + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>mkdir-bin</id> + <phase>initialize</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <exportAntProperties>true</exportAntProperties> + <target> + <mkdir dir="target/dist/apache-yetus-${project.version}/lib/${project.artifactId}"/> + <mkdir dir="target/dist/apache-yetus-${project.version}/bin"/> + </target> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.yetus</groupId> + <artifactId>yetus-assemblies</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>build</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>dist/apache-yetus-${project.version}</finalName> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <descriptorRefs> + <descriptorRef>script-bundle</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + + <execution> + <id>module-dist</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <attach>true</attach> + <descriptorRefs> + <descriptorRef>module-dist</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + + </executions> + </plugin> + + <plugin> + <groupId>org.apache.yetus</groupId> + <artifactId>yetus-minimaven-plugin</artifactId> + <version>${project.version}</version> + <executions> + <execution> + <id>qbt.sh</id> + <phase>compile</phase> + <goals> + <goal>symlink</goal> + </goals> + <configuration> + <target>test-patch.sh</target> + <newLink>dist/apache-yetus-${project.version}/lib/${project.artifactId}/qbt.sh</newLink> + </configuration> + </execution> + <execution> + <id>bins4libs</id> + <phase>prepare-package</phase> + <goals> + <goal>bin4libs</goal> + </goals> + <configuration> + <libdir>lib/precommit</libdir> + <basedir>${project.build.directory}/dist/apache-yetus-${project.version}</basedir> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-deploy-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>src/main/python/unit-test-filter-file.example</exclude> + </excludes> + </configuration> + </plugin> + + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/smart-apply-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/smart-apply-patch.sh b/precommit/smart-apply-patch.sh deleted file mode 100755 index a5dd09b..0000000 --- a/precommit/smart-apply-patch.sh +++ /dev/null @@ -1,333 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Make sure that bash version meets the pre-requisite - -if [[ -z "${BASH_VERSINFO[0]}" ]] \ - || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ - || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then - echo "bash v3.2+ is required. Sorry." - exit 1 -fi - -this="${BASH_SOURCE-$0}" -BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) -#shellcheck disable=SC2034 -QATESTMODE=false - -# dummy functions -function add_vote_table -{ - true -} - -function add_footer_table -{ - true -} - -function big_console_header -{ - true -} - -function add_test -{ - true -} - -## @description Clean the filesystem as appropriate and then exit -## @audience private -## @stability evolving -## @replaceable no -## @param runresult -function cleanup_and_exit -{ - local result=$1 - - if [[ ${PATCH_DIR} =~ ^/tmp/yetus - && -d ${PATCH_DIR} ]]; then - rm -rf "${PATCH_DIR}" - fi - - # shellcheck disable=SC2086 - exit ${result} -} - -## @description Setup the default global variables -## @audience public -## @stability stable -## @replaceable no -function setup_defaults -{ - common_defaults -} - -## @description Print the usage information -## @audience public -## @stability stable -## @replaceable no -function yetus_usage -{ - echo "smart-apply-patch.sh [OPTIONS] patch" - echo "" - echo "Where:" - echo " patch is a file, URL, or bugsystem-compatible location of the patch file" - echo "" - echo "Options:" - echo "" - yetus_add_option "--committer" "Apply patches like a boss." - yetus_add_option "--debug" "If set, then output some extra stuff to stderr" - yetus_add_option "--dry-run" "Check for patch viability without applying" - yetus_add_option "--list-plugins" "List all installed plug-ins and then exit" - yetus_add_option "--modulelist=<list>" "Specify additional modules to test (comma delimited)" - yetus_add_option "--offline" "Avoid connecting to the Internet" - yetus_add_option "--patch-dir=<dir>" "The directory for working and output files (default '/tmp/yetus-(random))" - yetus_add_option "--personality=<file>" "The personality file to load" - yetus_add_option "--plugins=<list>" "Specify which plug-ins to add/delete (comma delimited; use 'all' for all found)" - yetus_add_option "--project=<name>" "The short name for project currently using test-patch (default 'yetus')" - yetus_add_option "--skip-system-plugins" "Do not load plugins from ${BINDIR}/test-patch.d" - yetus_add_option "--user-plugins=<dir>" "A directory of user provided plugins. see test-patch.d for examples (default empty)" - yetus_add_option "--version" "Print release version information and exit" - yetus_add_option "--gpg-sign" "GPG sign the commit using gpg keys" - yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" - yetus_reset_usage - - echo "" - echo "Shell binary overrides:" - yetus_add_option "--awk-cmd=<cmd>" "The 'awk' command to use (default 'awk')" - yetus_add_option "--curl-cmd=<cmd>" "The 'curl' command to use (default 'curl')" - yetus_add_option "--diff-cmd=<cmd>" "The GNU-compatible 'diff' command to use (default 'diff')" - yetus_add_option "--file-cmd=<cmd>" "The 'file' command to use (default 'file')" - yetus_add_option "--git-cmd=<cmd>" "The 'git' command to use (default 'git')" - yetus_add_option "--grep-cmd=<cmd>" "The 'grep' command to use (default 'grep')" - yetus_add_option "--patch-cmd=<cmd>" "The 'patch' command to use (default 'patch')" - yetus_add_option "--sed-cmd=<cmd>" "The 'sed' command to use (default 'sed')" - yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" - yetus_reset_usage - - echo "" - importplugins - - unset TESTFORMATS - unset TESTTYPES - unset BUILDTOOLS - - for plugin in ${BUGSYSTEMS}; do - if declare -f ${plugin}_usage >/dev/null 2>&1; then - echo "" - echo "${plugin} plugin usage options:" - "${plugin}_usage" - yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" - yetus_reset_usage - fi - done -} - -## @description Interpret the command line parameters -## @audience private -## @stability stable -## @replaceable no -## @param $@ -## @return May exit on failure -function parse_args -{ - local i - - common_args "$@" - - for i in "$@"; do - case ${i} in - --committer) - COMMITMODE=true - ;; - --gpg-sign) - GPGSIGN=true - ;; - --dry-run) - PATCH_DRYRUNMODE=true - ;; - --*) - ## PATCH_OR_ISSUE can't be a --. So this is probably - ## a plugin thing. - continue - ;; - *) - PATCH_OR_ISSUE=${i#*=} - ;; - esac - done - - if [[ ! -d ${PATCH_DIR} ]]; then - mkdir -p "${PATCH_DIR}" - if [[ $? != 0 ]] ; then - yetus_error "ERROR: Unable to create ${PATCH_DIR}" - cleanup_and_exit 1 - fi - fi -} - -## @description git am dryrun -## @replaceable no -## @audience private -## @stability evolving -function gitam_dryrun -{ - - # there is no dryrun method for git-am, so just - # use apply instead. - gitapply_dryrun "$@" - - if [[ ${PATCH_METHOD} = "gitapply" ]]; then - PATCH_METHOD="gitam" - fi -} - -## @description git am signoff -## @replaceable no -## @audience private -## @stability evolving -function gitam_apply -{ - declare patchfile=$1 - declare gpg=$2 - - if [[ ${gpg} = true ]]; then - EXTRA_ARGS="-S" - fi - - echo "Applying the patch:" - yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-am.log" \ - "${GIT}" am --signoff ${EXTRA_ARGS} --whitespace=fix "-p${PATCH_LEVEL}" "${patchfile}" - RESULT=$? - ${GREP} -v "^Checking" "${PATCH_DIR}/apply-patch-git-am.log" - - # fallback - if [[ ${RESULT} -gt 0 && ${PATCH_SYSTEM} == 'jira' ]]; then - echo "Use git apply and commit with the information from jira." - gitapply_and_commit "${patchfile}" - fi -} - -## @description get author and summary from jira and commit it. -## if the author and the summary contains " or *, -## the function does not work correctly because -## the characters are used for delimiters. -## @replaceable no -## @audience private -## @stability evolving -function gitapply_and_commit -{ - declare patchfile=$1 - declare jsontmpfile - declare assigneeline - declare assigneefile - declare name - declare email - declare author - declare summary - - yetus_debug "gitapply_and_commit: fetching ${JIRA_URL}/rest/api/2/issue/${PATCH_OR_ISSUE}" - if ! jira_http_fetch "rest/api/2/issue/${PATCH_OR_ISSUE}" "${PATCH_DIR}/issue"; then - yetus_debug "gitapply_and_commit: not a JIRA." - return 1 - fi - - jsontmpfile="${PATCH_DIR}/jsontmpfile" - # cannot set " as delimiter for cut command in script, so replace " with * - tr ',' '\n' < "${PATCH_DIR}/issue" | ${SED} 's/\"/*/g' > "${jsontmpfile}" - - assigneeline=$(${GREP} -n -E "^\*assignee\*:" "${jsontmpfile}" | cut -f1 -d":") - assigneefile="${PATCH_DIR}/assigneefile" - tail -n +"${assigneeline}" "${jsontmpfile}" | head -n 20 > "${assigneefile}" - - name=$(${GREP} "displayName" "${assigneefile}" | cut -f4 -d"*") - email=$(${GREP} "emailAddress" "${assigneefile}" | cut -f4 -d"*" \ - | ${SED} 's/ at /@/g' | ${SED} 's/ dot /./g') - author="${name} <${email}>" - summary=$(${GREP} -E "^\*summary\*:" "${jsontmpfile}" | cut -f4 -d"*") - gitapply_apply "${patchfile}" - ${GIT} add --all - echo "Committing with author: ${author}, summary: ${summary}" - yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-am-fallback.log" \ - "${GIT}" commit ${EXTRA_ARGS} --signoff -m "${PATCH_OR_ISSUE}. ${summary}" \ - --author="${author}" -} - -## @description import core library routines -## @audience private -## @stability evolving -function import_core -{ - declare filename - - for filename in "${BINDIR}/core.d"/*; do - # shellcheck disable=SC1091 - # shellcheck source=core.d/01-common.sh - . "${filename}" - done -} - -trap "cleanup_and_exit 1" HUP INT QUIT TERM - -import_core - -setup_defaults - -parse_args "$@" - -importplugins -yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list" -unset BUILDTOOLS -unset TESTTYPES -unset TESTFORMATS - -parse_args_plugins "$@" - -plugins_initialize - -locate_patch - -if [[ ${COMMITMODE} = true ]]; then - status=$(${GIT} status --porcelain) - if [[ "$status" != "" ]] ; then - yetus_error "ERROR: Can't use --committer option in a workspace that contains the following modifications:" - yetus_error "${status}" - cleanup_and_exit 1 - fi - PATCH_METHODS=("gitam" "${PATCH_METHODS[@]}") -fi - -patchfile_dryrun_driver "${PATCH_DIR}/patch" -RESULT=$? - -if [[ ${RESULT} -gt 0 ]]; then - yetus_error "ERROR: Aborting! ${PATCH_OR_ISSUE} cannot be verified." - cleanup_and_exit ${RESULT} -fi - -if [[ ${PATCH_DRYRUNMODE} == false ]]; then - patchfile_apply_driver "${PATCH_DIR}/patch" "${GPGSIGN}" - RESULT=$? -fi - -if [[ ${COMMITMODE} = true - && ${PATCH_METHOD} != "gitam" ]]; then - yetus_debug "Running git add -A" - git add -A -fi - -cleanup_and_exit ${RESULT} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/python/jenkins-admin.py ---------------------------------------------------------------------- diff --git a/precommit/src/main/python/jenkins-admin.py b/precommit/src/main/python/jenkins-admin.py new file mode 100755 index 0000000..f6ffdfe --- /dev/null +++ b/precommit/src/main/python/jenkins-admin.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from optparse import OptionParser +from tempfile import NamedTemporaryFile +from xml.etree import ElementTree +import base64 +import httplib +import os +import re +import sys +import urllib2 + +def httpGet(resource, ignoreError=False, username=None, password=None): + request = urllib2.Request(resource) + if username and password: + base64string = base64.b64encode('%s:%s' % (username, password)) + request.add_header("Authorization", "Basic %s" % base64string) + try: + response = urllib2.urlopen(request) + except urllib2.HTTPError, http_err: + code = http_err.code + print '%s returns HTTP error %d: %s' \ + % (resource, code, http_err.reason) + if ignoreError: + return '' + else: + print 'Aborting.' + sys.exit(1) + except urllib2.URLError, url_err: + print 'Error contacting %s: %s' % (resource, url_err.reason) + if ignoreError: + return '' + else: + raise url_err + except httplib.BadStatusLine, err: + if ignoreError: + return '' + else: + raise err + return response.read() + + +# returns a map of (project, issue) => attachment id + +def parseJiraData(fileName): + tree = ElementTree.parse(fileName) + root = tree.getroot() + jiraPattern = re.compile('([A-Z]+)\-([0-9]+)') + result = {} + for item in root.findall('./channel/item'): + jirakey = item.find('key') + if jirakey is None: + continue + jiraissue = jirakey.text + matcher = jiraPattern.match(jiraissue) + if not matcher: + continue + jiraissue = (matcher.group(1), matcher.group(2)) + attachmentIds = [] + for jiraattachment in item.findall('./attachments/attachment'): + attachmentId = jiraattachment.get('id') + try: + attachmentIds.append(int(attachmentId)) + except ValueError: + pass + if attachmentIds: + attachmentIds.sort() + result[jiraissue] = attachmentIds[-1] + return result + + +if __name__ == '__main__': + parser = OptionParser(prog = 'jenkins-admin') + if os.getenv('JENKINS_URL'): + parser.set_defaults(jenkinsUrl=os.getenv('JENKINS_URL')) + if os.getenv('JOB_NAME'): + parser.set_defaults(jenkinsJobName=os.getenv('JOB_NAME')) + else: + parser.set_defaults(jenkinsJobName='PreCommit-Admin') + + parser.set_defaults(jenkinsJobTemplate='PreCommit-{project}') + parser.add_option('--initialize', action='store_true', + dest='jenkinsInit', + help='Start a new patch_tested.txt file') + parser.add_option('--jenkins-jobname', type='string', + dest='jenkinsJobName', + help='PreCommit-Admin JobName', metavar='JOB_NAME') + parser.add_option('--jenkins-project-template', type='string', + dest='jenkinsJobTemplate', + help='Template for project jobs', + metavar='TEMPLATE') + parser.add_option('--jenkins-token', type='string', + dest='jenkinsToken', help='Jenkins Token', + metavar='TOKEN') + parser.add_option('--jenkins-url', type='string', dest='jenkinsUrl' + , help='Jenkins base URL', metavar='URL') + parser.add_option( + '--jenkins-url-override', + type='string', + dest='jenkinsUrlOverrides', + action='append', + help='Project specific Jenkins base URL', + metavar='PROJECT=URL', + ) + parser.add_option('--jira-filter', type='string', dest='jiraFilter', + help='JIRA filter URL', metavar='URL') + parser.add_option('--jira-user', type='string', dest='jiraUser', + help='JIRA username') + parser.add_option('--jira-password', type='string', dest='jiraPassword', + help='JIRA password') + parser.add_option('--live', dest='live', action='store_true', + help='Submit Job to jenkins') + parser.add_option('--max-history', dest='history', type='int', + help='Maximum history to store', default=5000) + parser.add_option( + '-V', + '--version', + dest='release_version', + action='store_true', + default=False, + help="display version information for jenkins-admin and exit.") + + (options, args) = parser.parse_args() + + # Handle the version string right away and exit + if options.release_version: + with open( + os.path.join( + os.path.dirname(__file__), "../../VERSION"), 'r') as ver_file: + print ver_file.read() + sys.exit(0) + + tokenFrag = '' + if options.jenkinsToken: + tokenFrag = 'token=%s' % options.jenkinsToken + else: + tokenFrag = 'token={project}-token' + if not options.jiraFilter: + parser.error('ERROR: --jira-filter is a required argument.') + if not options.jenkinsUrl: + parser.error('ERROR: --jenkins-url or the JENKINS_URL environment variable is required.' + ) + if options.history < 0: + parser.error('ERROR: --max-history must be 0 or a positive integer.' + ) + jenkinsUrlOverrides = {} + if options.jenkinsUrlOverrides: + for override in options.jenkinsUrlOverrides: + if '=' not in override: + parser.error('Invalid Jenkins Url Override: ' + + override) + (project, url) = override.split('=', 1) + jenkinsUrlOverrides[project.upper()] = url + tempFile = NamedTemporaryFile(delete=False) + try: + jobLogHistory = None + if not options.jenkinsInit: + jobLogHistory = httpGet(options.jenkinsUrl + + '/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt' + % options.jenkinsJobName, True) + + # if we don't have a successful build available try the last build + + if not jobLogHistory: + jobLogHistory = httpGet(options.jenkinsUrl + + '/job/%s/lastCompletedBuild/artifact/patch_tested.txt' + % options.jenkinsJobName) + jobLogHistory = jobLogHistory.strip().split('\n') + if 'TESTED ISSUES' not in jobLogHistory[0]: + print 'Downloaded patch_tested.txt control file may be corrupted. Failing.' + sys.exit(1) + + # we are either going to write a new one or rewrite the old one + + jobLog = open('patch_tested.txt', 'w+') + + if jobLogHistory: + if len(jobLogHistory) > options.history: + jobLogHistory = [jobLogHistory[0]] \ + + jobLogHistory[len(jobLogHistory) + - options.history:] + for jobHistoryRecord in jobLogHistory: + jobLog.write(jobHistoryRecord + '\n') + else: + jobLog.write('TESTED ISSUES\n') + jobLog.flush() + rssData = httpGet(options.jiraFilter,False,options.jiraUser,options.jiraPassword) + tempFile.write(rssData) + tempFile.flush() + for (key, attachment) in parseJiraData(tempFile.name).items(): + (project, issue) = key + if jenkinsUrlOverrides.has_key(project): + url = jenkinsUrlOverrides[project] + else: + url = options.jenkinsUrl + + jenkinsUrlTemplate = url + '/job/' \ + + options.jenkinsJobTemplate \ + + '/buildWithParameters?' + tokenFrag \ + + '&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}' + + urlArgs = { + 'project': project, + 'issue': issue, + 'attachment': attachment, + } + jenkinsUrl = jenkinsUrlTemplate.format(**urlArgs) + + # submit job + + jobName = '%s-%s,%s' % (project, issue, attachment) + if not jobLogHistory or jobName not in jobLogHistory: + print jobName + ' has not been processed, submitting' + jobLog.write(jobName + '\n') + jobLog.flush() + if options.live: + httpGet(jenkinsUrl, True) + else: + print 'GET ' + jenkinsUrl + else: + print jobName + ' has been processed, ignoring' + jobLog.close() + finally: + if options.live: + os.remove(tempFile.name) + else: + print 'JIRA Data is located: ' + tempFile.name http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/python/unit-test-filter-file.example ---------------------------------------------------------------------- diff --git a/precommit/src/main/python/unit-test-filter-file.example b/precommit/src/main/python/unit-test-filter-file.example new file mode 100644 index 0000000..dc38859 --- /dev/null +++ b/precommit/src/main/python/unit-test-filter-file.example @@ -0,0 +1,2 @@ +[package].[class].[method] +![package].[class].[method] http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/coprocs.d/README.md ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/coprocs.d/README.md b/precommit/src/main/shell/coprocs.d/README.md new file mode 100755 index 0000000..e960cad --- /dev/null +++ b/precommit/src/main/shell/coprocs.d/README.md @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################# +## +## bash v4+ coproc routines +## +############################# + +# bash v3 and lower will treat coproc commands as syntax errors +# therefore, ALL functions which activate coprocs are located +# here \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/coprocs.d/e_a_r_helper.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/coprocs.d/e_a_r_helper.sh b/precommit/src/main/shell/coprocs.d/e_a_r_helper.sh new file mode 100755 index 0000000..3df8293 --- /dev/null +++ b/precommit/src/main/shell/coprocs.d/e_a_r_helper.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SHELLDOC-IGNORE + +## @description helper function for echo_and_redirect +## @audience private +## @stability evolving +## @replaceable no +function e_a_r_helper +{ + declare logfile=$1 + shift + declare params=("${@}") + + echo "Launching yrr_coproc" >> "${COPROC_LOGFILE}" + # shellcheck disable=SC2034 + coproc yrr_coproc { + ulimit -Su "${PROC_LIMIT}" + yetus_run_and_redirect "${logfile}" "${params[@]}" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/coprocs.d/process_counter.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/coprocs.d/process_counter.sh b/precommit/src/main/shell/coprocs.d/process_counter.sh new file mode 100755 index 0000000..4f944bb --- /dev/null +++ b/precommit/src/main/shell/coprocs.d/process_counter.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SHELLDOC-IGNORE + +function process_counter_coproc_start +{ + if [[ "${OSTYPE}" = Linux && "${DOCKERMODE}" = true ]]; then + # this is really only even remotely close to + # accurate under Docker, for the time being. + + echo "Launching process_counter_coproc" >> "${COPROC_LOGFILE}" + # shellcheck disable=SC2034 + coproc process_counter_coproc { + declare threadcount + declare maxthreadcount + declare cmd + + sleep 2 + while true; do + threadcount=$(ps -L -u "${USER_ID}" -o lwp 2>/dev/null | wc -l) + if [[ ${threadcount} -gt ${maxthreadcount} ]]; then + maxthreadcount="${threadcount}" + echo "${maxthreadcount}" > "${PATCH_DIR}/threadcounter.txt" + fi + read -r -t 2 cmd + case "${cmd}" in + exit) + exit 0 + ;; + esac + done + } + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/coprocs.d/reaper.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/coprocs.d/reaper.sh b/precommit/src/main/shell/coprocs.d/reaper.sh new file mode 100755 index 0000000..4674130 --- /dev/null +++ b/precommit/src/main/shell/coprocs.d/reaper.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SHELLDOC-IGNORE + +function reaper_coproc_start +{ + if [[ "${REAPER_MODE}" != "off" ]]; then + + echo "Launching reaper_coproc" >> "${COPROC_LOGFILE}" + + # shellcheck disable=SC2034 + coproc reaper_coproc { + reaper_coproc_func + } + fi +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/core.d/00-yetuslib.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/core.d/00-yetuslib.sh b/precommit/src/main/shell/core.d/00-yetuslib.sh new file mode 100755 index 0000000..983dfe6 --- /dev/null +++ b/precommit/src/main/shell/core.d/00-yetuslib.sh @@ -0,0 +1,331 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# we need to declare this globally as an array, which can only +# be done outside of a function +declare -a YETUS_OPTION_USAGE + +## @description Print a message to stderr +## @audience public +## @stability stable +## @replaceable no +## @param string +function yetus_error +{ + echo "$*" 1>&2 +} + +## @description Print a message to stderr if --debug is turned on +## @audience public +## @stability stable +## @replaceable no +## @param string +function yetus_debug +{ + if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then + echo "[$(date) DEBUG]: $*" 1>&2 + fi +} + +## @description Given variable $1 delete $2 from it +## @audience public +## @stability stable +## @replaceable no +function yetus_delete_entry +{ + if [[ ${!1} =~ \ ${2}\ ]] ; then + yetus_debug "Removing ${2} from ${1}" + eval "${1}"=\""${!1// ${2} }"\" + fi +} + +## @description Given variable $1 add $2 to it +## @audience public +## @stability stable +## @replaceable no +function yetus_add_entry +{ + if [[ ! ${!1} =~ \ ${2}\ ]] ; then + yetus_debug "Adding ${2} to ${1}" + #shellcheck disable=SC2140 + eval "${1}"=\""${!1} ${2} "\" + fi +} + +## @description Given variable $1 determine if $2 is in it +## @audience public +## @stability stable +## @replaceable no +## @return 0 = yes, 1 = no +function yetus_verify_entry +{ + [[ ${!1} =~ \ ${2}\ ]] +} + +## @description run the command, sending stdout and stderr to the given filename +## @audience public +## @stability stable +## @param filename +## @param command +## @param [..] +## @replaceable no +## @return $? +function yetus_run_and_redirect +{ + declare logfile=$1 + shift + + # to the log + { + date + echo "cd $(pwd)" + echo "${*}" + } >> "${logfile}" + # run the actual command + "${@}" >> "${logfile}" 2>&1 +} + +## @description Given a filename or dir, return the absolute version of it +## @audience public +## @stability stable +## @param fsobj +## @replaceable no +## @return 0 success +## @return 1 failure +## @return stdout abspath +function yetus_abs +{ + declare obj=$1 + declare dir + declare fn + + if [[ ! -e ${obj} ]]; then + return 1 + elif [[ -d ${obj} ]]; then + dir=${obj} + else + dir=$(dirname -- "${obj}") + fn=$(basename -- "${obj}") + fn="/${fn}" + fi + + dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P) + if [[ $? = 0 ]]; then + echo "${dir}${fn}" + return 0 + fi + return 1 +} + +## @description Add a header to the usage output +## @audience public +## @stability evolving +## @replaceable no +## @param header +function yetus_add_header +{ + declare text=$1 + + #shellcheck disable=SC2034 + YETUS_USAGE_HEADER="${text}" +} + +## @description Add an option to the usage output +## @audience public +## @stability evolving +## @replaceable no +## @param subcommand +## @param subcommanddesc +function yetus_add_option +{ + declare option=$1 + declare text=$2 + + YETUS_OPTION_USAGE[${YETUS_OPTION_USAGE_COUNTER}]="${option}@${text}" + ((YETUS_OPTION_USAGE_COUNTER=YETUS_OPTION_USAGE_COUNTER+1)) +} + +## @description Reset the usage information to blank +## @audience private +## @stability evolving +## @replaceable no +function yetus_reset_usage +{ + # shellcheck disable=SC2034 + YETUS_OPTION_USAGE=() + YETUS_OPTION_USAGE_COUNTER=0 +} + +## @description Print a screen-size aware two-column output +## @audience public +## @stability evolving +## @replaceable no +## @param array +function yetus_generic_columnprinter +{ + declare -a input=("$@") + declare -i i=0 + declare -i counter=0 + declare line + declare text + declare option + declare giventext + declare -i maxoptsize + declare -i foldsize + declare -a tmpa + declare numcols + + if [[ -n "${COLUMNS}" ]]; then + numcols=${COLUMNS} + else + numcols=$(tput cols) 2>/dev/null + fi + + if [[ -z "${numcols}" + || ! "${numcols}" =~ ^[0-9]+$ ]]; then + numcols=75 + else + ((numcols=numcols-5)) + fi + + while read -r line; do + tmpa[${counter}]=${line} + ((counter=counter+1)) + option=$(echo "${line}" | cut -f1 -d'@') + if [[ ${#option} -gt ${maxoptsize} ]]; then + maxoptsize=${#option} + fi + done < <(for text in "${input[@]}"; do + echo "${text}" + done | sort) + + i=0 + ((foldsize=numcols-maxoptsize)) + + until [[ $i -eq ${#tmpa[@]} ]]; do + option=$(echo "${tmpa[$i]}" | cut -f1 -d'@') + giventext=$(echo "${tmpa[$i]}" | cut -f2 -d'@') + + while read -r line; do + printf "%-${maxoptsize}s %-s\n" "${option}" "${line}" + option=" " + done < <(echo "${giventext}"| fold -s -w ${foldsize}) + ((i=i+1)) + done +} + +## @description Convert a comma-delimited string to an array +## @audience public +## @stability evolving +## @replaceable no +## @param arrayname +## @param string +function yetus_comma_to_array +{ + declare var=$1 + declare string=$2 + + oldifs="${IFS}" + IFS=',' read -r -a "${var}" <<< "${string}" + IFS="${oldifs}" +} + +## @description Check if an array has a given value +## @audience public +## @stability stable +## @replaceable yes +## @param element +## @param array +## @returns 0 = yes +## @returns 1 = no +function yetus_array_contains +{ + declare element=$1 + shift + declare val + + if [[ "$#" -eq 0 ]]; then + return 1 + fi + + for val in "${@}"; do + if [[ "${val}" == "${element}" ]]; then + return 0 + fi + done + return 1 +} + +## @description Add the element if it is not +## @description present in the given array +## @audience public +## @stability stable +## @replaceable yes +## @param arrayname +## @param element +function yetus_add_array_element +{ + declare arrname=$1 + declare add=$2 + + declare arrref="${arrname}[@]" + declare array=("${!arrref}") + + if ! yetus_array_contains "${add}" "${array[@]}"; then + # shellcheck disable=SC1083,SC2086 + eval "${arrname}"=\(\"\${array[@]}\" \"${add}\" \) + yetus_debug "$1 accepted $2" + else + yetus_debug "$1 declined $2" + fi +} + +## @description Sort an array by its elements +## @audience public +## @stability stable +## @replaceable yes +## @param arrayvar +function yetus_sort_array +{ + declare arrname=$1 + declare arrref="${arrname}[@]" + declare array=("${!arrref}") + + declare globstatus + declare oifs + declare -a sa + + globstatus=$(set -o | grep noglob | awk '{print $NF}') + + if [[ -n ${IFS} ]]; then + oifs=${IFS} + fi + set -f + # shellcheck disable=SC2034 + IFS=$'\n' sa=($(sort <<<"${array[*]}")) + # shellcheck disable=SC1083 + eval "${arrname}"=\(\"\${sa[@]}\"\) + + if [[ -n "${oifs}" ]]; then + IFS=${oifs} + else + unset IFS + fi + + if [[ "${globstatus}" = off ]]; then + set +f + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/core.d/01-common.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/core.d/01-common.sh b/precommit/src/main/shell/core.d/01-common.sh new file mode 100755 index 0000000..0d89106 --- /dev/null +++ b/precommit/src/main/shell/core.d/01-common.sh @@ -0,0 +1,643 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description Setup the default global variables +## @audience public +## @stability stable +## @replaceable no +function common_defaults +{ + #shellcheck disable=SC2034 + BASEDIR=$(pwd) + BUGSYSTEMS="" + BUILDTOOL="nobuild" + BUILDTOOLS="" + #shellcheck disable=SC2034 + EXEC_MODES="" + #shellcheck disable=SC2034 + JENKINS=false + LOAD_SYSTEM_PLUGINS=true + #shellcheck disable=SC2034 + OFFLINE=false + OSTYPE=$(uname -s) + #shellcheck disable=SC2034 + PATCH_BRANCH="" + PATCH_BRANCH_DEFAULT="master" + #shellcheck disable=SC2034 + PATCH_DRYRUNMODE=false + PATCH_DIR=/tmp + while [[ -e ${PATCH_DIR} ]]; do + PATCH_DIR=/tmp/yetus-${RANDOM}.${RANDOM} + done + #shellcheck disable=SC2034 + PATCH_METHOD="" + #shellcheck disable=SC2034 + PATCH_METHODS=("gitapply" "patchcmd") + #shellcheck disable=SC2034 + PATCH_LEVEL=0 + #shellcheck disable=SC2034 + PATCH_SYSTEM="" + PROJECT_NAME=unknown + RESULT=0 + #shellcheck disable=SC2034 + ROBOT=false + #shellcheck disable=SC2034 + SENTINEL=false + #shellcheck disable=SC2034 + TESTTYPES="" + TESTFORMATS="" + USER_PLUGIN_DIR="" + + #shellcheck disable=SC2034 + YETUS_SHELL_SCRIPT_DEBUG=false + + # Solaris needs POSIX and GNU, not SVID + case ${OSTYPE} in + SunOS) + AWK=${AWK:-/usr/xpg4/bin/awk} + CURL=${CURL:-curl} + DIFF=${DIFF:-/usr/gnu/bin/diff} + FILE=${FILE:-file} + GIT=${GIT:-git} + GREP=${GREP:-/usr/xpg4/bin/grep} + PATCH=${PATCH:-/usr/gnu/bin/patch} + SED=${SED:-/usr/xpg4/bin/sed} + ;; + *) + AWK=${AWK:-awk} + CURL=${CURL:-curl} + DIFF=${DIFF:-diff} + FILE=${FILE:-file} + GIT=${GIT:-git} + GREP=${GREP:-grep} + PATCH=${PATCH:-patch} + SED=${SED:-sed} + ;; + esac + + RSYNC=${RSYNC:-rsync} +} + +## @description Interpret the common command line parameters used by test-patch, +## @description smart-apply-patch, and the bug system plugins +## @audience private +## @stability stable +## @replaceable no +## @param $@ +## @return May exit on failure +function common_args +{ + declare i + declare showhelp=false + declare showversion=false + declare version + + for i in "$@"; do + case ${i} in + --awk-cmd=*) + AWK=${i#*=} + ;; + --basedir=*) + #shellcheck disable=SC2034 + BASEDIR=${i#*=} + ;; + --branch=*) + #shellcheck disable=SC2034 + PATCH_BRANCH=${i#*=} + ;; + --branch-default=*) + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=${i#*=} + ;; + --curl-cmd=*) + CURL=${i#*=} + ;; + --debug) + #shellcheck disable=SC2034 + YETUS_SHELL_SCRIPT_DEBUG=true + ;; + --diff-cmd=*) + DIFF=${i#*=} + ;; + --file-cmd=*) + FILE=${i#*=} + ;; + --git-cmd=*) + GIT=${i#*=} + ;; + --grep-cmd=*) + GREP=${i#*=} + ;; + --help|-help|-h|help|--h|--\?|-\?|\?) + showhelp=true + ;; + --list-plugins) + list_plugins + exit 0 + ;; + --offline) + #shellcheck disable=SC2034 + OFFLINE=true + ;; + --patch-cmd=*) + PATCH=${i#*=} + ;; + --patch-dir=*) + PATCH_DIR=${i#*=} + ;; + --plugins=*) + ENABLED_PLUGINS=${i#*=} + ENABLED_PLUGINS=${ENABLED_PLUGINS//,/ } + ;; + --project=*) + PROJECT_NAME=${i#*=} + ;; + --rsync-cmd=*) + RSYNC=${i#*=} + ;; + --skip-system-plugins) + LOAD_SYSTEM_PLUGINS=false + ;; + --sed-cmd=*) + SED=${i#*=} + ;; + --user-plugins=*) + USER_PLUGIN_DIR=${i#*=} + ;; + --version) + showversion=true + ;; + *) + ;; + esac + done + + set_yetus_version + + if [[ ${showhelp} == true ]]; then + yetus_usage + exit 0 + fi + if [[ ${showversion} == true ]]; then + echo "${VERSION}" + exit 0 + fi + + # Absolutely require v1.7.3 or higher + # versions lower than this either have bugs with + # git apply or don't support all the + # expected options + version=$(${GIT} --version) + + if [[ $? != 0 ]]; then + yetus_error "ERROR: ${GIT} failed during version detection." + exit 1 + fi + + # shellcheck disable=SC2016 + version=$(echo "${version}" | ${AWK} '{print $NF}') + if [[ ${version} =~ ^0 + || ${version} =~ ^1.[0-6] + || ${version} =~ ^1.7.[0-2]$ + ]]; then + yetus_error "ERROR: ${GIT} v1.7.3 or higher is required (found ${version})." + exit 1 + fi +} + +## @description List all installed plug-ins, regardless of whether +## @description they have been enabled +## @audience public +## @stability evolving +## @replaceable no +function list_plugins +{ + declare plugintype + declare name + + ENABLED_PLUGINS="all" + importplugins + + printf "Reminder: every plug-in may be enabled via 'all'.\n\n" + for plugintype in BUILDTOOLS TESTTYPES BUGSYSTEMS TESTFORMATS; do + printf "%s:\n\t" ${plugintype} + for name in ${!plugintype}; do + printf "%s " ${name} + done + echo "" + done +} + +## @description Let plugins also get a copy of the arguments +## @audience private +## @stability evolving +## @replaceable no +function parse_args_plugins +{ + declare plugin + + for plugin in ${TESTTYPES} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOLS}; do + if declare -f ${plugin}_parse_args >/dev/null 2>&1; then + yetus_debug "Running ${plugin}_parse_args" + #shellcheck disable=SC2086 + ${plugin}_parse_args "$@" + (( RESULT = RESULT + $? )) + fi + done +} + +## @description Initialize all enabled plugins +## @audience private +## @stability evolving +## @replaceable no +function plugins_initialize +{ + declare plugin + + for plugin in ${TESTTYPES} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOL}; do + if declare -f ${plugin}_initialize >/dev/null 2>&1; then + yetus_debug "Running ${plugin}_initialize" + #shellcheck disable=SC2086 + ${plugin}_initialize + (( RESULT = RESULT + $? )) + fi + done +} + +## @description Determine if a plugin was enabled by the user +## @description ENABLED_PLUGINS must be defined +## @audience public +## @stability stable +## @replaceable yes +## @param test +function verify_plugin_enabled +{ + declare toadd=$1 + declare bar + declare idx + declare strip + declare stridx + + yetus_debug "Testing if $1 has been enabled by user" + + bar="" + for idx in ${ENABLED_PLUGINS}; do + stridx=${idx// } + yetus_debug "verify_plugin_enabled: processing ${stridx}" + case ${stridx} in + all) + bar=${toadd} + ;; + -*) + strip=${stridx#-} + if [[ ${strip} = "${toadd}" ]]; then + bar="" + fi + ;; + +*|*) + strip=${stridx#+} + if [[ ${strip} = "${toadd}" ]]; then + bar=${toadd} + fi + ;; + esac + done + + if [[ -n ${bar} ]]; then + yetus_debug "Post-parsing: checking ${bar} = ${toadd}" + fi + [[ ${bar} = "${toadd}" ]] +} + +## @description Personality-defined plug-in list +## @audience public +## @stability stable +## @replaceable yes +## @param plug-in list string +function personality_plugins +{ + if [[ -z "${ENABLED_PLUGINS}" ]]; then + ENABLED_PLUGINS="$1" + ENABLED_PLUGINS=${ENABLED_PLUGINS//,/ } + yetus_debug "Using personality plug-in list: ${ENABLED_PLUGINS}" + fi +} + +## @description Add the given test type +## @audience public +## @stability stable +## @replaceable yes +## @param test +function add_test +{ + if verify_plugin_enabled "${1}"; then + yetus_add_entry NEEDED_TESTS "${1}" + fi +} + +## @description Remove the given test type +## @audience public +## @stability stable +## @replaceable yes +## @param test +function delete_test +{ + yetus_delete_entry NEEDED_TESTS "${1}" +} + +## @description Verify if a given test was requested +## @audience public +## @stability stable +## @replaceable yes +## @param test +## @return 0 = yes +## @return 1 = no +function verify_needed_test +{ + yetus_verify_entry NEEDED_TESTS "${1}" +} + +## @description Add the given test type +## @audience public +## @stability stable +## @replaceable yes +## @param plugin +function add_test_type +{ + if verify_plugin_enabled "${1}"; then + yetus_add_entry TESTTYPES "${1}" + fi +} + +## @description Remove the given test type +## @audience public +## @stability stable +## @replaceable yes +## @param plugin +function delete_test_type +{ + yetus_delete_entry TESTTYPES "${1}" +} + +## @description Add the given bugsystem type +## @audience public +## @stability stable +## @replaceable yes +## @param bugsystem +function add_bugsystem +{ + if verify_plugin_enabled "${1}"; then + yetus_add_entry BUGSYSTEMS "${1}" + fi +} + +## @description Remove the given bugsystem type +## @audience public +## @stability stable +## @replaceable yes +## @param bugsystem +function delete_bugsystem +{ + yetus_delete_entry BUGSYSTEMS "${1}" +} + +## @description Add the given test format type +## @audience public +## @stability stable +## @replaceable yes +## @param test format +function add_test_format +{ + if verify_plugin_enabled "${1}"; then + yetus_add_entry TESTFORMATS "${1}" + fi +} + +## @description Remove the given test format type +## @audience public +## @stability stable +## @replaceable yes +## @param test format +function delete_test_format +{ + yetus_delete_entry TESTFORMATS "${1}" +} + +## @description Add the given build tool type +## @audience public +## @stability stable +## @replaceable yes +## @param build tool +function add_build_tool +{ + if verify_plugin_enabled "${1}"; then + yetus_add_entry BUILDTOOLS "${1}" + fi +} + +## @description Remove the given build tool type +## @audience public +## @stability stable +## @replaceable yes +## @param build tool +function delete_build_tool +{ + yetus_delete_entry BUILDTOOLS "${1}" +} + +## @description Import content from test-patch.d and optionally +## @description from user provided plugin directory +## @audience private +## @stability evolving +## @replaceable no +function importplugins +{ + local i + local plugin + local files=() + + if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then + if [[ -d "${BINDIR}/test-patch.d" ]]; then + files=(${BINDIR}/test-patch.d/*.sh) + fi + fi + + if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then + yetus_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}" + files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh) + fi + + if [[ -n ${PERSONALITY} && ! -f ${PERSONALITY} ]]; then + yetus_error "ERROR: Can't find ${PERSONALITY} to import." + unset PERSONALITY + fi + + if [[ -z ${PERSONALITY} + && -f "${BINDIR}/personality/${PROJECT_NAME}.sh" + && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then + yetus_debug "Using project personality." + PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh" + fi + + if [[ -n ${PERSONALITY} && -f ${PERSONALITY} ]]; then + yetus_debug "Importing ${PERSONALITY}" + # shellcheck disable=SC1090 + . "${PERSONALITY}" + fi + + for i in "${files[@]}"; do + if [[ -f ${i} ]]; then + yetus_debug "Importing ${i}" + #shellcheck disable=SC1090 + . "${i}" + fi + done + + if declare -f personality_globals > /dev/null; then + personality_globals + fi +} + +## @description Print the plugin's usage info +## @audience public +## @stability evolving +## @replaceable no +## @param array +function plugin_usage_output +{ + echo "" + echo "${YETUS_USAGE_HEADER}" + echo "" +} + +## @description Verifies the existence of a command +## @audience private +## @stability evolving +## @replaceable no +## @param commandname +## @param commandpath +## @return 0 = ok +## @return 1 = error +function verify_command +{ + local cmd_name="$1" + local cmd_path="$2" + + if [[ -z ${cmd_path} ]]; then + yetus_error "executable for '${cmd_name}' was not specified." + return 1 + fi + if [[ ! "${cmd_path}" =~ / ]]; then + cmd_path=$(command -v "${cmd_path}") + fi + if [[ ! -f ${cmd_path} ]]; then + yetus_error "executable '${cmd_path}' for '${cmd_name}' does not exist." + return 1 + fi + if [[ ! -x ${cmd_path} ]]; then + yetus_error "executable '${cmd_path}' for '${cmd_name}' is not executable." + return 1 + fi + return 0 +} + +## @description Faster dirname, given the assumption that +## @description dirs are always absolute (e.g., start with /) +## @description DO NOT USE with relative paths or where +## @description assumption may not be valid! +## @audience private +## @stability evolving +## @replaceable no +## @param fileobj +function faster_dirname +{ + declare o=$1 + + if [[ "${o}" =~ / ]]; then + echo "${o%/*}" + else + echo . + fi +} + +## @description Set the USER_NAME, USER_ID, and GROUP_ID env vars +## @audience private +## @stability evolving +## @replaceable no +function determine_user +{ + # On the Apache Jenkins hosts, $USER is pretty much untrustable because + # something sets it to an account that doesn't actually exist. + # Instead, we need to try and override it with something that's + # probably close to reality. + if [[ ${TESTPATCHMODE} =~ jenkins ]]; then + USER=$(id | cut -f2 -d\( | cut -f1 -d\)) + fi + + USER_NAME=${SUDO_USER:=$USER} + # shellcheck disable=SC2034 + USER_ID=$(id -u "${USER_NAME}") + # shellcheck disable=SC2034 + GROUP_ID=$(id -g "${USER_NAME}") +} + +## @description Kill a process id +## @audience private +## @stability evolving +## @replaceable yes +## @param pid +function pid_kill +{ + declare pid=$1 + declare cmd + declare kill_timeout=3 + + kill "${pid}" >/dev/null 2>&1 + sleep "${kill_timeout}" + if kill -0 "${pid}" > /dev/null 2>&1; then + yetus_error "WARNING: ${pid} did not stop gracefully after ${kill_timeout} second(s): Trying to kill with kill -9" + kill -9 "${pid}" >/dev/null 2>&1 + fi + if ps -p "${pid}" > /dev/null 2>&1; then + cmd=$(ps -o args "${pid}") + yetus_error "ERROR: Unable to kill ${pid}: ${cmd}" + fi +} + +## @description set VERSION to the current version if not set +## @audience private +## @stability evolving +## @replaceable yes +function set_yetus_version +{ + + if [[ -n "${VERSION}" ]]; then + return + fi + + if [[ -f "${BINDIR}/../VERSION" ]]; then + # old src version file + VERSION=$(cat "${BINDIR}/../VERSION") + elif [[ -f "${BINDIR}/VERSION" ]]; then + # dist version file + VERSION=$(cat "${BINDIR}/VERSION") + elif [[ -f "${BINDIR}/../../../pom.xml" ]]; then + # this way we have no dependency on Maven being installed + VERSION=$(${GREP} "<version>" "${BINDIR}/../../../pom.xml" 2>/dev/null \ + | head -1 \ + | ${SED} -e 's|^ *<version>||' -e 's|</version>.*$||' 2>/dev/null) + fi +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/core.d/builtin-bugsystem.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/core.d/builtin-bugsystem.sh b/precommit/src/main/shell/core.d/builtin-bugsystem.sh new file mode 100755 index 0000000..0aeddf1 --- /dev/null +++ b/precommit/src/main/shell/core.d/builtin-bugsystem.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This bug system handles the output on the screen. + +add_bugsystem console + +CONSOLE_USE_BUILD_URL=false + +## @description Print out the finished details on the console +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +## @return 0 on success +## @return 1 on failure +function console_finalreport +{ + declare result=$1 + shift + declare i=0 + declare ourstring + declare vote + declare subs + declare ela + declare comment + declare commentfile1="${PATCH_DIR}/comment.1" + declare commentfile2="${PATCH_DIR}/comment.2" + declare normaltop + declare line + declare seccoladj=0 + declare spcfx=${PATCH_DIR}/spcl.txt + declare calctime + + if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then + exec 6>&1 + exec >"${CONSOLE_REPORT_FILE}" + fi + + if [[ ${result} == 0 ]]; then + if [[ ${ROBOT} == false ]]; then + if declare -f ${PROJECT_NAME}_console_success >/dev/null; then + "${PROJECT_NAME}_console_success" > "${spcfx}" + else + { + printf "IF9fX18gICAgICAgICAgICAgICAgICAgICAgICAgICAgICBfIAovIF9fX3wg"; + printf "XyAgIF8gIF9fXyBfX18gX19fICBfX18gX19ffCB8ClxfX18gXHwgfCB8IHwv"; + printf "IF9fLyBfXy8gXyBcLyBfXy8gX198IHwKIF9fXykgfCB8X3wgfCAoX3wgKF98"; + printf "ICBfXy9cX18gXF9fIFxffAp8X19fXy8gXF9fLF98XF9fX1xfX19cX19ffHxf"; + printf "X18vX19fKF8pCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg"; + printf "ICAK"; + } > "${spcfx}" + fi + fi + printf "\n\n+1 overall\n\n" + else + if [[ ${ROBOT} == false ]]; then + if declare -f ${PROJECT_NAME}_console_failure >/dev/null; then + "${PROJECT_NAME}_console_failure" > "${spcfx}" + else + { + printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf"; + printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf"; + printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98"; + printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg"; + printf "ICAgICAgICAgICAgICAgICAK" + } > "${spcfx}" + fi + fi + printf "\n\n-1 overall\n\n" + fi + + if [[ -f ${spcfx} ]]; then + if which base64 >/dev/null 2>&1; then + base64 --decode "${spcfx}" 2>/dev/null + elif which openssl >/dev/null 2>&1; then + openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null + fi + echo + echo + rm "${spcfx}" + fi + + seccoladj=$(findlargest 2 "${TP_VOTE_TABLE[@]}") + if [[ ${seccoladj} -lt 10 ]]; then + seccoladj=10 + fi + + seccoladj=$((seccoladj + 2 )) + i=0 + until [[ $i -eq ${#TP_HEADER[@]} ]]; do + printf "%s\n" "${TP_HEADER[${i}]}" + ((i=i+1)) + done + + printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment" + echo "============================================================================" + i=0 + until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + subs=$(echo "${ourstring}" | cut -f3 -d\|) + ela=$(echo "${ourstring}" | cut -f4 -d\|) + calctime=$(clock_display "${ela}") + comment=$(echo "${ourstring}" | cut -f5 -d\|) + + echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}" + normaltop=$(head -1 "${commentfile1}") + ${SED} -e '1d' "${commentfile1}" > "${commentfile2}" + + if [[ "${vote}" = "H" ]]; then + printf "| | %*s | |%-s\n" ${seccoladj} " " "${normaltop}" + else + printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \ + "${subs}" "${calctime}" "${normaltop}" + fi + while read -r line; do + printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}" + done < "${commentfile2}" + + ((i=i+1)) + rm "${commentfile2}" "${commentfile1}" 2>/dev/null + done + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + seccoladj=$(findlargest 1 "${TP_TEST_TABLE[@]}") + printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason" + i=0 + until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do + ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + subs=$(echo "${ourstring}" | cut -f3 -d\|) + printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}" + ((i=i+1)) + done + fi + + printf "\n\n|| Subsystem || Report/Notes ||\n" + echo "============================================================================" + i=0 + + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + if [[ "${CONSOLE_USE_BUILD_URL}" = true && + -n "${BUILD_URL}" ]]; then + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") + else + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${PATCH_DIR},g") + fi + printf "%s\n" "${comment}" + ((i=i+1)) + done + + if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then + exec 1>&6 6>&- + cat "${CONSOLE_REPORT_FILE}" + fi +} + + +## @description Give access to the brief report file in docker mode +## @audience private +## @stability evolving +## @replaceable no +function console_docker_support +{ + if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then + DOCKER_EXTRAARGS+=("-v" "${CONSOLE_REPORT_FILE}:/testptch/console.txt") + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/core.d/builtin-personality.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/core.d/builtin-personality.sh b/precommit/src/main/shell/core.d/builtin-personality.sh new file mode 100755 index 0000000..c473d66 --- /dev/null +++ b/precommit/src/main/shell/core.d/builtin-personality.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description Generate a list of all personality modules for a given +## @description buildtool for the system to invoke +## @audience private +## @stability evolving +## @replaceable no +function personality_modules +{ + if declare -f "${BUILDTOOL}_builtin_personality_modules" >/dev/null; then + "${BUILDTOOL}_builtin_personality_modules" "$@" + fi +} + +## @description Generate a list of all personality file tests for a given +## @description buildtool for the system to invoke +## @audience private +## @stability evolving +## @replaceable no +function personality_file_tests +{ + if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then + "${BUILDTOOL}_builtin_personality_file_tests" "$@" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/core.d/docker.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh new file mode 100755 index 0000000..83efc3b --- /dev/null +++ b/precommit/src/main/shell/core.d/docker.sh @@ -0,0 +1,661 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DOCKERMODE=false +DOCKERCMD=$(command -v docker) +DOCKER_ID=${RANDOM} +DOCKER_DESTRUCTIVE=true +DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile" +DOCKERFAIL="fallback,continue,fail" +DOCKERSUPPORT=false +DOCKER_ENABLE_PRIVILEGED=true +DOCKER_CLEANUP_CMD=false +DOCKER_MEMORY="4g" + +declare -a DOCKER_EXTRAARGS + +#### +#### IMPORTANT +#### +#### If these times are updated, the documentation needs to +#### be changed too! + +# created, stopped, exited, running, for 24 hours +DOCKER_CONTAINER_PURGE=("86400" "86400" "86400" "86400" ) + +# keep images for 1 week +DOCKER_IMAGE_PURGE=604800 + +## @description Docker-specific usage +## @stability stable +## @audience private +## @replaceable no +function docker_usage +{ + if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then + yetus_add_option "--docker" "Spawn a docker container" + fi + yetus_add_option "--dockercmd=<file>" "Command to use as docker executable (default: '${DOCKERCMD}')" + if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then + yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default: '${DOCKERFILE_DEFAULT}')" + yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method order (default: ${DOCKERFAIL})" + yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default: '${DOCKER_ENABLE_PRIVILEGED}')" + fi + yetus_add_option "--dockerdelrep" "In Docker mode, only report image/container deletions, not act on them" + if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then + yetus_add_option "--dockermemlimit=<num>" "Limit a Docker container's memory usage (default: ${DOCKER_MEMORY})" + fi +} + +## @description Docker-specific argument parsing +## @stability stable +## @audience private +## @replaceable no +## @params arguments +function docker_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --docker) + DOCKERSUPPORT=true + ;; + --dockercmd=*) + #shellcheck disable=SC2034 + DOCKERCMD=${i#*=} + ;; + --dockerdelrep) + DOCKER_DESTRUCTIVE=false + ;; + --dockerfile=*) + DOCKERFILE=${i#*=} + ;; + --dockermemlimit=*) + DOCKER_MEMORY=${i#*=} + ;; + --dockermode) + DOCKERMODE=true + ;; + --dockeronfail=*) + DOCKERFAIL=${i#*=} + ;; + --dockerprivd=*) + DOCKER_ENABLE_PRIVILEGED=${i#*=} + ;; + esac + done +} + +## @description Docker initialization pre- and post- re-exec +## @stability stable +## @audience private +## @replaceable no +function docker_initialize +{ + declare dockvers + + # --docker and --dockermode are mutually + # exclusive. --docker is used by the user to + # re-exec test-patch in Docker mode. + # --dockermode is used by launch-test-patch (which is + # run as the Docker EXEC in the Dockerfile, + # see elsewhere for more info) to tell test-patch that + # it has been restarted already. launch-test-patch + # also strips --docker from the command line so that we + # don't end up in a loop if the docker image + # also has the docker command in it + + # we are already in docker mode + if [[ "${DOCKERMODE}" == true ]]; then + # DOCKER_VERSION is set by our creator. + add_footer_table "Docker" "${DOCKER_VERSION}" + return + fi + + # docker mode hasn't been requested + if [[ "${DOCKERSUPPORT}" != true ]]; then + return + fi + + # turn DOCKERFAIL into a string composed of numbers + # to ease interpretation: 123, 213, 321, ... whatever + # some of these combos are non-sensical but that's ok. + # we'll treat non-sense as effectively errors. + DOCKERFAIL=${DOCKERFAIL//,/ } + DOCKERFAIL=${DOCKERFAIL//fallback/1} + DOCKERFAIL=${DOCKERFAIL//continue/2} + DOCKERFAIL=${DOCKERFAIL//fail/3} + DOCKERFAIL=${DOCKERFAIL//[[:blank:]]/} + + if ! docker_exeverify; then + if [[ "${DOCKERFAIL}" =~ ^12 + || "${DOCKERFAIL}" =~ ^2 ]]; then + add_vote_table 0 docker "Docker command '${DOCKERCMD}' not found/broken. Disabling docker." + DOCKERSUPPORT=false + else + add_vote_table -1 docker "Docker command '${DOCKERCMD}' not found/broken." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi + fi + + dockvers=$(docker_version Client) + if [[ "${dockvers}" =~ ^0 + || "${dockvers}" =~ ^1\.[0-5]$ || "${dockvers}" =~ ^1\.[0-5]\. ]]; then + if [[ "${DOCKERFAIL}" =~ ^12 + || "${DOCKERFAIL}" =~ ^2 ]]; then + add_vote_table 0 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 1.6.0). Disabling docker." + DOCKERSUPPORT=false + else + add_vote_table -1 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 1.6.0). Disabling docker." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi + fi +} + +## @description Verify dockerfile exists +## @audience private +## @stability evolving +## @replaceable no +## @return exits on failure if configured +function docker_fileverify +{ + if [[ ${DOCKERMODE} = false && + ${DOCKERSUPPORT} = true ]]; then + if [[ -n "${DOCKERFILE}" ]]; then + pushd "${STARTINGDIR}" >/dev/null + if [[ -f ${DOCKERFILE} ]]; then + DOCKERFILE=$(yetus_abs "${DOCKERFILE}") + else + if [[ "${DOCKERFAIL}" =~ ^1 ]]; then + yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, falling back to built-in." + add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, falling back to built-in." + DOCKERFILE=${DOCKERFILE_DEFAULT} + elif [[ "${DOCKERFAIL}" =~ ^2 ]]; then + yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, disabling docker." + add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, disabling docker." + DOCKERSUPPORT=false + else + yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found." + add_vote_table -1 docker "Dockerfile '${DOCKERFILE}' not found." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi + fi + popd >/dev/null + else + DOCKERFILE=${DOCKERFILE_DEFAULT} + fi + fi +} + +## @description Verify docker exists +## @audience private +## @stability evolving +## @replaceable no +## @return 1 if docker is broken +## @return 0 if docker is working +function docker_exeverify +{ + if ! verify_command "Docker" "${DOCKERCMD}"; then + return 1 + fi + + if ! ${DOCKERCMD} info >/dev/null 2>&1; then + yetus_error "Docker is not functioning properly. Daemon down/unreachable?" + return 1 + fi + return 0 +} + +## @description Run docker with some arguments, and +## @description optionally send to debug. +## @description some destructive commands require +## @description DOCKER_DESTRUCTIVE to be set to true +## @audience private +## @stability evolving +## @replaceable no +## @param args +function dockercmd +{ + declare subcmd=$1 + shift + + yetus_debug "dockercmd: ${DOCKERCMD} ${subcmd} $*" + if [[ ${subcmd} == rm + || ${subcmd} == rmi + || ${subcmd} == stop + || ${subcmd} == kill ]]; then + if [[ "${DOCKER_DESTRUCTIVE}" == false ]]; then + yetus_error "Safemode: not running ${DOCKERCMD} ${subcmd} $*" + return + fi + fi + "${DOCKERCMD}" "${subcmd}" "$@" +} + +## @description Convet docker's time format to ctime +## @audience private +## @stability evolving +## @replaceable no +## @param time +function dockerdate_to_ctime +{ + declare mytime=$1 + + if [[ "${mytime}" = "0001-01-01T00:00:00Z" ]]; then + mytime="1970-01-01T00:00:00" + fi + + # believe it or not, date is not even close to standardized... + if [[ $(uname -s) == Linux ]]; then + + # GNU date + date -d "${mytime}" "+%s" + else + + # BSD date; docker gives us two different format because fun + if ! date -j -f "%FT%T%z" "${mytime}" "+%s" 2>/dev/null; then + date -j -f "%FT%T" "${mytime}" "+%s" + fi + fi +} + +## @description Stop and delete all defunct containers +## @audience private +## @stability evolving +## @replaceable no +function docker_container_maintenance +{ + declare line + declare id + declare name + declare status + declare tmptime + declare createtime + declare starttime + declare stoptime + declare remove + declare difftime + declare data + + if [[ "${ROBOT}" = false ]]; then + return + fi + + big_console_header "Docker Container Maintenance" + + dockercmd ps -a + + data=$(dockercmd ps -qa) + + if [[ -z "${data}" ]]; then + return + fi + + while read -r line; do + id=$(echo "${line}" | cut -f1 -d, ) + status=$(echo "${line}" | cut -f2 -d, ) + tmptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. ) + createtime=$(dockerdate_to_ctime "${tmptime}") + tmptime=$(echo "${line}" | cut -f4 -d, | cut -f1 -d. ) + starttime=$(dockerdate_to_ctime "${tmptime}") + tmptime=$(echo "${line}" | cut -f5 -d, | cut -f1 -d. ) + stoptime=$(dockerdate_to_ctime "${tmptime}") + name=$(echo "${line}" | cut -f6 -d, ) + curtime=$("${AWK}" 'BEGIN {srand(); print srand()}') + remove=false + + case ${status} in + created) + ((difftime = curtime - createtime)) + if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[0]} ]]; then + remove=true + fi + ;; + stopped) + ((difftime = curtime - stoptime)) + if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[1]} ]]; then + remove=true + fi + ;; + exited | dead) + ((difftime = curtime - stoptime)) + if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[2]} ]]; then + remove=true + fi + ;; + running) + ((difftime = curtime - starttime)) + if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[3]} + && "${SENTINEL}" = true ]]; then + remove=true + echo "Attempting to kill docker container ${name} [${id}]" + dockercmd kill "${id}" + fi + ;; + *) + ;; + esac + + if [[ "${remove}" == true ]]; then + echo "Attempting to remove docker container ${name} [${id}]" + dockercmd rm "${id}" + fi + + done < <( + # shellcheck disable=SC2086 + dockercmd inspect \ + --format '{{.Id}},{{.State.Status}},{{.Created}},{{.State.StartedAt}},{{.State.FinishedAt}},{{.Name}}' \ + ${data}) +} + +## @description Delete images after ${DOCKER_IMAGE_PURGE} +## @audience private +## @stability evolving +## @replaceable no +function docker_image_maintenance_helper +{ + declare id + declare tmptime + declare createtime + declare difftime + declare name + + if [[ "${ROBOT}" = false ]]; then + return + fi + + if [[ -z "$*" ]]; then + return + fi + + for id in "$@"; do + tmptime=$(dockercmd inspect --format '{{.Created}}' "${id}" | cut -f1 -d. ) + createtime=$(dockerdate_to_ctime "${tmptime}") + curtime=$(date "+%s") + + ((difftime = curtime - createtime)) + if [[ ${difftime} -gt ${DOCKER_IMAGE_PURGE} ]]; then + echo "Attempting to remove docker image ${id}" + dockercmd rmi "${id}" + fi + done +} + + +## @description get sentinel-level docker images +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_get_sentinel_images +{ + #shellcheck disable=SC2016 + dockercmd images \ + | tail -n +2 \ + | "${GREP}" -v hours \ + | "${AWK}" '{print $1":"$2}' \ + | "${GREP}" -v "<none>:<none>" +} + +## @description Remove untagged/unused images +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_image_maintenance +{ + declare id + + if [[ "${ROBOT}" = false ]]; then + return + fi + + big_console_header "Removing old images" + + dockercmd images + + echo "Untagged images:" + + #shellcheck disable=SC2046 + docker_image_maintenance_helper $(dockercmd images --filter "dangling=true" -q --no-trunc) + + echo "Apache Yetus images:" + + # removing this by image id doesn't always work without a force + # in the situations that, for whatever reason, docker decided + # to use the same image. this was a rare problem with older + # releases of yetus. at some point, we should revisit this + # in the mean time, we're going to reconstruct the + # repostory:tag and send that to get removed. + + #shellcheck disable=SC2046,SC2016 + docker_image_maintenance_helper $(dockercmd images | ${GREP} -e ^yetus | grep tp- | ${AWK} '{print $1":"$2}') + #shellcheck disable=SC2046,SC2016 + docker_image_maintenance_helper $(dockercmd images | ${GREP} -e ^yetus | ${GREP} -v hours | ${AWK} '{print $1":"$2}') + + if [[ "${SENTINEL}" = false ]]; then + return + fi + + echo "Other images:" + #shellcheck disable=SC2046 + docker_image_maintenance_helper $(docker_get_sentinel_images) +} + + +## @description Perform pre-run maintenance to free up +## @description resources. With --jenkins, it is a lot +## @description more destructive. +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_cleanup +{ + + docker_image_maintenance + + docker_container_maintenance +} + +## @description Determine the revision of a dockerfile +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_getfilerev +{ + ${GREP} 'YETUS_PRIVATE: gitrev=' \ + "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \ + | cut -f2 -d= +} + +function docker_version +{ + declare vertype=$1 + declare val + declare ret + + # new version command + val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null) + ret=$? + + if [[ ${ret} != 0 ]];then + # old version command + val=$(dockercmd version | ${GREP} "${vertype} version" | cut -f2 -d: | tr -d ' ') + fi + + echo "${val}" +} + +## @description Start a test patch docker container +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_run_image +{ + declare dockerfilerev + declare baseimagename + declare patchimagename="yetus/${PROJECT_NAME}:tp-${DOCKER_ID}" + declare containername="yetus_tp-${DOCKER_ID}" + declare client + declare server + declare retval + declare elapsed + + dockerfilerev=$(docker_getfilerev) + + baseimagename="yetus/${PROJECT_NAME}:${dockerfilerev}" + + # make a base image, if it isn't available + big_console_header "Building base image: ${baseimagename}" + start_clock + dockercmd build \ + -t "${baseimagename}" \ + "${PATCH_DIR}/precommit/test-patch-docker" + retval=$? + + #shellcheck disable=SC2046 + elapsed=$(clock_display $(stop_clock)) + + echo "" + echo "Total Elapsed time: ${elapsed}" + echo "" + + if [[ ${retval} != 0 ]]; then + yetus_error "ERROR: Docker failed to build image." + add_vote_table -1 docker "Docker failed to build ${baseimagename}." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi + + big_console_header "Building ${BUILDMODE} image: ${patchimagename}" + start_clock + # using the base image, make one that is patch specific + dockercmd build \ + -t "${patchimagename}" \ + - <<PatchSpecificDocker +FROM ${baseimagename} +LABEL org.apache.yetus="" +LABEL org.apache.yetus.testpatch.patch="tp-${DOCKER_ID}" +LABEL org.apache.yetus.testpatch.project=${PROJECT_NAME} +RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} || true +RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} || true +RUN chown -R ${USER_NAME} /home/${USER_NAME} || true +ENV HOME /home/${USER_NAME} +USER ${USER_NAME} +PatchSpecificDocker + + retval=$? + + #shellcheck disable=SC2046 + elapsed=$(clock_display $(stop_clock)) + + echo "" + echo "Total Elapsed time: ${elapsed}" + echo "" + + if [[ ${retval} != 0 ]]; then + yetus_error "ERROR: Docker failed to build image." + add_vote_table -1 docker "Docker failed to build ${patchimagename}." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi + + if [[ "${DOCKER_ENABLE_PRIVILEGED}" = true ]]; then + DOCKER_EXTRAARGS+=("--privileged") + fi + + if [[ -n "${DOCKER_MEMORY}" ]]; then + DOCKER_EXTRAARGS+=("-m" "${DOCKER_MEMORY}") + fi + + client=$(docker_version Client) + server=$(docker_version Server) + + dockerversion="Client=${client} Server=${server}" + + + # make the kernel prefer to kill us if we run out of RAM + DOCKER_EXTRAARGS+=("--oom-score-adj" "500") + + DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile") + DOCKER_EXTRAARGS+=(-v "${PWD}:/testptch/${PROJECT_NAME}") + DOCKER_EXTRAARGS+=(-u "${USER_NAME}") + DOCKER_EXTRAARGS+=(-w "/testptch/${PROJECT_NAME}") + DOCKER_EXTRAARGS+=("--env=BASEDIR=/testptch/${PROJECT_NAME}") + DOCKER_EXTRAARGS+=("--env=DOCKER_VERSION=${dockerversion} Image:${baseimagename}") + DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}") + DOCKER_EXTRAARGS+=("--env=PATCH_SYSTEM=${PATCH_SYSTEM}") + DOCKER_EXTRAARGS+=("--env=PROJECT_NAME=${PROJECT_NAME}") + DOCKER_EXTRAARGS+=("--env=TESTPATCHMODE=${TESTPATCHMODE}") + DOCKER_EXTRAARGS+=(--name "${containername}") + + + trap 'docker_signal_handler' SIGTERM + trap 'docker_signal_handler' SIGINT + + if [[ ${PATCH_DIR} =~ ^/ ]]; then + dockercmd run --rm=true -i \ + "${DOCKER_EXTRAARGS[@]}" \ + -v "${PATCH_DIR}:/testptch/patchprocess" \ + --env=PATCH_DIR=/testptch/patchprocess \ + "${patchimagename}" & + else + dockercmd run --rm=true -i \ + "${DOCKER_EXTRAARGS[@]}" \ + --env=PATCH_DIR="${PATCH_DIR}" \ + "${patchimagename}" & + fi + + wait ${!} + cleanup_and_exit $? +} + +## @description docker kill the container on SIGTERM +## @audience private +## @stability evolving +## @replaceable no +function docker_signal_handler +{ + declare cid + + cid=$(cat "${PATCH_DIR}/cidfile") + + yetus_error "ERROR: Caught signal. Killing docker container:" + dockercmd kill "${cid}" + yetus_error "ERROR: Exiting." + cleanup_and_exit 143 # 128 + 15 -- SIGTERM +} + +## @description Switch over to a Docker container +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_handler +{ + PATCH_DIR=$(relative_dir "${PATCH_DIR}") + + docker_cleanup + determine_user + docker_run_image +}
