http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/build.sh ---------------------------------------------------------------------- diff --git a/build.sh b/build.sh deleted file mode 100755 index a6f2dc9..0000000 --- a/build.sh +++ /dev/null @@ -1,283 +0,0 @@ -#!/bin/bash -e -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Temporary script for building tarballs. See YETUS-125 to help -# create a more sustainable build system. -# -# Pass --release to get release checks -# Pass --deploy to deploy maven snapshot artifacts -# -# Presumes you have -# * maven 3.2.0+ -# * jdk 1.7+ (1.7 in --release) -# * ruby + gems needed to run middleman -# * python + python-dateutil - -## @description Verify that all required dependencies exist -## @audience private -## @stability evolving -## @replaceable no -## @param true iff this is a release build -## @return 1 - Some dependencies are missing -## @return 0 - All dependencies exist -function detect_dependencies -{ - declare is_release=$1 - local exit_code=0 - if ! [ -x "$(command -v java)" ]; then - echo "Java not found! Must install JDK version >= 1.7" >&2 - exit_code=1 - fi - if ! [ -x "$(command -v mvn)" ]; then - echo "Apache Maven not found! Must install version >= 3.2.0" >&2 - echo "Download it at https://maven.apache.org/download.cgi"; >&2 - exit_code=1 - fi - if ! [ -x "$(command -v bundle)" ]; then - echo "building docs requires a Ruby executable bundle." >&2 - echo "Install it by executing 'gem install bundler && bundle install'" >&2 - exit_code=1 - fi - - ! python -c 'import dateutil.parser' 2>/dev/null - if [ "$?" -eq "0" ]; then - echo "Building release docs requires the python-dateutil module" >&2 - echo "Install it by executing 'pip install python-dateutil'" >&2 - exit_code=1 - fi - - if ! [ -x "$(command -v tar)" ]; then - echo "Building archives requires the 'tar' command." >&2 - exit_code=1 - fi - - if [ "${is_release}" = "true" ] && ! [ -x "$(command -v pax)" ]; then - echo "building the release source archive requires the 'pax' command." >&2 - exit_code=1 - fi - - if [[ "${exit_code}" -ne "0" ]]; then - echo "Some dependencies are missing. Exit now." >&2 - fi - return ${exit_code} -} - -YETUS_VERSION=$(cat VERSION) -RAT_DOWNLOAD_URL=https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.11/apache-rat-0.11.jar - -release=false -offline=false -for arg in "$@"; do - case ${arg} in - --release) - release=true - ;; - --offline) - offline=true - ;; - --deploy) - deploy=true - ;; - esac -done - -echo "working on version '${YETUS_VERSION}'" - -detect_dependencies "${release}" -mkdir -p target - -if [ "${offline}" != "true" ]; then - JIRA_VERSION="${YETUS_VERSION%%-SNAPSHOT}" - echo "generating release docs." - # Note that we use the bare python here instead of the wrapper script, since we - # create said script. - release-doc-maker/releasedocmaker.py --lint=all --license --outputdir target \ - --project YETUS "--version=${JIRA_VERSION}" \ - --projecttitle="Apache Yetus" --usetoday -else - echo "in offline mode, skipping release notes." -fi - -MAVEN_ARGS=() -if [ "${offline}" = "true" ]; then - MAVEN_ARGS=("${MAVEN_ARGS[@]}" --offline) -fi - -if [ "${release}" = "true" ]; then - MAVEN_ARGS=("${MAVEN_ARGS[@]}" -Papache-release) - echo "hard reseting working directory." - git reset --hard HEAD - - if [ ! -f target/rat.jar ]; then - if [ "${offline}" != "true" ]; then - echo "downloading rat jar file to '$(pwd)/target/'" - curl -o target/rat.jar "${RAT_DOWNLOAD_URL}" - else - echo "in offline mode, can't retrieve rat jar. will skip license check." - fi - fi - echo "creating source tarball at '$(pwd)/target/'" - rm "target/yetus-${YETUS_VERSION}-src".tar* 2>/dev/null || true - pax -w -f "target/yetus-${YETUS_VERSION}-src.tar" -s "/target/yetus-${YETUS_VERSION}/" target/RELEASENOTES.md target/CHANGELOG.md - current=$(basename "$(pwd)") - #shellcheck disable=SC2038 - (cd ..; find "${current}" \( -name target -o -name publish -o -name .git \) -prune -o ! -type d -print | xargs pax -w -a -f "${current}/target/yetus-${YETUS_VERSION}-src.tar" -s "/${current}/yetus-${YETUS_VERSION}/") - gzip "target/yetus-${YETUS_VERSION}-src.tar" -fi - -echo "running maven builds for java components" -# build java components -if [[ "${deploy}" = true ]]; then - mvncmd=deploy -else - mvncmd=install -fi -mvn "${MAVEN_ARGS[@]}" "${mvncmd}" --file yetus-project/pom.xml -mvn "${MAVEN_ARGS[@]}" -Pinclude-jdiff-module "${mvncmd}" javadoc:aggregate --file audience-annotations-component/pom.xml - -echo "building documentation" -# build docs after javadocs -docs_out=$(cd asf-site-src && bundle exec middleman build) -echo "${docs_out}" - -bin_tarball="target/bin-dir/yetus-${YETUS_VERSION}" -echo "creating staging area for convenience binary at '$(pwd)/${bin_tarball}'" -rm -rf "${bin_tarball}" 2>/dev/null || true -mkdir -p "${bin_tarball}" - -for i in LICENSE NOTICE; do - lines=$(grep -n 'Apache Yetus Source' "${i}" | cut -f1 -d:) - if [[ -z "${lines}" ]]; then - cp -p "${i}" "${bin_tarball}" - else - ((lines=lines-2)) - head -n "${lines}" "${i}" > "${bin_tarball}/${i}" - fi -done - -cp target/RELEASENOTES.md target/CHANGELOG.md "${bin_tarball}" -cp -r asf-site-src/publish/documentation/in-progress "${bin_tarball}/docs" - -mkdir -p "${bin_tarball}/lib" -cp VERSION "${bin_tarball}/lib/" - -mkdir -p "${bin_tarball}/lib/yetus-project" -cp yetus-project/pom.xml "${bin_tarball}/lib/yetus-project/yetus-project-${YETUS_VERSION}.pom" - -mkdir -p "${bin_tarball}/lib/audience-annotations" -cp audience-annotations-component/audience-annotations/target/audience-annotations-*.jar \ - audience-annotations-component/audience-annotations-jdiff/target/audience-annotations-jdiff-*.jar \ - "${bin_tarball}/lib/audience-annotations/" - -cp -r shelldocs "${bin_tarball}/lib/" - -cp -r release-doc-maker "${bin_tarball}/lib/" - -cp -r precommit "${bin_tarball}/lib/" -ln -s test-patch.sh "${bin_tarball}/lib/precommit/qbt.sh" - -mkdir -p "${bin_tarball}/bin" - -# Make a special version of the shell wrapper for releasedocmaker -# that maintains the ability to have '--lint' mean '--lint=all' -cat >"${bin_tarball}/bin/releasedocmaker" <<EOF -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -args=() -for arg in "\${@}"; do - if [ "\${arg}" = "-n" ] || [ "\${arg}" = "--lint" ]; then - args=("\${args[@]}" "--lint=all") - else - args=("\${args[@]}" "\${arg}") - fi -done - -exec "\$(dirname -- "\${BASH_SOURCE-0}")/../lib/release-doc-maker/releasedocmaker.py" "\${args[@]}" -EOF -chmod +x "${bin_tarball}/bin/releasedocmaker" - -for utility in shelldocs/shelldocs.py \ - precommit/docker-cleanup.sh \ - precommit/qbt.sh \ - precommit/smart-apply-patch.sh \ - precommit/test-patch.sh \ - precommit/jenkins/jenkins-admin.py -do - wrapper=${utility##*/} - wrapper=${wrapper%.*} - cat >"${bin_tarball}/bin/${wrapper}" <<EOF -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -exec "\$(dirname -- "\${BASH_SOURCE-0}")/../lib/${utility}" "\${@}" -EOF - chmod +x "${bin_tarball}/bin/${wrapper}" -done - -bin_file="target/yetus-${YETUS_VERSION}-bin.tar.gz" -echo "creating convenience binary in '$(pwd)/target'" -rm "${bin_file}" 2>/dev/null || true -tar -C "$(dirname "${bin_tarball}")" -czf "${bin_file}" "$(basename "${bin_tarball}")" - -if [ "${release}" = "true" ] && [ -f target/rat.jar ]; then - echo "checking asf licensing requirements for source tarball '$(pwd)/target/yetus-${YETUS_VERSION}-src.tar.gz'." - rm -rf target/source-unpack 2>/dev/null || true - mkdir target/source-unpack - tar -C target/source-unpack -xzf "target/yetus-${YETUS_VERSION}-src.tar.gz" - java -jar target/rat.jar -E .rat-excludes -d target/source-unpack - - echo "checking asf licensing requirements for convenience binary '$(pwd)/${bin_file}'." - rm -rf target/bin-unpack 2>/dev/null || true - mkdir target/bin-unpack - tar -C target/bin-unpack -xzf "${bin_file}" - java -jar target/rat.jar -E .rat-excludes -d target/bin-unpack -fi -echo "All Done!" -echo "Find your output at:" -if [ "${release}" = "true" ] && [ -f "target/yetus-${YETUS_VERSION}-src.tar.gz" ]; then - echo " $(pwd)/target/yetus-${YETUS_VERSION}-src.tar.gz" -fi -if [ -f "${bin_file}" ]; then - echo " $(pwd)/${bin_file}" -fi
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/hooks/build ---------------------------------------------------------------------- diff --git a/hooks/build b/hooks/build new file mode 100755 index 0000000..21876dc --- /dev/null +++ b/hooks/build @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -n "${DOCKERFILE_PATH}" ]]; then + docker build -t "${IMAGE_NAME}" \ + --build-arg DOCKER_TAG="${DOCKER_TAG}" \ + --build-arg DOCKER_REPO="${DOCKER_REPO}" \ + -f "${DOCKERFILE_PATH}" +else + docker build -t "${IMAGE_NAME}" \ + --build-arg DOCKER_TAG="${DOCKER_TAG}" \ + --build-arg DOCKER_REPO="${DOCKER_REPO}" \ + . +fi http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..334c807 --- /dev/null +++ b/pom.xml @@ -0,0 +1,336 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache</groupId> + <artifactId>apache</artifactId> + <version>21</version> + <relativePath/> + <!-- no parent resolution --> + </parent> + <groupId>org.apache.yetus</groupId> + <artifactId>yetus-project</artifactId> + <version>0.9.0-SNAPSHOT</version> + <description>Project-wide definitions for Apache Yetus</description> + <name>Apache Yetus Project</name> + <packaging>pom</packaging> + + <url>https://yetus.apache.org</url> + <inceptionYear>2015</inceptionYear> + <mailingLists> + <mailingList> + <name>Apache Yetus Dev List</name> + <subscribe>[email protected]</subscribe> + <unsubscribe>[email protected]</unsubscribe> + <post>[email protected]</post> + <archive>https://mail-archives.apache.org/mod_mbox/yetus-dev/</archive> + </mailingList> + </mailingLists> + + <properties> + <maven.min.version>3.2.0</maven.min.version> + <maven.api.version>3.2</maven.api.version> + <maven.plugin.api.version>3.5.4</maven.plugin.api.version> + <java.min.version>1.8</java.min.version> + <maven.compiler.source>1.8</maven.compiler.source> + <maven.compiler.target>1.8</maven.compiler.target> + <extra.enforcer.version>1.0-beta-9</extra.enforcer.version> + + <build-helper-maven-plugin.version>3.0.0</build-helper-maven-plugin.version> + <checkstyle.version>8.14</checkstyle.version> + <commons.io.version>2.6</commons.io.version> + <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version> + <jdiff.version>1.0.9</jdiff.version> + <jython-compile-maven-plugin.version>2.0</jython-compile-maven-plugin.version> + <jython-shaded.version>2.7.1</jython-shaded.version> + <maven-checkstyle-plugin.version>3.0.0</maven-checkstyle-plugin.version> + <maven-project-info-reports-plugin.version>3.0.0</maven-project-info-reports-plugin.version> + <spotbugs-maven-plugin.version>3.1.7</spotbugs-maven-plugin.version> + + <sourceReleaseAssemblyDescriptor>source-release-tar</sourceReleaseAssemblyDescriptor> + + </properties> + + <scm> + <connection>scm:git:git://git.apache.org/yetus.git</connection> + <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/yetus.git</developerConnection> + <url>https://git-wip-us.apache.org/repos/asf?p=yetus.git</url> + </scm> + <issueManagement> + <system>JIRA</system> + <url>https://issues.apache.org/jira/browse/YETUS</url> + </issueManagement> + + <build> + + <pluginManagement> + <plugins> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>${exec-maven-plugin.version}</version> + </plugin> + + <plugin> + <groupId>com.github.spotbugs</groupId> + <artifactId>spotbugs-maven-plugin</artifactId> + <version>${spotbugs-maven-plugin.version}</version> + </plugin> + + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <dependencies> + <dependency> + <groupId>com.puppycrawl.tools</groupId> + <artifactId>checkstyle</artifactId> + <version>${checkstyle.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>validate</id> + <phase>validate</phase> + <configuration> + <configLocation>checkstyle.xml</configLocation> + <encoding>UTF-8</encoding> + <consoleOutput>true</consoleOutput> + <failsOnError>true</failsOnError> + <linkXRef>false</linkXRef> + </configuration> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <configuration> + <failOnError>false</failOnError> + </configuration> + </plugin> + + <!-- plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <version>${apache-rat-plugin.version}</version> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin --> + + + </plugins> + </pluginManagement> + + + <plugins> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.codehaus.mojo</groupId> + <artifactId>extra-enforcer-rules</artifactId> + <version>${extra.enforcer.version}</version> + </dependency> + </dependencies> + <!-- version set by parent --> + <executions> + <execution> + <id>enforce</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <!-- The earliest maven version we verify builds for via ASF Jenkins --> + <!-- Might be obviated by move to gradle --> + <requireMavenVersion> + <version>[${maven.min.version},)</version> + <message>Maven is out of date. + Yetus requires at least version ${maven.min.version} of Maven to properly build from source. + You appear to be using an older version. You can use either "mvn -version" or + "mvn enforcer:display-info" to verify what version is active. + See the contributor guide on building for more information: ${project.url}/contribute/ + </message> + </requireMavenVersion> + <!-- The earliest JVM version we verify builds for via ASF Jenkins --> + <requireJavaVersion> + <version>[${java.min.version},)</version> + <message>Java is out of date. + Yetus requires at least version ${java.min.version} of the JDK to properly build from source. + You appear to be using an older version. You can use either "mvn -version" or + "mvn enforcer:display-info" to verify what version is active. + See the contributor guide on building for more information: ${project.url}/contribute/ + </message> + </requireJavaVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludeSubprojects>false</excludeSubprojects> + <excludes> + <exclude>.pylintrc</exclude> + <exclude>.rubocop.yml</exclude> + </excludes> + </configuration> + </plugin> + + </plugins> + </build> + + <profiles> + <profile> + <id>gpg2</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-gpg-plugin</artifactId> + <configuration> + <executable>gpg2</executable> + </configuration> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>jdk1.8</id> + <activation> + <jdk>1.8</jdk> + </activation> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>jdk.tools</groupId> + <artifactId>jdk.tools</artifactId> + <version>1.8</version> + <scope>system</scope> + <systemPath>${java.home}/../lib/tools.jar</systemPath> + </dependency> + </dependencies> + </dependencyManagement> + </profile> + </profiles> + + <reporting> + <plugins> + <plugin> + <artifactId>maven-project-info-reports-plugin</artifactId> + <reportSets> + <reportSet> + <configuration> + <skip>true</skip> + </configuration> + </reportSet> + </reportSets> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <inherited>false</inherited> + <reportSets> + <reportSet> + <id>aggregate</id> + <configuration> + <maxmemory>1024m</maxmemory> + <quiet>true</quiet> + <verbose>false</verbose> + <source>${maven.compile.source}</source> + <charset>${maven.compile.encoding}</charset> + <reportOutputDirectory>${project.build.directory}/site</reportOutputDirectory> + <destDir>yetus-project/api</destDir> + <doclet>org.apache.yetus.audience.tools.IncludePublicAnnotationsStandardDoclet</doclet> + <docletArtifacts> + <docletArtifact> + <groupId>org.apache.yetus</groupId> + <artifactId>audience-annotations</artifactId> + <version>${project.version}</version> + </docletArtifact> + </docletArtifacts> + <useStandardDocletOptions>true</useStandardDocletOptions> + + <!-- switch on dependency-driven aggregation --> + <includeDependencySources>false</includeDependencySources> + + <dependencySourceIncludes> + <!-- include ONLY dependencies I control --> + <dependencySourceInclude>org.apache.hadoop:hadoop-annotations</dependencySourceInclude> + </dependencySourceIncludes> + + </configuration> + <reports> + <report>aggregate</report> + </reports> + </reportSet> + </reportSets> + </plugin> + + <!-- plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <version>${maven-checkstyle-plugin.version}</version> + <reportSets> + <reportSet> + <reports> + <report>checkstyle</report> + </reports> + </reportSet> + </reportSets> + </plugin --> + </plugins> + </reporting> + + <modules> + <module>yetus-minimaven-plugin</module> + <module>yetus-assemblies</module> + <module>audience-annotations-component</module> + <module>precommit</module> + <module>releasedocmaker</module> + <module>shelldocs</module> + <module>asf-site-src</module> + <module>yetus-dist</module> + <module>yetus-maven-plugin</module> + </modules> + +</project> http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/README.md ---------------------------------------------------------------------- diff --git a/precommit/coprocs.d/README.md b/precommit/coprocs.d/README.md deleted file mode 100755 index e960cad..0000000 --- a/precommit/coprocs.d/README.md +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -############################# -## -## bash v4+ coproc routines -## -############################# - -# bash v3 and lower will treat coproc commands as syntax errors -# therefore, ALL functions which activate coprocs are located -# here \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/e_a_r_helper.sh ---------------------------------------------------------------------- diff --git a/precommit/coprocs.d/e_a_r_helper.sh b/precommit/coprocs.d/e_a_r_helper.sh deleted file mode 100755 index 3df8293..0000000 --- a/precommit/coprocs.d/e_a_r_helper.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# SHELLDOC-IGNORE - -## @description helper function for echo_and_redirect -## @audience private -## @stability evolving -## @replaceable no -function e_a_r_helper -{ - declare logfile=$1 - shift - declare params=("${@}") - - echo "Launching yrr_coproc" >> "${COPROC_LOGFILE}" - # shellcheck disable=SC2034 - coproc yrr_coproc { - ulimit -Su "${PROC_LIMIT}" - yetus_run_and_redirect "${logfile}" "${params[@]}" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/process_counter.sh ---------------------------------------------------------------------- diff --git a/precommit/coprocs.d/process_counter.sh b/precommit/coprocs.d/process_counter.sh deleted file mode 100755 index 4f944bb..0000000 --- a/precommit/coprocs.d/process_counter.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# SHELLDOC-IGNORE - -function process_counter_coproc_start -{ - if [[ "${OSTYPE}" = Linux && "${DOCKERMODE}" = true ]]; then - # this is really only even remotely close to - # accurate under Docker, for the time being. - - echo "Launching process_counter_coproc" >> "${COPROC_LOGFILE}" - # shellcheck disable=SC2034 - coproc process_counter_coproc { - declare threadcount - declare maxthreadcount - declare cmd - - sleep 2 - while true; do - threadcount=$(ps -L -u "${USER_ID}" -o lwp 2>/dev/null | wc -l) - if [[ ${threadcount} -gt ${maxthreadcount} ]]; then - maxthreadcount="${threadcount}" - echo "${maxthreadcount}" > "${PATCH_DIR}/threadcounter.txt" - fi - read -r -t 2 cmd - case "${cmd}" in - exit) - exit 0 - ;; - esac - done - } - fi -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/reaper.sh ---------------------------------------------------------------------- diff --git a/precommit/coprocs.d/reaper.sh b/precommit/coprocs.d/reaper.sh deleted file mode 100755 index 4674130..0000000 --- a/precommit/coprocs.d/reaper.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# SHELLDOC-IGNORE - -function reaper_coproc_start -{ - if [[ "${REAPER_MODE}" != "off" ]]; then - - echo "Launching reaper_coproc" >> "${COPROC_LOGFILE}" - - # shellcheck disable=SC2034 - coproc reaper_coproc { - reaper_coproc_func - } - fi -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/00-yetuslib.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/00-yetuslib.sh b/precommit/core.d/00-yetuslib.sh deleted file mode 100755 index 983dfe6..0000000 --- a/precommit/core.d/00-yetuslib.sh +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# we need to declare this globally as an array, which can only -# be done outside of a function -declare -a YETUS_OPTION_USAGE - -## @description Print a message to stderr -## @audience public -## @stability stable -## @replaceable no -## @param string -function yetus_error -{ - echo "$*" 1>&2 -} - -## @description Print a message to stderr if --debug is turned on -## @audience public -## @stability stable -## @replaceable no -## @param string -function yetus_debug -{ - if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then - echo "[$(date) DEBUG]: $*" 1>&2 - fi -} - -## @description Given variable $1 delete $2 from it -## @audience public -## @stability stable -## @replaceable no -function yetus_delete_entry -{ - if [[ ${!1} =~ \ ${2}\ ]] ; then - yetus_debug "Removing ${2} from ${1}" - eval "${1}"=\""${!1// ${2} }"\" - fi -} - -## @description Given variable $1 add $2 to it -## @audience public -## @stability stable -## @replaceable no -function yetus_add_entry -{ - if [[ ! ${!1} =~ \ ${2}\ ]] ; then - yetus_debug "Adding ${2} to ${1}" - #shellcheck disable=SC2140 - eval "${1}"=\""${!1} ${2} "\" - fi -} - -## @description Given variable $1 determine if $2 is in it -## @audience public -## @stability stable -## @replaceable no -## @return 0 = yes, 1 = no -function yetus_verify_entry -{ - [[ ${!1} =~ \ ${2}\ ]] -} - -## @description run the command, sending stdout and stderr to the given filename -## @audience public -## @stability stable -## @param filename -## @param command -## @param [..] -## @replaceable no -## @return $? -function yetus_run_and_redirect -{ - declare logfile=$1 - shift - - # to the log - { - date - echo "cd $(pwd)" - echo "${*}" - } >> "${logfile}" - # run the actual command - "${@}" >> "${logfile}" 2>&1 -} - -## @description Given a filename or dir, return the absolute version of it -## @audience public -## @stability stable -## @param fsobj -## @replaceable no -## @return 0 success -## @return 1 failure -## @return stdout abspath -function yetus_abs -{ - declare obj=$1 - declare dir - declare fn - - if [[ ! -e ${obj} ]]; then - return 1 - elif [[ -d ${obj} ]]; then - dir=${obj} - else - dir=$(dirname -- "${obj}") - fn=$(basename -- "${obj}") - fn="/${fn}" - fi - - dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P) - if [[ $? = 0 ]]; then - echo "${dir}${fn}" - return 0 - fi - return 1 -} - -## @description Add a header to the usage output -## @audience public -## @stability evolving -## @replaceable no -## @param header -function yetus_add_header -{ - declare text=$1 - - #shellcheck disable=SC2034 - YETUS_USAGE_HEADER="${text}" -} - -## @description Add an option to the usage output -## @audience public -## @stability evolving -## @replaceable no -## @param subcommand -## @param subcommanddesc -function yetus_add_option -{ - declare option=$1 - declare text=$2 - - YETUS_OPTION_USAGE[${YETUS_OPTION_USAGE_COUNTER}]="${option}@${text}" - ((YETUS_OPTION_USAGE_COUNTER=YETUS_OPTION_USAGE_COUNTER+1)) -} - -## @description Reset the usage information to blank -## @audience private -## @stability evolving -## @replaceable no -function yetus_reset_usage -{ - # shellcheck disable=SC2034 - YETUS_OPTION_USAGE=() - YETUS_OPTION_USAGE_COUNTER=0 -} - -## @description Print a screen-size aware two-column output -## @audience public -## @stability evolving -## @replaceable no -## @param array -function yetus_generic_columnprinter -{ - declare -a input=("$@") - declare -i i=0 - declare -i counter=0 - declare line - declare text - declare option - declare giventext - declare -i maxoptsize - declare -i foldsize - declare -a tmpa - declare numcols - - if [[ -n "${COLUMNS}" ]]; then - numcols=${COLUMNS} - else - numcols=$(tput cols) 2>/dev/null - fi - - if [[ -z "${numcols}" - || ! "${numcols}" =~ ^[0-9]+$ ]]; then - numcols=75 - else - ((numcols=numcols-5)) - fi - - while read -r line; do - tmpa[${counter}]=${line} - ((counter=counter+1)) - option=$(echo "${line}" | cut -f1 -d'@') - if [[ ${#option} -gt ${maxoptsize} ]]; then - maxoptsize=${#option} - fi - done < <(for text in "${input[@]}"; do - echo "${text}" - done | sort) - - i=0 - ((foldsize=numcols-maxoptsize)) - - until [[ $i -eq ${#tmpa[@]} ]]; do - option=$(echo "${tmpa[$i]}" | cut -f1 -d'@') - giventext=$(echo "${tmpa[$i]}" | cut -f2 -d'@') - - while read -r line; do - printf "%-${maxoptsize}s %-s\n" "${option}" "${line}" - option=" " - done < <(echo "${giventext}"| fold -s -w ${foldsize}) - ((i=i+1)) - done -} - -## @description Convert a comma-delimited string to an array -## @audience public -## @stability evolving -## @replaceable no -## @param arrayname -## @param string -function yetus_comma_to_array -{ - declare var=$1 - declare string=$2 - - oldifs="${IFS}" - IFS=',' read -r -a "${var}" <<< "${string}" - IFS="${oldifs}" -} - -## @description Check if an array has a given value -## @audience public -## @stability stable -## @replaceable yes -## @param element -## @param array -## @returns 0 = yes -## @returns 1 = no -function yetus_array_contains -{ - declare element=$1 - shift - declare val - - if [[ "$#" -eq 0 ]]; then - return 1 - fi - - for val in "${@}"; do - if [[ "${val}" == "${element}" ]]; then - return 0 - fi - done - return 1 -} - -## @description Add the element if it is not -## @description present in the given array -## @audience public -## @stability stable -## @replaceable yes -## @param arrayname -## @param element -function yetus_add_array_element -{ - declare arrname=$1 - declare add=$2 - - declare arrref="${arrname}[@]" - declare array=("${!arrref}") - - if ! yetus_array_contains "${add}" "${array[@]}"; then - # shellcheck disable=SC1083,SC2086 - eval "${arrname}"=\(\"\${array[@]}\" \"${add}\" \) - yetus_debug "$1 accepted $2" - else - yetus_debug "$1 declined $2" - fi -} - -## @description Sort an array by its elements -## @audience public -## @stability stable -## @replaceable yes -## @param arrayvar -function yetus_sort_array -{ - declare arrname=$1 - declare arrref="${arrname}[@]" - declare array=("${!arrref}") - - declare globstatus - declare oifs - declare -a sa - - globstatus=$(set -o | grep noglob | awk '{print $NF}') - - if [[ -n ${IFS} ]]; then - oifs=${IFS} - fi - set -f - # shellcheck disable=SC2034 - IFS=$'\n' sa=($(sort <<<"${array[*]}")) - # shellcheck disable=SC1083 - eval "${arrname}"=\(\"\${sa[@]}\"\) - - if [[ -n "${oifs}" ]]; then - IFS=${oifs} - else - unset IFS - fi - - if [[ "${globstatus}" = off ]]; then - set +f - fi -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/01-common.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/01-common.sh b/precommit/core.d/01-common.sh deleted file mode 100755 index e8dee09..0000000 --- a/precommit/core.d/01-common.sh +++ /dev/null @@ -1,615 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description Setup the default global variables -## @audience public -## @stability stable -## @replaceable no -function common_defaults -{ - #shellcheck disable=SC2034 - BASEDIR=$(pwd) - BUGSYSTEMS="" - BUILDTOOL="nobuild" - BUILDTOOLS="" - #shellcheck disable=SC2034 - EXEC_MODES="" - #shellcheck disable=SC2034 - JENKINS=false - LOAD_SYSTEM_PLUGINS=true - #shellcheck disable=SC2034 - OFFLINE=false - OSTYPE=$(uname -s) - #shellcheck disable=SC2034 - PATCH_BRANCH="" - PATCH_BRANCH_DEFAULT="master" - #shellcheck disable=SC2034 - PATCH_DRYRUNMODE=false - PATCH_DIR=/tmp - while [[ -e ${PATCH_DIR} ]]; do - PATCH_DIR=/tmp/yetus-${RANDOM}.${RANDOM} - done - #shellcheck disable=SC2034 - PATCH_METHOD="" - #shellcheck disable=SC2034 - PATCH_METHODS=("gitapply" "patchcmd") - #shellcheck disable=SC2034 - PATCH_LEVEL=0 - #shellcheck disable=SC2034 - PATCH_SYSTEM="" - PROJECT_NAME=unknown - RESULT=0 - #shellcheck disable=SC2034 - ROBOT=false - #shellcheck disable=SC2034 - SENTINEL=false - #shellcheck disable=SC2034 - TESTTYPES="" - TESTFORMATS="" - USER_PLUGIN_DIR="" - - #shellcheck disable=SC2034 - YETUS_SHELL_SCRIPT_DEBUG=false - - # Solaris needs POSIX and GNU, not SVID - case ${OSTYPE} in - SunOS) - AWK=${AWK:-/usr/xpg4/bin/awk} - CURL=${CURL:-curl} - DIFF=${DIFF:-/usr/gnu/bin/diff} - FILE=${FILE:-file} - GIT=${GIT:-git} - GREP=${GREP:-/usr/xpg4/bin/grep} - PATCH=${PATCH:-/usr/gnu/bin/patch} - SED=${SED:-/usr/xpg4/bin/sed} - ;; - *) - AWK=${AWK:-awk} - CURL=${CURL:-curl} - DIFF=${DIFF:-diff} - FILE=${FILE:-file} - GIT=${GIT:-git} - GREP=${GREP:-grep} - PATCH=${PATCH:-patch} - SED=${SED:-sed} - ;; - esac - - RSYNC=${RSYNC:-rsync} -} - -## @description Interpret the common command line parameters used by test-patch, -## @description smart-apply-patch, and the bug system plugins -## @audience private -## @stability stable -## @replaceable no -## @param $@ -## @return May exit on failure -function common_args -{ - declare i - declare showhelp=false - declare showversion=false - declare version - - for i in "$@"; do - case ${i} in - --awk-cmd=*) - AWK=${i#*=} - ;; - --basedir=*) - #shellcheck disable=SC2034 - BASEDIR=${i#*=} - ;; - --branch=*) - #shellcheck disable=SC2034 - PATCH_BRANCH=${i#*=} - ;; - --branch-default=*) - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=${i#*=} - ;; - --curl-cmd=*) - CURL=${i#*=} - ;; - --debug) - #shellcheck disable=SC2034 - YETUS_SHELL_SCRIPT_DEBUG=true - ;; - --diff-cmd=*) - DIFF=${i#*=} - ;; - --file-cmd=*) - FILE=${i#*=} - ;; - --git-cmd=*) - GIT=${i#*=} - ;; - --grep-cmd=*) - GREP=${i#*=} - ;; - --help|-help|-h|help|--h|--\?|-\?|\?) - showhelp=true - ;; - --list-plugins) - list_plugins - exit 0 - ;; - --offline) - #shellcheck disable=SC2034 - OFFLINE=true - ;; - --patch-cmd=*) - PATCH=${i#*=} - ;; - --patch-dir=*) - PATCH_DIR=${i#*=} - ;; - --plugins=*) - ENABLED_PLUGINS=${i#*=} - ENABLED_PLUGINS=${ENABLED_PLUGINS//,/ } - ;; - --project=*) - PROJECT_NAME=${i#*=} - ;; - --rsync-cmd=*) - RSYNC=${i#*=} - ;; - --skip-system-plugins) - LOAD_SYSTEM_PLUGINS=false - ;; - --sed-cmd=*) - SED=${i#*=} - ;; - --user-plugins=*) - USER_PLUGIN_DIR=${i#*=} - ;; - --version) - showversion=true - ;; - *) - ;; - esac - done - if [[ ${showhelp} == true ]]; then - yetus_usage - exit 0 - fi - if [[ ${showversion} == true ]]; then - cat "${BINDIR}/../VERSION" - exit 0 - fi - - # Absolutely require v1.7.3 or higher - # versions lower than this either have bugs with - # git apply or don't support all the - # expected options - version=$(${GIT} --version) - - if [[ $? != 0 ]]; then - yetus_error "ERROR: ${GIT} failed during version detection." - exit 1 - fi - - # shellcheck disable=SC2016 - version=$(echo "${version}" | ${AWK} '{print $NF}') - if [[ ${version} =~ ^0 - || ${version} =~ ^1.[0-6] - || ${version} =~ ^1.7.[0-2]$ - ]]; then - yetus_error "ERROR: ${GIT} v1.7.3 or higher is required (found ${version})." - exit 1 - fi -} - -## @description List all installed plug-ins, regardless of whether -## @description they have been enabled -## @audience public -## @stability evolving -## @replaceable no -function list_plugins -{ - declare plugintype - declare name - - ENABLED_PLUGINS="all" - importplugins - - printf "Reminder: every plug-in may be enabled via 'all'.\n\n" - for plugintype in BUILDTOOLS TESTTYPES BUGSYSTEMS TESTFORMATS; do - printf "%s:\n\t" ${plugintype} - for name in ${!plugintype}; do - printf "%s " ${name} - done - echo "" - done -} - -## @description Let plugins also get a copy of the arguments -## @audience private -## @stability evolving -## @replaceable no -function parse_args_plugins -{ - declare plugin - - for plugin in ${TESTTYPES} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOLS}; do - if declare -f ${plugin}_parse_args >/dev/null 2>&1; then - yetus_debug "Running ${plugin}_parse_args" - #shellcheck disable=SC2086 - ${plugin}_parse_args "$@" - (( RESULT = RESULT + $? )) - fi - done -} - -## @description Initialize all enabled plugins -## @audience private -## @stability evolving -## @replaceable no -function plugins_initialize -{ - declare plugin - - for plugin in ${TESTTYPES} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOL}; do - if declare -f ${plugin}_initialize >/dev/null 2>&1; then - yetus_debug "Running ${plugin}_initialize" - #shellcheck disable=SC2086 - ${plugin}_initialize - (( RESULT = RESULT + $? )) - fi - done -} - -## @description Determine if a plugin was enabled by the user -## @description ENABLED_PLUGINS must be defined -## @audience public -## @stability stable -## @replaceable yes -## @param test -function verify_plugin_enabled -{ - declare toadd=$1 - declare bar - declare idx - declare strip - declare stridx - - yetus_debug "Testing if $1 has been enabled by user" - - bar="" - for idx in ${ENABLED_PLUGINS}; do - stridx=${idx// } - yetus_debug "verify_plugin_enabled: processing ${stridx}" - case ${stridx} in - all) - bar=${toadd} - ;; - -*) - strip=${stridx#-} - if [[ ${strip} = "${toadd}" ]]; then - bar="" - fi - ;; - +*|*) - strip=${stridx#+} - if [[ ${strip} = "${toadd}" ]]; then - bar=${toadd} - fi - ;; - esac - done - - if [[ -n ${bar} ]]; then - yetus_debug "Post-parsing: checking ${bar} = ${toadd}" - fi - [[ ${bar} = "${toadd}" ]] -} - -## @description Personality-defined plug-in list -## @audience public -## @stability stable -## @replaceable yes -## @param plug-in list string -function personality_plugins -{ - if [[ -z "${ENABLED_PLUGINS}" ]]; then - ENABLED_PLUGINS="$1" - ENABLED_PLUGINS=${ENABLED_PLUGINS//,/ } - yetus_debug "Using personality plug-in list: ${ENABLED_PLUGINS}" - fi -} - -## @description Add the given test type -## @audience public -## @stability stable -## @replaceable yes -## @param test -function add_test -{ - if verify_plugin_enabled "${1}"; then - yetus_add_entry NEEDED_TESTS "${1}" - fi -} - -## @description Remove the given test type -## @audience public -## @stability stable -## @replaceable yes -## @param test -function delete_test -{ - yetus_delete_entry NEEDED_TESTS "${1}" -} - -## @description Verify if a given test was requested -## @audience public -## @stability stable -## @replaceable yes -## @param test -## @return 0 = yes -## @return 1 = no -function verify_needed_test -{ - yetus_verify_entry NEEDED_TESTS "${1}" -} - -## @description Add the given test type -## @audience public -## @stability stable -## @replaceable yes -## @param plugin -function add_test_type -{ - if verify_plugin_enabled "${1}"; then - yetus_add_entry TESTTYPES "${1}" - fi -} - -## @description Remove the given test type -## @audience public -## @stability stable -## @replaceable yes -## @param plugin -function delete_test_type -{ - yetus_delete_entry TESTTYPES "${1}" -} - -## @description Add the given bugsystem type -## @audience public -## @stability stable -## @replaceable yes -## @param bugsystem -function add_bugsystem -{ - if verify_plugin_enabled "${1}"; then - yetus_add_entry BUGSYSTEMS "${1}" - fi -} - -## @description Remove the given bugsystem type -## @audience public -## @stability stable -## @replaceable yes -## @param bugsystem -function delete_bugsystem -{ - yetus_delete_entry BUGSYSTEMS "${1}" -} - -## @description Add the given test format type -## @audience public -## @stability stable -## @replaceable yes -## @param test format -function add_test_format -{ - if verify_plugin_enabled "${1}"; then - yetus_add_entry TESTFORMATS "${1}" - fi -} - -## @description Remove the given test format type -## @audience public -## @stability stable -## @replaceable yes -## @param test format -function delete_test_format -{ - yetus_delete_entry TESTFORMATS "${1}" -} - -## @description Add the given build tool type -## @audience public -## @stability stable -## @replaceable yes -## @param build tool -function add_build_tool -{ - if verify_plugin_enabled "${1}"; then - yetus_add_entry BUILDTOOLS "${1}" - fi -} - -## @description Remove the given build tool type -## @audience public -## @stability stable -## @replaceable yes -## @param build tool -function delete_build_tool -{ - yetus_delete_entry BUILDTOOLS "${1}" -} - -## @description Import content from test-patch.d and optionally -## @description from user provided plugin directory -## @audience private -## @stability evolving -## @replaceable no -function importplugins -{ - local i - local plugin - local files=() - - if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then - if [[ -d "${BINDIR}/test-patch.d" ]]; then - files=(${BINDIR}/test-patch.d/*.sh) - fi - fi - - if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then - yetus_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}" - files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh) - fi - - if [[ -n ${PERSONALITY} && ! -f ${PERSONALITY} ]]; then - yetus_error "ERROR: Can't find ${PERSONALITY} to import." - unset PERSONALITY - fi - - if [[ -z ${PERSONALITY} - && -f "${BINDIR}/personality/${PROJECT_NAME}.sh" - && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then - yetus_debug "Using project personality." - PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh" - fi - - if [[ -n ${PERSONALITY} && -f ${PERSONALITY} ]]; then - yetus_debug "Importing ${PERSONALITY}" - # shellcheck disable=SC1090 - . "${PERSONALITY}" - fi - - for i in "${files[@]}"; do - if [[ -f ${i} ]]; then - yetus_debug "Importing ${i}" - #shellcheck disable=SC1090 - . "${i}" - fi - done - - if declare -f personality_globals > /dev/null; then - personality_globals - fi -} - -## @description Print the plugin's usage info -## @audience public -## @stability evolving -## @replaceable no -## @param array -function plugin_usage_output -{ - echo "" - echo "${YETUS_USAGE_HEADER}" - echo "" -} - -## @description Verifies the existence of a command -## @audience private -## @stability evolving -## @replaceable no -## @param commandname -## @param commandpath -## @return 0 = ok -## @return 1 = error -function verify_command -{ - local cmd_name="$1" - local cmd_path="$2" - - if [[ -z ${cmd_path} ]]; then - yetus_error "executable for '${cmd_name}' was not specified." - return 1 - fi - if [[ ! "${cmd_path}" =~ / ]]; then - cmd_path=$(command -v "${cmd_path}") - fi - if [[ ! -f ${cmd_path} ]]; then - yetus_error "executable '${cmd_path}' for '${cmd_name}' does not exist." - return 1 - fi - if [[ ! -x ${cmd_path} ]]; then - yetus_error "executable '${cmd_path}' for '${cmd_name}' is not executable." - return 1 - fi - return 0 -} - -## @description Faster dirname, given the assumption that -## @description dirs are always absolute (e.g., start with /) -## @description DO NOT USE with relative paths or where -## @description assumption may not be valid! -## @audience private -## @stability evolving -## @replaceable no -## @param fileobj -function faster_dirname -{ - declare o=$1 - - if [[ "${o}" =~ / ]]; then - echo "${o%/*}" - else - echo . - fi -} - -## @description Set the USER_NAME, USER_ID, and GROUP_ID env vars -## @audience private -## @stability evolving -## @replaceable no -function determine_user -{ - # On the Apache Jenkins hosts, $USER is pretty much untrustable because - # something sets it to an account that doesn't actually exist. - # Instead, we need to try and override it with something that's - # probably close to reality. - if [[ ${TESTPATCHMODE} =~ jenkins ]]; then - USER=$(id | cut -f2 -d\( | cut -f1 -d\)) - fi - - USER_NAME=${SUDO_USER:=$USER} - # shellcheck disable=SC2034 - USER_ID=$(id -u "${USER_NAME}") - # shellcheck disable=SC2034 - GROUP_ID=$(id -g "${USER_NAME}") -} - -## @description Kill a process id -## @audience private -## @stability evolving -## @replaceable yes -## @param pid -function pid_kill -{ - declare pid=$1 - declare cmd - declare kill_timeout=3 - - kill "${pid}" >/dev/null 2>&1 - sleep "${kill_timeout}" - if kill -0 "${pid}" > /dev/null 2>&1; then - yetus_error "WARNING: ${pid} did not stop gracefully after ${kill_timeout} second(s): Trying to kill with kill -9" - kill -9 "${pid}" >/dev/null 2>&1 - fi - if ps -p "${pid}" > /dev/null 2>&1; then - cmd=$(ps -o args "${pid}") - yetus_error "ERROR: Unable to kill ${pid}: ${cmd}" - fi -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/builtin-bugsystem.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/builtin-bugsystem.sh b/precommit/core.d/builtin-bugsystem.sh deleted file mode 100755 index 0aeddf1..0000000 --- a/precommit/core.d/builtin-bugsystem.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# This bug system handles the output on the screen. - -add_bugsystem console - -CONSOLE_USE_BUILD_URL=false - -## @description Print out the finished details on the console -## @audience private -## @stability evolving -## @replaceable no -## @param runresult -## @return 0 on success -## @return 1 on failure -function console_finalreport -{ - declare result=$1 - shift - declare i=0 - declare ourstring - declare vote - declare subs - declare ela - declare comment - declare commentfile1="${PATCH_DIR}/comment.1" - declare commentfile2="${PATCH_DIR}/comment.2" - declare normaltop - declare line - declare seccoladj=0 - declare spcfx=${PATCH_DIR}/spcl.txt - declare calctime - - if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then - exec 6>&1 - exec >"${CONSOLE_REPORT_FILE}" - fi - - if [[ ${result} == 0 ]]; then - if [[ ${ROBOT} == false ]]; then - if declare -f ${PROJECT_NAME}_console_success >/dev/null; then - "${PROJECT_NAME}_console_success" > "${spcfx}" - else - { - printf "IF9fX18gICAgICAgICAgICAgICAgICAgICAgICAgICAgICBfIAovIF9fX3wg"; - printf "XyAgIF8gIF9fXyBfX18gX19fICBfX18gX19ffCB8ClxfX18gXHwgfCB8IHwv"; - printf "IF9fLyBfXy8gXyBcLyBfXy8gX198IHwKIF9fXykgfCB8X3wgfCAoX3wgKF98"; - printf "ICBfXy9cX18gXF9fIFxffAp8X19fXy8gXF9fLF98XF9fX1xfX19cX19ffHxf"; - printf "X18vX19fKF8pCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg"; - printf "ICAK"; - } > "${spcfx}" - fi - fi - printf "\n\n+1 overall\n\n" - else - if [[ ${ROBOT} == false ]]; then - if declare -f ${PROJECT_NAME}_console_failure >/dev/null; then - "${PROJECT_NAME}_console_failure" > "${spcfx}" - else - { - printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf"; - printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf"; - printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98"; - printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg"; - printf "ICAgICAgICAgICAgICAgICAK" - } > "${spcfx}" - fi - fi - printf "\n\n-1 overall\n\n" - fi - - if [[ -f ${spcfx} ]]; then - if which base64 >/dev/null 2>&1; then - base64 --decode "${spcfx}" 2>/dev/null - elif which openssl >/dev/null 2>&1; then - openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null - fi - echo - echo - rm "${spcfx}" - fi - - seccoladj=$(findlargest 2 "${TP_VOTE_TABLE[@]}") - if [[ ${seccoladj} -lt 10 ]]; then - seccoladj=10 - fi - - seccoladj=$((seccoladj + 2 )) - i=0 - until [[ $i -eq ${#TP_HEADER[@]} ]]; do - printf "%s\n" "${TP_HEADER[${i}]}" - ((i=i+1)) - done - - printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment" - echo "============================================================================" - i=0 - until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do - ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') - vote=$(echo "${ourstring}" | cut -f2 -d\|) - subs=$(echo "${ourstring}" | cut -f3 -d\|) - ela=$(echo "${ourstring}" | cut -f4 -d\|) - calctime=$(clock_display "${ela}") - comment=$(echo "${ourstring}" | cut -f5 -d\|) - - echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}" - normaltop=$(head -1 "${commentfile1}") - ${SED} -e '1d' "${commentfile1}" > "${commentfile2}" - - if [[ "${vote}" = "H" ]]; then - printf "| | %*s | |%-s\n" ${seccoladj} " " "${normaltop}" - else - printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \ - "${subs}" "${calctime}" "${normaltop}" - fi - while read -r line; do - printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}" - done < "${commentfile2}" - - ((i=i+1)) - rm "${commentfile2}" "${commentfile1}" 2>/dev/null - done - - if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then - seccoladj=$(findlargest 1 "${TP_TEST_TABLE[@]}") - printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason" - i=0 - until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do - ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ') - vote=$(echo "${ourstring}" | cut -f2 -d\|) - subs=$(echo "${ourstring}" | cut -f3 -d\|) - printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}" - ((i=i+1)) - done - fi - - printf "\n\n|| Subsystem || Report/Notes ||\n" - echo "============================================================================" - i=0 - - until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do - if [[ "${CONSOLE_USE_BUILD_URL}" = true && - -n "${BUILD_URL}" ]]; then - comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | - ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") - else - comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | - ${SED} -e "s,@@BASE@@,${PATCH_DIR},g") - fi - printf "%s\n" "${comment}" - ((i=i+1)) - done - - if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then - exec 1>&6 6>&- - cat "${CONSOLE_REPORT_FILE}" - fi -} - - -## @description Give access to the brief report file in docker mode -## @audience private -## @stability evolving -## @replaceable no -function console_docker_support -{ - if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then - DOCKER_EXTRAARGS+=("-v" "${CONSOLE_REPORT_FILE}:/testptch/console.txt") - fi -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/builtin-personality.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/builtin-personality.sh b/precommit/core.d/builtin-personality.sh deleted file mode 100755 index c473d66..0000000 --- a/precommit/core.d/builtin-personality.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description Generate a list of all personality modules for a given -## @description buildtool for the system to invoke -## @audience private -## @stability evolving -## @replaceable no -function personality_modules -{ - if declare -f "${BUILDTOOL}_builtin_personality_modules" >/dev/null; then - "${BUILDTOOL}_builtin_personality_modules" "$@" - fi -} - -## @description Generate a list of all personality file tests for a given -## @description buildtool for the system to invoke -## @audience private -## @stability evolving -## @replaceable no -function personality_file_tests -{ - if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then - "${BUILDTOOL}_builtin_personality_file_tests" "$@" - fi -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/docker.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh deleted file mode 100755 index 83efc3b..0000000 --- a/precommit/core.d/docker.sh +++ /dev/null @@ -1,661 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -DOCKERMODE=false -DOCKERCMD=$(command -v docker) -DOCKER_ID=${RANDOM} -DOCKER_DESTRUCTIVE=true -DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile" -DOCKERFAIL="fallback,continue,fail" -DOCKERSUPPORT=false -DOCKER_ENABLE_PRIVILEGED=true -DOCKER_CLEANUP_CMD=false -DOCKER_MEMORY="4g" - -declare -a DOCKER_EXTRAARGS - -#### -#### IMPORTANT -#### -#### If these times are updated, the documentation needs to -#### be changed too! - -# created, stopped, exited, running, for 24 hours -DOCKER_CONTAINER_PURGE=("86400" "86400" "86400" "86400" ) - -# keep images for 1 week -DOCKER_IMAGE_PURGE=604800 - -## @description Docker-specific usage -## @stability stable -## @audience private -## @replaceable no -function docker_usage -{ - if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then - yetus_add_option "--docker" "Spawn a docker container" - fi - yetus_add_option "--dockercmd=<file>" "Command to use as docker executable (default: '${DOCKERCMD}')" - if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then - yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default: '${DOCKERFILE_DEFAULT}')" - yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method order (default: ${DOCKERFAIL})" - yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default: '${DOCKER_ENABLE_PRIVILEGED}')" - fi - yetus_add_option "--dockerdelrep" "In Docker mode, only report image/container deletions, not act on them" - if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then - yetus_add_option "--dockermemlimit=<num>" "Limit a Docker container's memory usage (default: ${DOCKER_MEMORY})" - fi -} - -## @description Docker-specific argument parsing -## @stability stable -## @audience private -## @replaceable no -## @params arguments -function docker_parse_args -{ - declare i - - for i in "$@"; do - case ${i} in - --docker) - DOCKERSUPPORT=true - ;; - --dockercmd=*) - #shellcheck disable=SC2034 - DOCKERCMD=${i#*=} - ;; - --dockerdelrep) - DOCKER_DESTRUCTIVE=false - ;; - --dockerfile=*) - DOCKERFILE=${i#*=} - ;; - --dockermemlimit=*) - DOCKER_MEMORY=${i#*=} - ;; - --dockermode) - DOCKERMODE=true - ;; - --dockeronfail=*) - DOCKERFAIL=${i#*=} - ;; - --dockerprivd=*) - DOCKER_ENABLE_PRIVILEGED=${i#*=} - ;; - esac - done -} - -## @description Docker initialization pre- and post- re-exec -## @stability stable -## @audience private -## @replaceable no -function docker_initialize -{ - declare dockvers - - # --docker and --dockermode are mutually - # exclusive. --docker is used by the user to - # re-exec test-patch in Docker mode. - # --dockermode is used by launch-test-patch (which is - # run as the Docker EXEC in the Dockerfile, - # see elsewhere for more info) to tell test-patch that - # it has been restarted already. launch-test-patch - # also strips --docker from the command line so that we - # don't end up in a loop if the docker image - # also has the docker command in it - - # we are already in docker mode - if [[ "${DOCKERMODE}" == true ]]; then - # DOCKER_VERSION is set by our creator. - add_footer_table "Docker" "${DOCKER_VERSION}" - return - fi - - # docker mode hasn't been requested - if [[ "${DOCKERSUPPORT}" != true ]]; then - return - fi - - # turn DOCKERFAIL into a string composed of numbers - # to ease interpretation: 123, 213, 321, ... whatever - # some of these combos are non-sensical but that's ok. - # we'll treat non-sense as effectively errors. - DOCKERFAIL=${DOCKERFAIL//,/ } - DOCKERFAIL=${DOCKERFAIL//fallback/1} - DOCKERFAIL=${DOCKERFAIL//continue/2} - DOCKERFAIL=${DOCKERFAIL//fail/3} - DOCKERFAIL=${DOCKERFAIL//[[:blank:]]/} - - if ! docker_exeverify; then - if [[ "${DOCKERFAIL}" =~ ^12 - || "${DOCKERFAIL}" =~ ^2 ]]; then - add_vote_table 0 docker "Docker command '${DOCKERCMD}' not found/broken. Disabling docker." - DOCKERSUPPORT=false - else - add_vote_table -1 docker "Docker command '${DOCKERCMD}' not found/broken." - bugsystem_finalreport 1 - cleanup_and_exit 1 - fi - fi - - dockvers=$(docker_version Client) - if [[ "${dockvers}" =~ ^0 - || "${dockvers}" =~ ^1\.[0-5]$ || "${dockvers}" =~ ^1\.[0-5]\. ]]; then - if [[ "${DOCKERFAIL}" =~ ^12 - || "${DOCKERFAIL}" =~ ^2 ]]; then - add_vote_table 0 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 1.6.0). Disabling docker." - DOCKERSUPPORT=false - else - add_vote_table -1 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 1.6.0). Disabling docker." - bugsystem_finalreport 1 - cleanup_and_exit 1 - fi - fi -} - -## @description Verify dockerfile exists -## @audience private -## @stability evolving -## @replaceable no -## @return exits on failure if configured -function docker_fileverify -{ - if [[ ${DOCKERMODE} = false && - ${DOCKERSUPPORT} = true ]]; then - if [[ -n "${DOCKERFILE}" ]]; then - pushd "${STARTINGDIR}" >/dev/null - if [[ -f ${DOCKERFILE} ]]; then - DOCKERFILE=$(yetus_abs "${DOCKERFILE}") - else - if [[ "${DOCKERFAIL}" =~ ^1 ]]; then - yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, falling back to built-in." - add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, falling back to built-in." - DOCKERFILE=${DOCKERFILE_DEFAULT} - elif [[ "${DOCKERFAIL}" =~ ^2 ]]; then - yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, disabling docker." - add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, disabling docker." - DOCKERSUPPORT=false - else - yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found." - add_vote_table -1 docker "Dockerfile '${DOCKERFILE}' not found." - bugsystem_finalreport 1 - cleanup_and_exit 1 - fi - fi - popd >/dev/null - else - DOCKERFILE=${DOCKERFILE_DEFAULT} - fi - fi -} - -## @description Verify docker exists -## @audience private -## @stability evolving -## @replaceable no -## @return 1 if docker is broken -## @return 0 if docker is working -function docker_exeverify -{ - if ! verify_command "Docker" "${DOCKERCMD}"; then - return 1 - fi - - if ! ${DOCKERCMD} info >/dev/null 2>&1; then - yetus_error "Docker is not functioning properly. Daemon down/unreachable?" - return 1 - fi - return 0 -} - -## @description Run docker with some arguments, and -## @description optionally send to debug. -## @description some destructive commands require -## @description DOCKER_DESTRUCTIVE to be set to true -## @audience private -## @stability evolving -## @replaceable no -## @param args -function dockercmd -{ - declare subcmd=$1 - shift - - yetus_debug "dockercmd: ${DOCKERCMD} ${subcmd} $*" - if [[ ${subcmd} == rm - || ${subcmd} == rmi - || ${subcmd} == stop - || ${subcmd} == kill ]]; then - if [[ "${DOCKER_DESTRUCTIVE}" == false ]]; then - yetus_error "Safemode: not running ${DOCKERCMD} ${subcmd} $*" - return - fi - fi - "${DOCKERCMD}" "${subcmd}" "$@" -} - -## @description Convet docker's time format to ctime -## @audience private -## @stability evolving -## @replaceable no -## @param time -function dockerdate_to_ctime -{ - declare mytime=$1 - - if [[ "${mytime}" = "0001-01-01T00:00:00Z" ]]; then - mytime="1970-01-01T00:00:00" - fi - - # believe it or not, date is not even close to standardized... - if [[ $(uname -s) == Linux ]]; then - - # GNU date - date -d "${mytime}" "+%s" - else - - # BSD date; docker gives us two different format because fun - if ! date -j -f "%FT%T%z" "${mytime}" "+%s" 2>/dev/null; then - date -j -f "%FT%T" "${mytime}" "+%s" - fi - fi -} - -## @description Stop and delete all defunct containers -## @audience private -## @stability evolving -## @replaceable no -function docker_container_maintenance -{ - declare line - declare id - declare name - declare status - declare tmptime - declare createtime - declare starttime - declare stoptime - declare remove - declare difftime - declare data - - if [[ "${ROBOT}" = false ]]; then - return - fi - - big_console_header "Docker Container Maintenance" - - dockercmd ps -a - - data=$(dockercmd ps -qa) - - if [[ -z "${data}" ]]; then - return - fi - - while read -r line; do - id=$(echo "${line}" | cut -f1 -d, ) - status=$(echo "${line}" | cut -f2 -d, ) - tmptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. ) - createtime=$(dockerdate_to_ctime "${tmptime}") - tmptime=$(echo "${line}" | cut -f4 -d, | cut -f1 -d. ) - starttime=$(dockerdate_to_ctime "${tmptime}") - tmptime=$(echo "${line}" | cut -f5 -d, | cut -f1 -d. ) - stoptime=$(dockerdate_to_ctime "${tmptime}") - name=$(echo "${line}" | cut -f6 -d, ) - curtime=$("${AWK}" 'BEGIN {srand(); print srand()}') - remove=false - - case ${status} in - created) - ((difftime = curtime - createtime)) - if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[0]} ]]; then - remove=true - fi - ;; - stopped) - ((difftime = curtime - stoptime)) - if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[1]} ]]; then - remove=true - fi - ;; - exited | dead) - ((difftime = curtime - stoptime)) - if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[2]} ]]; then - remove=true - fi - ;; - running) - ((difftime = curtime - starttime)) - if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[3]} - && "${SENTINEL}" = true ]]; then - remove=true - echo "Attempting to kill docker container ${name} [${id}]" - dockercmd kill "${id}" - fi - ;; - *) - ;; - esac - - if [[ "${remove}" == true ]]; then - echo "Attempting to remove docker container ${name} [${id}]" - dockercmd rm "${id}" - fi - - done < <( - # shellcheck disable=SC2086 - dockercmd inspect \ - --format '{{.Id}},{{.State.Status}},{{.Created}},{{.State.StartedAt}},{{.State.FinishedAt}},{{.Name}}' \ - ${data}) -} - -## @description Delete images after ${DOCKER_IMAGE_PURGE} -## @audience private -## @stability evolving -## @replaceable no -function docker_image_maintenance_helper -{ - declare id - declare tmptime - declare createtime - declare difftime - declare name - - if [[ "${ROBOT}" = false ]]; then - return - fi - - if [[ -z "$*" ]]; then - return - fi - - for id in "$@"; do - tmptime=$(dockercmd inspect --format '{{.Created}}' "${id}" | cut -f1 -d. ) - createtime=$(dockerdate_to_ctime "${tmptime}") - curtime=$(date "+%s") - - ((difftime = curtime - createtime)) - if [[ ${difftime} -gt ${DOCKER_IMAGE_PURGE} ]]; then - echo "Attempting to remove docker image ${id}" - dockercmd rmi "${id}" - fi - done -} - - -## @description get sentinel-level docker images -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_get_sentinel_images -{ - #shellcheck disable=SC2016 - dockercmd images \ - | tail -n +2 \ - | "${GREP}" -v hours \ - | "${AWK}" '{print $1":"$2}' \ - | "${GREP}" -v "<none>:<none>" -} - -## @description Remove untagged/unused images -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_image_maintenance -{ - declare id - - if [[ "${ROBOT}" = false ]]; then - return - fi - - big_console_header "Removing old images" - - dockercmd images - - echo "Untagged images:" - - #shellcheck disable=SC2046 - docker_image_maintenance_helper $(dockercmd images --filter "dangling=true" -q --no-trunc) - - echo "Apache Yetus images:" - - # removing this by image id doesn't always work without a force - # in the situations that, for whatever reason, docker decided - # to use the same image. this was a rare problem with older - # releases of yetus. at some point, we should revisit this - # in the mean time, we're going to reconstruct the - # repostory:tag and send that to get removed. - - #shellcheck disable=SC2046,SC2016 - docker_image_maintenance_helper $(dockercmd images | ${GREP} -e ^yetus | grep tp- | ${AWK} '{print $1":"$2}') - #shellcheck disable=SC2046,SC2016 - docker_image_maintenance_helper $(dockercmd images | ${GREP} -e ^yetus | ${GREP} -v hours | ${AWK} '{print $1":"$2}') - - if [[ "${SENTINEL}" = false ]]; then - return - fi - - echo "Other images:" - #shellcheck disable=SC2046 - docker_image_maintenance_helper $(docker_get_sentinel_images) -} - - -## @description Perform pre-run maintenance to free up -## @description resources. With --jenkins, it is a lot -## @description more destructive. -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_cleanup -{ - - docker_image_maintenance - - docker_container_maintenance -} - -## @description Determine the revision of a dockerfile -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_getfilerev -{ - ${GREP} 'YETUS_PRIVATE: gitrev=' \ - "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \ - | cut -f2 -d= -} - -function docker_version -{ - declare vertype=$1 - declare val - declare ret - - # new version command - val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null) - ret=$? - - if [[ ${ret} != 0 ]];then - # old version command - val=$(dockercmd version | ${GREP} "${vertype} version" | cut -f2 -d: | tr -d ' ') - fi - - echo "${val}" -} - -## @description Start a test patch docker container -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_run_image -{ - declare dockerfilerev - declare baseimagename - declare patchimagename="yetus/${PROJECT_NAME}:tp-${DOCKER_ID}" - declare containername="yetus_tp-${DOCKER_ID}" - declare client - declare server - declare retval - declare elapsed - - dockerfilerev=$(docker_getfilerev) - - baseimagename="yetus/${PROJECT_NAME}:${dockerfilerev}" - - # make a base image, if it isn't available - big_console_header "Building base image: ${baseimagename}" - start_clock - dockercmd build \ - -t "${baseimagename}" \ - "${PATCH_DIR}/precommit/test-patch-docker" - retval=$? - - #shellcheck disable=SC2046 - elapsed=$(clock_display $(stop_clock)) - - echo "" - echo "Total Elapsed time: ${elapsed}" - echo "" - - if [[ ${retval} != 0 ]]; then - yetus_error "ERROR: Docker failed to build image." - add_vote_table -1 docker "Docker failed to build ${baseimagename}." - bugsystem_finalreport 1 - cleanup_and_exit 1 - fi - - big_console_header "Building ${BUILDMODE} image: ${patchimagename}" - start_clock - # using the base image, make one that is patch specific - dockercmd build \ - -t "${patchimagename}" \ - - <<PatchSpecificDocker -FROM ${baseimagename} -LABEL org.apache.yetus="" -LABEL org.apache.yetus.testpatch.patch="tp-${DOCKER_ID}" -LABEL org.apache.yetus.testpatch.project=${PROJECT_NAME} -RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} || true -RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} || true -RUN chown -R ${USER_NAME} /home/${USER_NAME} || true -ENV HOME /home/${USER_NAME} -USER ${USER_NAME} -PatchSpecificDocker - - retval=$? - - #shellcheck disable=SC2046 - elapsed=$(clock_display $(stop_clock)) - - echo "" - echo "Total Elapsed time: ${elapsed}" - echo "" - - if [[ ${retval} != 0 ]]; then - yetus_error "ERROR: Docker failed to build image." - add_vote_table -1 docker "Docker failed to build ${patchimagename}." - bugsystem_finalreport 1 - cleanup_and_exit 1 - fi - - if [[ "${DOCKER_ENABLE_PRIVILEGED}" = true ]]; then - DOCKER_EXTRAARGS+=("--privileged") - fi - - if [[ -n "${DOCKER_MEMORY}" ]]; then - DOCKER_EXTRAARGS+=("-m" "${DOCKER_MEMORY}") - fi - - client=$(docker_version Client) - server=$(docker_version Server) - - dockerversion="Client=${client} Server=${server}" - - - # make the kernel prefer to kill us if we run out of RAM - DOCKER_EXTRAARGS+=("--oom-score-adj" "500") - - DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile") - DOCKER_EXTRAARGS+=(-v "${PWD}:/testptch/${PROJECT_NAME}") - DOCKER_EXTRAARGS+=(-u "${USER_NAME}") - DOCKER_EXTRAARGS+=(-w "/testptch/${PROJECT_NAME}") - DOCKER_EXTRAARGS+=("--env=BASEDIR=/testptch/${PROJECT_NAME}") - DOCKER_EXTRAARGS+=("--env=DOCKER_VERSION=${dockerversion} Image:${baseimagename}") - DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}") - DOCKER_EXTRAARGS+=("--env=PATCH_SYSTEM=${PATCH_SYSTEM}") - DOCKER_EXTRAARGS+=("--env=PROJECT_NAME=${PROJECT_NAME}") - DOCKER_EXTRAARGS+=("--env=TESTPATCHMODE=${TESTPATCHMODE}") - DOCKER_EXTRAARGS+=(--name "${containername}") - - - trap 'docker_signal_handler' SIGTERM - trap 'docker_signal_handler' SIGINT - - if [[ ${PATCH_DIR} =~ ^/ ]]; then - dockercmd run --rm=true -i \ - "${DOCKER_EXTRAARGS[@]}" \ - -v "${PATCH_DIR}:/testptch/patchprocess" \ - --env=PATCH_DIR=/testptch/patchprocess \ - "${patchimagename}" & - else - dockercmd run --rm=true -i \ - "${DOCKER_EXTRAARGS[@]}" \ - --env=PATCH_DIR="${PATCH_DIR}" \ - "${patchimagename}" & - fi - - wait ${!} - cleanup_and_exit $? -} - -## @description docker kill the container on SIGTERM -## @audience private -## @stability evolving -## @replaceable no -function docker_signal_handler -{ - declare cid - - cid=$(cat "${PATCH_DIR}/cidfile") - - yetus_error "ERROR: Caught signal. Killing docker container:" - dockercmd kill "${cid}" - yetus_error "ERROR: Exiting." - cleanup_and_exit 143 # 128 + 15 -- SIGTERM -} - -## @description Switch over to a Docker container -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_handler -{ - PATCH_DIR=$(relative_dir "${PATCH_DIR}") - - docker_cleanup - determine_user - docker_run_image -}
