Repository: yetus Updated Branches: refs/heads/master 5a2c02552 -> 7615f35f9
YETUS-640. add hadolint support Signed-off-by: Jack Bearden <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/7615f35f Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/7615f35f Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/7615f35f Branch: refs/heads/master Commit: 7615f35f9835fbe144697b6c89e313d70316e99e Parents: 5a2c025 Author: Allen Wittenauer <[email protected]> Authored: Tue Jul 10 16:06:02 2018 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Thu Aug 23 17:05:44 2018 -0700 ---------------------------------------------------------------------- .hadolint.yaml | 17 +++ precommit/test-patch-docker/Dockerfile | 87 ++++++++---- precommit/test-patch.d/hadolint.sh | 204 ++++++++++++++++++++++++++++ 3 files changed, 284 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/7615f35f/.hadolint.yaml ---------------------------------------------------------------------- diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 0000000..6645586 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ignored: + - DL3008 http://git-wip-us.apache.org/repos/asf/yetus/blob/7615f35f/precommit/test-patch-docker/Dockerfile ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile b/precommit/test-patch-docker/Dockerfile index 2f54bda..8494d5c 100644 --- a/precommit/test-patch-docker/Dockerfile +++ b/precommit/test-patch-docker/Dockerfile @@ -26,6 +26,7 @@ FROM ubuntu:xenial WORKDIR /root +SHELL ["/bin/bash", "-o", "pipefail", "-c"] ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true @@ -41,7 +42,9 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ pinentry-curses \ pkg-config \ rsync \ - software-properties-common + software-properties-common \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ### # Set the locale @@ -58,16 +61,20 @@ ENV LC_ALL en_US.UTF-8 #### # OpenJDK 8 #### -RUN apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless +RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ####### # OpenJDK 9 # w/workaround for # https://bugs.launchpad.net/ubuntu/+source/openjdk-9/+bug/1593191 ####### -RUN apt-get -o Dpkg::Options::="--force-overwrite" \ +RUN apt-get -q update && apt-get -o Dpkg::Options::="--force-overwrite" \ -q install --no-install-recommends -y \ - openjdk-9-jdk-headless + openjdk-9-jdk-headless \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ####### # Set default Java @@ -78,72 +85,91 @@ RUN apt-get -o Dpkg::Options::="--force-overwrite" \ # RUN update-java-alternatives --set java-1.8.0-openjdk-amd64 RUN update-alternatives --get-selections | grep -i jdk | \ - while read line; do \ - alternative=$(echo $line | awk '{print $1}'); \ - path=$(echo $line | awk '{print $3}'); \ - newpath=$(echo $path | sed -e 's/java-9/java-8/'); \ - update-alternatives --set $alternative $newpath; \ + while read -r line; do \ + alternative=$(echo "$line" | awk '{print $1}'); \ + path=$(echo "$line" | awk '{print $3}'); \ + newpath=${path//java-9/java-8}; \ + update-alternatives --set "$alternative" "$newpath"; \ done #### # Install ant #### -RUN apt-get -q install --no-install-recommends -y ant +RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* #### # Install GNU automake, GNU make, and related #### -RUN apt-get -q install --no-install-recommends -y autoconf automake libtool make +RUN apt-get -q update && apt-get -q install --no-install-recommends -y autoconf automake libtool make \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* #### # Install bats (TAP-capable unit testing for shell scripts) #### -RUN apt-get -q install --no-install-recommends -y bats +RUN apt-get -q update && apt-get -q install --no-install-recommends -y bats \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* #### # Install cmake #### -RUN apt-get -q install --no-install-recommends -y cmake +RUN apt-get -q update && apt-get -q install --no-install-recommends -y cmake \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ###### # Install findbugs ###### -RUN apt-get -q install --no-install-recommends -y findbugs +RUN apt-get -q update && apt-get -q install --no-install-recommends -y findbugs \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ENV FINDBUGS_HOME /usr #### # Install GNU C/C++ #### -RUN apt-get -q install --no-install-recommends -y g++ gcc libc-dev +RUN apt-get -q update && apt-get -q install --no-install-recommends -y g++ gcc libc-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ###### # Install maven ###### -RUN apt-get -q install --no-install-recommends -y maven +RUN apt-get -q update && apt-get -q install --no-install-recommends -y maven \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ###### # Install perl ###### -RUN apt-get -q install --no-install-recommends -y perl libperl-critic-perl +RUN apt-get -q update && apt-get -q install --no-install-recommends -y perl libperl-critic-perl \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ###### # Install python and pylint ###### -RUN apt-get -q install --no-install-recommends -y python \ +RUN apt-get -q update && apt-get -q install --no-install-recommends -y python \ python2.7 \ python-pip \ python-pkg-resources \ python-setuptools \ - python-wheel -RUN pip2 install --upgrade pip==9.0.3 -RUN pip install -v pylint==1.9.2 + python-wheel \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +RUN pip2 install -v pylint==1.9.2 #### # Install ruby, rubocop, and ruby-lint ### RUN echo 'gem: --no-rdoc --no-ri' >> /root/.gemrc -RUN apt-get -q install --no-install-recommends -y ruby ruby-dev +RUN apt-get -q update && apt-get -q install --no-install-recommends -y ruby ruby-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* RUN gem install rake RUN gem install rubocop RUN gem install ruby-lint @@ -152,8 +178,19 @@ RUN gem install ruby-lint # Install shellcheck (shell script lint) #### RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y shellcheck +RUN apt-get -q update && apt-get -q install --no-install-recommends -y shellcheck \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +### +# Install hadolint +#### +RUN curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint && \ + chmod a+rx /bin/hadolint && \ + shasum -a 512 /bin/hadolint | \ + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' #### # YETUS CUT HERE @@ -161,4 +198,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y shellchec # include other requirements not needed by your development # (but not build) environment ### -RUN apt-get -q install --no-install-recommends -y sudo vim +RUN apt-get -q update && apt-get -q install --no-install-recommends -y sudo vim \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* http://git-wip-us.apache.org/repos/asf/yetus/blob/7615f35f/precommit/test-patch.d/hadolint.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/hadolint.sh b/precommit/test-patch.d/hadolint.sh new file mode 100755 index 0000000..2392e29 --- /dev/null +++ b/precommit/test-patch.d/hadolint.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no public APIs here +# SHELLDOC-IGNORE + +add_test_type hadolint + +HADOLINT_TIMER=0 +HADOLINT=${HADOLINT:-$(command -v hadolint 2>/dev/null)} + +# files that are going to get hadolint'd +HADOLINT_CHECKFILES=() + +function hadolint_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ Dockerfile$ ]]; then + add_test hadolint + yetus_add_array_element HADOLINT_CHECKFILES "${filename}" + fi +} + +function hadolint_precheck +{ + declare langs + + if ! verify_command "hadolint" "${HADOLINT}"; then + add_vote_table 0 hadolint "hadolint was not available." + delete_test hadolint + fi + + if [[ -z "${LANG}" ]]; then + langs=$(locale -a) + if [[ ${langs} =~ C.UTF-8 ]]; then + yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing C.UTF-8." + export LANG=C.UTF-8 + export LC_ALL=C.UTF-8 + elif [[ ${langs} =~ en_US.UTF-8 ]]; then + yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing en_US.UTF-8." + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + else + for i in ${langs}; do + if [[ "${i}" =~ UTF-8 ]]; then + yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing ${i}." + export LANG="${i}" + export LC_ALL="${i}" + break + fi + done + fi + fi + + if [[ ! "${LANG}" =~ UTF-8 ]]; then + yetus_error "WARNING: hadolint may fail without UTF-8 locale setting." + fi +} + +function hadolint_logic +{ + declare repostatus=$1 + declare i + + pushd "${BASEDIR}" >/dev/null || return 1 + + for i in "${HADOLINT_CHECKFILES[@]}"; do + if [[ -f "${i}" ]]; then + echo " * ${i}" + "${HADOLINT}" "${i}" >> "${PATCH_DIR}/${repostatus}-hadolint-result.txt" + fi + done + popd > /dev/null || return 1 +} + +function hadolint_preapply +{ + if ! verify_needed_test hadolint; then + return 0 + fi + + big_console_header "hadolint plugin: ${PATCH_BRANCH}" + + start_clock + + hadolint_logic branch + + # keep track of how much as elapsed for us already + HADOLINT_TIMER=$(stop_clock) + return 0 +} + +## filename:line\sCODE Text +function hadolint_calcdiffs +{ + declare branch=$1 + declare patch=$2 + declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} + declare j + + # first, pull out just the errors + # shellcheck disable=SC2016 + ${AWK} -F: '{print $NF}' "${branch}" | cut -d' ' -f2- > "${tmp}.branch" + + # shellcheck disable=SC2016 + ${AWK} -F: '{print $NF}' "${patch}" | cut -d' ' -f2- > "${tmp}.patch" + + ${DIFF} --unchanged-line-format="" \ + --old-line-format="" \ + --new-line-format="%dn " \ + "${tmp}.branch" \ + "${tmp}.patch" > "${tmp}.lined" + + # now, pull out those lines of the raw output + # shellcheck disable=SC2013 + for j in $(cat "${tmp}.lined"); do + # shellcheck disable=SC2086 + head -${j} "${patch}" | tail -1 + done + + rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null +} + +function hadolint_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test hadolint; then + return 0 + fi + + big_console_header "hadolint plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${HADOLINT_TIMER}" + + hadolint_logic patch + + calcdiffs \ + "${PATCH_DIR}/branch-hadolint-result.txt" \ + "${PATCH_DIR}/patch-hadolint-result.txt" \ + hadolint \ + > "${PATCH_DIR}/diff-patch-hadolint.txt" + + # shellcheck disable=SC2016 + numPrepatch=$(wc -l "${PATCH_DIR}/branch-hadolint-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + numPostpatch=$(wc -l "${PATCH_DIR}/patch-hadolint-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-hadolint.txt" | ${AWK} '{print $1}') + + + ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch)) + + statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" ) + + if [[ ${diffPostpatch} -gt 0 ]] ; then + add_vote_table -1 hadolint "${BUILDMODEMSG} ${statstring}" + add_footer_table hadolint "@@BASE@@/diff-patch-hadolint.txt" + bugsystem_linecomments "hadolint" "${PATCH_DIR}/diff-patch-hadolint.txt" + return 1 + elif [[ ${fixedpatch} -gt 0 ]]; then + add_vote_table +1 hadolint "${BUILDMODEMSG} ${statstring}" + return 0 + fi + + add_vote_table +1 hadolint "There were no new hadolint issues." + return 0 +} + +function hadolint_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + hadolint_preapply + else + hadolint_postapply + fi +}
