This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/master by this push:
new 3db66f6 YETUS-788. expose patch analysis
3db66f6 is described below
commit 3db66f617bb87df6bcc378bb39a9de0af6c67190
Author: Allen Wittenauer <[email protected]>
AuthorDate: Fri Feb 1 14:00:21 2019 -0800
YETUS-788. expose patch analysis
Signed-off-by: Allen Wittenauer <[email protected]>
---
precommit/src/main/shell/core.d/01-common.sh | 16 ++
precommit/src/main/shell/core.d/change-analysis.sh | 236 +++++++++++++++++++++
precommit/src/main/shell/smart-apply-patch.sh | 89 +++++++-
precommit/src/main/shell/test-patch.sh | 225 --------------------
4 files changed, 340 insertions(+), 226 deletions(-)
diff --git a/precommit/src/main/shell/core.d/01-common.sh
b/precommit/src/main/shell/core.d/01-common.sh
index 6c58021..0692a9c 100755
--- a/precommit/src/main/shell/core.d/01-common.sh
+++ b/precommit/src/main/shell/core.d/01-common.sh
@@ -712,4 +712,20 @@ function guess_build_tool
fi
echo "Setting build tool to ${BUILDTOOL}"
+}
+
+## @description Convert the given module name to a file fragment
+## @audience public
+## @stability stable
+## @replaceable no
+## @param module
+function module_file_fragment
+{
+ local mod=$1
+ if [[ ${mod} = \. ]]; then
+ echo root
+ else
+ #shellcheck disable=SC1003
+ echo "$1" | tr '/' '_' | tr '\\' '_'
+ fi
}
\ No newline at end of file
diff --git a/precommit/src/main/shell/core.d/change-analysis.sh
b/precommit/src/main/shell/core.d/change-analysis.sh
new file mode 100755
index 0000000..4b1fb13
--- /dev/null
+++ b/precommit/src/main/shell/core.d/change-analysis.sh
@@ -0,0 +1,236 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+## @description Locate the build file for a given directory
+## @audience private
+## @stability stable
+## @replaceable no
+## @return directory containing the buildfile. Nothing returned if not
found.
+## @param buildfile
+## @param directory
+function find_buildfile_dir
+{
+ local buildfile=$1
+ local dir=$2
+
+ yetus_debug "Find ${buildfile} dir for: ${dir}"
+
+ while builtin true; do
+ if [[ -f "${dir}/${buildfile}" ]];then
+ echo "${dir}"
+ yetus_debug "Found: ${dir}"
+ return 0
+ elif [[ ${dir} == "." || ${dir} == "/" ]]; then
+ yetus_debug "ERROR: ${buildfile} is not found."
+ return 1
+ else
+ dir=$(faster_dirname "${dir}")
+ fi
+ done
+}
+
+## @description List of files that ${PATCH_DIR}/patch modifies
+## @audience private
+## @stability stable
+## @replaceable no
+## @return None; sets ${CHANGED_FILES[@]}
+function find_changed_files
+{
+ declare line
+
+ BUILDMODE=${BUILDMODE:-patch}
+
+ pushd "${BASEDIR}" >/dev/null || return 1
+
+ case "${BUILDMODE}" in
+ full)
+ echo "Building a list of all files in the source tree"
+ while IFS= read -r; do CHANGED_FILES+=("$REPLY"); done < <("${GIT}"
ls-files)
+ ;;
+ patch)
+ # get a list of all of the files that have been changed,
+ # except for /dev/null (which would be present for new files).
+ # Additionally, remove any a/ b/ patterns at the front of the patch
filenames.
+ # shellcheck disable=SC2016
+ while read -r line; do
+ CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}")
+ done < <(
+ "${AWK}" 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print
s}}
+ /^diff --git / { p($3); p($4) }
+ /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
+ ;;
+ esac
+ popd >/dev/null || return 1
+}
+
+## @description Check for directories to skip during
+## @description changed module calcuation
+## @description requires $MODULE_SKIPDIRS to be set
+## @audience private
+## @stability stable
+## @replaceable no
+## @param directory
+## @return 0 for use
+## @return 1 for skip
+function module_skipdir
+{
+ local dir=${1}
+ local i
+
+ yetus_debug "Checking skipdirs for ${dir}"
+
+ if [[ -z ${MODULE_SKIPDIRS} ]]; then
+ yetus_debug "Skipping skipdirs"
+ return 0
+ fi
+
+ while builtin true; do
+ for i in ${MODULE_SKIPDIRS}; do
+ if [[ ${dir} = "${i}" ]];then
+ yetus_debug "Found a skip: ${dir}"
+ return 1
+ fi
+ done
+ if [[ ${dir} == "." || ${dir} == "/" ]]; then
+ return 0
+ else
+ dir=$(faster_dirname "${dir}")
+ yetus_debug "Trying to skip: ${dir}"
+ fi
+ done
+}
+
+## @description Find the modules of the build that ${PATCH_DIR}/patch modifies
+## @audience private
+## @stability stable
+## @replaceable no
+## @param repostatus
+## @return None; sets ${CHANGED_MODULES[@]}
+function find_changed_modules
+{
+ declare repostatus=$1
+ declare i
+ declare builddir
+ declare module
+ declare prev_builddir
+ declare i=1
+ declare dir
+ declare dirt
+ declare buildfile
+ declare -a tmpmods
+ declare retval
+
+ if declare -f "${BUILDTOOL}_buildfile" >/dev/null; then
+ buildfile=$("${BUILDTOOL}_buildfile")
+ retval=$?
+ else
+ yetus_error "ERROR: build tool plugin is broken"
+ bugsystem_finalreport 1
+ cleanup_and_exit 1
+ fi
+
+ if [[ ${retval} != 0 ]]; then
+ yetus_error "ERROR: Unsupported build tool."
+ bugsystem_finalreport 1
+ cleanup_and_exit 1
+ fi
+
+ pushd "${BASEDIR}" >/dev/null || return 1
+
+ # Empty string indicates the build system wants to disable module detection
+ if [[ -z ${buildfile} ]]; then
+ CHANGED_MODULES=(".")
+ else
+
+ # Now find all the modules that were changed
+ for i in "${CHANGED_FILES[@]}"; do
+
+ # TODO: optimize this
+ if [[ "${BUILDMODE}" = full && ! "${i}" =~ ${buildfile} ]]; then
+ continue
+ fi
+
+ dirt=$(dirname "${i}")
+
+ if ! module_skipdir "${dirt}"; then
+ continue
+ fi
+
+ builddir=$(find_buildfile_dir "${buildfile}" "${dirt}")
+ if [[ -z ${builddir} ]]; then
+ yetus_error "ERROR: ${buildfile} is not found. Make sure the target is
a ${BUILDTOOL}-based project."
+ bugsystem_finalreport 1
+ cleanup_and_exit 1
+ fi
+ CHANGED_MODULES+=("${builddir}")
+ done
+ fi
+
+ CHANGED_MODULES+=("${USER_MODULE_LIST[@]}")
+
+ yetus_sort_and_unique_array CHANGED_MODULES
+
+ yetus_debug "Locate the union of ${CHANGED_MODULES[*]}"
+
+ count=${#CHANGED_MODULES[@]}
+ if [[ ${count} -lt 2 ]]; then
+ yetus_debug "Only one entry, so keeping it ${CHANGED_MODULES[0]}"
+ # shellcheck disable=SC2034
+ CHANGED_UNION_MODULES="${CHANGED_MODULES[0]}"
+ else
+ i=1
+
+ # BUG - fix me
+ # shellcheck disable=SC2207
+ while [[ ${i} -lt 100 ]]; do
+ tmpmods=()
+ for j in "${CHANGED_MODULES[@]}"; do
+ tmpmods+=($(echo "${j}" | cut -f1-${i} -d/))
+ done
+ tmpmods=($(printf '%s\n' "${tmpmods[@]}" | sort -u))
+
+ module=${tmpmods[0]}
+ count=${#tmpmods[@]}
+ if [[ ${count} -eq 1
+ && -f ${module}/${buildfile} ]]; then
+ prev_builddir=${module}
+ elif [[ ${count} -gt 1 ]]; then
+ builddir=${prev_builddir}
+ break
+ fi
+ ((i=i+1))
+ done
+
+ if [[ -z ${builddir} ]]; then
+ builddir="."
+ fi
+
+ yetus_debug "Finding union of ${builddir}"
+ builddir=$(find_buildfile_dir "${buildfile}" "${builddir}" || true)
+
+ #shellcheck disable=SC2034
+ CHANGED_UNION_MODULES="${builddir}"
+ fi
+
+ # some build tools may want to change these and/or
+ # make other changes based upon these results
+ if declare -f "${BUILDTOOL}_reorder_modules" >/dev/null; then
+ "${BUILDTOOL}_reorder_modules" "${repostatus}"
+ fi
+ popd >/dev/null || return 1
+
+}
diff --git a/precommit/src/main/shell/smart-apply-patch.sh
b/precommit/src/main/shell/smart-apply-patch.sh
index bdee4cb..0a5a742 100755
--- a/precommit/src/main/shell/smart-apply-patch.sh
+++ b/precommit/src/main/shell/smart-apply-patch.sh
@@ -27,6 +27,8 @@ this="${BASH_SOURCE-$0}"
BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
#shellcheck disable=SC2034
QATESTMODE=false
+STARTINGDIR=$(pwd)
+
## @description disable function
## @stability stable
@@ -64,6 +66,16 @@ function add_test
true
}
+
+## @description disable function
+## @stability stable
+## @audience private
+## @replaceable no
+function bugsystem_finalreport
+{
+ true
+}
+
## @description Clean the filesystem as appropriate and then exit
## @audience private
## @stability evolving
@@ -135,6 +147,15 @@ function yetus_usage
yetus_reset_usage
echo ""
+ echo "Patch reporting:"
+ yetus_add_option "--build-tool=<tool>" "Override the build tool"
+ yetus_add_option "--changedfilesreport=<name>" "List of files that this
patch modifies"
+ yetus_add_option "--changedmodulesreport=<name>" "List of modules that this
patch modifies"
+ yetus_add_option "--changedunionreport=<name>" "Union of modules that this
patch modifies"
+ yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}"
+ yetus_reset_usage
+
+ echo ""
importplugins
unset TESTFORMATS
@@ -166,6 +187,9 @@ function parse_args
for i in "$@"; do
case ${i} in
+ --build-tool=*)
+ BUILDTOOL=${i#*=}
+ ;;
--committer)
COMMITMODE=true
;;
@@ -175,6 +199,15 @@ function parse_args
--dry-run)
PATCH_DRYRUNMODE=true
;;
+ --changedfilesreport=*)
+ FILEREPORT=${i#*=}
+ ;;
+ --changedmodulesreport=*)
+ MODULEREPORT=${i#*=}
+ ;;
+ --changedunionreport=*)
+ UNIONREPORT=${i#*=}
+ ;;
--*)
## PATCH_OR_ISSUE can't be a --. So this is probably
## a plugin thing.
@@ -192,6 +225,55 @@ function parse_args
cleanup_and_exit 1
fi
fi
+
+ # we need absolute dir for ${BASEDIR}
+ cd "${STARTINGDIR}" || cleanup_and_exit 1
+ BASEDIR=$(yetus_abs "${BASEDIR}")
+}
+
+## @description generate reports on the given patch file
+## @replaceable no
+## @audience private
+## @stability evolving
+function patch_reports
+{
+ declare i
+
+ if [[ -z "${FILEREPORT}" ]] && [[ -z "${MODULEREPORT}" ]] && [[ -z
"${UNIONREPORT}" ]]; then
+ return
+ fi
+
+ set -x
+
+ find_changed_files
+
+ if [[ -n "${FILEREPORT}" ]]; then
+ : > "${FILEREPORT}"
+ for i in "${CHANGED_FILES[@]}"; do
+ echo "${i}" >> "${FILEREPORT}"
+ done
+ fi
+
+ if [[ -n "${MODULEREPORT}" ]] || [[ -n "${UNIONREPORT}" ]]; then
+ if [[ -z "${BUILDTOOL}" ]]; then
+ guess_build_tool
+ fi
+
+ unset -f "${BUILDTOOL}_reorder_modules"
+
+ find_changed_modules
+
+ if [[ -n "${MODULEREPORT}" ]]; then
+ : > "${MODULEREPORT}"
+ for i in "${CHANGED_MODULES[@]}"; do
+ echo "${i}" >> "${MODULEREPORT}"
+ done
+ fi
+
+ if [[ -n "${UNIONREPORT}" ]]; then
+ cat "${CHANGED_UNION_MODULES}" > "${UNIONREPORT}"
+ fi
+ fi
}
## @description git am dryrun
@@ -305,7 +387,6 @@ parse_args "$@"
importplugins
yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed
plug-in list"
-unset BUILDTOOLS
unset TESTTYPES
unset TESTFORMATS
@@ -333,6 +414,8 @@ if [[ ${RESULT} -gt 0 ]]; then
cleanup_and_exit ${RESULT}
fi
+pushd "${BASEDIR}" >/dev/null || exit 1
+
if [[ ${PATCH_DRYRUNMODE} == false ]]; then
patchfile_apply_driver "${PATCH_DIR}/patch" "${GPGSIGN}"
RESULT=$?
@@ -344,4 +427,8 @@ if [[ ${COMMITMODE} = true
git add -A
fi
+patch_reports
+
+popd >/dev/null || exit 1
+
cleanup_and_exit ${RESULT}
diff --git a/precommit/src/main/shell/test-patch.sh
b/precommit/src/main/shell/test-patch.sh
index dd0bae5..b84fb57 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -103,21 +103,6 @@ function setup_defaults
yetus_add_entry JDK_TEST_LIST unit
}
-## @description Convert the given module name to a file fragment
-## @audience public
-## @stability stable
-## @replaceable no
-## @param module
-function module_file_fragment
-{
- local mod=$1
- if [[ ${mod} = \. ]]; then
- echo root
- else
- #shellcheck disable=SC1003
- echo "$1" | tr '/' '_' | tr '\\' '_'
- fi
-}
## @description Convert time in seconds to m + s
## @audience public
@@ -1080,216 +1065,6 @@ function set_buildmode
fi
}
-## @description Locate the build file for a given directory
-## @audience private
-## @stability stable
-## @replaceable no
-## @return directory containing the buildfile. Nothing returned if not
found.
-## @param buildfile
-## @param directory
-function find_buildfile_dir
-{
- local buildfile=$1
- local dir=$2
-
- yetus_debug "Find ${buildfile} dir for: ${dir}"
-
- while builtin true; do
- if [[ -f "${dir}/${buildfile}" ]];then
- echo "${dir}"
- yetus_debug "Found: ${dir}"
- return 0
- elif [[ ${dir} == "." || ${dir} == "/" ]]; then
- yetus_debug "ERROR: ${buildfile} is not found."
- return 1
- else
- dir=$(faster_dirname "${dir}")
- fi
- done
-}
-
-## @description List of files that ${PATCH_DIR}/patch modifies
-## @audience private
-## @stability stable
-## @replaceable no
-## @return None; sets ${CHANGED_FILES[@]}
-function find_changed_files
-{
- declare line
-
- case "${BUILDMODE}" in
- full)
- echo "Building a list of all files in the source tree"
- while IFS= read -r; do CHANGED_FILES+=("$REPLY"); done < <(git ls-files)
- ;;
- patch)
- # get a list of all of the files that have been changed,
- # except for /dev/null (which would be present for new files).
- # Additionally, remove any a/ b/ patterns at the front of the patch
filenames.
- # shellcheck disable=SC2016
- while read -r line; do
- CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}")
- done < <(
- ${AWK} 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print s}}
- /^diff --git / { p($3); p($4) }
- /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
- ;;
- esac
-}
-
-## @description Check for directories to skip during
-## @description changed module calcuation
-## @audience private
-## @stability stable
-## @replaceable no
-## @param directory
-## @return 0 for use
-## @return 1 for skip
-function module_skipdir
-{
- local dir=${1}
- local i
-
- yetus_debug "Checking skipdirs for ${dir}"
-
- if [[ -z ${MODULE_SKIPDIRS} ]]; then
- yetus_debug "Skipping skipdirs"
- return 0
- fi
-
- while builtin true; do
- for i in ${MODULE_SKIPDIRS}; do
- if [[ ${dir} = "${i}" ]];then
- yetus_debug "Found a skip: ${dir}"
- return 1
- fi
- done
- if [[ ${dir} == "." || ${dir} == "/" ]]; then
- return 0
- else
- dir=$(faster_dirname "${dir}")
- yetus_debug "Trying to skip: ${dir}"
- fi
- done
-}
-
-## @description Find the modules of the build that ${PATCH_DIR}/patch modifies
-## @audience private
-## @stability stable
-## @replaceable no
-## @param repostatus
-## @return None; sets ${CHANGED_MODULES[@]}
-function find_changed_modules
-{
- declare repostatus=$1
- declare i
- declare builddir
- declare module
- declare prev_builddir
- declare i=1
- declare dir
- declare dirt
- declare buildfile
- declare -a tmpmods
- declare retval
-
- if declare -f "${BUILDTOOL}_buildfile" >/dev/null; then
- buildfile=$("${BUILDTOOL}_buildfile")
- retval=$?
- else
- yetus_error "ERROR: build tool plugin is broken"
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
-
- if [[ ${retval} != 0 ]]; then
- yetus_error "ERROR: Unsupported build tool."
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
-
- # Empty string indicates the build system wants to disable module detection
- if [[ -z ${buildfile} ]]; then
- CHANGED_MODULES=(".")
- else
-
- # Now find all the modules that were changed
- for i in "${CHANGED_FILES[@]}"; do
-
- # TODO: optimize this
- if [[ "${BUILDMODE}" = full && ! "${i}" =~ ${buildfile} ]]; then
- continue
- fi
-
- dirt=$(dirname "${i}")
-
- if ! module_skipdir "${dirt}"; then
- continue
- fi
-
- builddir=$(find_buildfile_dir "${buildfile}" "${dirt}")
- if [[ -z ${builddir} ]]; then
- yetus_error "ERROR: ${buildfile} is not found. Make sure the target is
a ${BUILDTOOL}-based project."
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
- CHANGED_MODULES+=("${builddir}")
- done
- fi
-
- CHANGED_MODULES+=("${USER_MODULE_LIST[@]}")
-
- yetus_sort_and_unique_array CHANGED_MODULES
-
- yetus_debug "Locate the union of ${CHANGED_MODULES[*]}"
-
- count=${#CHANGED_MODULES[@]}
- if [[ ${count} -lt 2 ]]; then
- yetus_debug "Only one entry, so keeping it ${CHANGED_MODULES[0]}"
- # shellcheck disable=SC2034
- CHANGED_UNION_MODULES="${CHANGED_MODULES[0]}"
- else
- i=1
-
- # BUG - fix me
- # shellcheck disable=SC2207
- while [[ ${i} -lt 100 ]]; do
- tmpmods=()
- for j in "${CHANGED_MODULES[@]}"; do
- tmpmods+=($(echo "${j}" | cut -f1-${i} -d/))
- done
- tmpmods=($(printf '%s\n' "${tmpmods[@]}" | sort -u))
-
- module=${tmpmods[0]}
- count=${#tmpmods[@]}
- if [[ ${count} -eq 1
- && -f ${module}/${buildfile} ]]; then
- prev_builddir=${module}
- elif [[ ${count} -gt 1 ]]; then
- builddir=${prev_builddir}
- break
- fi
- ((i=i+1))
- done
-
- if [[ -z ${builddir} ]]; then
- builddir="."
- fi
-
- yetus_debug "Finding union of ${builddir}"
- builddir=$(find_buildfile_dir "${buildfile}" "${builddir}" || true)
-
- #shellcheck disable=SC2034
- CHANGED_UNION_MODULES="${builddir}"
- fi
-
- # some build tools may want to change these and/or
- # make other changes based upon these results
- if declare -f "${BUILDTOOL}_reorder_modules" >/dev/null; then
- "${BUILDTOOL}_reorder_modules" "${repostatus}"
- fi
-}
-
## @description check if repo requires creds to do remote operations
## @description also sets and uses GIT_OFFLINE as appropriate
## @audience private
