Repository: yetus Updated Branches: refs/heads/master 6ce890bcd -> e330e3fdf
YETUS-280. build systems should be able to predetermine module order Signed-off-by: Sean Busbey <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/e330e3fd Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/e330e3fd Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/e330e3fd Branch: refs/heads/master Commit: e330e3fdf111ce0ac22076db59cc2f34780a347c Parents: 6ce890b Author: Allen Wittenauer <[email protected]> Authored: Tue Jan 12 14:06:37 2016 -0800 Committer: Allen Wittenauer <[email protected]> Committed: Wed Jan 13 08:57:31 2016 -0800 ---------------------------------------------------------------------- precommit/personality/hadoop.sh | 8 --- precommit/test-patch.d/maven.sh | 130 +++++++++++++++++++++++++++++++++++ precommit/test-patch.sh | 8 ++- 3 files changed, 137 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/e330e3fd/precommit/personality/hadoop.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/hadoop.sh b/precommit/personality/hadoop.sh index 0656491..0ccf137 100755 --- a/precommit/personality/hadoop.sh +++ b/precommit/personality/hadoop.sh @@ -63,15 +63,7 @@ function hadoop_module_manipulation fi done - # ${startingmodules} is already sorted and uniq'd. - # let's remove child modules if we're going to - # touch their parent. passed_modules=${startingmodules} - for module in ${startingmodules}; do - yetus_debug "Stripping ${module}" - # shellcheck disable=SC2086 - passed_modules=$(echo ${passed_modules} | tr ' ' '\n' | ${GREP} -v ${module}/ ) - done yetus_debug "hmm pre-ordering: ${startingmodules}" http://git-wip-us.apache.org/repos/asf/yetus/blob/e330e3fd/precommit/test-patch.d/maven.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/maven.sh b/precommit/test-patch.d/maven.sh index 7068ea6..861c31c 100755 --- a/precommit/test-patch.d/maven.sh +++ b/precommit/test-patch.d/maven.sh @@ -15,6 +15,8 @@ # limitations under the License. declare -a MAVEN_ARGS=("--batch-mode") +declare -a MAVEN_ALREADY=() +declare -a MAVEN_NEWLIST=() if [[ -z "${MAVEN_HOME:-}" ]]; then MAVEN=mvn @@ -24,6 +26,7 @@ fi MAVEN_CUSTOM_REPOS=false MAVEN_CUSTOM_REPOS_DIR="${HOME}/yetus-m2" +MAVEN_DEPENDENCY_ORDER=true add_test_type mvnsite add_test_type mvneclipse @@ -34,6 +37,7 @@ function maven_usage yetus_add_option "--mvn-cmd=<cmd>" "The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')" yetus_add_option "--mvn-custom-repos" "Use per-project maven repos" yetus_add_option "--mvn-custom-repos-dir=dir" "Location of repos, default is '${MAVEN_CUSTOM_REPOS_DIR}'" + yetus_add_option "--mvn-deps-order=<bool>" "Disable maven's auto-dependency module ordering (Default: '${MAVEN_DEPENDENCY_ORDER}')" yetus_add_option "--mvn-settings=file" "File to use for settings.xml" } @@ -52,6 +56,9 @@ function maven_parse_args --mvn-custom-repos-dir=*) MAVEN_CUSTOM_REPOS_DIR=${i#*=} ;; + --mvn-deps-order=*) + MAVEN_DEPENDENCY_ORDER=${i#*=} + ;; --mvn-settings=*) MAVEN_SETTINGS=${i#*=} if [[ -f ${MAVEN_SETTINGS} ]]; then @@ -524,3 +531,126 @@ function maven_docker_support >> "${PATCH_DIR}/buildtool-docker-params.txt" fi } + +## @description worker for maven reordering +## @audience private +## @stability evolving +## @replaceable no +## @param repostatus +## @param module +function maven_reorder_module_process +{ + declare repostatus=$1 + declare module=$2 + declare indexd + declare indexm + declare deplist + declare fn + declare basemod + + yetus_debug "Maven: check order: ${module}" + + # this is a recursive function; no need to process + # modules that we have already seen + if [[ " ${MAVEN_ALREADY[@]} " =~ \ ${module}\ ]]; then + return + fi + + # immediately place a module in the list to avoid + # loops + MAVEN_ALREADY=("${MAVEN_ALREADY[@]}" "${module}") + + pushd "${BASEDIR}/${module}" >/dev/null + fn=$(module_file_fragment "${module}") + + # get the dependency list from maven + #shellcheck disable=SC2046 + echo_and_redirect "${PATCH_DIR}/maven-${repostatus}-dependencylist-${fn}.txt" \ + $("${BUILDTOOL}_executor") dependency:list \ + -DoutputFile="${PATCH_DIR}/${repostatus}-mvndeporder-${fn}.txt" + + # if this fails, there's really not much we can do other than tell the + # user that it did. So we'll press on and just hope for the best + + if [[ $? != 0 ]]; then + add_vote_table -1 mvndep "${repostatus}'s ${module} dependency:list failed" + add_footer_table mvndep "${fn}: @@BASE@@/maven-${repostatus}-dependencylist-${fn}.txt" + fi + + # take the list from maven, and pull out the artifactids. this means + # we have an assumption that the module names we are given map + # 1:1 to the artifactid + deplist=$(cut -f2 -d: "${PATCH_DIR}/${repostatus}-mvndeporder-${fn}.txt") + + # loop through the generated dependency artificatid list + # and try to match it to any of the other modules we are working on + # if it is a match, process it. dependencies that aren't being hit + # by the patch should be irrelevant since maven will pick them up + # from the repo caches. if a dependency really does matter, then + # the personality will have a chance to manipulate this list later + for indexd in ${deplist}; do + for indexm in ${CHANGED_MODULES}; do + + # modules could be foo/bar, where bar is the artifactid + # so get the basename and compare that too + basemod=$(basename "${indexm}") + if [[ " ${indexd} " = " ${indexm} " + || " ${indexd} " = " ${basemod} " ]]; then + maven_reorder_module_process "${repostatus}" "${indexm}" + fi + done + done + popd >/dev/null + + # at this point, any modules that we care about should have been placed + # before us. now add the one we've been working on. if the artificatids + # don't match, then we're effectively back to the old behavior of + # using fs globs and the personality will have to do the hard work + yetus_debug "Maven: placing ${module}" + MAVEN_NEWLIST=("${MAVEN_NEWLIST[@]}" " ${module} ") +} + +## @description take a stab at reordering modules based upon +## @description maven dependency order +## @audience private +## @stability evolving +## @replaceable no +## @param repostatus +## @param module +function maven_reorder_modules +{ + declare repostatus=$1 + declare index + + # sidenote: this code does not get called when there is only one + # module, so no reason to shortcut that + + if [[ "${MAVEN_DEPENDENCY_ORDER}" != "true" ]]; then + return + fi + + big_console_header "Determining Maven Dependency Order" + + start_clock + + # this will get called again, so always make sure we start fresh + MAVEN_ALREADY=() + MAVEN_NEWLIST=() + + yetus_debug "Maven: start re-ordering modules" + yetus_debug "Starting list: ${CHANGED_MODULES}" + + # sort the list. parents will likely get us most of the + # dependency list sooner/more correct, esp if . is in the list + # shellcheck disable=SC2086 + for index in $(echo ${CHANGED_MODULES} | tr ' ' '\n' | sort); do + maven_reorder_module_process "${repostatus}" "${index}" + done + + CHANGED_MODULES="${MAVEN_NEWLIST[*]}" + + yetus_debug "Maven: finish re-ordering modules" + yetus_debug "Finished list: ${CHANGED_MODULES}" + + add_vote_table 0 mvndep "Maven dependency ordering for ${repostatus}" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/e330e3fd/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index f2f9c59..a71c299 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -1026,9 +1026,11 @@ function module_skipdir ## @audience private ## @stability stable ## @replaceable no +## @param repostatus ## @return None; sets ${CHANGED_MODULES} function find_changed_modules { + local repostatus=$1 local i local changed_dirs local builddirs @@ -1115,6 +1117,10 @@ function find_changed_modules #shellcheck disable=SC2034 CHANGED_UNION_MODULES="${builddir}" + + if declare -f ${BUILDTOOL}_reorder_modules >/dev/null; then + "${BUILDTOOL}_reorder_modules" "${repostatus}" + fi } ## @description git checkout the appropriate branch to test. Additionally, this calls @@ -2506,7 +2512,7 @@ function compile_cycle declare result=0 declare plugin - find_changed_modules + find_changed_modules "${codebase}" for plugin in ${PROJECT_NAME} ${BUILDTOOL} ${TESTTYPES} ${TESTFORMATS}; do if declare -f ${plugin}_precompile >/dev/null 2>&1; then
