YETUS-5. Support per-instance maven repos (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/965184ed Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/965184ed Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/965184ed Branch: refs/heads/master Commit: 965184edfed7014c5e2510fc9b308e7fa4488888 Parents: c44ce1e Author: Allen Wittenauer <[email protected]> Authored: Sun Sep 27 06:37:48 2015 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Wed Oct 7 08:22:38 2015 -0700 ---------------------------------------------------------------------- .../latest/precommit-buildtools.md | 26 +++++ .../test-patch-docker/test-patch-docker.sh | 10 +- dev-support/test-patch.d/ant.sh | 18 +++ dev-support/test-patch.d/gradle.sh | 15 +++ dev-support/test-patch.d/maven.sh | 114 ++++++++++++++++++- dev-support/test-patch.sh | 20 +++- 6 files changed, 199 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/965184ed/asf-site-src/source/documentation/latest/precommit-buildtools.md ---------------------------------------------------------------------- diff --git a/asf-site-src/source/documentation/latest/precommit-buildtools.md b/asf-site-src/source/documentation/latest/precommit-buildtools.md index d14f34b..cce1ddd 100644 --- a/asf-site-src/source/documentation/latest/precommit-buildtools.md +++ b/asf-site-src/source/documentation/latest/precommit-buildtools.md @@ -73,22 +73,48 @@ For example, the gradle build tool does not have a standard way to execute check - Certain language test plug-ins require assistance from the build tool to count problems in the compile log due to some tools having custom handling for those languages. The test plug-in name should be in the (test) part of the function name. +* pluginname\_docker\_support + + - If this build tool requires extra settings on the `docker run` command line, this function should be defined and write those options into a file called `${PATCH_DIR}/buildtool-docker-params.txt`. This is particularly useful for things like mounting volumes for repository caches. + + **WARNING**: Be aware that directories that do not exist MAY be created by root by Docker itself under certain conditions. It is HIGHLY recommend that `pluginname_initialize` be used to create the necessary directories prior to be used in the `docker run` command. + # Ant Specific ## Command Arguments test-patch always passes -noinput to Ant. This forces ant to be non-interactive. +## Docker Mode + +In Docker mode, the `${HOME}/.ivy2` directory is shared amongst all invocations. + # Gradle Specific The gradle plug-in always rebuilds the gradlew file and uses gradlew as the method to execute commands. +In Docker mode, the `${HOME}/.gradle` directory is shared amongst all invocations. + # Maven Specific ## Command Arguments test-patch always passes --batch-mode to maven to force it into non-interactive mode. Additionally, some tests will also force -fae in order to get all of messages/errors during that mode. Some tests are executed with -DskipTests. Additional arguments should be handled via the personality. +## Per-instance Repositories + +Under many common configurations, maven (as of 3.3.3 and lower) may not properly handle being executed by multiple processes simultaneously, especially given that some tests require the `mvn install` command to be used. + +To assist, `test-patch` supports a `--mvn-custom-repo` option to set the `-Dmaven.repo.local` value to a per-instance repository directory keyed to the project and branch being used for the test. If the `--jenkins` flag is also passed, the instance will be tied to the Jenkins `${EXECUTOR_NUMBER}` value. Otherwise, the instance value will be randomly generated via `${RANDOM}`. If the repository has not been used in 30 days, it will be automatically deleted when any test run for that project (regardless of branch!). + +By default, `test-patch` uses `${HOME}/yetus-m2` as the base directory to store these custom maven repositories. That location may be changed via the `--mvn-custom-repos-dir` option. + +The location of the `settings.xml` may be changed via the `--mvn-settings` option. + +## Docker Mode + +In Docker mode, `${HOME}/.m2` is shared amongst all invocations. If `--mvn-custom-repos` is used, all of `--mvn-custom-repos-dir` is shared with all invocations. The per-instance directory will be calculated and configured after Docker has launched. + ## Test Profile By default, test-patch will pass -Ptest-patch to Maven. This will allow you to configure special actions that should only happen when running underneath test-patch. http://git-wip-us.apache.org/repos/asf/yetus/blob/965184ed/dev-support/test-patch-docker/test-patch-docker.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch-docker/test-patch-docker.sh b/dev-support/test-patch-docker/test-patch-docker.sh index f8e46b4..6a212b9 100755 --- a/dev-support/test-patch-docker/test-patch-docker.sh +++ b/dev-support/test-patch-docker/test-patch-docker.sh @@ -347,11 +347,17 @@ ENV HOME /home/${USER_NAME} USER ${USER_NAME} PatchSpecificDocker + if [[ -f "${PATCH_DIR}/buildtool-docker-params.txt" ]]; then + extraargs=$(cat "${PATCH_DIR}/buildtool-docker-params.txt") + else + extraargs="" + fi + if [[ ${PATCH_DIR} =~ ^/ ]]; then dockercmd run --rm=true -i \ + ${extraargs} \ -v "${PWD}:/testptch/${PROJECT_NAME}" \ -v "${PATCH_DIR}:/testptch/patchprocess" \ - -v "${HOME}/.m2:${HOME}/.m2" \ -u "${USER_NAME}" \ -w "/testptch/${PROJECT_NAME}" \ --env=BASEDIR="/testptch/${PROJECT_NAME}" \ @@ -363,8 +369,8 @@ PatchSpecificDocker "test-patch-tp-${PROJECT_NAME}-${DID}" else dockercmd run --rm=true -i \ + ${extraargs} \ -v "${PWD}:/testptch/${PROJECT_NAME}" \ - -v "${HOME}/.m2:${HOME}/.m2" \ -u "${USER_NAME}" \ -w "/testptch/${PROJECT_NAME}" \ --env=BASEDIR="/testptch/${PROJECT_NAME}" \ http://git-wip-us.apache.org/repos/asf/yetus/blob/965184ed/dev-support/test-patch.d/ant.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/ant.sh b/dev-support/test-patch.d/ant.sh index 8d1f364..9d931a5 100755 --- a/dev-support/test-patch.d/ant.sh +++ b/dev-support/test-patch.d/ant.sh @@ -48,6 +48,19 @@ function ant_parse_args fi } +function ant_initialize +{ + # we need to do this before docker kicks in + if [[ -e "${HOME}/.ivy2" + && ! -d "${HOME}/.ivy2" ]]; then + yetus_error "ERROR: ${HOME}/.ivy2 is not a directory." + return 1 + elif [[ ! -e "${HOME}/.ivy2" ]]; then + yetus_debug "Creating ${HOME}/.ivy2" + mkdir -p "${HOME}/.ivy2" + fi +} + function ant_buildfile { echo "build.xml" @@ -181,3 +194,8 @@ function ant_builtin_personality_file_tests add_test findbugs fi } + +function ant_docker_support +{ + echo "-v ${HOME}/.ivy2:${HOME}/.ivy2" > "${PATCH_DIR}/buildtool-docker-params.txt" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/965184ed/dev-support/test-patch.d/gradle.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/gradle.sh b/dev-support/test-patch.d/gradle.sh index fbf3529..56da8d5 100755 --- a/dev-support/test-patch.d/gradle.sh +++ b/dev-support/test-patch.d/gradle.sh @@ -55,6 +55,16 @@ function gradle_initialize # shellcheck disable=SC2034 BUILDTOOLCWD=false fi + + # we need to do this before docker kicks in + if [[ -e "${HOME}/.gradle" + && ! -d "${HOME}/.gradle" ]]; then + yetus_error "ERROR: ${HOME}/.gradle is not a directory." + return 1 + elif [[ ! -e "${HOME}/.gradle" ]]; then + yetus_debug "Creating ${HOME}/.gradle" + mkdir -p "${HOME}/.gradle" + fi } function gradle_buildfile @@ -246,3 +256,8 @@ function gradle_builtin_personality_file_tests add_test findbugs fi } + +function gradle_docker_support +{ + echo "-v ${HOME}/.gradle:${HOME}/.gradle" > "${PATCH_DIR}/buildtool-docker-params.txt" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/965184ed/dev-support/test-patch.d/maven.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/maven.sh b/dev-support/test-patch.d/maven.sh index 94ac383..d4f056e 100755 --- a/dev-support/test-patch.d/maven.sh +++ b/dev-support/test-patch.d/maven.sh @@ -22,6 +22,9 @@ else MAVEN=${MAVEN_HOME}/bin/mvn fi +MAVEN_CUSTOM_REPOS=false +MAVEN_CUSTOM_REPOS_DIR="${HOME}/yetus-m2" + add_plugin mvnsite add_plugin mvneclipse add_build_tool maven @@ -29,7 +32,10 @@ add_build_tool maven function maven_usage { echo "maven specific:" - echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')" + echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')" + echo "--mvn-custom-repos Use per-project maven repos" + echo "--mvn-custom-repos-dir=dir Location of repos, default is \'${MAVEN_CUSTOM_REPOS_DIR}\'" + echo "--mvn-settings=file File to use for settings.xml" } function maven_parse_args @@ -41,6 +47,20 @@ function maven_parse_args --mvn-cmd=*) MAVEN=${i#*=} ;; + --mvn-custom-repos) + MAVEN_CUSTOM_REPOS=true + ;; + --mvn-custom-repos-dir=*) + MAVEN_CUSTOM_REPOS_DIR=${i#*=} + ;; + --mvn-settings=*) + MAVEN_SETTINGS=${i#*=} + if [[ -f ${MAVEN_SETTINGS} ]]; then + MAVEN_ARGS=("${MAVEN_ARGS[@]}" "--settings=${MAVEN_SETTINGS}") + else + yetus_error "WARNING: ${MAVEN_SETTINGS} not found. Ignorning." + fi + ;; esac done @@ -49,6 +69,88 @@ function maven_parse_args fi } +function maven_initialize +{ + # we need to do this before docker does it as root + + if [[ ! ${MAVEN_CUSTOM_REPOS_DIR} =~ ^/ ]]; then + yetus_error "ERROR: --mvn-custom-repos-dir must be an absolute path." + return 1 + fi + + if [[ ${MAVEN_CUSTOM_REPOS} = true ]]; then + MAVEN_LOCAL_REPO="${MAVEN_CUSTOM_REPOS_DIR}" + if [[ -e "${MAVEN_CUSTOM_REPOS_DIR}" + && ! -d "${MAVEN_CUSTOM_REPOS_DIR}" ]]; then + yetus_error "ERROR: ${MAVEN_CUSTOM_REPOS_DIR} is not a directory." + return 1 + elif [[ ! -d "${MAVEN_CUSTOM_REPOS_DIR}" ]]; then + yetus_debug "Creating ${MAVEN_CUSTOM_REPOS_DIR}" + mkdir -p "${MAVEN_CUSTOM_REPOS_DIR}" + fi + fi + + if [[ -e "${HOME}/.m2" + && ! -d "${HOME}/.m2" ]]; then + yetus_error "ERROR: ${HOME}/.m2 is not a directory." + return 1 + elif [[ ! -e "${HOME}/.m2" ]]; then + yetus_debug "Creating ${HOME}/.m2" + mkdir -p "${HOME}/.m2" + fi +} + +function maven_precheck +{ + declare logfile="${PATCH_DIR}/mvnrepoclean.log" + declare line + + if [[ ! ${MAVEN_CUSTOM_REPOS_DIR} =~ ^/ ]]; then + yetus_error "ERROR: --mvn-custom-repos-dir must be an absolute path." + return 1 + fi + + if [[ ${MAVEN_CUSTOM_REPOS} = true ]]; then + MAVEN_LOCAL_REPO="${MAVEN_CUSTOM_REPOS_DIR}/${PROJECT_NAME}-${PATCH_BRANCH}-${INSTANCE}" + if [[ -e "${MAVEN_LOCAL_REPO}" + && ! -d "${MAVEN_LOCAL_REPO}" ]]; then + yetus_error "ERROR: ${MAVEN_LOCAL_REPO} is not a directory." + return 1 + fi + + if [[ ! -d "${MAVEN_LOCAL_REPO}" ]]; then + yetus_debug "Creating ${MAVEN_LOCAL_REPO}" + mkdir -p "${MAVEN_LOCAL_REPO}" + if [[ $? -ne 0 ]]; then + yetus_error "ERROR: Unable to create ${MAVEN_LOCAL_REPO}" + return 1 + fi + fi + touch "${MAVEN_LOCAL_REPO}" + + # if we have a local settings.xml file, we copy it. + if [[ -f "${HOME}/.m2/settings.xml" ]]; then + cp -p "${HOME}/.m2/settings.xml" "${MAVEN_LOCAL_REPO}" + fi + MAVEN_ARGS=("${MAVEN_ARGS[@]}" "-Dmaven.repo.local=${MAVEN_LOCAL_REPO}") + + # let's do some cleanup while we're here + + find "${MAVEN_CUSTOM_REPOS_DIR}" \ + -name '*-*-*' \ + -type d \ + -mtime +30 \ + -maxdepth 1 \ + -print \ + > "${logfile}" + + while read -r line; do + echo "Removing old maven repo ${line}" + rm -rf "${line}" + done < "${logfile}" + fi +} + function maven_buildfile { echo "pom.xml" @@ -325,4 +427,14 @@ function maven_precompile return 1 fi return 0 +} + +function maven_docker_support +{ + echo "-v ${HOME}/.m2:${HOME}/.m2" > "${PATCH_DIR}/buildtool-docker-params.txt" + + if [[ ${MAVEN_CUSTOM_REPOS} = true ]]; then + echo "-v ${MAVEN_CUSTOM_REPOS_DIR}:${MAVEN_CUSTOM_REPOS_DIR}" \ + >> "${PATCH_DIR}/buildtool-docker-params.txt" + fi } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/965184ed/dev-support/test-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 60caedc..086a0a4 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -60,6 +60,7 @@ function setup_defaults DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile-startstub" HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"; JENKINS=false + INSTANCE=${RANDOM} BASEDIR=$(pwd) RELOCATE_PATCH_DIR=false @@ -763,7 +764,7 @@ function testpatch_usage echo echo "Jenkins-only options:" - echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" + echo "--jenkins Jenkins mode" echo "--build-url Set the build location web page" echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup." @@ -859,6 +860,7 @@ function parse_args --jenkins) JENKINS=true TEST_PARALLEL=${TEST_PARALLEL:-true} + INSTANCE=${EXECUTOR_NUMBER:-RANDOM} ;; --linecomments=*) BUGLINECOMMENTS=${i#*=} @@ -938,6 +940,10 @@ function parse_args --tpglobaltimer=*) GLOBALTIMER=${i#*=} ;; + --tpinstance=*) + INSTANCE=${i#*=} + EXECUTOR_NUMBER=${INSTANCE} + ;; --tpreexectimer=*) REEXECLAUNCHTIMER=${i#*=} ;; @@ -1734,6 +1740,7 @@ function check_reexec local copy=false local testdir local person + local debugflag if [[ ${REEXECED} == true ]]; then big_console_header "Re-exec mode detected. Continuing." @@ -1791,6 +1798,10 @@ function check_reexec # if we are doing docker, then we re-exec, but underneath the # container + if declare -f ${BUILDTOOL}_docker_support >/dev/null; then + "${BUILDTOOL}_docker_support" + fi + client=$(docker version | grep 'Client version' | cut -f2 -d: | tr -d ' ') server=$(docker version | grep 'Server version' | cut -f2 -d: | tr -d ' ') @@ -1802,6 +1813,7 @@ function check_reexec fi TESTPATCHMODE="--tpglobaltimer=${GLOBALTIMER} ${TESTPATCHMODE}" TESTPATCHMODE="--tpreexectimer=${TIMER} ${TESTPATCHMODE}" + TESTPATCHMODE="--tpinstance=${INSTANCE} ${TESTPATCHMODE}" TESTPATCHMODE="--personality=\'${PERSONALITY}\' ${TESTPATCHMODE}" TESTPATCHMODE="--plugins=\'${USER_PLUGIN_DIR}\' ${TESTPATCHMODE}" TESTPATCHMODE=" ${TESTPATCHMODE}" @@ -1809,9 +1821,14 @@ function check_reexec patchdir=$(relative_dir "${PATCH_DIR}") + if [[ ${TP_SHELL_SCRIPT_DEBUG} = true ]]; then + debugflag="--debug" + fi + cd "${BASEDIR}" #shellcheck disable=SC2093 exec bash "${PATCH_DIR}/precommit/test-patch-docker/test-patch-docker.sh" \ + ${debugflag} \ --dockerversion="${dockerversion}" \ --java-home="${JAVA_HOME}" \ --patch-dir="${patchdir}" \ @@ -1832,6 +1849,7 @@ function check_reexec --tpglobaltimer="${GLOBALTIMER}" \ --tpreexectimer="${TIMER}" \ --personality="${PERSONALITY}" \ + --tpinstance="${INSTANCE}" \ --plugins="${USER_PLUGIN_DIR}" fi }
