Repository: hive Updated Branches: refs/heads/master 05e251036 -> c90eed20d
HIVE-14734: Detect ptest profile and submit to ptest-server from jenkins-execute-build.sh (Sergio Pena, reviewed by Siddarth Seth) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c90eed20 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c90eed20 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c90eed20 Branch: refs/heads/master Commit: c90eed20d84cf1c86dd6dd6a5dc3f56e0e48d4ff Parents: 05e2510 Author: Sergio Pena <sergio.p...@cloudera.com> Authored: Fri Sep 16 23:18:31 2016 -0500 Committer: Sergio Pena <sergio.p...@cloudera.com> Committed: Sat Sep 17 16:16:39 2016 -0500 ---------------------------------------------------------------------- dev-support/jenkins-common.sh | 63 ++++++++++++ dev-support/jenkins-execute-build.sh | 157 ++++++++++++++++++++---------- 2 files changed, 166 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c90eed20/dev-support/jenkins-common.sh ---------------------------------------------------------------------- diff --git a/dev-support/jenkins-common.sh b/dev-support/jenkins-common.sh index 214ca1c..0467d11 100644 --- a/dev-support/jenkins-common.sh +++ b/dev-support/jenkins-common.sh @@ -13,6 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +JIRA_ROOT_URL="https://issues.apache.org"; + fail() { echo "$@" 1>&2 exit 1 @@ -48,6 +51,66 @@ get_branch_name() { return 0 } +# Gets the attachment identifier of a JIRA attachment file +get_attachment_id() { + local jira_attachment_url="$1" + + basename $(dirname $jira_attachment_url) +} + +# Fetchs JIRA information, and save it in a file specified as a parameter +initialize_jira_info() { + local jira_issue="$1" + local output_file="$2" + + curl -s -S --location --retry 3 "${JIRA_ROOT_URL}/jira/browse/${jira_issue}" > $output_file +} + +# Checks if a JIRA is in 'Patch Available' status +is_patch_available() { + grep -q "Patch Available" $1 +} + +# Checks if a JIRA has 'NO PRECOMMIT TESTS' enabled +is_check_no_precommit_tests_set() { + grep -q "NO PRECOMMIT TESTS" $1 +} + +# Gets the URL for the JIRA patch attached +get_jira_patch_url() { + grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' $1 | grep -v -e 'htm[l]*$' | sort | tail -1 | \ + grep -o '/jira/secure/attachment/[0-9]*/[^"]*' +} + +# Checks if the patch was already tested +is_patch_already_tested() { + local attachment_id="$1" + local test_handle="$2" + local jira_info_file="$3" + + grep -q "ATTACHMENT ID: $attachment_id $test_handle" $jira_info_file +} + +# Gets the branch profile attached to the patch +get_branch_profile() { + local jira_patch_url="$1" + local branch_name=`get_branch_name $(basename $jira_patch_url)` + + if [ -n "$branch_name" ]; then + shopt -s nocasematch + if [[ $branch_name =~ (mr1|mr2)$ ]]; then + echo $branch_name + else + echo ${branch_name}-mr2 + fi + fi +} + +# Checks if a JIRA patch has 'CLEAR LIBRARY CACHE' enabled +is_clear_cache_set() { + grep -q "CLEAR LIBRARY CACHE" $1 +} + # Parses the JIRA/patch to find relavent information. # Exports two variables of import: # * BUILD_PROFILE - the profile which the ptest server understands http://git-wip-us.apache.org/repos/asf/hive/blob/c90eed20/dev-support/jenkins-execute-build.sh ---------------------------------------------------------------------- diff --git a/dev-support/jenkins-execute-build.sh b/dev-support/jenkins-execute-build.sh index 3b41b0b..a9935e1 100644 --- a/dev-support/jenkins-execute-build.sh +++ b/dev-support/jenkins-execute-build.sh @@ -13,67 +13,116 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + set -e -. jenkins-common.sh -test -n "$BRANCH" || fail "BRANCH must be specified" -test -n "$API_ENDPOINT" || fail "API_ENDPOINT must be specified" -test -n "$LOG_ENDPOINT" || fail "LOG_ENDPOINT must be specified" -test -n "$API_PASSWORD" || fail "API_PASSWORD must be specified" -if [[ -n "$ISSUE_NUM" ]] -then - export JIRA_NAME="HIVE-${ISSUE_NUM}" -fi -export ROOT=$PWD -export JIRA_ROOT_URL="https://issues.apache.org"; -export BUILD_TAG="${BUILD_TAG##jenkins-}" -if [[ -n "$JIRA_NAME" ]] -then - echo $JIRA_NAME -fi set -x -env -if [[ -n "$JIRA_NAME" ]] -then - process_jira -fi +. jenkins-common.sh -profile=$BUILD_PROFILE -if [[ -z "$profile" ]] -then - profile=$DEFAULT_BUILD_PROFILE -fi -if [[ -z "$profile" ]] -then - fail "Could not find build profile" -fi +# Build the ptest framework locally +build_ptest_client() { + [ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR is not defined." + + test -d $PTEST_BUILD_DIR || mkdir -p $PTEST_BUILD_DIR + cd $PTEST_BUILD_DIR && rm -rf hive + + git clone --depth 1 https://github.com/apache/hive.git + cd hive/testutils/ptest2 + mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$MVN_REPO_LOCAL +} + +# Call the ptest server to run all tests +call_ptest_server() { + [ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR is not defined." + + read -s -p "JIRA password: " JIRA_PASSWORD + + build_ptest_client || return 1 + + local PTEST_CLASSPATH="$PTEST_BUILD_DIR/hive/testutils/ptest2/target/hive-ptest-1.0-classes.jar:$PTEST_BUILD_DIR/hive/testutils/ptest2/target/lib/*" + + java -cp "$PTEST_CLASSPATH" org.apache.hive.ptest.api.client.PTestClient --command testStart \ + --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD" "$@" +} + +# Unpack all test results +unpack_test_results() { + [ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR is not defined." + + cd "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" + if [[ -f test-results.tar.gz ]]; then + rm -rf $PTEST_BUILD_DIR/test-results/ + tar zxf test-results.tar.gz -C $PTEST_BUILD_DIR + fi +} + +# Check required arguments +test -n "$TEST_HANDLE" || fail "TEST_HANDLE must be specified and cannot be empty." +test -n "$PTEST_API_ENDPOINT" || fail "PTEST_API_ENDPOINT must be specified and cannot be empty." +test -n "$PTEST_LOG_ENDPOINT" || fail "PTEST_LOG_ENDPOINT must be specified and cannot be empty." -test -d hive/build/ || mkdir -p hive/build/ -cd hive/build/ -rm -rf hive -git clone --depth 1 https://github.com/apache/hive.git -cd hive/testutils/ptest2 +# WORKSPACE is an environment variable created by Jenkins, and it is the directory where the build is executed. +# If not set, then default to $HOME +MVN_REPO_LOCAL=${WORKSPACE:-$HOME}/.m2/repository -mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$WORKSPACE/.m2 -set +e +# Directory where to build the ptest framework +PTEST_BUILD_DIR="$PWD/hive/build" + +# Default profile in case a patch does not have a profile assigned, or +# if this script is executed without JIRA_ISSUE defined +DEFAULT_BUILD_PROFILE="master-mr2" + +# Optional parameters that might be passed to the ptest client command optionalArgs=() -if [[ -n "$JIRA_NAME" ]] -then - optionalArgs=(--patch "${JIRA_ROOT_URL}${PATCH_URL}" --jira "$JIRA_NAME") + +if [ -n "$JIRA_ISSUE" ]; then + JIRA_INFO_FILE=`mktemp` + trap "rm -f $JIRA_INFO_FILE" EXIT + + initialize_jira_info $JIRA_ISSUE $JIRA_INFO_FILE || fail "Error: Cannot initialize JIRA information." + + if ! is_patch_available $JIRA_INFO_FILE; then + fail "$JIRA_ISSUE is not 'Patch Available'." + fi + + if is_check_no_precommit_tests_set $JIRA_INFO_FILE; then + fail "$JIRA_ISSUE has tag NO PRECOMMIT TESTS" + fi + + JIRA_PATCH_URL=`get_jira_patch_url $JIRA_INFO_FILE` + if [ -z "$JIRA_PATCH_URL" ]; then + fail "Unable to find attachment for $JIRA_ISSUE" + fi + + attachment_id=`get_attachment_id $JIRA_PATCH_URL` + if is_patch_already_tested "$attachment_id" "$TEST_HANDLE" "$JIRA_INFO_FILE"; then + fail "attachment $attachment_id is already tested for $JIRA_ISSUE" + fi + + BUILD_PROFILE=`get_branch_profile $JIRA_PATCH_URL $JIRA_INFO_FILE` + if [ -z "$BUILD_PROFILE" ]; then + BUILD_PROFILE="$DEFAULT_BUILD_PROFILE" + fi + + if is_clear_cache_set $JIRA_INFO_FILE; then + optionalArgs+=(--clearLibraryCache) + fi + + optionalArgs+=(--patch "${JIRA_ROOT_URL}${JIRA_PATCH_URL}" --jira "$JIRA_ISSUE") + + echo "ISSUE: $JIRA_ISSUE PROFILE: $BUILD_PROFILE" +else + # If not JIRA is specified, then use a default profile + BUILD_PROFILE="$DEFAULT_BUILD_PROFILE" + + echo "ISSUE: unspecified PROFILE: $BUILD_PROFILE" fi -java -cp "target/hive-ptest-1.0-classes.jar:target/lib/*" org.apache.hive.ptest.api.client.PTestClient --endpoint "$API_ENDPOINT" \ - --logsEndpoint "$LOG_ENDPOINT" \ - --command testStart \ - --profile $profile \ - --password $API_PASSWORD \ - --outputDir target/ \ - --testHandle "$BUILD_TAG" \ - ${optionalArgs[@]} ${BUILD_OPTS} "$@" + +call_ptest_server --testHandle "$TEST_HANDLE" --endpoint "$PTEST_API_ENDPOINT" --logsEndpoint "$PTEST_LOG_ENDPOINT" \ + --profile "$BUILD_PROFILE" ${optionalArgs[@]} "$@" + ret=$? -cd target/ -if [[ -f test-results.tar.gz ]] -then - rm -rf $ROOT/hive/build/test-results/ - tar zxf test-results.tar.gz -C $ROOT/hive/build/ -fi + +unpack_test_results + exit $ret
