David Caro has uploaded a new change for review. Change subject: Refactored and improved cleanup on jenkins.sh ......................................................................
Refactored and improved cleanup on jenkins.sh Change-Id: I20dc835ff7edf90f8e301da2ea86a8baf7d9bf07 Signed-off-by: David Caro <[email protected]> --- M jenkins.sh 1 file changed, 219 insertions(+), 87 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-node-iso refs/changes/07/25707/1 diff --git a/jenkins.sh b/jenkins.sh index 7b191e3..2c2fd4f 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -ex # oVirt node iso jenkins build script # # Copyright (C) 2008 Red Hat, Inc. @@ -18,101 +18,233 @@ # MA 02110-1301, USA. A copy of the GNU General Public License is # also available at http://www.gnu.org/copyleft/gpl.html. -set -e -set -v - -#Cleanup -test -f Makefile && make -k distclean -rm -rf ${WORKSPACE}/ovirt-node-tools ${WORKSPACE}/*iso ${WORKSPACE}/rpmbuild ${WORKSPACE}/manifest* ${WORKSPACE}/old_artifacts ${WORKSPACE}/ovirt-node-recipe - -OVIRT_CACHE_DIR=${WORKSPACE}/ovirt-cache -OVIRT_LOCAL_REPO=file://${OVIRT_CACHE_DIR}/ovirt +## Make sure WORKSPACE is not empty +WORKSPACE="${WORKSPACE?No WORKSPACE env var, are you running in jenkins?}" +BUILD_NUMBER="${BUILD_NUMBER?No BUILD_NUMBER env var, are you running in jenkins?}" +JOB_URL="${JOB_URL?No JOB_URL env var, are you running in jenkins?}" +MANIFEST_LOG="${WORKSPACE}/ovirt-node-iso.mini-manifest.txt" +OVIRT_CACHE_DIR="${WORKSPACE}/ovirt-cache" +OVIRT_LOCAL_REPO="file://${OVIRT_CACHE_DIR}/ovirt" +BUILD_TYPE="${BUILD_TYPE:-STABLE}" +export HOME="${WORKSPACE}" export OVIRT_CACHE_DIR OVIRT_LOCAL_REPO -if ls ${OVIRT_CACHE_DIR}/ovirt/noarch/ovirt-node-recipe*rpm >/dev/null 2>&1; then - RECIPE_RPM_NAME=ovirt-node-recipe -elif ls ${OVIRT_CACHE_DIR}/ovirt/noarch/ovirt-node-tools*rpm >/dev/null 2>&1; then - RECIPE_RPM_NAME=ovirt-node-tools -else +## Avoid non matching globs from getting printed +shopt -s nullglob + +log() { + echo "$@" >> "$MANIFEST_LOG" +} + +pre_cleanup() +{ + [[ -f Makefile ]] && make -k distclean + rm -rf "${WORKSPACE}"/ovirt-node-tools \ + "${WORKSPACE}"/*iso \ + "${WORKSPACE}"/rpmbuild \ + "${WORKSPACE}"/manifest* \ + "${WORKSPACE}"/old_artifacts \ + "${WORKSPACE}"/ovirt-node-recipe + +} + +post_cleanup() +{ + [[ -f Makefile ]] && make -k distclean + rm -rf \ + "$WORKSPACE/rpmbuild" \ + "$WORKSPACE/autom4te.cache" \ + "$WORKSPACE/gluster-3.4" \ + "$WORKSPACE/local" \ + "$WORKSPACE/node-stable-repo" \ + "$WORKSPACE/ovirt-cache" \ + "$WORKSPACE/ovirt-node-recipe" \ + "$WORKSPACE/ovirt-node-tools" \ + "$WORKSPACE/*iso" \ + "$WORKSPACE/old_artifacts" \ + "$WORKSPACE/epel" \ + "$WORKSPACE/centos" \ + "$WORKSPACE/centos-updates" +} + +get_recipe() +{ + local cache_dir=${1:?} + local -a recipes tools + recipes=("${OVIRT_CACHE_DIR}"/ovirt/noarch/ovirt-node-recipe*rpm) + tools=("${OVIRT_CACHE_DIR}"/ovirt/noarch/ovirt-node-tools*rpm) + if [[ -n $recipes ]]; then + echo -n "ovirt-node-recipe" + elif [[ -n $tools ]]; then + echo -n "ovirt-node-tools" + else + return 4 + fi +} + +extract_recipes() +{ + local recipe_dir="${1?}" + local recipe_rpm="${2?}" + # ovirt-node recipe rpm should be copied to ${OVIRT_CACHE_DIR}/ovirt/noarch + mkdir "${recipe_dir}" + pushd "${recipe_dir}" &>/dev/null + rpm2cpio "${recipe_rpm}" | pax -r + popd &>/dev/null + return 0 +} + +generate_iso() +{ + local recipes_dir="${1?}" + local custom_build_number="${2?}" + ./autogen.sh --with-recipe="${recipes_dir}" \ + --with-build-number="${custom_build_number}" + ## add the repo-creator to the path + PATH="$PATH:${recipes_dir}/../../sbin" + sudo -E env PATH=$PATH make BUILD_TYPE="$BUILD_TYPE" iso publish \ + 1>make.stdout.log 2>make.stderr.log + ## restore any file permissions + sudo -E chown -R $USER:$USER . +} + +get_manifests_from_iso() +{ + local iso_file="${1?}" + local dst_dir="${2?}" + # Get iso details + iso_dir=$(mktemp -d) + sudo mount -o loop "${iso_file}" "$iso_dir" + cp "$iso_dir"/isolinux/manifest-srpm.txt \ + "$iso_dir"/isolinux/manifest-rpm.txt \ + "$iso_dir"/isolinux/manifest-file.txt.bz2 \ + "${dst_dir}" + chmod 666 "${dst_dir}"/manifest-*txt* + sudo umount "${iso_dir}" + rmdir "${iso_dir}" +} + +get_old_artifacts() +{ + local dst_dir="${1?}" + [[ -e "$dst_dir" ]] && rm -rf "$dst_dir" + mkdir -p "$dst_dir" + pushd "$dst_dir" &>/dev/null + wget "${JOB_URL}/lastSuccessfulBuild/artifact/*zip*/archive.zip" \ + || return 1 + unzip archive.zip + popd &>/dev/null +} + + +write_build_info() +{ + local recipe_name="${1?}" + local recipe_rpm="${2}" + local iso_file size human_size old_size old_human_size + ## get iso details + iso_file="${WORKSPACE}/$(make verrel).iso" + get_manifests_from_iso "$iso_file" "$WORKSPACE" + egrep '^kernel|kvm|libvirt|^vdsm|^ovirt-node|^fence-agents' manifest-srpm.txt \ + | sed 's/\.src\.rpm//' > "$MANIFEST_LOG" + + log "======================================================" + log "${recipe_name} used: ${recipe_rpm##*/} " + + # Check size of iso and report in mini-manifest.txt + log "======================================================" + size=$(du -k ${iso_file}) + human_size=$(du -h ${iso_file}) + log " Iso Size: ${size%% *} (${human_size%% *})" + + old_size="" + old_human_size="" + if true; then #get_old_artifacts "$WORKSPACE/old_artifacts"; then + old_isos=("${WORKSPACE}"/old_artifacts/archive/ovirt-node-iso*iso) + old_iso="${old_isos[0]}" + if [[ -e "$old_iso" ]]; then + old_size=$(du -k "$old_iso") + old_human_size=$(du -h "$old_iso") + log "Old Iso Size: ${old_size%% *} (${old_human_size%% *})" + else + log "No old iso found for compairson" + fi + else + log "No previous build archive found for old iso compairson" + fi + # md5 and sha256sums + md5="$(md5sum ${iso_file})" + sha="$(sha256sum ${iso_file})" + log "MD5SUM: ${md5%% *}" + log "SHA256SUM: ${sha%% *}" + log "======================================================" + log "livecd-tools version: $(rpm -qa livecd-tools)" +} + +gather_artifacts() +{ + local dst_dir + dst_dir="$WORKSPACE/exported-artifacts" + [[ -e "$dst_dir" ]] && rm -rf "$dst_dir" + mkdir -p "$dst_dir" + mv "$WORKSPACE/ovirt-cache/ovirt/noarch/scm_hash.txt" \ + "$dst_dir/ovirt-node_scm_hash.txt" + cp "$WORKSPACE/.git/HEAD" "$dst_dir/scm_hash.txt" + to_archive=( + "$WORKSPACE"/ovirt-node-iso*iso + "$WORKSPACE"/ovirt-cache/ovirt/*/ovirt-node-iso*rpm + "$WORKSPACE"/manifest* + "$WORKSPACE"/ovirt-node-iso.mini-manifest.txt + "$WORKSPACE"/*log + ) + mv "${to_archive[@]}" "$dst_dir" +} + + +################## MAIN +## Make sure we are in that WORKSPACE +cd "$WORKSPACE" + +## And that we don't have old files +pre_cleanup + +recipe=$(get_recipe $OVIRT_CACHE_DIR) +if [[ $? -ne 0 ]]; then echo "ERROR: no recipe rpm found" exit 4 fi -OVIRT_NODE_RECIPE_RPM=$(ls -t ${OVIRT_CACHE_DIR}/ovirt/noarch/${RECIPE_RPM_NAME}* | head -n1) -export HOME=${WORKSPACE} - -createrepo ${OVIRT_CACHE_DIR}/ovirt - -# ovirt-node recipe rpm should be copied to ${OVIRT_CACHE_DIR}/ovirt/noarch -mkdir ${WORKSPACE}/${RECIPE_RPM_NAME} -cd ${WORKSPACE}/${RECIPE_RPM_NAME} -rpm2cpio ${OVIRT_NODE_RECIPE_RPM} | pax -r -OVIRT_NODE_RECIPE_RPM=$(basename ${OVIRT_NODE_RECIPE_RPM}) -ONT_NAME=$(echo $OVIRT_NODE_RECIPE_RPM | sed -r 's/^([a-zA-Z0-9\-]+)-([a-zA-Z0-9\.]+)-([a-zA-Z0-9\.]+).noarch.rpm$/\1/') -ONT_VERSION=$(echo $OVIRT_NODE_RECIPE_RPM | sed -r 's/^([a-zA-Z0-9\-]+)-([a-zA-Z0-9\.]+)-([a-zA-Z0-9\.]+).noarch.rpm$/\2/') -ONT_RELEASE=$(echo $OVIRT_NODE_RECIPE_RPM | sed -r 's/^([a-zA-Z0-9\-]+)-([a-zA-Z0-9\.]+)-([a-zA-Z0-9\.]+).noarch.rpm$/\3/') -ONT_BUILD_NUMBER=$(echo $ONT_RELEASE | sed -r 's/^[0-9]+\.(.*)\.[ef][lc][0-9]+$/\1./') -if [ "$ONT_BUILD_NUMBER" = "$ONT_RELEASE" ]; then - ONT_BUILD_NUMBER="" -fi -cd ${WORKSPACE} - -if [ -z "${BUILD_TYPE}" ]; then - BUILD_TYPE="STABLE" +recipe_rpms=("${OVIRT_CACHE_DIR}/ovirt/noarch/${recipe}"*) +## get only the first one +recipe_rpm="${recipe_rpms[0]}" +[[ -f "$recipe_rpm" ]] \ +|| { + echo "Unable to find previous build rpms at" \ + "${OVIRT_CACHE_DIR}/ovirt/noarch/${recipe}" + exit 1 +} +## get the info from the rpm +read -r recipe_name recipe_version recipe_release <<<$( + rpm -qp \ + --queryformat "%{name}\n%{version}\n%{release}" \ + "$recipe_rpm" +) +recipe_build_number="${recipe_release%.*}" +recipe_build_number="${recipe_build_number#*.}" +if [[ "$recipe_build_number" == "$release" ]]; then + recipe_build_number="" fi -RECIPE_DIR=${WORKSPACE}/${RECIPE_RPM_NAME}/usr/share/${RECIPE_RPM_NAME} -cp ${WORKSPACE}/${RECIPE_RPM_NAME}/usr/sbin/node-creator ${WORKSPACE} +extract_recipes "${WORKSPACE}/${recipe_name}" "$recipe_rpm" -./autogen.sh --with-recipe=${RECIPE_DIR} --with-build-number=${ONT_BUILD_NUMBER}${BUILD_NUMBER} +## prepare repo +createrepo "${OVIRT_CACHE_DIR}"/ovirt -make BUILD_TYPE=$BUILD_TYPE iso publish +recipes_dir="${WORKSPACE}/${recipe_name}/usr/share/${recipe_name}" +generate_iso "$recipes_dir" "${recipe_build_number}${BUILD_NUMBER}" -ISO_NAME=$(make verrel).iso -# Get iso details -ISO_DIR=$(mktemp -d) -sudo mount -o loop ${ISO_NAME} $ISO_DIR -cp $ISO_DIR/isolinux/manifest-srpm.txt ${WORKSPACE} -cp $ISO_DIR/isolinux/manifest-rpm.txt ${WORKSPACE} -cp $ISO_DIR/isolinux/manifest-file.txt.bz2 ${WORKSPACE} -chmod 666 ${WORKSPACE}/manifest-*txt* -sudo umount ${ISO_DIR} -rmdir ${ISO_DIR} -egrep '^kernel|kvm|libvirt|^vdsm|^ovirt-node|^fence-agents' manifest-srpm.txt | sed 's/\.src\.rpm//' > ovirt-node-iso.mini-manifest.txt +write_build_info "$recipe_name" "$recipe_rpm" -# Add additional information to mini-manifest.txt -echo "======================================================" >> ovirt-node-iso.mini-manifest.txt -echo "${RECIPE_RPM_NAME} used: $(basename ${OVIRT_NODE_RECIPE_RPM}) " >> ovirt-node-iso.mini-manifest.txt +gather_artifacts - -# Check size of iso and report in mini-manifest.txt -echo "======================================================" >> ovirt-node-iso.mini-manifest.txt -size=$(ls -l ${ISO_NAME} | awk '{print $5}') -human_size=$(ls -lh ${ISO_NAME} | awk '{print $5}') -echo " Iso Size: $size ($human_size)" >> ovirt-node-iso.mini-manifest.txt - -old_size="" -old_human_size="" -mkdir -p old_artifacts -cd old_artifacts -if wget ${JOB_URL}/lastSuccessfulBuild/artifact/*zip*/archive.zip; then - unzip archive.zip - cd $WORKSPACE - if [ -e ${WORKSPACE}/old_artifacts/archive/ovirt-node-iso*iso ]; then - old_size=$(ls -l ${WORKSPACE}/old_artifacts/archive/ovirt-node-iso*iso | awk '{print $5}') - old_human_size=$(ls -lh ${WORKSPACE}/old_artifacts/archive/ovirt-node-iso*iso | awk '{print $5}') - echo "Old Iso Size: $old_size ($old_human_size)" >> ovirt-node-iso.mini-manifest.txt - else - echo "No old iso found for compairson">> ovirt-node-iso.mini-manifest.txt - fi -else - cd $WORKSPACE - echo "No previous build archive found for old iso compairson">> ovirt-node-iso.mini-manifest.txt -fi -rm -rf old_artifacts -# md5 and sha256sums -echo "MD5SUM: $(md5sum ${ISO_NAME} |awk '{print $1}')" >> ovirt-node-iso.mini-manifest.txt -echo "SHA256SUM: $(sha256sum ${ISO_NAME} |awk '{print $1}')" >> ovirt-node-iso.mini-manifest.txt - -echo "======================================================" >> ovirt-node-iso.mini-manifest.txt -echo "livecd-tools version: $(rpm -qa livecd-tools)" >> ovirt-node-iso.mini-manifest.txt - +## free space in the slave +post_cleanup -- To view, visit http://gerrit.ovirt.org/25707 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I20dc835ff7edf90f8e301da2ea86a8baf7d9bf07 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-node-iso Gerrit-Branch: master Gerrit-Owner: David Caro <[email protected]> _______________________________________________ node-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/node-patches
