mrutkows closed pull request #25: Add the support to git clone all source code based on configurations URL: https://github.com/apache/incubator-openwhisk-release/pull/25
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/.travis.yml b/.travis.yml index 4147f1a..bda6e54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,14 +5,19 @@ group: deprecated-2017Q3 services: - docker +env: + global: + - secure: S4eRZSGj/pfsicg50UXrTmwRurzgUrd0idQH5nKvc+O+lwAYv3SIhbsufUHseQEUSfoO0jFCyoCbWDtFMdsW3t2eGPdciViek/LaocxxG2Br539VDVXhS+NAwqiRGbCqKHCe2aNqIdo8vCvUaj/BfFJNUK4Fhpw6iBix0pvlnRkYBX7hHphU88N69gmnMOo1oZwEwaobaQeXOc/8OwH2aPD8lCmNk41IvvLFAwIhXrP4Thb+xBh4+oSlqcJyDCUouIfYFd4B1pHQehAVu0kBoUiKZHfKUyT+Q/V8hzBR3nJ/IzzSLNOYcTTYXBbL5x/SKe/LiBio4yLdPuRMTZl/xKarhShHJg3JKRvfushaMJDKJfLrbcCPUDnSCK1v0D4zZc0DceOBawhipSPkam3kNER/ImZcpTj79u/VKBrlwbVwjKTyZ4d/4/N2cP4jyvjmwW6fM4N3kM/V+S+EoGR5EiFTKAiPBzKaRi+/+VUO8ha9TsdTu+vkDRUjDFuYLSod0aj8XHUoZiL7NF8qS1PK6AFfTRpet1NM6BDOWYh15GmuZ/noyXTZ5eJv/8/ecfLMEp+DXAZ+YVI6a9/xhxE1klI88zHIoVMmhwd76jcvHVLolKa8fX9Dv5rZeOcPLoR2/jiteoApOGD0ehnkJH76gUMyBQO+q3yzRzz1VQvjOyA= + - secure: zqevjGa1iX3I6ZM1upNvNhEVBRq5sYpfK1pAzj6t7nTyCNvZoftTeMtDpP6eo6xAo+PatycEubwZ/pwac3V142AAgYrLXwqlojWWMQmx11BoTa3n1uJTtQyBr/5qjOLYs8VE5eDFKcXvXLQ5IDtLSZiYFGxmWQGInUNQlBaP9MQIRb2w+UgZBrp+1jRuaFs9I0+eLz3XLsyp9DeUa70lm49QiniKEktJdcUEQpQR0906k9DaRsvA3BIVsLtBwmUSg5+v8tsy13yKsPZgWNYfAjLU2DZHUQBQVj22FeSTMGOiCFSdRejhhFu9HpR7eJGoKMQGRzdTDLWZechVcVim849eVBjS6VfNBLAHOlyxIxy1Qf530610BYW2bBcaU3PhDjIcW8B+Z7EFZmYdpJRaT4CT7metLKTszy6W/splqG+3XP2XH4KmN8YPSikNiPagGP4beFIHxmCcNopHtJGhD4R8w6xqCKvNbtjq3GiUyU3ju+kA1JL0IxAttHTFF1WVTHjcXstopfXw+9HIUETm++OoKN6fxZ7Ysq7RvetkeJabL9Sqi1IsMJ3oDUdLoktTxYcE8A0gTpnLZRa2IdMJZ0ee2ZsMGieeAqfcJQ+61BMjxBTfrC9k+9gAj3zjgWsXOBMqePuQ8SwIT0H7/QgH5V/tP1wYvRUaJdJJoVoXJ2I= + jobs: include: - - stage: verify source code - script: ./tools/travis/verify_source_code.sh - - stage: deploy - script: ./tools/travis/setup.sh + #- stage: verify source code + # script: ./tools/travis/verify_source_code.sh + #- stage: deploy + # script: ./tools/travis/setup.sh - stage: package - script: ./tools/travis/package_source_code.sh + script: ./tools/travis/package_source_code.sh $(dirname "$TRAVIS_BUILD_DIR") $SVN_USERNAME $SVN_PASSWORD after_failure: - echo "failed build" diff --git a/tools/clean_remote_artifacts.sh b/tools/clean_remote_artifacts.sh new file mode 100755 index 0000000..8e31d97 --- /dev/null +++ b/tools/clean_remote_artifacts.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e + +echo "Clean the remote artifacts" + +SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)" +source "$SCRIPTDIR/util.sh" + +CONFIG=$(read_file $SCRIPTDIR/config.json) +version_key="version" +version_major=$(json_by_key "$CONFIG" ${version_key}.major) +version_minor=$(json_by_key "$CONFIG" ${version_key}.minor) + +version=$version_major-$version_minor +REMOTE_PATH="openwhisk-$version" +STAGE_URL=$(json_by_key "$CONFIG" "stage_url") +CURRENT_VERSION_URL="$STAGE_URL/${REMOTE_PATH}/" +CREDENTIALS="" + +SVN_USERNAME=$1 +SVN_PASSWORD=$2 + +if [ ! -z "$SVN_USERNAME" ] && [ ! -z "$SVN_PASSWORD" ];then + CREDENTIALS="--username $SVN_USERNAME --password $SVN_PASSWORD --non-interactive" +fi + +if [[ `wget -S --spider $CURRENT_VERSION_URL 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then + svn delete $CURRENT_VERSION_URL -m "Removing Apache OpenWhisk release ${version} from staging." $CREDENTIALS +fi + + + diff --git a/tools/config.json b/tools/config.json new file mode 100644 index 0000000..5b9e217 --- /dev/null +++ b/tools/config.json @@ -0,0 +1,27 @@ +{ + "stage_url": "https://dist.apache.org/repos/dist/dev/incubator/openwhisk", + "version": { + "major": "incubator", + "minor": "rc1" + }, + "RepoList": [ + "incubator-openwhisk", + "incubator-openwhisk-wskdeploy", + "incubator-openwhisk-cli" + ], + "incubator_openwhisk": { + "hash": "a891e06", + "repository": "https://github.com/apache/incubator-openwhisk.git", + "branch": "master" + }, + "incubator_openwhisk_wskdeploy": { + "hash": "1425456", + "repository": "https://github.com/apache/incubator-openwhisk-wskdeploy.git", + "branch": "master" + }, + "incubator_openwhisk_cli": { + "hash": "90db37e", + "repository": "https://github.com/houshengbo/incubator-openwhisk-cli.git", + "branch": "add-zip-source" + } +} diff --git a/tools/download_source_code.sh b/tools/download_source_code.sh new file mode 100755 index 0000000..8837139 --- /dev/null +++ b/tools/download_source_code.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -e + +WORK_DIR=${1:-"$HOME"} +OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources" + +SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)" +source "$SCRIPTDIR/util.sh" + +clone_depth=${2:-"100"} + +function git_clone_repo() { + PROJECT_NAME=$1 + HASH=$2 + REPO=$3 + BRANCH=$4 + git clone --depth $clone_depth -b $BRANCH $REPO $OPENWHISK_SOURCE_DIR/$PROJECT_NAME + cd $OPENWHISK_SOURCE_DIR/$PROJECT_NAME + git reset --hard $HASH + rm -rf .git +} + +rm -rf $OPENWHISK_SOURCE_DIR/* + +CONFIG=$(read_file $SCRIPTDIR/config.json) +repos=$(echo $(json_by_key "$CONFIG" "RepoList") | sed 's/[][]//g') + +for repo in $(echo $repos | sed "s/,/ /g") +do + repo_name=$(echo "$repo" | sed -e 's/^"//' -e 's/"$//') + HASH_KEY=${repo_name//-/_}.hash + REPO_KEY=${repo_name//-/_}.repository + REPO_BRANCH=${repo_name//-/_}.branch + HASH=$(json_by_key "$CONFIG" $HASH_KEY) + REPO=$(json_by_key "$CONFIG" $REPO_KEY) + BRANCH=$(json_by_key "$CONFIG" $REPO_BRANCH) + if [ "$HASH" != "null" ]; then + echo "The hash for $repo_name is $HASH" + git_clone_repo $repo_name $HASH $REPO $BRANCH + fi +done diff --git a/tools/install_dependencies.sh b/tools/install_dependencies.sh new file mode 100755 index 0000000..a4cbbac --- /dev/null +++ b/tools/install_dependencies.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +sysOS=`uname -s` +if [ $sysOS == "Darwin" ];then + echo "This is MacOS." + brew install jq +elif [ $sysOS == "Linux" ];then + echo "This is Linux." + if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then + export DISTRO=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) + else + export DISTRO=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) + fi + if [ $DISTRO == "Ubuntu" ];then + echo "This is Ubuntu." + sudo apt-get install jq + fi +else + echo "Other OS: $sysOS. Please manually install jq library." +fi diff --git a/tools/package_source_code.sh b/tools/package_source_code.sh new file mode 100755 index 0000000..d97ad5d --- /dev/null +++ b/tools/package_source_code.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -e + +echo "Package the artifacts." + +SVN_USERNAME=$2 +SVN_PASSWORD=$3 +CREDENTIALS="" + +if [ ! -z "$SVN_USERNAME" ] && [ ! -z "$SVN_PASSWORD" ];then + CREDENTIALS="--username $SVN_USERNAME --password $SVN_PASSWORD --non-interactive" +fi + +WORK_DIR=${1:-"$HOME"} + +OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources" +OPENWHISK_SVN="$OPENWHISK_SOURCE_DIR/openwhisk" + +SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)" +source "$SCRIPTDIR/util.sh" + +CONFIG=$(read_file $SCRIPTDIR/config.json) +repos=$(echo $(json_by_key "$CONFIG" "RepoList") | sed 's/[][]//g') +version_key="version" +version_major=$(json_by_key "$CONFIG" ${version_key}.major) +version_minor=$(json_by_key "$CONFIG" ${version_key}.minor) + +version=$version_major-$version_minor +CURRENT_VERSION_DIR="$OPENWHISK_SVN/openwhisk-$version" +echo $version + +STAGE_URL=$(json_by_key "$CONFIG" "stage_url") +echo $STAGE_URL + +# Create a subversion directory for openwhisk to stage all the packages +rm -rf $OPENWHISK_SVN + +mkdir -p $OPENWHISK_SOURCE_DIR +cd $OPENWHISK_SOURCE_DIR +svn co $STAGE_URL $OPENWHISK_SVN $CREDENTIALS +mkdir -p $CURRENT_VERSION_DIR +rm -rf $CURRENT_VERSION_DIR/* + +for repo in $(echo $repos | sed "s/,/ /g") +do + repo_name=$(echo "$repo" | sed -e 's/^"//' -e 's/"$//') + echo $repo_name + if [ $repo_name != "incubator-openwhisk" ]; then + # Only wskdeploy has both the gradle tasks implemented: cleanBuild to clean the build folder, and taredSources + # to package the source code. + cd $OPENWHISK_SOURCE_DIR/$repo_name + ./gradlew cleanBuild + ./gradlew taredSources -PprojectVersion=$version + # Copy all the source code packages into $CURRENT_VERSION_DIR + cp -a build/. $CURRENT_VERSION_DIR + fi +done + diff --git a/tools/travis/config.json b/tools/travis/config.json deleted file mode 100644 index e4077f0..0000000 --- a/tools/travis/config.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "RepoList": [ - "incubator-openwhisk", - "incubator-openwhisk-catalog", - "incubator-openwhisk-runtime-nodejs", - "incubator-openwhisk-cli", - "incubator-openwhisk-wskdeploy" - ], - "incubator_openwhisk": { - "hash": "a891e06" - }, - "incubator_openwhisk_catalog": { - "hash": "ad24d4c" - }, - "incubator_openwhisk_runtime_nodejs": { - "hash": "1c6c5db" - }, - "incubator_openwhisk_cli": { - "hash": "7b78091" - }, - "incubator_openwhisk_wskdeploy": { - "hash": "9840db4" - } -} diff --git a/tools/travis/download_source_code.sh b/tools/travis/download_source_code.sh deleted file mode 100755 index 336746e..0000000 --- a/tools/travis/download_source_code.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -set -e - -sudo apt-get install jq - -HOMEDIR="$(dirname "$TRAVIS_BUILD_DIR")" -OPENWHISKDIR="$HOMEDIR/openwhisk" -source "$TRAVIS_BUILD_DIR/tools/travis/util.sh" - -clone_depth=${1:-"100"} - -function git_clone_repo() { - PROJECT_NAME=$1 - HASH=$2 - git clone --depth $clone_depth https://github.com/apache/$PROJECT_NAME.git $OPENWHISKDIR/$PROJECT_NAME - cd $OPENWHISKDIR/$PROJECT_NAME - git reset --hard $HASH - rm -rf .git -} - -rm -rf $OPENWHISKDIR/* - -CONFIG=$(read_file $TRAVIS_BUILD_DIR/tools/travis/config.json) -repos=$(echo $(json_by_key "$CONFIG" "RepoList") | sed 's/[][]//g') - -for repo in $(echo $repos | sed "s/,/ /g") -do - repo_name=$(echo "$repo" | sed -e 's/^"//' -e 's/"$//') - HASH_KEY=${repo_name//-/_}.hash - HASH=$(json_by_key "$CONFIG" $HASH_KEY) - if [ "$HASH" != "null" ]; then - echo "The hash for $repo_name is $HASH" - git_clone_repo $repo_name $HASH - fi -done diff --git a/tools/travis/package_source_code.sh b/tools/travis/package_source_code.sh index 62df67e..305c967 100755 --- a/tools/travis/package_source_code.sh +++ b/tools/travis/package_source_code.sh @@ -2,10 +2,24 @@ set -e -cd $TRAVIS_BUILD_DIR -./tools/travis/download_source_code.sh +WORK_DIR=${1:-"$(dirname "$TRAVIS_BUILD_DIR")"} +OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources" -# TODO: add release note and license file if necessary -echo "add release note and license file if necessary" +SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)" +PARENTDIR="$(dirname "$SCRIPTDIR")" -# TODO: package all the source code and upload them into the designated directory +SVN_USERNAME=$2 +SVN_PASSWORD=$3 + +"$PARENTDIR/install_dependencies.sh" +"$PARENTDIR/download_source_code.sh" $WORK_DIR + +if [ "$TRAVIS_EVENT_TYPE" == "push" ] ; then + "$PARENTDIR/clean_remote_artifacts.sh" $SVN_USERNAME $SVN_PASSWORD +fi + +"$PARENTDIR/package_source_code.sh" $WORK_DIR $SVN_USERNAME $SVN_PASSWORD + +if [ "$TRAVIS_EVENT_TYPE" == "push" ] ; then + "$PARENTDIR/upload_artifacts.sh" $WORK_DIR $SVN_USERNAME $SVN_PASSWORD +fi diff --git a/tools/travis/setup.sh b/tools/travis/setup.sh index d79c58c..1ccdaca 100755 --- a/tools/travis/setup.sh +++ b/tools/travis/setup.sh @@ -5,8 +5,11 @@ set -e echo "hello openwhisk release" # Define the parent directory of TRAVIS_BUILD_DIR(the directory of the current repository) as HOMEDIR -HOMEDIR="$(dirname "$TRAVIS_BUILD_DIR")" -OPENWHISKDIR="$HOMEDIR/openwhisk" +WORK_DIR=${1:-"$(dirname "$TRAVIS_BUILD_DIR")"} +OPENWHISKDIR="$WORK_DIR/openwhisk_sources" +SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)" +PARENTDIR="$(dirname "$SCRIPTDIR")" + SUFFIX="$TRAVIS_BUILD_NUMBER" PR_NUM="$TRAVIS_PULL_REQUEST" PREFIX="build_testing" @@ -19,8 +22,7 @@ cd $OPENWHISKDIR echo "list images in setup before build" docker images -cd $TRAVIS_BUILD_DIR -./tools/travis/download_source_code.sh +"$PARENTDIR/download_source_code.sh" $WORK_DIR cd $OPENWHISKDIR/incubator-openwhisk ./tools/travis/setup.sh @@ -38,8 +40,8 @@ $ANSIBLE_CMD openwhisk.yml # Build the binaries for CLI cd $OPENWHISKDIR/incubator-openwhisk-cli -./gradlew buildBinaries -PcrossCompileCLI=true +#./gradlew buildBinaries -PcrossCompileCLI=true # Build the binaries for wskdeploy cd $OPENWHISKDIR/incubator-openwhisk-wskdeploy -./gradlew distDocker -PcrossCompileWSKDEPLOY=true +#./gradlew distDocker -PcrossCompileWSKDEPLOY=true diff --git a/tools/travis/verify_source_code.sh b/tools/travis/verify_source_code.sh index ef1e1a6..8b14faf 100755 --- a/tools/travis/verify_source_code.sh +++ b/tools/travis/verify_source_code.sh @@ -2,8 +2,16 @@ set -e -cd $TRAVIS_BUILD_DIR -./tools/travis/download_source_code.sh +WORK_DIR=${1:-"$(dirname "$TRAVIS_BUILD_DIR")"} +OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources" + +SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)" +PARENTDIR="$(dirname "$SCRIPTDIR")" + +echo $PARENTDIR +"$PARENTDIR/install_dependencies.sh" +"$PARENTDIR/download_source_code.sh" $WORK_DIR +"$PARENTDIR/verify_source_code.sh" $WORK_DIR # TODO: verify the source code headers echo "TODO: verify the source code headers" diff --git a/tools/upload_artifacts.sh b/tools/upload_artifacts.sh new file mode 100755 index 0000000..821865a --- /dev/null +++ b/tools/upload_artifacts.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e + +echo "Upload the artifacts." + +WORK_DIR=${1:-"$HOME"} + +SVN_USERNAME=$2 +SVN_PASSWORD=$3 +CREDENTIALS="" + +if [ ! -z "$SVN_USERNAME" ] && [ ! -z "$SVN_PASSWORD" ];then + CREDENTIALS="--username $SVN_USERNAME --password $SVN_PASSWORD --non-interactive" +fi + +OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources" +OPENWHISK_SVN="$OPENWHISK_SOURCE_DIR/openwhisk" + +SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)" +source "$SCRIPTDIR/util.sh" + +CONFIG=$(read_file $SCRIPTDIR/config.json) +version_key="version" +version_major=$(json_by_key "$CONFIG" ${version_key}.major) +version_minor=$(json_by_key "$CONFIG" ${version_key}.minor) + +version=$version_major-$version_minor +CURRENT_VERSION_DIR="$OPENWHISK_SVN/openwhisk-$version" + +cd $OPENWHISK_SVN +svn add --force "openwhisk-$version" $CREDENTIALS +svn commit -m "Staging Apache OpenWhisk release ${version}." $CREDENTIALS diff --git a/tools/travis/util.sh b/tools/util.sh similarity index 100% rename from tools/travis/util.sh rename to tools/util.sh diff --git a/tools/verify_source_code.sh b/tools/verify_source_code.sh new file mode 100755 index 0000000..7ad4a9d --- /dev/null +++ b/tools/verify_source_code.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e + +WORK_DIR=${1:-"$HOME"} +OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources" + +SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)" + +# TODO: verify the source code headers +echo "TODO: verify the source code headers" ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services