[
https://issues.apache.org/jira/browse/BEAM-7620?focusedWorklogId=286207&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-286207
]
ASF GitHub Bot logged work on BEAM-7620:
----------------------------------------
Author: ASF GitHub Bot
Created on: 31/Jul/19 20:35
Start Date: 31/Jul/19 20:35
Worklog Time Spent: 10m
Work Description: alanmyrvold commented on pull request #8929:
[BEAM-7620] Make run_rc_validation.sh non-interactive
URL: https://github.com/apache/beam/pull/8929#discussion_r309419346
##########
File path: release/src/main/scripts/run_rc_validation.sh
##########
@@ -36,223 +51,186 @@ function clean_up(){
echo "* Restored ~/.bashrc"
fi
- rm -rf ~/{LOCAL_CLONE_DIR}
- echo "* Deleted working dir ~/{LOCAL_CLONE_DIR}"
+ rm -rf ${LOCAL_BEAM_DIR}
+ echo "* Deleted workspace ${LOCAL_BEAM_DIR}"
}
+trap clean_up EXIT
-RELEASE=
-REPO_URL=
-RC_NUM=
-RELEASE_BRANCH=
-WORKING_BRANCH=
-LOCAL_CLONE_DIR=rc_validations
-BEAM_ROOT_DIR=beam
+RELEASE_BRANCH=release-${RELEASE_VER}
+WORKING_BRANCH=release-${RELEASE}-RC${RC_NUM}_validations
GIT_REPO_URL=https://github.com/apache/beam.git
PYTHON_RC_DOWNLOAD_URL=https://dist.apache.org/repos/dist/dev/beam
HUB_VERSION=2.5.0
HUB_ARTIFACTS_NAME=hub-linux-amd64-${HUB_VERSION}
-declare -a DEFAULT_PYTHON_VERSIONS_TO_VALIDATE=("python2.7" "python3.5")
+declare -a PYTHON_VERSIONS_TO_VALIDATE=("python2.7" "python3.5")
-echo "[Input Required] Please enter the release version: "
-read RELEASE
-RELEASE_BRANCH=release-${RELEASE}
-WORKING_BRANCH=release-${RELEASE}-RC${RC_NUM}_validations
-echo "[Input Required] Please enter the release candidate number(e.g. 1): "
-read RC_NUM
-echo "[Input Required] Please copy the repo URL from the vote email sent out
by Release Manager:"
-echo "The URL should look like:
https://repository.apache.org/content/repositories/orgapachebeam-0000";
-read REPO_URL
-
-echo "====================Checking Environment Variables================="
-echo "running validations on release ${RELEASE} RC${RC_NUM}."
+echo ""
+echo "====================Checking Environment & Variables================="
+echo "running validations on release ${RELEASE_VER} RC${RC_NUM}."
echo "repo URL for this RC: ${REPO_URL}"
-echo "[Confirmation Required] Do you confirm all information above are
correct? [y|N]"
+echo "using workspace: ${LOCAL_BEAM_DIR}"
+echo "validate Python versions: "$(IFS=$' '; echo
"${PYTHON_VERSIONS_TO_VALIDATE[*]}")
+echo ""
+echo "All environment and workflow configurations from file script.config:"
+for i in "${CONFIG_NAME_LIST[@]}"; do
+ echo "$i = ${!i}"
+done
+echo "[Confirmation Required] Are they all provided and correctly set? [y|N]"
read confirmation
if [[ $confirmation != "y" ]]; then
- echo "Please rerun this script and make sure you have the right inputs."
+ echo "Please rerun this script and make sure you have the right
configurations."
exit
fi
-echo "=================Checking hub========================"
+echo "-----------------Checking hub-----------------"
if [[ -z `which hub` ]]; then
- echo "There is no hub installed on your machine."
- echo "Would you like to install hub with root permission? [y|N]"
- read confirmation
- if [[ $confirmation != "y" ]]; then
- echo "Refused to install hub. Cannot proceed into next setp."
- exit
+ if [[ "${INSTALL_HUB}" = true ]]; then
+ echo "-----------------Installing hub-----------------"
+ wget
https://github.com/github/hub/releases/download/v${HUB_VERSION}/${HUB_ARTIFACTS_NAME}.tgz
+ tar zvxvf ${HUB_ARTIFACTS_NAME}.tgz
+ sudo ./${HUB_ARTIFACTS_NAME}/install
+ echo "eval "$(hub alias -s)"" >> ~/.bashrc
+ rm -rf ${HUB_ARTIFACTS_NAME}*
+ else
+ echo "Hub is not installed. Validation on Python Leaderboard & GameStates
will be skipped."
fi
- echo "=================Installing hub======================="
- wget
https://github.com/github/hub/releases/download/v${HUB_VERSION}/${HUB_ARTIFACTS_NAME}.tgz
- tar zvxvf ${HUB_ARTIFACTS_NAME}.tgz
- sudo ./${HUB_ARTIFACTS_NAME}/install
- echo "eval "$(hub alias -s)"" >> ~/.bashrc
- rm -rf ${HUB_ARTIFACTS_NAME}*
fi
hub version
+echo "-----------------Checking Google Cloud SDK-----------------"
+if [[ -z `which gcloud` ]]; then
+ echo "You don't have Google Cloud SDK installed."
+ if [[ "${INSTALL_GCLOUD}" = true ]]; then
+ echo "-----------------Installing Google Cloud SDK-----------------"
+ sudo apt-get install google-cloud-sdk
-echo "====================Cloning Beam Release Branch===================="
-cd ~
-if [[ -d ${LOCAL_CLONE_DIR} ]]; then
- rm -rf ${LOCAL_CLONE_DIR}
+ echo "[gcloud Login Required] Please login into your gcp account: "
+ gcloud auth login
+ gcloud config set project ${USER_GCP_PROJECT}
+
+ echo "-----------------Setting Up Service Account-----------------"
+ SERVICE_ACCOUNT_KEY_JSON=${USER}_json_key.json
+ gcloud iam service-accounts keys create ${SERVICE_ACCOUNT_KEY_JSON}
--iam-account ${USER_SERVICE_ACCOUNT_EMAIL}
+ export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/${SERVICE_ACCOUNT_KEY_JSON}
+ fi
fi
+gcloud --version
-mkdir ${LOCAL_CLONE_DIR}
-cd ${LOCAL_CLONE_DIR}
-git clone ${GIT_REPO_URL}
-cd ${BEAM_ROOT_DIR}
-git checkout ${RELEASE_BRANCH}
-git checkout -b ${WORKING_BRANCH}
+echo ""
+echo ""
+echo "====================Cloning Beam Release Branch===================="
+if [[ -d ${LOCAL_BEAM_DIR} ]]; then
+ rm -rf ${LOCAL_BEAM_DIR}
+fi
+echo "* Creating local Beam workspace: ${LOCAL_BEAM_DIR}"
+mkdir -p ${LOCAL_BEAM_DIR}
+git clone ${GIT_REPO_URL} ${LOCAL_BEAM_DIR}
+cd ${LOCAL_BEAM_DIR}
+git checkout -b ${WORKING_BRANCH} origin/${RELEASE_BRANCH}
+
+echo ""
echo "====================Starting Java Quickstart======================="
echo "[Current task] Java quickstart with direct runner"
-echo "[Confirmation Required] Do you want to start this task? [y|N]"
-read confirmation
-if [[ $confirmation = "y" ]]; then
+if [[ "$java_quickstart_direct" = true ]]; then
echo "*************************************************************"
echo "* Running Java Quickstart with DirectRunner"
echo "*************************************************************"
./gradlew :runners:direct-java:runQuickstartJavaDirect \
-Prepourl=${REPO_URL} \
- -Pver=${RELEASE}
+ -Pver=${RELEASE_VER}
+else
+ echo "* Skip Java quickstart with direct runner"
fi
echo "[Current task] Java quickstart with Apex local runner"
-echo "[Confirmation Required] Do you want to start this task? [y|N]"
-read confirmation
-if [[ $confirmation = "y" ]]; then
+if [[ "$java_quickstart_apex_local" = true ]]; then
echo "*************************************************************"
echo "* Running Java Quickstart with Apex local runner"
echo "*************************************************************"
./gradlew :runners:apex:runQuickstartJavaApex \
-Prepourl=${REPO_URL} \
- -Pver=${RELEASE}
+ -Pver=${RELEASE_VER}
+else
+ echo "* Skip Java quickstart with Apex local runner"
fi
echo "[Current task] Java quickstart with Flink local runner"
-echo "[Confirmation Required] Do you want to start this task? [y|N]"
-read confirmation
-if [[ $confirmation = "y" ]]; then
+if [[ "$java_quickstart_flink_local" = true ]]; then
echo "*************************************************************"
echo "* Running Java Quickstart with Flink local runner"
echo "*************************************************************"
./gradlew :runners:flink:1.5:runQuickstartJavaFlinkLocal \
-Prepourl=${REPO_URL} \
- -Pver=${RELEASE}
+ -Pver=${RELEASE_VER}
+else
+ echo "* Skip Java quickstart with Flink local runner"
fi
echo "[Current task] Java quickstart with Spark local runner"
-echo "[Confirmation Required] Do you want to start this task? [y|N]"
-read confirmation
-if [[ $confirmation = "y" ]]; then
+if [[ "$java_quickstart_spark_local" = true ]]; then
echo "*************************************************************"
echo "* Running Java Quickstart with Spark local runner"
echo "*************************************************************"
./gradlew :runners:spark:runQuickstartJavaSpark \
-Prepourl=${REPO_URL} \
- -Pver=${RELEASE}
+ -Pver=${RELEASE_VER}
+else
+ echo "* Skip Java quickstart with Spark local runner"
fi
-echo "====================Checking Google Cloud SDK======================"
-if [[ -z `which gcloud` ]]; then
- echo "You don't have Google Cloud SDK installed."
- echo " Do you want to install gcloud with sudo permission? [y|N]"
- read confirmation
- if [[ $confirmation != 'y' ]]; then
- echo "Exit script without running rest validations."
- exit
- fi
- sudo apt-get install google-cloud-sdk
-fi
-gcloud --version
-
echo "[Current task] Java quickstart with Dataflow runner"
-echo "[Confirmation Required] Do you want to start this task? [y|N]"
-read confirmation
-if [[ $confirmation = "y" ]]; then
- echo "[GCP Project Required] Please input your GCP project:"
- read USER_GCP_PROJECT
- echo "[GCP GCS Bucket Required] Please input your GCS bucket: "
- read USER_GCS_BUCKET
- echo "[gcloud Login Required] Please login into your gcp account: "
- gcloud auth application-default login
-
GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json
-
+if [[ "$java_quickstart_dataflow" = true && ! -z `which gcloud` ]]; then
echo "*************************************************************"
echo "* Running Java Quickstart with DataflowRunner"
echo "*************************************************************"
- gcloud auth application-default login
- gcloud config set project ${USER_GCP_PROJECT}
./gradlew :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow \
-Prepourl=${REPO_URL} \
- -Pver=${RELEASE} \
+ -Pver=${RELEASE_VER} \
-PgcpProject=${USER_GCP_PROJECT} \
-PgcsBucket=${USER_GCS_BUCKET}
+else
+ echo "* Skip Java quickstart with Dataflow runner. Google Cloud SDK is
required."
fi
-echo "===================Starting Java Mobile Game====================="
-echo "[Confirmation Required] This task asks for GCP resources."
-echo "Do you want to proceed? [y|N]"
-read confirmation
-if [[ $confirmation = "y" ]]; then
- echo "[GCP Project Required] Please input your GCP project:"
- read USER_GCP_PROJECT
- echo "[GCP GCS Bucket Required] Please input your GCS bucket: "
- read USER_GCS_BUCKET
- MOBILE_GAME_DATASET=${USER}_java_validations
- MOBILE_GAME_PUBSUB_TOPIC=leader_board-${USER}-java-topic-1
- echo "Please review following GCP sources setup: "
+echo ""
+echo "====================Starting Java Mobile Game====================="
+if [[ "$java_mobile_game" = true ]]; then
+ MOBILE_GAME_DATASET=${USER}_java_validations_$(date +%m%d)_$RANDOM
+ MOBILE_GAME_PUBSUB_TOPIC=leader_board-${USER}-java-topic-$(date
+%m%d)_$RANDOM
echo "Using GCP project: ${USER_GCP_PROJECT}"
echo "Will create BigQuery dataset: ${MOBILE_GAME_DATASET}"
echo "Will create Pubsub topic: ${MOBILE_GAME_PUBSUB_TOPIC}"
- echo "[Confirmation Required] Do you want to run validations with
configurations above? [y|N]"
- read confirmation
- if [[ $confirmation = "y" ]]; then
- gcloud auth login
- gcloud config set project ${USER_GCP_PROJECT}
- echo "-----------------Setting Up Service Account------------------------"
- echo "Please go to GCP IAM console under your
project(${USER_GCP_PROJECT})."
- echo "Create a service account as project owner, if you don't have one."
- echo "[Input Required] Please enter your service account email:"
- read USER_SERVICE_ACCOUNT_EMAIL
- SERVICE_ACCOUNT_KEY_JSON=${USER}_json_key.json
- gcloud iam service-accounts keys create ${SERVICE_ACCOUNT_KEY_JSON}
--iam-account ${USER_SERVICE_ACCOUNT_EMAIL}
- export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/${SERVICE_ACCOUNT_KEY_JSON}
- echo "-------------------Creating BigQuery Dataset-----------------------"
- bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
-
- echo "----------------------Creating Pubsub Topic------------------------"
- gcloud alpha pubsub topics delete
projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC}
- gcloud alpha pubsub topics create --project=${USER_GCP_PROJECT}
${MOBILE_GAME_PUBSUB_TOPIC}
-
- echo
"**************************************************************************"
- echo "* Java mobile game validations: UserScore, HourlyTeamScore,
Leaderboard"
- echo
"**************************************************************************"
- ./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE} \
- -PgcpProject=${USER_GCP_PROJECT} \
- -PgcsBucket=${USER_GCS_BUCKET} \
- -PbqDataset=${MOBILE_GAME_DATASET}
-PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC}
- fi
+ echo "-----------------Creating BigQuery Dataset-----------------"
+ bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
+
+ echo "-----------------Creating Pubsub Topic-----------------"
+ gcloud alpha pubsub topics create --project=${USER_GCP_PROJECT}
${MOBILE_GAME_PUBSUB_TOPIC}
Review comment:
Does this need to be "gcloud alpha", or can "alpha" be dropped?
https://cloud.google.com/sdk/gcloud/reference/pubsub/topics/create
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 286207)
Time Spent: 1h 20m (was: 1h 10m)
> Make release scripts non-interactive
> ------------------------------------
>
> Key: BEAM-7620
> URL: https://issues.apache.org/jira/browse/BEAM-7620
> Project: Beam
> Issue Type: Task
> Components: build-system
> Reporter: Mark Liu
> Assignee: Mark Liu
> Priority: Major
> Time Spent: 1h 20m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.14#76016)
