This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8d75633 [BEAM-13220] Update release instructions.
new 5fc55f3 Merge pull request #16065 from ibzib/BEAM-13220
8d75633 is described below
commit 8d75633d0c082b1151405cf364c9370bfc95f596
Author: Kyle Weaver <[email protected]>
AuthorDate: Wed Nov 24 15:24:04 2021 -0800
[BEAM-13220] Update release instructions.
The biggest change here is to run Java wordcount and mobile gaming examples
on Jenkins instead of the bash script, since getting the script to work
properly took too long.
---
.../main/scripts/publish_github_release_notes.sh | 2 +-
release/src/main/scripts/run_rc_validation.sh | 128 +--------------------
release/src/main/scripts/script.config | 2 -
.../site/content/en/contribute/release-guide.md | 20 ++--
4 files changed, 14 insertions(+), 138 deletions(-)
diff --git a/release/src/main/scripts/publish_github_release_notes.sh
b/release/src/main/scripts/publish_github_release_notes.sh
index 7f58648..09fec15 100755
--- a/release/src/main/scripts/publish_github_release_notes.sh
+++ b/release/src/main/scripts/publish_github_release_notes.sh
@@ -33,7 +33,7 @@ RELEASE_NOTES=$(
)
# Escape notes' content to work with JSON
-ESCAPED_NOTES=$(printf '%s' "${RELEASE_NOTES}" | python -c 'import json,sys;
print(json.dumps(sys.stdin.read()))')
+ESCAPED_NOTES=$(printf '%s' "${RELEASE_NOTES}" | python3 -c 'import json,sys;
print(json.dumps(sys.stdin.read()))')
# Build JSON for the API request
REQUEST_JSON="$(cat <<-EOF
diff --git a/release/src/main/scripts/run_rc_validation.sh
b/release/src/main/scripts/run_rc_validation.sh
index baff09b..28ba2eb 100755
--- a/release/src/main/scripts/run_rc_validation.sh
+++ b/release/src/main/scripts/run_rc_validation.sh
@@ -207,133 +207,6 @@ kubectl version
echo ""
-echo ""
-echo "====================Starting Java Quickstart======================="
-echo "[Current task] Java quickstart with direct runner"
-if [[ "$java_quickstart_direct" = true ]]; then
- echo "*************************************************************"
- echo "* Running Java Quickstart with DirectRunner"
- echo "*************************************************************"
- ./gradlew :runners:direct-java:runQuickstartJavaDirect \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER}
-else
- echo "* Skip Java quickstart with direct runner"
-fi
-
-echo "[Current task] Java quickstart with Flink local runner"
-if [[ "$java_quickstart_flink_local" = true ]]; then
- echo "*************************************************************"
- echo "* Running Java Quickstart with Flink local runner"
- echo "*************************************************************"
- ./gradlew :runners:flink:1.13:runQuickstartJavaFlinkLocal \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER}
-else
- echo "* Skip Java quickstart with Flink local runner"
-fi
-
-echo "[Current task] Java quickstart with Spark local runner"
-if [[ "$java_quickstart_spark_local" = true ]]; then
- echo "*************************************************************"
- echo "* Running Java Quickstart with Spark local runner"
- echo "*************************************************************"
- ./gradlew :runners:spark:2:runQuickstartJavaSpark \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER}
-else
- echo "* Skip Java quickstart with Spark local runner"
-fi
-
-echo "[Current task] Java quickstart with Dataflow runner"
-if [[ "$java_quickstart_dataflow" = true && ! -z `which gcloud` ]]; then
- echo "*************************************************************"
- echo "* Running Java Quickstart with DataflowRunner"
- echo "*************************************************************"
- ./gradlew :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER} \
- -PgcpProject=${USER_GCP_PROJECT} \
- -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix
-else
- echo "* Skip Java quickstart with Dataflow runner. Google Cloud SDK is
required."
-fi
-
-echo "[Current task] Java quickstart with Twister2 local runner"
-if [[ "$java_quickstart_twister2_local" = true ]]; then
- echo "*************************************************************"
- echo "* Running Java Quickstart with Twister2 local runner"
- echo "*************************************************************"
- ./gradlew :runners:twister2:runQuickstartJavaTwister2 \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER}
-else
- echo "* Skip Java quickstart with Twister2 local runner"
-fi
-
-echo ""
-echo "====================Starting Java Mobile Game====================="
-if [[ ("$java_mobile_game_direct" = true || "$java_mobile_game_dataflow" =
true) \
- && ! -z `which gcloud` ]]; then
- MOBILE_GAME_DATASET=${USER}_java_validations_$(date +%m%d)_$RANDOM
- MOBILE_GAME_PUBSUB_TOPIC=leader_board-${USER}-java-topic-$(date
+%m%d)_$RANDOM
- echo "Using GCP project: ${USER_GCP_PROJECT}"
- echo "Will create BigQuery dataset: ${MOBILE_GAME_DATASET}"
- echo "Will create Pubsub topic: ${MOBILE_GAME_PUBSUB_TOPIC}"
-
- echo "-----------------Creating BigQuery Dataset-----------------"
- bq mk --project_id=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
-
- echo "-----------------Creating Pubsub Topic-----------------"
- gcloud pubsub topics create --project=${USER_GCP_PROJECT}
${MOBILE_GAME_PUBSUB_TOPIC}
-
- if [[ "$java_mobile_game_direct" = true ]]; then
- echo
"**************************************************************************"
- echo "* Java mobile game on DirectRunner: UserScore, HourlyTeamScore,
Leaderboard"
- echo
"**************************************************************************"
- ./gradlew :runners:direct-java:runMobileGamingJavaDirect \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER} \
- -PgcpProject=${USER_GCP_PROJECT} \
- -PbqDataset=${MOBILE_GAME_DATASET} \
- -PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC} \
- -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix
- else
- echo "* Skip Java Mobile Game on DirectRunner."
- fi
-
- if [[ "$java_mobile_game_dataflow" = true ]]; then
- echo
"**************************************************************************"
- echo "* Java mobile game on DataflowRunner: UserScore, HourlyTeamScore,
Leaderboard"
- echo
"**************************************************************************"
- ./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER} \
- -PgcpProject=${USER_GCP_PROJECT} \
- -PbqDataset=${MOBILE_GAME_DATASET} \
- -PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC} \
- -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix
-
- echo
"**************************************************************************"
- echo "* Java mobile game on DataflowRunner using Beam GCP BOM: UserScore,
HourlyTeamScore, Leaderboard"
- echo
"**************************************************************************"
- ./gradlew
:runners:google-cloud-dataflow-java:runMobileGamingJavaDataflowBom \
- -Prepourl=${REPO_URL} \
- -Pver=${RELEASE_VER} \
- -PgcpProject=${USER_GCP_PROJECT} \
- -PbqDataset=${MOBILE_GAME_DATASET} \
- -PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC} \
- -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix
- else
- echo "* Skip Java Mobile Game on DataflowRunner."
- fi
-
- echo "-----------------Cleaning up BigQuery & Pubsub-----------------"
- bq rm -r -f --project_id=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- gcloud pubsub topics delete
projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC}
-fi
-
-echo ""
echo "====================Starting Python Quickstart and
MobileGame==================="
echo "This task will create a PR against apache/beam, trigger a jenkins job to
run:"
echo "1. Python quickstart validations(batch & streaming)"
@@ -357,6 +230,7 @@ else
echo "* Skip Python Quickstart and MobileGame. Hub is required."
fi
+# TODO(BEAM-13220) Run the remaining tests on Jenkins.
echo ""
echo "====================Starting Python Leaderboard & GameStates
Validations==============="
if [[ ("$python_leaderboard_direct" = true \
diff --git a/release/src/main/scripts/script.config
b/release/src/main/scripts/script.config
index 97e9dd3..52836f9 100755
--- a/release/src/main/scripts/script.config
+++ b/release/src/main/scripts/script.config
@@ -41,7 +41,6 @@ RC_VALIDATE_CONFIGS=(
INSTALL_KUBECTL
LOCAL_BEAM_DIR
java_quickstart_direct
- java_quickstart_apex_local
java_quickstart_flink_local
java_quickstart_spark_local
java_quickstart_dataflow
@@ -153,7 +152,6 @@ LOCAL_BEAM_DIR="$(mktemp -d -t
beam-release.${RELEASE_VER}.XXXXXX)"
# Whether to start certain validation pipeline.
##############################################################################
java_quickstart_direct=true
-java_quickstart_apex_local=true
java_quickstart_flink_local=true
java_quickstart_spark_local=true
java_quickstart_dataflow=true
diff --git a/website/www/site/content/en/contribute/release-guide.md
b/website/www/site/content/en/contribute/release-guide.md
index 748af95..8a8b607 100644
--- a/website/www/site/content/en/contribute/release-guide.md
+++ b/website/www/site/content/en/contribute/release-guide.md
@@ -135,11 +135,11 @@ __NOTE__: When generating the key, please make sure you
choose the key type as _
This will list your GPG keys. One of these should reflect your Apache
account, for example:
--------------------------------------------------
- pub 2048R/845E6689 2016-02-23
+ pub rsa4096/845E6689845E6689 2016-02-23
uid Nomen Nescio <[email protected]>
- sub 2048R/BA4D50BE 2016-02-23
+ sub rsa4096/BA4D50BEBA4D50BE 2016-02-23
- Here, the key ID is the 8-digit hex string in the `pub` line: `845E6689`.
+ Here, the key ID is the 16-digit hex string in the `pub` line:
`845E6689845E6689`.
#### Access to Apache Nexus repository
@@ -548,7 +548,7 @@ The final state of the repository should match this diagram:
./beam/release/src/main/scripts/choose_rc_commit.sh \
--release "${RELEASE_VERSION}" \
--rc "${RC_NUM}" \
- --commit "${COMMIT_REF}" \
+ --commit "${COMMIT_REF}" \
--clone \
--push-tag
@@ -569,7 +569,7 @@ Before you start, run this command to make sure you'll be
using the latest docke
* **Usage**
- ./beam/release/src/main/scripts/build_release_candidate.sh
+ ./beam/release/src/main/scripts/build_release_candidate.sh --release
"${RELEASE_VERSION}" --rc "${RC_NUM}" --github-user "${GITHUB_USER}"
* **The script will:**
1. Clone the repo at the selected RC tag.
@@ -839,8 +839,6 @@ versions to run all of the tests. See Python installation
tips in [Developer Wik
```
* **Tasks included**
- 1. Run Java quickstart with Direct Runner, Flink local runner, Spark local
runner and Dataflow runner.
- 1. Run Java Mobile Games(UserScore, HourlyTeamScore, Leaderboard) with
Dataflow runner.
1. Create a PR to trigger python validation job, including
* Python quickstart in batch and streaming mode with direct runner and
Dataflow runner.
* Python Mobile Games(UserScore, HourlyTeamScore) with direct runner and
Dataflow runner.
@@ -854,6 +852,12 @@ versions to run all of the tests. See Python installation
tips in [Developer Wik
* **Tasks you need to do manually**
1. Check whether validations succeed by following console output
instructions.
1. Terminate streaming jobs and java injector.
+ 1. Run Java quickstart (wordcount) and mobile game examples with the staged
artifacts. The easiest way to do this is by running the tests on Jenkins.
+ 1. Log in to Jenkins.
+ 1. Go to https://ci-beam.apache.org/job/beam_PostRelease_NightlySnapshot/.
+ 1. Click "Build with Parameters".
+ 1. Set `snapshot_version` to `2.xx.0`, and set `snapshot_url` to point to
the staged artifacts in Maven central
(https://repository.apache.org/content/repositories/orgapachebeam-NNNN/).
+ 1. Click "Build".
1. Sign up [spreadsheet](https://s.apache.org/beam-release-validation).
1. Vote in the release thread.
@@ -1184,7 +1188,7 @@ After pushing the tag, the tag should be visible on
Github's [Tags](https://gith
Once the tag is uploaded, publish the release notes to Github, as follows:
```
-cd beam/release/src/main/scripts && ./publish_github_release_notes.sh
+./beam/release/src/main/scripts/publish_github_release_notes.sh
```
Note this script reads the release notes from the blog post, so you should
make sure to run this from master _after_ merging the blog post PR.
