This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 60307b436ad Fix PostRelease Nightly Snapshot job (#35079)
60307b436ad is described below
commit 60307b436ad26cc262c281cf44c0001af6c79ea4
Author: Vitaly Terentyev <[email protected]>
AuthorDate: Thu May 29 17:12:15 2025 +0400
Fix PostRelease Nightly Snapshot job (#35079)
* Add retry loop
* Remove typo
---
.../main/groovy/mobilegaming-java-dataflow.groovy | 40 ++++++++++++++++++++--
.../groovy/mobilegaming-java-dataflowbom.groovy | 40 ++++++++++++++++++++--
.../main/groovy/quickstart-java-dataflow.groovy | 20 ++++++++++-
3 files changed, 95 insertions(+), 5 deletions(-)
diff --git a/release/src/main/groovy/mobilegaming-java-dataflow.groovy
b/release/src/main/groovy/mobilegaming-java-dataflow.groovy
index 9ce93d31c14..a2164aa6a01 100644
--- a/release/src/main/groovy/mobilegaming-java-dataflow.groovy
+++ b/release/src/main/groovy/mobilegaming-java-dataflow.groovy
@@ -39,7 +39,26 @@ mobileGamingCommands = new MobileGamingCommands(testScripts:
t, testRunId: UUID.
t.intent("Running: UserScore example on DataflowRunner")
t.run(mobileGamingCommands.createPipelineCommand("UserScore", runner))
-command_output_text = t.run "gsutil cat
gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}* |
grep user19_BananaWallaby"
+
+int retries = 5
+int waitTime = 15 // seconds
+def outputPath =
"gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}"
+def outputFound = false
+for (int i = 0; i < retries; i++) {
+ def files = t.run("gsutil ls ${outputPath}*")
+ if (files?.trim()) {
+ outputFound = true
+ break
+ }
+ t.intent("Output not found yet. Waiting ${waitTime}s...")
+ Thread.sleep(waitTime * 1000)
+}
+
+if (!outputFound) {
+ throw new RuntimeException("No output files found for HourlyTeamScore after
${retries * waitTime} seconds.")
+}
+
+command_output_text = t.run "gsutil cat ${outputPath}* | grep
user19_BananaWallaby"
t.see "total_score: 231, user: user19_BananaWallaby", command_output_text
t.success("UserScore successfully run on DataflowRunner.")
t.run "gsutil rm
gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}*"
@@ -53,7 +72,24 @@ mobileGamingCommands = new MobileGamingCommands(testScripts:
t, testRunId: UUID.
t.intent("Running: HourlyTeamScore example on DataflowRunner")
t.run(mobileGamingCommands.createPipelineCommand("HourlyTeamScore", runner))
-command_output_text = t.run "gsutil cat
gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*
| grep AzureBilby "
+
+outputPath =
"gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}"
+outputFound = false
+for (int i = 0; i < retries; i++) {
+ def files = t.run("gsutil ls ${outputPath}*")
+ if (files?.trim()) {
+ outputFound = true
+ break
+ }
+ t.intent("Output not found yet. Waiting ${waitTime}s...")
+ Thread.sleep(waitTime * 1000)
+}
+
+if (!outputFound) {
+ throw new RuntimeException("No output files found for UserScore after
${retries * waitTime} seconds.")
+}
+
+command_output_text = t.run "gsutil cat ${outputPath}* | grep AzureBilby "
t.see "total_score: 2788, team: AzureBilby", command_output_text
t.success("HourlyTeamScore successfully run on DataflowRunner.")
t.run "gsutil rm
gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*"
diff --git a/release/src/main/groovy/mobilegaming-java-dataflowbom.groovy
b/release/src/main/groovy/mobilegaming-java-dataflowbom.groovy
index a0fd0f13c75..e156357c19d 100644
--- a/release/src/main/groovy/mobilegaming-java-dataflowbom.groovy
+++ b/release/src/main/groovy/mobilegaming-java-dataflowbom.groovy
@@ -39,7 +39,26 @@ mobileGamingCommands = new MobileGamingCommands(testScripts:
t, testRunId: UUID.
t.intent("Running: UserScore example with Beam GCP BOM on DataflowRunner")
t.run(mobileGamingCommands.createPipelineCommand("UserScore", runner))
-command_output_text = t.run "gsutil cat
gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}* |
grep user19_BananaWallaby"
+
+int retries = 5
+int waitTime = 15 // seconds
+def outputPath =
"gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}"
+def outputFound = false
+for (int i = 0; i < retries; i++) {
+ def files = t.run("gsutil ls ${outputPath}*")
+ if (files?.trim()) {
+ outputFound = true
+ break
+ }
+ t.intent("Output not found yet. Waiting ${waitTime}s...")
+ Thread.sleep(waitTime * 1000)
+}
+
+if (!outputFound) {
+ throw new RuntimeException("No output files found for UserScore after
${retries * waitTime} seconds.")
+}
+
+command_output_text = t.run "gsutil cat ${outputPath}* | grep
user19_BananaWallaby"
t.see "total_score: 231, user: user19_BananaWallaby", command_output_text
t.success("UserScore successfully run on DataflowRunner.")
t.run "gsutil rm
gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}*"
@@ -53,7 +72,24 @@ mobileGamingCommands = new MobileGamingCommands(testScripts:
t, testRunId: UUID.
t.intent("Running: HourlyTeamScore example with Beam GCP BOM on
DataflowRunner")
t.run(mobileGamingCommands.createPipelineCommand("HourlyTeamScore", runner))
-command_output_text = t.run "gsutil cat
gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*
| grep AzureBilby "
+
+outputPath =
"gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}"
+outputFound = false
+for (int i = 0; i < retries; i++) {
+ def files = t.run("gsutil ls ${outputPath}*")
+ if (files?.trim()) {
+ outputFound = true
+ break
+ }
+ t.intent("Output not found yet. Waiting ${waitTime}s...")
+ Thread.sleep(waitTime * 1000)
+}
+
+if (!outputFound) {
+ throw new RuntimeException("No output files found for HourlyTeamScore
after ${retries * waitTime} seconds.")
+}
+
+command_output_text = t.run "gsutil cat ${outputPath}* | grep AzureBilby "
t.see "total_score: 2788, team: AzureBilby", command_output_text
t.success("HourlyTeamScore successfully run on DataflowRunner.")
t.run "gsutil rm
gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*"
diff --git a/release/src/main/groovy/quickstart-java-dataflow.groovy
b/release/src/main/groovy/quickstart-java-dataflow.groovy
index 6364cdef1f3..fb84fedca0b 100644
--- a/release/src/main/groovy/quickstart-java-dataflow.groovy
+++ b/release/src/main/groovy/quickstart-java-dataflow.groovy
@@ -49,8 +49,26 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Dataflow'
--inputFile=gs://apache-beam-samples/shakespeare/*" \
-Pdataflow-runner"""
+ int retries = 5
+ int waitTime = 15 // seconds
+ def outputPath = "gs://${t.gcsBucket()}/count"
+ def outputFound = false
+ for (int i = 0; i < retries; i++) {
+ def files = t.run("gsutil ls ${outputPath}*")
+ if (files?.trim()) {
+ outputFound = true
+ break
+ }
+ t.intent("Output not found yet. Waiting ${waitTime}s...")
+ Thread.sleep(waitTime * 1000)
+ }
+
+ if (!outputFound) {
+ throw new RuntimeException("No output files found for WordCount after
${retries * waitTime} seconds.")
+ }
+
// Verify wordcount text
- String result = t.run """gsutil cat gs://${t.gcsBucket()}/count* | grep
Montague:"""
+ String result = t.run """gsutil cat ${outputPath}* | grep Montague:"""
t.see "Montague: 47", result
// Remove count files
