[
https://issues.apache.org/jira/browse/BEAM-3339?focusedWorklogId=92321&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92321
]
ASF GitHub Bot logged work on BEAM-3339:
----------------------------------------
Author: ASF GitHub Bot
Created on: 19/Apr/18 01:04
Start Date: 19/Apr/18 01:04
Worklog Time Spent: 10m
Work Description: tgroh closed pull request #4788: [BEAM-3339] Mobile
gaming automation for Java nightly snapshot on core runners
URL: https://github.com/apache/beam/pull/4788
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
index 59bc243680d..29c7ae11eff 100644
--- a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
@@ -48,14 +48,14 @@ job('beam_PostRelease_NightlySnapshot') {
// Allows triggering this build against pull requests.
common_job_properties.enablePhraseTriggeringFromPullRequest(
delegate,
- './gradlew :release:runQuickstartsJava',
+ './gradlew :release:runJavaExamplesValidationTask',
'Run Dataflow PostRelease')
steps {
// Run a quickstart from
https://beam.apache.org/get-started/quickstart-java
gradle {
rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':release:runQuickstartsJava')
+ tasks(':release:runJavaExamplesValidationTask')
common_job_properties.setGradleSwitches(delegate)
switches('-Pver=$snapshot_version -Prepourl=$snapshot_url')
}
diff --git a/build_rules.gradle b/build_rules.gradle
index 8f54fe0f478..cd8c162561e 100644
--- a/build_rules.gradle
+++ b/build_rules.gradle
@@ -977,26 +977,35 @@ ext.applyAvroNature = {
apply plugin: "com.commercehub.gradle.plugin.avro"
}
-// A class defining the set of configurable properties for
createJavaQuickstartValidationTask
-class JavaQuickstartConfiguration {
- // Name for the quickstart is required.
- // Used both for the test name runQuickstartJava${name}
- // and also for the script name, quickstart-java-${name}.toLowerCase().
- String name
-
- // gcpProject sets the gcpProject argument when executing the quickstart.
+// A class defining the set of configurable properties for
createJavaExamplesArchetypeValidationTask
+class JavaExamplesArchetypeValidationConfiguration {
+ // Type [Quickstart, MobileGaming] for the postrelease validation is
required.
+ // Used both for the test name run${type}Java${runner}
+ // and also for the script name, ${type}-java-${runner}.toLowerCase().
+ String type
+
+ // runner [Direct, Dataflow, Spark, Flink, FlinkLocal, Apex]
+ String runner
+
+ // gcpProject sets the gcpProject argument when executing examples.
String gcpProject
- // gcsBucket sets the gcsProject argument when executing the quickstart.
+ // gcsBucket sets the gcsProject argument when executing examples.
String gcsBucket
+
+ // bqDataset sets the BigQuery Dataset when executing mobile-gaming examples
+ String bqDataset
+
+ // pubsubTopic sets topics when executing streaming pipelines
+ String pubsubTopic
}
// Creates a task to run the quickstart for a runner.
// Releases version and URL, can be overriden for a RC release with
-// ./gradlew :release:runQuickstartJava -Pver=2.3.0
-Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-1027
-ext.createJavaQuickstartValidationTask = {
- JavaQuickstartConfiguration config = it as JavaQuickstartConfiguration
- def taskName = "runQuickstartJava${config.name}"
+// ./gradlew :release:runJavaExamplesValidationTask -Pver=2.3.0
-Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-1027
+ext.createJavaExamplesArchetypeValidationTask = {
+ JavaExamplesArchetypeValidationConfiguration config = it as
JavaExamplesArchetypeValidationConfiguration
+ def taskName = "run${config.type}Java${config.runner}"
println "Generating :${taskName}"
def releaseVersion = project.findProperty('ver') ?: version
def releaseRepo = project.findProperty('repourl') ?:
'https://repository.apache.org/content/repositories/snapshots'
@@ -1007,9 +1016,15 @@ ext.createJavaQuickstartValidationTask = {
if (config.gcsBucket) {
argsNeeded.add("--gcsBucket=${config.gcsBucket}")
}
+ if (config.bqDataset) {
+ argsNeeded.add("--bqDataset=${config.bqDataset}")
+ }
+ if (config.pubsubTopic) {
+ argsNeeded.add("--pubsubTopic=${config.pubsubTopic}")
+ }
project.evaluationDependsOn(':release')
task "${taskName}" (dependsOn: ':release:classes', type: JavaExec) {
- main = "quickstart-java-${config.name}".toLowerCase()
+ main = "${config.type}-java-${config.runner}".toLowerCase()
classpath = project(':release').sourceSets.main.runtimeClasspath
args argsNeeded
}
diff --git
a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java
b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java
index 626f677e846..3e1b9d30ea9 100644
---
a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java
+++
b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java
@@ -90,6 +90,8 @@
private static final int THREAD_SLEEP_MS = 500;
// Lists used to generate random team names.
+ // If COLORS is changed, please also make changes in
+ // release/src/main/groovy/MobileGamingCommands.COLORS
private static final ArrayList<String> COLORS =
new ArrayList<>(
Arrays.asList(
diff --git a/release/build.gradle b/release/build.gradle
index ebde4cf74da..3710b1a8f7b 100644
--- a/release/build.gradle
+++ b/release/build.gradle
@@ -27,10 +27,12 @@ dependencies {
compile 'commons-cli:commons-cli:1.2'
}
-task runQuickstartsJava {
+task runJavaExamplesValidationTask {
dependsOn ":beam-runners-direct-java:runQuickstartJavaDirect"
dependsOn
":beam-runners-google-cloud-dataflow-java:runQuickstartJavaDataflow"
dependsOn ":beam-runners-apex:runQuickstartJavaApex"
dependsOn ":beam-runners-spark:runQuickstartJavaSpark"
dependsOn ":beam-runners-flink_2.11:runQuickstartJavaFlinkLocal"
+ dependsOn ":beam-runners-direct-java:runMobileGamingJavaDirect"
+ dependsOn
":beam-runners-google-cloud-dataflow-java:runMobileGamingJavaDataflow"
}
diff --git a/release/src/main/groovy/MobileGamingCommands.groovy
b/release/src/main/groovy/MobileGamingCommands.groovy
new file mode 100644
index 00000000000..335738189f6
--- /dev/null
+++ b/release/src/main/groovy/MobileGamingCommands.groovy
@@ -0,0 +1,144 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+class MobileGamingCommands {
+
+ private TestScripts testScripts
+
+ public static final RUNNERS = [DirectRunner: "direct-runner",
+ DataflowRunner: "dataflow-runner",
+ SparkRunner: "spark-runner",
+ ApexRunner: "apex-runner",
+ FlinkRunner: "flink-runner"]
+
+ public static final EXECUTION_TIMEOUT_IN_MINUTES = 20
+
+ // Lists used to verify team names generated in the LeaderBoard example.
+ // This list should be kept sync with COLORS in
org.apache.beam.examples.complete.game.injector.Injector.
+ public static final COLORS = new ArrayList<>(Arrays.asList(
+ "Magenta",
+ "AliceBlue",
+ "Almond",
+ "Amaranth",
+ "Amber",
+ "Amethyst",
+ "AndroidGreen",
+ "AntiqueBrass",
+ "Fuchsia",
+ "Ruby",
+ "AppleGreen",
+ "Apricot",
+ "Aqua",
+ "ArmyGreen",
+ "Asparagus",
+ "Auburn",
+ "Azure",
+ "Banana",
+ "Beige",
+ "Bisque",
+ "BarnRed",
+ "BattleshipGrey"))
+
+ public String getUserScoreOutputName(String runner){
+ return "java-userscore-result-${RUNNERS[runner]}.txt"
+ }
+
+ public String getHourlyTeamScoreOutputName(String runner){
+ return "java-hourlyteamscore-result-${RUNNERS[runner]}.txt"
+ }
+
+ public String createPipelineCommand(String exampleName, String runner){
+ return """mvn compile exec:java -q \
+ -Dexec.mainClass=org.apache.beam.examples.complete.game.${exampleName} \
+ -Dexec.args=\"${getArgs(exampleName, runner)}\" \
+ -P${RUNNERS[runner]}"""
+ }
+
+ public String createInjectorCommand(){
+ return """mvn compile exec:java \
+
-Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \
+ -Dexec.args=\"${testScripts.gcpProject()} ${testScripts.pubsubTopic()}
none\""""
+ }
+
+
+ private String getArgs(String exampleName, String runner){
+ def args
+ switch (exampleName) {
+ case "UserScore":
+ args = getUserScoreArgs(runner)
+ break
+ case "HourlyTeamScore":
+ args = getHourlyTeamScoreArgs(runner)
+ break
+ case "LeaderBoard":
+ args = getLeaderBoardArgs(runner)
+ break
+ case "GameStats":
+ args = getGameStatsArgs(runner)
+ break
+ default:
+ testScripts.error("Cannot find example ${exampleName} in archetypes.")
+ }
+
+ StringBuilder exampleArgs = new
StringBuilder("--tempLocation=gs://${testScripts.gcsBucket()}/tmp
--runner=${runner} ")
+ args.each{argName, argValue ->
exampleArgs.append("--${argName}=${argValue} ")}
+ return exampleArgs
+ }
+
+ private Map getUserScoreArgs(String runner){
+ if(runner == "DataflowRunner"){
+ return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+ project: testScripts.gcpProject(),
+ output:
"gs://${testScripts.gcsBucket()}/${getUserScoreOutputName(runner)}"]
+ }
+ return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+ output: "${getUserScoreOutputName(runner)}"]
+ }
+
+ private Map getHourlyTeamScoreArgs(String runner){
+ if(runner == "DataflowRunner"){
+ return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+ project: testScripts.gcpProject(),
+ output:
"gs://${testScripts.gcsBucket()}/${getHourlyTeamScoreOutputName(runner)}"]
+ }
+ return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+ output: "${getHourlyTeamScoreOutputName(runner)}"]
+ }
+
+ private Map getLeaderBoardArgs(String runner){
+ return [project: testScripts.gcpProject(),
+ dataset: testScripts.bqDataset(),
+ topic:
"projects/${testScripts.gcpProject()}/topics/${testScripts.pubsubTopic()}",
+ output: "gs://${testScripts.gcsBucket()}/java-leaderboard-result.txt",
+ leaderBoardTableName: "leaderboard_${runner}",
+ teamWindowDuration: 5]
+ }
+
+ private Map getGameStatsArgs(String runner){
+ return [project: testScripts.gcpProject(),
+ dataset: testScripts.bqDataset(),
+ topic:
"projects/${testScripts.gcpProject()}/topics/${testScripts.pubsubTopic()}",
+ output: "gs://${testScripts.gcsBucket()}/java-leaderboard-result.txt",
+ fixedWindowDuration: 5,
+ userActivityWindowDuration: 5,
+ sessionGap: 1,
+ gameStatsTablePrefix: "gamestats_${runner}"]
+ }
+}
diff --git a/release/src/main/groovy/QuickstartArchetype.groovy
b/release/src/main/groovy/QuickstartArchetype.groovy
index c0119ee9816..eb3fa43a5a9 100644
--- a/release/src/main/groovy/QuickstartArchetype.groovy
+++ b/release/src/main/groovy/QuickstartArchetype.groovy
@@ -20,7 +20,7 @@
class QuickstartArchetype {
def static generate(TestScripts t) {
// Generate a maven project from the snapshot repository
- t.run """mvn archetype:generate \
+ String output_text = t.run """mvn archetype:generate \
-DarchetypeGroupId=org.apache.beam \
-DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
-DarchetypeVersion=${t.ver()} \
@@ -31,12 +31,14 @@ class QuickstartArchetype {
-DinteractiveMode=false"""
// Check if it was generated
- t.see "[INFO] BUILD SUCCESS"
+ t.see "[INFO] BUILD SUCCESS", output_text
t.run "cd word-count-beam"
- t.run "ls"
- t.see "pom.xml"
- t.see "src"
- t.run "ls src/main/java/org/apache/beam/examples/"
- t.see "WordCount.java"
+ output_text = t.run "ls"
+ t.see "pom.xml", output_text
+ t.see "src", output_text
+ String wordcounts = t.run "ls src/main/java/org/apache/beam/examples/"
+ t.see "WordCount.java", wordcounts
+ String games = t.run "ls
src/main/java/org/apache/beam/examples/complete/game/"
+ t.see "UserScore.java", games
}
}
diff --git a/release/src/main/groovy/TestScripts.groovy
b/release/src/main/groovy/TestScripts.groovy
index 914614d35a6..63743a216d4 100644
--- a/release/src/main/groovy/TestScripts.groovy
+++ b/release/src/main/groovy/TestScripts.groovy
@@ -29,11 +29,12 @@ class TestScripts {
class var {
static File startDir
static File curDir
- static String lastText
static String repoUrl
static String ver
static String gcpProject
static String gcsBucket
+ static String bqDataset
+ static String pubsubTopic
}
def TestScripts(String[] args) {
@@ -42,6 +43,9 @@ class TestScripts {
cli.repourl(args:1, 'Repository URL')
cli.gcpProject(args:1, 'Google Cloud Project')
cli.gcsBucket(args:1, 'Google Cloud Storage Bucket')
+ cli.bqDataset(args:1, "BigQuery Dataset")
+ cli.pubsubTopic(args:1, "PubSub Topic")
+
def options = cli.parse(args)
var.repoUrl = options.repourl
var.ver = options.ver
@@ -55,6 +59,14 @@ class TestScripts {
var.gcsBucket = options.gcsBucket
println "GCS Storage bucket: ${var.gcsBucket}"
}
+ if (options.bqDataset) {
+ var.bqDataset = options.bqDataset
+ println "BigQuery Dataset: ${var.bqDataset}"
+ }
+ if (options.pubsubTopic) {
+ var.pubsubTopic = options.pubsubTopic
+ println "PubSub Topic: ${var.pubsubTopic}"
+ }
}
def ver() {
@@ -69,42 +81,66 @@ class TestScripts {
return var.gcsBucket
}
+ def bqDataset() {
+ return var.bqDataset
+ }
+
+ def pubsubTopic() {
+ return var.pubsubTopic
+ }
+
// Both documents the overal scenario and creates a clean temp directory
def describe(String desc) {
var.startDir = File.createTempDir()
var.startDir.deleteOnExit()
var.curDir = var.startDir
- print "*****\n* Scenario: ${desc}\n*****\n"
+ print "**************************************\n* Scenario:
${desc}\n**************************************\n"
}
// Just document the intention of a set of steps
def intent(String desc) {
- print "\n*****\n* Test: ${desc}\n*****\n\n"
+ print "\n**************************************\n* Test:
${desc}\n**************************************\n\n"
}
+ def success(String desc) {
+ print "\n**************************************\n* SUCCESS:
${desc}\n**************************************\n\n"
+ }
// Run a command
- public void run(String cmd) {
+ public String run(String cmd) {
println cmd
if (cmd.startsWith("cd ")) {
_chdir(cmd.substring(3))
+ return ""
} else if (cmd.startsWith("mvn ")) {
- _mvn(cmd.substring(4))
+ return _mvn(cmd.substring(4))
} else {
- _execute(cmd)
+ return _execute(cmd)
}
}
- // Check for expected results in stdout of the last command
- public void see(String expected) {
- if (!var.lastText.contains(expected)) {
+ // Check for expected results in actual stdout from previous command, if
fails, log errors then exit.
+ public void see(String expected, String actual) {
+ if (!actual.contains(expected)) {
var.startDir.deleteDir()
- println "Cannot find ${expected} in ${var.lastText}"
- _error("Cannot find expected text")
+ println "Cannot find ${expected} in ${actual}"
+ error("Cannot find expected text")
}
println "Verified $expected"
}
+ // Check if there are one or more matches in stdout of the last command.
+ public boolean seeAnyOf(List<String> expecteds, String actual) {
+ for (String expected: expecteds) {
+ if(actual.contains(expected)) {
+ println "Verified $expected"
+ return true
+ }
+ }
+ println "Cannot find ${expecteds} in text"
+ return false
+ }
+
// Cleanup and print success
public void done() {
var.startDir.deleteDir()
@@ -113,37 +149,38 @@ class TestScripts {
}
// Run a single command, capture output, verify return code is 0
- private void _execute(String cmd) {
+ private String _execute(String cmd) {
def shell = "sh -c cmd".split(' ')
shell[2] = cmd
def pb = new ProcessBuilder(shell)
pb.directory(var.curDir)
pb.redirectErrorStream(true)
def proc = pb.start()
- var.lastText = ""
+ String output_text = ""
def text = StringBuilder.newInstance()
proc.inputStream.eachLine {
println it
text.append(it + "\n")
}
proc.waitFor()
- var.lastText = text.toString().trim()
+ output_text = text.toString().trim()
if (proc.exitValue() != 0) {
- println var.lastText
- _error("Failed command")
+ println output_text
+ error("Failed command")
}
+ return output_text
}
// Change directory
private void _chdir(String subdir) {
var.curDir = new File(var.curDir.absolutePath, subdir)
if (!var.curDir.exists()) {
- _error("Directory ${var.curDir} not found")
+ error("Directory ${var.curDir} not found")
}
}
// Run a maven command, setting up a new local repository and a
settings.xml with a custom repository
- private void _mvn(String args) {
+ private String _mvn(String args) {
def m2 = new File(var.startDir, ".m2/repository")
m2.mkdirs()
def settings = new File(var.startDir, "settings.xml")
@@ -171,11 +208,11 @@ class TestScripts {
println "Using maven ${maven_home}"
def mvnPath = "${maven_home}/bin"
def setPath = "export PATH=${mvnPath}:${path} && "
- _execute(setPath + cmd)
+ return _execute(setPath + cmd)
}
// Clean up and report error
- private void _error(String text) {
+ public void error(String text) {
var.startDir.deleteDir()
println "[ERROR] $text"
System.exit(1)
diff --git a/release/src/main/groovy/mobilegaming-java-dataflow.groovy
b/release/src/main/groovy/mobilegaming-java-dataflow.groovy
new file mode 100644
index 00000000000..1d721c86ef5
--- /dev/null
+++ b/release/src/main/groovy/mobilegaming-java-dataflow.groovy
@@ -0,0 +1,111 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+t = new TestScripts(args)
+mobileGamingCommands = new MobileGamingCommands(testScripts: t)
+
+/*
+ * Run the mobile game examples on Dataflow.
+ * https://beam.apache.org/get-started/mobile-gaming-example/
+ */
+
+t.describe ('Run Apache Beam Java SDK Mobile Gaming Examples - Dataflow')
+
+QuickstartArchetype.generate(t)
+
+def runner = "DataflowRunner"
+String command_output_text
+
+/**
+ * Run the UserScore example on DataflowRunner
+ * */
+
+t.intent("Running: UserScore example on DataflowRunner")
+t.run(mobileGamingCommands.createPipelineCommand("UserScore", runner))
+command_output_text = t.run "gsutil cat
gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}* |
grep user19_BananaWallaby"
+t.see "total_score: 231, user: user19_BananaWallaby", command_output_text
+t.success("UserScore successfully run on DataflowRunner.")
+t.run "gsutil rm
gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}*"
+
+
+/**
+ * Run the HourlyTeamScore example on DataflowRunner
+ * */
+
+t.intent("Running: HourlyTeamScore example on DataflowRunner")
+t.run(mobileGamingCommands.createPipelineCommand("HourlyTeamScore", runner))
+command_output_text = t.run "gsutil cat
gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*
| grep AzureBilby "
+t.see "total_score: 2788, team: AzureBilby", command_output_text
+t.success("HourlyTeamScore successfully run on DataflowRunner.")
+t.run "gsutil rm
gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*"
+
+
+/**
+ * Run the LeaderBoard example on DataflowRunner
+ * */
+
+t.intent("Running: LeaderBoard example on DataflowRunner")
+t.run("bq rm -f -t ${t.bqDataset()}.leaderboard_DataflowRunner_user")
+t.run("bq rm -f -t ${t.bqDataset()}.leaderboard_DataflowRunner_team")
+// It will take couple seconds to clean up tables.
+// This loop makes sure tables are completely deleted before running the
pipeline
+String tables = ""
+while({
+ sleep(3000)
+ tables = t.run ("bq query SELECT table_id FROM
${t.bqDataset()}.__TABLES_SUMMARY__")
+ tables.contains("leaderboard_${runner}_user") ||
tables.contains("leaderboard_${runner}_team")
+}());
+
+def InjectorThread = Thread.start() {
+ t.run(mobileGamingCommands.createInjectorCommand())
+}
+
+def LeaderBoardThread = Thread.start() {
+ t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner))
+}
+
+t.run("gcloud dataflow jobs list | grep pyflow-wordstream-candidate | grep
Running | cut -d' ' -f1")
+
+
+// verify outputs in BQ tables
+def startTime = System.currentTimeMillis()
+def isSuccess = false
+String query_result = ""
+while((System.currentTimeMillis() - startTime)/60000 <
mobileGamingCommands.EXECUTION_TIMEOUT_IN_MINUTES) {
+ tables = t.run "bq query SELECT table_id FROM
${t.bqDataset()}.__TABLES_SUMMARY__"
+ if(tables.contains("leaderboard_${runner}_user") &&
tables.contains("leaderboard_${runner}_team")){
+ query_result = t.run """bq query --batch "SELECT user FROM
[${t.gcpProject()}:${t.bqDataset()}.leaderboard_${runner}_user] LIMIT 10\""""
+ if(t.seeAnyOf(mobileGamingCommands.COLORS, query_result)){
+ isSuccess = true
+ break
+ }
+ }
+ println "Waiting for pipeline to produce more results..."
+ sleep(60000) // wait for 1 min
+}
+InjectorThread.stop()
+LeaderBoardThread.stop()
+t.run("gcloud dataflow jobs cancel \$(gcloud dataflow jobs list | grep
leaderboard-jenkins | grep Running | cut -d' ' -f1)")
+
+if(!isSuccess){
+ t.error("FAILED: Failed running LeaderBoard on DataflowRunner")
+}
+t.success("LeaderBoard successfully run on DataflowRunner.")
+
+t.done()
diff --git a/release/src/main/groovy/mobilegaming-java-direct.groovy
b/release/src/main/groovy/mobilegaming-java-direct.groovy
new file mode 100644
index 00000000000..b402ae9163a
--- /dev/null
+++ b/release/src/main/groovy/mobilegaming-java-direct.groovy
@@ -0,0 +1,105 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+t = new TestScripts(args)
+mobileGamingCommands = new MobileGamingCommands(testScripts: t)
+
+/*
+ * Run the mobile game examples on DirectRunner.
+ * https://beam.apache.org/get-started/mobile-gaming-example/
+ */
+
+t.describe ('Run Apache Beam Java SDK Mobile Gaming Examples - Direct')
+
+QuickstartArchetype.generate(t)
+
+def runner = "DirectRunner"
+String command_output_text
+
+/**
+ * Run the UserScore example with DirectRunner
+ * */
+
+t.intent("Running: UserScore example on DirectRunner")
+t.run(mobileGamingCommands.createPipelineCommand("UserScore", runner))
+command_output_text = t.run "grep user19_BananaWallaby
${mobileGamingCommands.getUserScoreOutputName(runner)}* "
+t.see "total_score: 231, user: user19_BananaWallaby", command_output_text
+t.success("UserScore successfully run on DirectRunners.")
+
+
+/**
+ * Run the HourlyTeamScore example with DirectRunner
+ * */
+
+t.intent("Running: HourlyTeamScore example on DirectRunner")
+t.run(mobileGamingCommands.createPipelineCommand("HourlyTeamScore", runner))
+command_output_text = t.run "grep AzureBilby
${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}* "
+t.see "total_score: 2788, team: AzureBilby", command_output_text
+t.success("HourlyTeamScore successfully run on DirectRunners.")
+
+
+/**
+ * Run the LeaderBoard example with DirectRunner
+ * */
+
+t.intent("Running: LeaderBoard example on DirectRunner")
+t.run("bq rm -f -t ${t.bqDataset()}.leaderboard_DirectRunner_user")
+t.run("bq rm -f -t ${t.bqDataset()}.leaderboard_DirectRunner_team")
+// It will take couple seconds to clean up tables.
+// This loop makes sure tables are completely deleted before running the
pipeline
+String tables = ""
+while({
+ sleep(3000)
+ tables = t.run ("bq query SELECT table_id FROM
${t.bqDataset()}.__TABLES_SUMMARY__")
+ tables.contains("leaderboard_${runner}_user") ||
tables.contains("leaderboard_${runner}_team")
+}());
+
+def InjectorThread = Thread.start() {
+ t.run(mobileGamingCommands.createInjectorCommand())
+}
+
+def LeaderBoardThread = Thread.start() {
+ t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner))
+}
+
+// verify outputs in BQ tables
+def startTime = System.currentTimeMillis()
+def isSuccess = false
+String query_result = ""
+while((System.currentTimeMillis() - startTime)/60000 <
mobileGamingCommands.EXECUTION_TIMEOUT_IN_MINUTES) {
+ tables = t.run "bq query SELECT table_id FROM
${t.bqDataset()}.__TABLES_SUMMARY__"
+ if(tables.contains("leaderboard_${runner}_user") &&
tables.contains("leaderboard_${runner}_team")){
+ query_result = t.run """bq query --batch "SELECT user FROM
[${t.gcpProject()}:${t.bqDataset()}.leaderboard_${runner}_user] LIMIT 10\""""
+ if(t.seeAnyOf(mobileGamingCommands.COLORS, query_result)){
+ isSuccess = true
+ break
+ }
+ }
+ println "Waiting for pipeline to produce more results..."
+ sleep(60000) // wait for 1 min
+}
+InjectorThread.stop()
+LeaderBoardThread.stop()
+
+if(!isSuccess){
+ t.error("FAILED: Failed running LeaderBoard on DirectRunner")
+}
+t.success("LeaderBoard successfully run on DirectRunner.")
+
+t.done()
diff --git a/release/src/main/groovy/quickstart-java-apex.groovy
b/release/src/main/groovy/quickstart-java-apex.groovy
index d220ca24cf3..3e8e00521a0 100644
--- a/release/src/main/groovy/quickstart-java-apex.groovy
+++ b/release/src/main/groovy/quickstart-java-apex.groovy
@@ -30,7 +30,7 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Apex'
t.intent 'Runs the WordCount Code with Apex runner'
// Run the wordcount example with the apex runner
- t.run """mvn compile exec:java \
+ t.run """mvn compile exec:java -q \
-Dexec.mainClass=org.apache.beam.examples.WordCount \
-Dexec.args="--inputFile=pom.xml \
--output=counts \
@@ -38,8 +38,8 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Apex'
-Papex-runner"""
// Verify text from the pom.xml input file
- t.run "grep Foundation counts*"
- t.see "Foundation: 1"
+ String result = t.run "grep Foundation counts*"
+ t.see "Foundation: 1", result
// Clean up
t.done()
diff --git a/release/src/main/groovy/quickstart-java-dataflow.groovy
b/release/src/main/groovy/quickstart-java-dataflow.groovy
index 80a9a06a68c..7dadff02d32 100644
--- a/release/src/main/groovy/quickstart-java-dataflow.groovy
+++ b/release/src/main/groovy/quickstart-java-dataflow.groovy
@@ -34,7 +34,7 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Dataflow'
t.run """gsutil rm gs://${t.gcsBucket()}/count* || echo 'No files'"""
// Run the wordcount example with the Dataflow runner
- t.run """mvn compile exec:java \
+ t.run """mvn compile exec:java -q \
-Dexec.mainClass=org.apache.beam.examples.WordCount \
-Dexec.args="--runner=DataflowRunner \
--project=${t.gcpProject()} \
@@ -44,8 +44,8 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Dataflow'
-Pdataflow-runner"""
// Verify wordcount text
- t.run """gsutil cat gs://${t.gcsBucket()}/count* | grep Montague:"""
- t.see "Montague: 47"
+ String result = t.run """gsutil cat gs://${t.gcsBucket()}/count* | grep
Montague:"""
+ t.see "Montague: 47", result
// Remove count files
t.run """gsutil rm gs://${t.gcsBucket()}/count*"""
diff --git a/release/src/main/groovy/quickstart-java-direct.groovy
b/release/src/main/groovy/quickstart-java-direct.groovy
index 948b6e871c2..33023b393ca 100644
--- a/release/src/main/groovy/quickstart-java-direct.groovy
+++ b/release/src/main/groovy/quickstart-java-direct.groovy
@@ -30,14 +30,14 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Direct'
t.intent 'Runs the WordCount Code with Direct runner'
// Run the wordcount example with the Direct runner
- t.run """mvn compile exec:java \
+ t.run """mvn compile exec:java -q \
-Dexec.mainClass=org.apache.beam.examples.WordCount \
-Dexec.args="--inputFile=pom.xml --output=counts" \
-Pdirect-runner"""
// Verify text from the pom.xml input file
- t.run "grep Foundation counts*"
- t.see "Foundation: 1"
+ String result = t.run "grep Foundation counts*"
+ t.see "Foundation: 1", result
// Clean up
t.done()
diff --git a/release/src/main/groovy/quickstart-java-flinklocal.groovy
b/release/src/main/groovy/quickstart-java-flinklocal.groovy
index 9da19b0e565..d8c3b0a3719 100644
--- a/release/src/main/groovy/quickstart-java-flinklocal.groovy
+++ b/release/src/main/groovy/quickstart-java-flinklocal.groovy
@@ -30,14 +30,14 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Flink
Local'
t.intent 'Runs the WordCount Code with Flink Local runner'
// Run the wordcount example with the flink local runner
- t.run """mvn compile exec:java \
+ t.run """mvn compile exec:java -q \
-Dexec.mainClass=org.apache.beam.examples.WordCount \
-Dexec.args="--inputFile=pom.xml --output=counts \
--runner=FlinkRunner" -Pflink-runner"""
// Verify text from the pom.xml input file
- t.run "grep Foundation counts*"
- t.see "Foundation: 1"
+ String result = t.run "grep Foundation counts*"
+ t.see "Foundation: 1", result
// Clean up
t.done()
diff --git a/release/src/main/groovy/quickstart-java-spark.groovy
b/release/src/main/groovy/quickstart-java-spark.groovy
index 671a4b1ee7d..3c5be754daa 100644
--- a/release/src/main/groovy/quickstart-java-spark.groovy
+++ b/release/src/main/groovy/quickstart-java-spark.groovy
@@ -30,14 +30,14 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Spark'
t.intent 'Runs the WordCount Code with Spark runner'
// Run the wordcount example with the spark runner
- t.run """mvn compile exec:java \
+ t.run """mvn compile exec:java -q \
-Dexec.mainClass=org.apache.beam.examples.WordCount \
-Dexec.args="--inputFile=pom.xml --output=counts \
--runner=SparkRunner" -Pspark-runner"""
// Verify text from the pom.xml input file
- t.run "grep Foundation counts*"
- t.see "Foundation: 1"
+ String result = t.run "grep Foundation counts*"
+ t.see "Foundation: 1", result
// Clean up
t.done()
diff --git a/runners/apex/build.gradle b/runners/apex/build.gradle
index 077a577ef53..8bcc30401fc 100644
--- a/runners/apex/build.gradle
+++ b/runners/apex/build.gradle
@@ -109,4 +109,4 @@ task validatesRunner {
}
// Generates :beam-runners-apex:runQuickstartJavaApex
-createJavaQuickstartValidationTask(name: 'Apex')
+createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner:'Apex')
diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle
index fd99721378a..5b36cdbba87 100644
--- a/runners/direct-java/build.gradle
+++ b/runners/direct-java/build.gradle
@@ -97,5 +97,18 @@ task needsRunnerTests(type: Test) {
}
}
+def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
+def gcsBucket = project.findProperty('gcsBucket') ?:
'temp-storage-for-release-validation-tests/nightly-snapshot-validation'
+def bqDataset = project.findProperty('bqDataset') ?:
'beam_postrelease_mobile_gaming'
+def pubsubTopic = project.findProperty('pubsubTopic') ?:
'java_mobile_gaming_topic'
+
// Generates :beam-runners-direct-java:runQuickstartJavaDirect
-createJavaQuickstartValidationTask(name: 'Direct')
+createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Direct')
+
+// Generates :beam-runners-direct-java:runMobileGamingJavaDirect
+createJavaExamplesArchetypeValidationTask(type: 'MobileGaming',
+ runner: 'Direct',
+ gcpProject: gcpProject,
+ gcsBucket: gcsBucket,
+ bqDataset: bqDataset,
+ pubsubTopic: pubsubTopic)
diff --git a/runners/flink/build.gradle b/runners/flink/build.gradle
index e7010ef5dcc..ea8bb62311f 100644
--- a/runners/flink/build.gradle
+++ b/runners/flink/build.gradle
@@ -132,4 +132,4 @@ task validatesRunner {
}
// Generates :beam-runners-flink_2.11:runQuickstartJavaFlinkLocal
-createJavaQuickstartValidationTask(name: 'FlinkLocal')
+createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner:
'FlinkLocal')
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 059069c2c82..f35c15263f9 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -136,11 +136,6 @@ task validatesRunner {
dependsOn validatesRunnerTest
}
-// Generates :beam-runners-google-cloud-dataflow-java:runQuickstartJavaDataflow
-def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
-def gcsBucket = project.findProperty('gcsBucket') ?:
'temp-storage-for-release-validation-tests/quickstart'
-createJavaQuickstartValidationTask(name: 'Dataflow', gcpProject: gcpProject,
gcsBucket: gcsBucket)
-
task googleCloudPlatformIntegrationTest(type: Test) {
group = "Verification"
def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
@@ -185,3 +180,22 @@ task postCommit {
dependsOn googleCloudPlatformIntegrationTest
dependsOn examplesJavaIntegrationTest
}
+
+def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
+def gcsBucket = project.findProperty('gcsBucket') ?:
'temp-storage-for-release-validation-tests/nightly-snapshot-validation'
+def bqDataset = project.findProperty('bqDataset') ?:
'beam_postrelease_mobile_gaming'
+def pubsubTopic = project.findProperty('pubsubTopic') ?:
'java_mobile_gaming_topic'
+
+// Generates :beam-runners-google-cloud-dataflow-java:runQuickstartJavaDataflow
+createJavaExamplesArchetypeValidationTask(type: 'Quickstart',
+ runner: 'Dataflow',
+ gcpProject: gcpProject,
+ gcsBucket: gcsBucket)
+
+// Generates
:beam-runners-google-cloud-dataflow-java:runMobileGamingJavaDataflow
+createJavaExamplesArchetypeValidationTask(type: 'MobileGaming',
+ runner: 'Dataflow',
+ gcpProject: gcpProject,
+ gcsBucket: gcsBucket,
+ bqDataset: bqDataset,
+ pubsubTopic: pubsubTopic)
diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle
index 6634be99c36..a4bc95e4831 100644
--- a/runners/spark/build.gradle
+++ b/runners/spark/build.gradle
@@ -161,4 +161,4 @@ task validatesRunner {
}
// Generates :beam-runners-spark:runQuickstartJavaSpark
-createJavaQuickstartValidationTask(name: 'Spark')
+createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Spark')
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 92321)
Time Spent: 93.5h (was: 93h 20m)
> Create post-release testing of the nightly snapshots
> ----------------------------------------------------
>
> Key: BEAM-3339
> URL: https://issues.apache.org/jira/browse/BEAM-3339
> Project: Beam
> Issue Type: Improvement
> Components: testing
> Reporter: Alan Myrvold
> Assignee: Jason Kuster
> Priority: Major
> Time Spent: 93.5h
> Remaining Estimate: 0h
>
> The nightly java snapshots in
> https://repository.apache.org/content/groups/snapshots/org/apache/beam should
> be verified by following the
> https://beam.apache.org/get-started/quickstart-java/ instructions, to verify
> that the release is usable.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
