This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 951a6dd [BEAM-3339] Fix failing post-release test by running groovy
from gradle, instead of as a command line + add apex, flink, and spark local
tests
new 54c39cd [BEAM-3339] Fix failing post-release test by running groovy
from gradle, instead of as a command line
951a6dd is described below
commit 951a6dd7d7a52b6ef517ebd3cd9bb420ed00b745
Author: Alan Myrvold <alan@Alans-MacBook.local>
AuthorDate: Wed Feb 7 21:13:48 2018 -0800
[BEAM-3339] Fix failing post-release test by running groovy from gradle,
instead of as a command line + add apex, flink, and spark local tests
---
.../job_beam_PostRelease_NightlySnapshot.groovy | 22 +++--
build_rules.gradle | 40 +++++++++-
release/build.gradle | 36 +++++++++
.../main/groovy/QuickstartArchetype.groovy} | 32 ++------
release/{ => src/main/groovy}/TestScripts.groovy | 93 ++++++++++++++++------
.../src/main/groovy/quickstart-java-apex.groovy | 45 +++++++++++
.../main/groovy/quickstart-java-dataflow.groovy | 54 +++++++++++++
.../main/groovy}/quickstart-java-direct.groovy | 31 ++------
.../main/groovy/quickstart-java-flinklocal.groovy | 43 ++++++++++
.../src/main/groovy/quickstart-java-spark.groovy | 43 ++++++++++
runners/apex/build.gradle | 3 +
runners/direct-java/build.gradle | 3 +
runners/flink/build.gradle | 3 +
runners/google-cloud-dataflow-java/build.gradle | 5 ++
runners/spark/build.gradle | 3 +
settings.gradle | 1 +
16 files changed, 372 insertions(+), 85 deletions(-)
diff --git a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
index 60abf9e..1da9d2c 100644
--- a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
@@ -31,10 +31,10 @@ job('beam_PostRelease_NightlySnapshot') {
parameters {
stringParam('snapshot_version',
- '2.3.0-SNAPSHOT',
+ '',
'Version of the repository snapshot to install')
stringParam('snapshot_url',
- 'https://repository.apache.org/content/repositories/snapshots',
+ '',
'Repository URL to install from')
}
@@ -42,11 +42,21 @@ job('beam_PostRelease_NightlySnapshot') {
common_job_properties.setPostCommit(
delegate,
'0 11 * * *',
- false,
- 'd...@beam.apache.org')
+ false)
+
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ './gradlew :release:runQuickstartsJava',
+ 'Run Dataflow PostRelease')
steps {
- // Run a quickstart from
https://beam.apache.org/get-started/quickstart-java/
- shell('cd ' + common_job_properties.checkoutDir + '/release && groovy
quickstart-java-direct.groovy')
+ // Run a quickstart from
https://beam.apache.org/get-started/quickstart-java
+ gradle {
+ rootBuildScriptDir(common_job_properties.checkoutDir)
+ tasks(':release:runQuickstartsJava')
+ switches('-Pver=$snapshot_version -Prepourl=$snapshot_url')
+ }
}
}
diff --git a/build_rules.gradle b/build_rules.gradle
index f7df03e..a8e521a 100644
--- a/build_rules.gradle
+++ b/build_rules.gradle
@@ -38,7 +38,7 @@ println "Applying build_rules.gradle to $project.name"
// We use the project.path as the group name to make this mapping unique since
// we have a few projects with the same name.
group = project.path
-version = "2.3.0-SNAPSHOT"
+version = "2.4.0-SNAPSHOT"
// Define the default set of repositories for all builds.
repositories {
@@ -484,3 +484,41 @@ ext.applyAvroNature = {
println "applyAvroNature with " + (it ? "$it" : "default configuration") + "
for project $project.name"
apply plugin: "com.commercehub.gradle.plugin.avro"
}
+
+// A class defining the set of configurable properties for
createJavaQuickstartValidationTask
+class JavaQuickstartConfiguration {
+ // Name for the quickstart is required.
+ // Used both for the test name runQuickstartJava${name}
+ // and also for the script name, quickstart-java-${name}.toLowerCase().
+ String name
+
+ // gcpProject sets the gcpProject argument when executing the quickstart.
+ String gcpProject
+
+ // gcsBucket sets the gcsProject argument when executing the quickstart.
+ String gcsBucket
+}
+
+// Creates a task to run the quickstart for a runner.
+// Releases version and URL, can be overriden for a RC release with
+// ./gradlew :release:runQuickstartJava -Pver=2.3.0
-Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-1027
+ext.createJavaQuickstartValidationTask = {
+ JavaQuickstartConfiguration config = it as JavaQuickstartConfiguration
+ def taskName = "runQuickstartJava${config.name}"
+ println "Generating :${taskName}"
+ def releaseVersion = project.findProperty('ver') ?: version
+ def releaseRepo = project.findProperty('repourl') ?:
'https://repository.apache.org/content/repositories/snapshots'
+ def argsNeeded = ["--ver=${releaseVersion}", "--repourl=${releaseRepo}"]
+ if (config.gcpProject) {
+ argsNeeded.add("--gcpProject=${config.gcpProject}")
+ }
+ if (config.gcsBucket) {
+ argsNeeded.add("--gcsBucket=${config.gcsBucket}")
+ }
+ project.evaluationDependsOn(':release')
+ task "${taskName}" (dependsOn: ':release:classes', type: JavaExec) {
+ main = "quickstart-java-${config.name}".toLowerCase()
+ classpath = project(':release').sourceSets.main.runtimeClasspath
+ args argsNeeded
+ }
+}
diff --git a/release/build.gradle b/release/build.gradle
new file mode 100644
index 0000000..95b3170
--- /dev/null
+++ b/release/build.gradle
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'groovy'
+
+repositories {
+ mavenCentral()
+}
+
+dependencies {
+ compile 'org.codehaus.groovy:groovy-all:2.4.13'
+ compile 'commons-cli:commons-cli:1.2'
+}
+
+task runQuickstartsJava {
+ dependsOn ":runners:direct-java:runQuickstartJavaDirect"
+ dependsOn ":runners:google-cloud-dataflow-java:runQuickstartJavaDataflow"
+ dependsOn ":runners:apex:runQuickstartJavaApex"
+ dependsOn ":runners:spark:runQuickstartJavaSpark"
+ dependsOn ":runners:flink:runQuickstartJavaFlinkLocal"
+}
diff --git a/release/quickstart-java-direct.groovy
b/release/src/main/groovy/QuickstartArchetype.groovy
similarity index 65%
copy from release/quickstart-java-direct.groovy
copy to release/src/main/groovy/QuickstartArchetype.groovy
index 2ac1558..c0119ee 100644
--- a/release/quickstart-java-direct.groovy
+++ b/release/src/main/groovy/QuickstartArchetype.groovy
@@ -17,22 +17,13 @@
* limitations under the License.
*/
-t = new TestScripts()
-
-/*
- * Run the direct quickstart from
https://beam.apache.org/get-started/quickstart-java/
- */
-
-t.describe 'Run Apache Beam Java SDK Quickstart - Direct'
-
- t.intent 'Gets the WordCount Code'
- ver = System.env.snapshot_version ?: "2.3.0-SNAPSHOT"
-
+class QuickstartArchetype {
+ def static generate(TestScripts t) {
// Generate a maven project from the snapshot repository
t.run """mvn archetype:generate \
-DarchetypeGroupId=org.apache.beam \
-DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
- -DarchetypeVersion=$ver \
+ -DarchetypeVersion=${t.ver()} \
-DgroupId=org.example \
-DartifactId=word-count-beam \
-Dversion="0.1" \
@@ -47,18 +38,5 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Direct'
t.see "src"
t.run "ls src/main/java/org/apache/beam/examples/"
t.see "WordCount.java"
-
- t.intent 'Runs the WordCount Code with Direct runner'
-
- // Run the workcount example with the direct runner
- t.run """mvn compile exec:java \
- -Dexec.mainClass=org.apache.beam.examples.WordCount \
- -Dexec.args="--inputFile=pom.xml --output=counts" \
- -Pdirect-runner"""
-
- // Verify text from the pom.xml input file
- t.run "grep Foundation counts*"
- t.see "Foundation: 1"
-
- // Clean up
- t.done()
+ }
+}
diff --git a/release/TestScripts.groovy
b/release/src/main/groovy/TestScripts.groovy
similarity index 65%
rename from release/TestScripts.groovy
rename to release/src/main/groovy/TestScripts.groovy
index d8fdcec..914614d 100644
--- a/release/TestScripts.groovy
+++ b/release/src/main/groovy/TestScripts.groovy
@@ -17,19 +17,58 @@
* limitations under the License.
*/
+import groovy.util.CliBuilder
+
/*
* Scripting functions to make writing a test similar to the quickstart
* instructions from https://beam.apache.org/get-started/quickstart-java/
*/
class TestScripts {
-
+
// Global state to maintain when running the steps
class var {
- static File startDir
+ static File startDir
static File curDir
static String lastText
+ static String repoUrl
+ static String ver
+ static String gcpProject
+ static String gcsBucket
+ }
+
+ def TestScripts(String[] args) {
+ def cli = new CliBuilder()
+ cli.ver(args:1, 'SDL Version')
+ cli.repourl(args:1, 'Repository URL')
+ cli.gcpProject(args:1, 'Google Cloud Project')
+ cli.gcsBucket(args:1, 'Google Cloud Storage Bucket')
+ def options = cli.parse(args)
+ var.repoUrl = options.repourl
+ var.ver = options.ver
+ println "Repository URL: ${var.repoUrl}"
+ println "Version: ${var.ver}"
+ if (options.gcpProject) {
+ var.gcpProject = options.gcpProject
+ println "GCS Project: ${var.gcpProject}"
+ }
+ if (options.gcsBucket) {
+ var.gcsBucket = options.gcsBucket
+ println "GCS Storage bucket: ${var.gcsBucket}"
+ }
+ }
+
+ def ver() {
+ return var.ver
+ }
+
+ def gcpProject() {
+ return var.gcpProject
+ }
+
+ def gcsBucket() {
+ return var.gcsBucket
}
-
+
// Both documents the overal scenario and creates a clean temp directory
def describe(String desc) {
var.startDir = File.createTempDir()
@@ -37,15 +76,16 @@ class TestScripts {
var.curDir = var.startDir
print "*****\n* Scenario: ${desc}\n*****\n"
}
-
+
// Just document the intention of a set of steps
- def intent(String desc) {
+ def intent(String desc) {
print "\n*****\n* Test: ${desc}\n*****\n\n"
}
-
-
+
+
// Run a command
public void run(String cmd) {
+ println cmd
if (cmd.startsWith("cd ")) {
_chdir(cmd.substring(3))
} else if (cmd.startsWith("mvn ")) {
@@ -54,7 +94,7 @@ class TestScripts {
_execute(cmd)
}
}
-
+
// Check for expected results in stdout of the last command
public void see(String expected) {
if (!var.lastText.contains(expected)) {
@@ -64,7 +104,7 @@ class TestScripts {
}
println "Verified $expected"
}
-
+
// Cleanup and print success
public void done() {
var.startDir.deleteDir()
@@ -78,57 +118,60 @@ class TestScripts {
shell[2] = cmd
def pb = new ProcessBuilder(shell)
pb.directory(var.curDir)
+ pb.redirectErrorStream(true)
def proc = pb.start()
var.lastText = ""
+ def text = StringBuilder.newInstance()
proc.inputStream.eachLine {
println it
- var.lastText += it + "\n";
+ text.append(it + "\n")
}
- var.lastText = var.lastText.trim()
proc.waitFor()
+ var.lastText = text.toString().trim()
if (proc.exitValue() != 0) {
println var.lastText
_error("Failed command")
}
}
-
+
// Change directory
private void _chdir(String subdir) {
var.curDir = new File(var.curDir.absolutePath, subdir)
if (!var.curDir.exists()) {
_error("Directory ${var.curDir} not found")
}
- _execute("pwd")
- if (var.lastText != var.curDir.absolutePath) {
- _error("Directory mismatch, ${var.lastText} !=
${var.curDir.absolutePath}")
-
- }
}
-
- // Run a maven command, setting up a new local repository and a
settings.xml with the snapshot repository
+
+ // Run a maven command, setting up a new local repository and a
settings.xml with a custom repository
private void _mvn(String args) {
def m2 = new File(var.startDir, ".m2/repository")
m2.mkdirs()
def settings = new File(var.startDir, "settings.xml")
- def repo = System.env.snapshot_repository ?:
"https://repository.apache.org/content/repositories/snapshots";
settings.write """
<settings>
<localRepository>${m2.absolutePath}</localRepository>
<profiles>
<profile>
- <id>snapshot</id>
+ <id>testrel</id>
<repositories>
<repository>
- <id>apache.snapshots</id>
- <url>${repo}</url>
+ <id>test.release</id>
+ <url>${var.repoUrl}</url>
</repository>
</repositories>
</profile>
</profiles>
</settings>
"""
- def cmd = "mvn ${args} -s${settings.absolutePath} -Psnapshot -B"
- _execute(cmd)
+ def cmd = "mvn ${args} -s ${settings.absolutePath} -Ptestrel -B"
+ String path = System.getenv("PATH");
+ // Set the path on jenkins executors to use a recent maven
+ // MAVEN_HOME is not set on some executors, so default to 3.5.2
+ String maven_home = System.getenv("MAVEN_HOME") ?:
'/home/jenkins/tools/maven/apache-maven-3.5.2'
+ println "Using maven ${maven_home}"
+ def mvnPath = "${maven_home}/bin"
+ def setPath = "export PATH=${mvnPath}:${path} && "
+ _execute(setPath + cmd)
}
// Clean up and report error
diff --git a/release/src/main/groovy/quickstart-java-apex.groovy
b/release/src/main/groovy/quickstart-java-apex.groovy
new file mode 100644
index 0000000..d220ca2
--- /dev/null
+++ b/release/src/main/groovy/quickstart-java-apex.groovy
@@ -0,0 +1,45 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+t = new TestScripts(args)
+
+/*
+ * Run the Apex quickstart from
https://beam.apache.org/get-started/quickstart-java/
+ */
+
+t.describe 'Run Apache Beam Java SDK Quickstart - Apex'
+
+ t.intent 'Gets the WordCount Example Code'
+ QuickstartArchetype.generate(t)
+
+ t.intent 'Runs the WordCount Code with Apex runner'
+ // Run the wordcount example with the apex runner
+ t.run """mvn compile exec:java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--inputFile=pom.xml \
+ --output=counts \
+ --runner=ApexRunner" \
+ -Papex-runner"""
+
+ // Verify text from the pom.xml input file
+ t.run "grep Foundation counts*"
+ t.see "Foundation: 1"
+
+ // Clean up
+ t.done()
diff --git a/release/src/main/groovy/quickstart-java-dataflow.groovy
b/release/src/main/groovy/quickstart-java-dataflow.groovy
new file mode 100644
index 0000000..80a9a06
--- /dev/null
+++ b/release/src/main/groovy/quickstart-java-dataflow.groovy
@@ -0,0 +1,54 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+t = new TestScripts(args)
+
+/*
+ * Run the Dataflow quickstart from
https://beam.apache.org/get-started/quickstart-java/
+ */
+
+t.describe 'Run Apache Beam Java SDK Quickstart - Dataflow'
+
+ t.intent 'Gets the WordCount Example Code'
+ QuickstartArchetype.generate(t)
+
+ t.intent 'Runs the WordCount Code with Dataflow runner'
+
+ // Remove any count files
+ t.run """gsutil rm gs://${t.gcsBucket()}/count* || echo 'No files'"""
+
+ // Run the wordcount example with the Dataflow runner
+ t.run """mvn compile exec:java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--runner=DataflowRunner \
+ --project=${t.gcpProject()} \
+ --gcpTempLocation=gs://${t.gcsBucket()}/tmp \
+ --output=gs://${t.gcsBucket()}/counts \
+ --inputFile=gs://apache-beam-samples/shakespeare/*" \
+ -Pdataflow-runner"""
+
+ // Verify wordcount text
+ t.run """gsutil cat gs://${t.gcsBucket()}/count* | grep Montague:"""
+ t.see "Montague: 47"
+
+ // Remove count files
+ t.run """gsutil rm gs://${t.gcsBucket()}/count*"""
+
+ // Clean up
+ t.done()
diff --git a/release/quickstart-java-direct.groovy
b/release/src/main/groovy/quickstart-java-direct.groovy
similarity index 59%
rename from release/quickstart-java-direct.groovy
rename to release/src/main/groovy/quickstart-java-direct.groovy
index 2ac1558..948b6e8 100644
--- a/release/quickstart-java-direct.groovy
+++ b/release/src/main/groovy/quickstart-java-direct.groovy
@@ -17,40 +17,19 @@
* limitations under the License.
*/
-t = new TestScripts()
+t = new TestScripts(args)
/*
- * Run the direct quickstart from
https://beam.apache.org/get-started/quickstart-java/
+ * Run the Direct quickstart from
https://beam.apache.org/get-started/quickstart-java/
*/
t.describe 'Run Apache Beam Java SDK Quickstart - Direct'
- t.intent 'Gets the WordCount Code'
- ver = System.env.snapshot_version ?: "2.3.0-SNAPSHOT"
-
- // Generate a maven project from the snapshot repository
- t.run """mvn archetype:generate \
- -DarchetypeGroupId=org.apache.beam \
- -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
- -DarchetypeVersion=$ver \
- -DgroupId=org.example \
- -DartifactId=word-count-beam \
- -Dversion="0.1" \
- -Dpackage=org.apache.beam.examples \
- -DinteractiveMode=false"""
-
- // Check if it was generated
- t.see "[INFO] BUILD SUCCESS"
- t.run "cd word-count-beam"
- t.run "ls"
- t.see "pom.xml"
- t.see "src"
- t.run "ls src/main/java/org/apache/beam/examples/"
- t.see "WordCount.java"
+ t.intent 'Gets the WordCount Example Code'
+ QuickstartArchetype.generate(t)
t.intent 'Runs the WordCount Code with Direct runner'
-
- // Run the workcount example with the direct runner
+ // Run the wordcount example with the Direct runner
t.run """mvn compile exec:java \
-Dexec.mainClass=org.apache.beam.examples.WordCount \
-Dexec.args="--inputFile=pom.xml --output=counts" \
diff --git a/release/src/main/groovy/quickstart-java-flinklocal.groovy
b/release/src/main/groovy/quickstart-java-flinklocal.groovy
new file mode 100644
index 0000000..9da19b0
--- /dev/null
+++ b/release/src/main/groovy/quickstart-java-flinklocal.groovy
@@ -0,0 +1,43 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+t = new TestScripts(args)
+
+/*
+ * Run the Flink local quickstart from
https://beam.apache.org/get-started/quickstart-java/
+ */
+
+t.describe 'Run Apache Beam Java SDK Quickstart - Flink Local'
+
+ t.intent 'Gets the WordCount Example Code'
+ QuickstartArchetype.generate(t)
+
+ t.intent 'Runs the WordCount Code with Flink Local runner'
+ // Run the wordcount example with the flink local runner
+ t.run """mvn compile exec:java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--inputFile=pom.xml --output=counts \
+ --runner=FlinkRunner" -Pflink-runner"""
+
+ // Verify text from the pom.xml input file
+ t.run "grep Foundation counts*"
+ t.see "Foundation: 1"
+
+ // Clean up
+ t.done()
diff --git a/release/src/main/groovy/quickstart-java-spark.groovy
b/release/src/main/groovy/quickstart-java-spark.groovy
new file mode 100644
index 0000000..671a4b1
--- /dev/null
+++ b/release/src/main/groovy/quickstart-java-spark.groovy
@@ -0,0 +1,43 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+t = new TestScripts(args)
+
+/*
+ * Run the Spark quickstart from
https://beam.apache.org/get-started/quickstart-java/
+ */
+
+t.describe 'Run Apache Beam Java SDK Quickstart - Spark'
+
+ t.intent 'Gets the WordCount Example Code'
+ QuickstartArchetype.generate(t)
+
+ t.intent 'Runs the WordCount Code with Spark runner'
+ // Run the wordcount example with the spark runner
+ t.run """mvn compile exec:java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--inputFile=pom.xml --output=counts \
+ --runner=SparkRunner" -Pspark-runner"""
+
+ // Verify text from the pom.xml input file
+ t.run "grep Foundation counts*"
+ t.see "Foundation: 1"
+
+ // Clean up
+ t.done()
diff --git a/runners/apex/build.gradle b/runners/apex/build.gradle
index 3668693..c002aed 100644
--- a/runners/apex/build.gradle
+++ b/runners/apex/build.gradle
@@ -72,3 +72,6 @@ task buildDependencyTree(type: DependencyReportTask) {
}
}
compileJava.dependsOn buildDependencyTree
+
+// Generates :runners:apex:runQuickstartJavaApex
+createJavaQuickstartValidationTask(name: 'Apex')
diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle
index f258048..954ffcf 100644
--- a/runners/direct-java/build.gradle
+++ b/runners/direct-java/build.gradle
@@ -69,3 +69,6 @@ shadowJar {
relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
relocate "javax.annotation", getJavaRelocatedPath("javax.annotation")
}
+
+// Generates :runners:direct-java:runQuickstartJavaDirect
+createJavaQuickstartValidationTask(name: 'Direct')
diff --git a/runners/flink/build.gradle b/runners/flink/build.gradle
index 7707ffc..64f4ef6 100644
--- a/runners/flink/build.gradle
+++ b/runners/flink/build.gradle
@@ -117,3 +117,6 @@ task validatesRunner {
dependsOn validatesRunnerBatch
dependsOn validatesRunnerStreaming
}
+
+// Generates :runners:flink:runQuickstartJavaFlinkLocal
+createJavaQuickstartValidationTask(name: 'FlinkLocal')
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index e059020..d7c1dbb 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -86,3 +86,8 @@ dependencies {
test {
systemProperties = [ "beamUseDummyRunner" : "true" ]
}
+
+// Generates :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow
+def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
+def gcsBucket = project.findProperty('gcsBucket') ?:
'temp-storage-for-release-validation-tests/quickstart'
+createJavaQuickstartValidationTask(name: 'Dataflow', gcpProject: gcpProject,
gcsBucket: gcsBucket)
diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle
index 47892e3..e1d2c0a 100644
--- a/runners/spark/build.gradle
+++ b/runners/spark/build.gradle
@@ -89,3 +89,6 @@ configurations.testRuntimeClasspath {
// Testing the Spark runner causes a StackOverflowError if slf4j-jdk14 is on
the classpath
exclude group: "org.slf4j", module: "slf4j-jdk14"
}
+
+// Generates :runners:spark:runQuickstartJavaSpark
+createJavaQuickstartValidationTask(name: 'Spark')
diff --git a/settings.gradle b/settings.gradle
index 7226273..09e68f5 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -20,6 +20,7 @@ include ":examples:java"
include ":model:fn-execution"
include ":model:job-management"
include ":model:pipeline"
+include ":release"
include ":runners:apex"
include ":runners:core-construction-java"
include ":runners:core-java"
--
To stop receiving notification emails like this one, please contact
lc...@apache.org.
