[
https://issues.apache.org/jira/browse/BEAM-6936?focusedWorklogId=222270&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-222270
]
ASF GitHub Bot logged work on BEAM-6936:
----------------------------------------
Author: ASF GitHub Bot
Created on: 03/Apr/19 11:45
Start Date: 03/Apr/19 11:45
Worklog Time Spent: 10m
Work Description: adude3141 commented on pull request #8183: [BEAM-6936]
Added Jenkins jobs running Java examples on Dataflow with Java 11
URL: https://github.com/apache/beam/pull/8183#discussion_r271701021
##########
File path: runners/google-cloud-dataflow-java/examples/build.gradle
##########
@@ -87,6 +87,61 @@ task preCommitFnApiWorker(type: Test) {
}
}
+task postCommitFnApiWorkerJava11(type: Test) {
+ dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+ dependsOn
":beam-runners-google-cloud-dataflow-java:buildAndPushDockerContainer"
+
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+ def preCommitBeamTestPipelineOptions = [
+ "--project=${gcpProject}",
+ "--tempRoot=${gcsTempRoot}",
+ "--runner=TestDataflowRunner",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=${dockerImageName}",
+ "--experiments=${fnapiExperiments}",
+ ]
+ testClassesDirs =
files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
+ include "**/WordCountIT.class"
+ include "**/WindowedWordCountIT.class"
+ forkEvery 1
+ maxParallelForks 4
+ systemProperty "beamTestPipelineOptions",
JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+ useJUnit {
+ excludeCategories 'org.apache.beam.sdk.testing.StreamingIT'
+ }
+}
+
+task postCommitLegacyWorkerJava11(type: Test) {
Review comment:
Maybe something like this will do?
```diff
diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle
b/runners/google-cloud-dataflow-java/examples/build.gradle
index 72059bb064..f591e5009b 100644
--- a/runners/google-cloud-dataflow-java/examples/build.gradle
+++ b/runners/google-cloud-dataflow-java/examples/build.gradle
@@ -42,25 +42,33 @@ def dockerImageName =
project(':beam-runners-google-cloud-dataflow-java').ext.do
// If -PuseExecutableStage is set, the
use_executable_stage_bundle_execution wil be enabled.
def fnapiExperiments = project.hasProperty('useExecutableStage') ?
'beam_fn_api,use_executable_stage_bundle_execution' : "beam_fn_api"
+// we require dataflowWorker, optional workerHarnessContainerImage and
optional additionalOptions
+def commonConfig = { dataflowWorkerJar, workerHarnessContainerImage = '',
additionalOptions = [] ->
+ // return the preevaluated configuration closure
+ return {
+ testClassesDirs =
files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
+ include "**/WordCountIT.class"
+ include "**/WindowedWordCountIT.class"
+ forkEvery 1
+ maxParallelForks 4
+
+ def preCommitBeamTestPipelineOptions = [
+ "--project=${gcpProject}",
+ "--tempRoot=${gcsTempRoot}",
+ "--runner=TestDataflowRunner",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=${workerHarnessContainerImage}"
+ ] + additionalOptions
+ systemProperty "beamTestPipelineOptions",
JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+ }
+}
+
task preCommitLegacyWorker(type: Test) {
dependsOn
":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
-
//Set workerHarnessContainerImage to empty to make Dataflow pick up the
non-versioned container
//image, which handles a staged worker jar.
- def preCommitBeamTestPipelineOptions = [
- "--project=${gcpProject}",
- "--tempRoot=${gcsTempRoot}",
- "--runner=TestDataflowRunner",
- "--dataflowWorkerJar=${dataflowWorkerJar}",
- "--workerHarnessContainerImage=",
- ]
- testClassesDirs =
files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
- include "**/WordCountIT.class"
- include "**/WindowedWordCountIT.class"
- forkEvery 1
- maxParallelForks 4
- systemProperty "beamTestPipelineOptions",
JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+ with commonConfig(dataflowWorkerJar)
}
task verifyFnApiWorker(type: Test) {
@@ -68,20 +76,7 @@ task verifyFnApiWorker(type: Test) {
dependsOn
":beam-runners-google-cloud-dataflow-java:buildAndPushDockerContainer"
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
- def preCommitBeamTestPipelineOptions = [
- "--project=${gcpProject}",
- "--tempRoot=${gcsTempRoot}",
- "--runner=TestDataflowRunner",
- "--dataflowWorkerJar=${dataflowWorkerJar}",
- "--workerHarnessContainerImage=${dockerImageName}",
- "--experiments=${fnapiExperiments}",
- ]
- testClassesDirs =
files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
- include "**/WordCountIT.class"
- include "**/WindowedWordCountIT.class"
- forkEvery 1
- maxParallelForks 4
- systemProperty "beamTestPipelineOptions",
JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+ with commonConfig(dataflowWorkerJar, dockerImageName,
"--experiments=${fnapiExperiments}")
useJUnit {
excludeCategories 'org.apache.beam.sdk.testing.StreamingIT'
}
@@ -90,24 +85,10 @@ task verifyFnApiWorker(type: Test) {
task postCommitLegacyWorkerJava11(type: Test) {
dependsOn
":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
-
//Set workerHarnessContainerImage to empty to make Dataflow pick up the
non-versioned container
//image, which handles a staged worker jar.
- def preCommitBeamTestPipelineOptions = [
- "--project=${gcpProject}",
- "--tempRoot=${gcsTempRoot}",
- "--runner=TestDataflowRunner",
- "--dataflowWorkerJar=${dataflowWorkerJar}",
- "--workerHarnessContainerImage=",
- ]
-
- testClassesDirs =
files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
- include "**/WordCountIT.class"
- include "**/WindowedWordCountIT.class"
- forkEvery 1
- maxParallelForks 4
+ with commonConfig(dataflowWorkerJar)
systemProperty "java.specification.version", "11"
- systemProperty "beamTestPipelineOptions",
JsonOutput.toJson(preCommitBeamTestPipelineOptions)
}
task java11PostCommit() {
```
But please bear with me. It is a fast hack and I did not run. Also, if you
decide to use, doublecheck that I did not accidentally deleted some required
configuration.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 222270)
Time Spent: 2h 10m (was: 2h)
> Add a Jenkins job running Java examples on Java 11 Dataflow
> -----------------------------------------------------------
>
> Key: BEAM-6936
> URL: https://issues.apache.org/jira/browse/BEAM-6936
> Project: Beam
> Issue Type: Sub-task
> Components: examples-java, testing
> Reporter: Michal Walenia
> Assignee: Michal Walenia
> Priority: Minor
> Time Spent: 2h 10m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
