[
https://issues.apache.org/jira/browse/BEAM-5730?focusedWorklogId=155616&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-155616
]
ASF GitHub Bot logged work on BEAM-5730:
----------------------------------------
Author: ASF GitHub Bot
Created on: 17/Oct/18 21:37
Start Date: 17/Oct/18 21:37
Worklog Time Spent: 10m
Work Description: lukecwik closed pull request #6694: [BEAM-5730] Migrate
ITs using DataflowRunner to use custom worker
URL: https://github.com/apache/beam/pull/6694
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 5e196e7060a..ebe9dcd1344 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -31,6 +31,7 @@ description = "Apache Beam :: Runners :: Google Cloud
Dataflow"
evaluationDependsOn(":beam-sdks-java-io-google-cloud-platform")
evaluationDependsOn(":beam-sdks-java-core")
evaluationDependsOn(":beam-examples-java")
+evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker")
processResources {
filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
@@ -102,14 +103,22 @@ test {
systemProperties = [ "beamUseDummyRunner" : "true" ]
}
+// For the following test tasks, set workerHarnessContainerImage to empty to
make Dataflow pick up
+// the non-versioned container image, which handles a staged worker jar.
task validatesRunnerTest(type: Test) {
group = "Verification"
+ dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-validates-runner-tests/'
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
+
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
"--tempRoot=${dataflowTempRoot}",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=",
+
])
// Increase test parallelism up to the number of Gradle workers. By default
this is equal
@@ -141,12 +150,16 @@ task validatesRunner {
task googleCloudPlatformIntegrationTest(type: Test) {
group = "Verification"
+ dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests'
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
"--tempRoot=${dataflowTempRoot}",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=",
])
include '**/*IT.class'
@@ -160,12 +173,17 @@ task googleCloudPlatformIntegrationTest(type: Test) {
task examplesJavaIntegrationTest(type: Test) {
group = "Verification"
+ dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests'
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
+
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
"--tempRoot=${dataflowTempRoot}",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=",
])
// The examples/java preCommit task already covers running
WordCountIT/WindowedWordCountIT so
@@ -181,12 +199,17 @@ task examplesJavaIntegrationTest(type: Test) {
task coreSDKJavaIntegrationTest(type: Test) {
group = "Verification"
+ dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests'
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
+
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
"--tempRoot=${dataflowTempRoot}",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=",
])
include '**/*IT.class'
diff --git a/runners/google-cloud-dataflow-java/examples-streaming/build.gradle
b/runners/google-cloud-dataflow-java/examples-streaming/build.gradle
index 9741166d45d..f52ec1dc65c 100644
--- a/runners/google-cloud-dataflow-java/examples-streaming/build.gradle
+++ b/runners/google-cloud-dataflow-java/examples-streaming/build.gradle
@@ -23,6 +23,7 @@ applyJavaNature(publish: false)
// Evaluate the given project before this one, to allow referencing
// its sourceSets.test.output directly.
evaluationDependsOn(":beam-examples-java")
+evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker")
configurations { dataflowStreamingRunnerPreCommit }
@@ -35,10 +36,17 @@ def gcpProject = project.findProperty('gcpProject') ?:
'apache-beam-testing'
def gcsTempRoot = project.findProperty('gcsTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests/'
task preCommit(type: Test) {
+ dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
+
+ // Set workerHarnessContainerImage to empty to make Dataflow pick up the
non-versioned container
+ // image, which handles a staged worker jar.
def preCommitBeamTestPipelineOptions = [
"--project=${gcpProject}",
"--tempRoot=${gcsTempRoot}",
"--runner=TestDataflowRunner",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=",
"--streaming=true",
]
testClassesDirs =
files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle
b/runners/google-cloud-dataflow-java/examples/build.gradle
index b9d5c478b30..6009ff9f284 100644
--- a/runners/google-cloud-dataflow-java/examples/build.gradle
+++ b/runners/google-cloud-dataflow-java/examples/build.gradle
@@ -23,6 +23,7 @@ applyJavaNature(publish: false)
// Evaluate the given project before this one, to allow referencing
// its sourceSets.test.output directly.
evaluationDependsOn(":beam-examples-java")
+evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker")
configurations { dataflowRunnerPreCommit }
@@ -31,14 +32,22 @@ dependencies {
testRuntimeOnly project(path: ":beam-examples-java", configuration:
"shadowTest")
testRuntimeOnly project(path: ":beam-runners-google-cloud-dataflow-java",
configuration: "shadow")
}
+
def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
def gcsTempRoot = project.findProperty('gcsTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests/'
task preCommit(type: Test) {
+ dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
+
+ //Set workerHarnessContainerImage to empty to make Dataflow pick up the
non-versioned container
+ //image, which handles a staged worker jar.
def preCommitBeamTestPipelineOptions = [
"--project=${gcpProject}",
"--tempRoot=${gcsTempRoot}",
"--runner=TestDataflowRunner",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=",
]
testClassesDirs =
files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
include "**/WordCountIT.class"
@@ -47,4 +56,3 @@ task preCommit(type: Test) {
maxParallelForks 4
systemProperty "beamTestPipelineOptions",
JsonOutput.toJson(preCommitBeamTestPipelineOptions)
}
-
diff --git
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 15740650017..680c348967d 100644
---
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -745,7 +745,7 @@ public DataflowPipelineJob run(Pipeline pipeline) {
List<String> experiments =
dataflowOptions.getExperiments() == null
? new ArrayList<>()
- : dataflowOptions.getExperiments();
+ : new ArrayList<>(dataflowOptions.getExperiments());
experiments.add("use_staged_dataflow_worker_jar");
dataflowOptions.setExperiments(experiments);
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 155616)
Time Spent: 2h 40m (was: 2.5h)
> Migrate Java test to use a staged worker jar
> --------------------------------------------
>
> Key: BEAM-5730
> URL: https://issues.apache.org/jira/browse/BEAM-5730
> Project: Beam
> Issue Type: Sub-task
> Components: runner-dataflow
> Reporter: Boyuan Zhang
> Assignee: Boyuan Zhang
> Priority: Major
> Time Spent: 2h 40m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
