[
https://issues.apache.org/jira/browse/BEAM-6028?focusedWorklogId=164170&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-164170
]
ASF GitHub Bot logged work on BEAM-6028:
----------------------------------------
Author: ASF GitHub Bot
Created on: 09/Nov/18 00:36
Start Date: 09/Nov/18 00:36
Worklog Time Spent: 10m
Work Description: swegner closed pull request #6992: [BEAM-6028] Revert
"Merge pull request #6979: Add helper task to print pipeline options for
Dataflow portability
URL: https://github.com/apache/beam/pull/6992
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 52b5b45a1bb..ab18b5381ee 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -117,12 +117,6 @@ def dockerImageContainer = "${dockerImageRoot}/java"
def dockerTag = new Date().format('yyyyMMddHHmmss')
def dockerImageName = "${dockerImageContainer}:${dockerTag}"
-def fnApiPipelineOptions = [
- "--dataflowWorkerJar=${dataflowFnApiWorkerJar}",
- "--workerHarnessContainerImage=${dockerImageContainer}:${dockerTag}",
- "--experiments=beam_fn_api",
-]
-
def commonExcludeCategories = [
'org.apache.beam.sdk.testing.LargeKeys$Above10MB',
'org.apache.beam.sdk.testing.UsesAttemptedMetrics',
@@ -175,7 +169,6 @@ task validatesRunnerLegacyWorkerTest(type: Test) {
task buildAndPushDockerContainer() {
dependsOn ":beam-sdks-java-container:docker"
- finalizedBy 'cleanUpDockerImages'
def defaultDockerImageName = containerImageName(name: "java")
doLast {
exec {
@@ -187,37 +180,20 @@ task buildAndPushDockerContainer() {
}
}
-afterEvaluate {
- // Ensure all tasks which use published docker images run before they are
cleaned up
- tasks.each { t ->
- if (t.dependsOn.contains(buildAndPushDockerContainer)) {
- cleanUpDockerImages.mustRunAfter t
- }
- }
-}
-
-task printFnApiPipelineOptions {
- group = "Help"
- description = "Prints to the console extra pipeline options needed to run
a Dataflow pipeline using portability"
-
- dependsOn
":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
- dependsOn buildAndPushDockerContainer
-
- doLast {
- println "To run a Dataflow job with portability, add the following
pipeline options to your command-line:"
- println fnApiPipelineOptions.join(' ')
- }
-}
-
task validatesRunnerFnApiWorkerTest(type: Test) {
group = "Verification"
dependsOn
":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
dependsOn buildAndPushDockerContainer
+
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
- "--tempRoot=${dataflowPostCommitTempRoot}"] + fnApiPipelineOptions
- )
+ "--tempRoot=${dataflowPostCommitTempRoot}",
+ "--dataflowWorkerJar=${dataflowFnApiWorkerJar}",
+
"--workerHarnessContainerImage=${dockerImageContainer}:${dockerTag}",
+ "--experiments=beam_fn_api",
+
+ ])
// Increase test parallelism up to the number of Gradle workers. By
default this is equal
// to the number of CPU cores, but can be increased by setting
--max-workers=N.
@@ -284,8 +260,11 @@ task googleCloudPlatformFnApiWorkerIntegrationTest(type:
Test) {
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
- "--tempRoot=${dataflowPostCommitTempRoot}"] + fnApiPipelineOptions
- )
+ "--tempRoot=${dataflowPostCommitTempRoot}",
+ "--dataflowWorkerJar=${dataflowFnApiWorkerJar}",
+
"--workerHarnessContainerImage=${dockerImageContainer}:${dockerTag}",
+ "--experiments=beam_fn_api",
+ ])
include '**/*IT.class'
exclude '**/BigQueryIOReadIT.class'
@@ -338,8 +317,11 @@ task examplesJavaFnApiWorkerIntegrationTest(type: Test) {
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
- "--tempRoot=${dataflowPostCommitTempRoot}"] + fnApiPipelineOptions
- )
+ "--tempRoot=${dataflowPostCommitTempRoot}",
+ "--dataflowWorkerJar=${dataflowFnApiWorkerJar}",
+
"--workerHarnessContainerImage=${dockerImageContainer}:${dockerTag}",
+ "--experiments=beam_fn_api",
+ ])
// The examples/java preCommit task already covers running
WordCountIT/WindowedWordCountIT so
// this postCommit integration test excludes them.
@@ -385,8 +367,11 @@ task coreSDKJavaFnApiWorkerIntegrationTest(type: Test) {
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
- "--tempRoot=${dataflowPostCommitTempRoot}"] + fnApiPipelineOptions
- )
+ "--tempRoot=${dataflowPostCommitTempRoot}",
+ "--dataflowWorkerJar=${dataflowFnApiWorkerJar}",
+
"--workerHarnessContainerImage=${dockerImageContainer}:${dockerTag}",
+ "--experiments=beam_fn_api",
+ ])
include '**/*IT.class'
maxParallelForks 4
@@ -409,16 +394,13 @@ task postCommitPortabilityApi {
dependsOn googleCloudPlatformFnApiWorkerIntegrationTest
dependsOn examplesJavaFnApiWorkerIntegrationTest
dependsOn coreSDKJavaFnApiWorkerIntegrationTest
-}
-
-// Clean up built images
-task cleanUpDockerImages() {
+ // Clean up docker image
doLast {
exec {
commandLine "docker", "rmi", "${dockerImageName}"
}
exec {
- commandLine "gcloud", "--quiet", "container", "images", "delete",
"--force-delete-tags", "${dockerImageName}"
+ commandLine "gcloud", "--quiet", "container", "images", "delete",
"${dockerImageName}"
}
}
}
diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle
b/runners/google-cloud-dataflow-java/examples/build.gradle
index 7cf6e853652..bee66be78bd 100644
--- a/runners/google-cloud-dataflow-java/examples/build.gradle
+++ b/runners/google-cloud-dataflow-java/examples/build.gradle
@@ -23,7 +23,6 @@ applyJavaNature(publish: false)
// Evaluate the given project before this one, to allow referencing
// its sourceSets.test.output directly.
evaluationDependsOn(":beam-examples-java")
-evaluationDependsOn(":beam-runners-google-cloud-dataflow-java")
evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker")
evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker")
evaluationDependsOn(":beam-sdks-java-container")
@@ -65,9 +64,21 @@ task preCommitLegacyWorker(type: Test) {
systemProperty "beamTestPipelineOptions",
JsonOutput.toJson(preCommitBeamTestPipelineOptions)
}
+task buildAndPushDockerContainer() {
+ dependsOn ":beam-sdks-java-container:docker"
+ doLast {
+ exec {
+ commandLine "docker", "tag", "${defaultDockerImageName}",
"${dockerImageName}"
+ }
+ exec {
+ commandLine "gcloud", "docker", "--", "push", "${dockerImageContainer}"
+ }
+ }
+}
+
task preCommitFnApiWorker(type: Test) {
dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
- dependsOn
":beam-runners-google-cloud-dataflow-java:buildAndPushDockerContainer"
+ dependsOn buildAndPushDockerContainer
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
def preCommitBeamTestPipelineOptions = [
@@ -95,13 +106,13 @@ task preCommit() {
task preCommitPortabilityApi() {
dependsOn preCommitFnApiWorker
-}
-
-afterEvaluate {
- // Ensure all tasks which use published docker images run before they are
cleaned up
- tasks.each { t ->
- if
(t.dependsOn.contains(":beam-runners-google-cloud-dataflow-java:buildAndPushDockerContainer"))
{
-
project(':beam-runners-google-cloud-dataflow-java').cleanUpDockerImages.mustRunAfter
t
+ // Clean up built images
+ doLast {
+ exec {
+ commandLine "docker", "rmi", "${dockerImageName}"
+ }
+ exec {
+ commandLine "gcloud", "--quiet", "container", "images", "delete",
"${dockerImageName}"
}
}
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 164170)
Time Spent: 1.5h (was: 1h 20m)
> beam_PreCommit_JavaPortabilityApi_Cron fails due to a timeout
> -------------------------------------------------------------
>
> Key: BEAM-6028
> URL: https://issues.apache.org/jira/browse/BEAM-6028
> Project: Beam
> Issue Type: Bug
> Components: sdk-java-harness
> Affects Versions: Not applicable
> Reporter: Chamikara Jayalath
> Assignee: Scott Wegner
> Priority: Critical
> Attachments: 1.png
>
> Time Spent: 1.5h
> Remaining Estimate: 0h
>
> Looks like last three runs failed (so probably perma red).
>
> For example,
> [https://builds.apache.org/job/beam_PreCommit_JavaPortabilityApi_Cron/26/]
> [https://builds.apache.org/job/beam_PreCommit_JavaPortabilityApi_Cron/26/console]
>
> Scott, looks like this first failed for your following commit. But not 100%
> sure if this is the reason.
> [https://github.com/apache/beam/commit/42984a821b3e73aee2966d11d7fb436b5ff22b68]
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
