[
https://issues.apache.org/jira/browse/BEAM-3250?focusedWorklogId=88405&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-88405
]
ASF GitHub Bot logged work on BEAM-3250:
----------------------------------------
Author: ASF GitHub Bot
Created on: 06/Apr/18 11:46
Start Date: 06/Apr/18 11:46
Worklog Time Spent: 10m
Work Description: lukecwik closed pull request #5029: [BEAM-3250] Migrate
Dataflow ValidatesRunner test to Gradle
URL: https://github.com/apache/beam/pull/5029
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
index 3230f9a5787..318c834957f 100644
---
a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
+++
b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
@@ -20,16 +20,22 @@ import common_job_properties
// This job runs the suite of ValidatesRunner tests against the Dataflow
// runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Dataflow') {
+job('beam_PostCommit_Java_ValidatesRunner_Dataflow_Gradle') {
description('Runs the ValidatesRunner suite on the Dataflow runner.')
+ previousNames('beam_PostCommit_Java_ValidatesRunner_Dataflow')
previousNames('beam_PostCommit_Java_RunnableOnService_Dataflow')
-
// Set common parameters. Sets a long (3 hour) timeout due to timeouts in
[BEAM-3775].
common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 180)
- // Set maven parameters.
- common_job_properties.setMavenConfig(delegate)
+ def gradle_switches = [
+ // Gradle log verbosity enough to diagnose basic build issues
+ "--info",
+ // Continue the build even if there is a failure to show as many potential
failures as possible.
+ '--continue',
+ // Until we verify the build cache is working appropriately, force
rerunning all tasks
+ '--rerun-tasks',
+ ]
// Sets that this is a PostCommit job.
common_job_properties.setPostCommit(delegate)
@@ -40,6 +46,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Dataflow') {
'Google Cloud Dataflow Runner ValidatesRunner Tests',
'Run Dataflow ValidatesRunner')
- // Maven goals for this job.
- goals('-B -e clean verify -am -pl runners/google-cloud-dataflow-java
-DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[
"--runner=TestDataflowRunner", "--project=apache-beam-testing",
"--tempRoot=gs://temp-storage-for-validates-runner-tests/" ]\'')
+ // Gradle goals for this job.
+ steps {
+ gradle {
+ rootBuildScriptDir(common_job_properties.checkoutDir)
+ tasks(':runners:google-cloud-dataflow-java:validatesRunner')
+ for (String gradle_switch : gradle_switches) {
+ switches(gradle_switch)
+ }
+ }
+ }
}
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index b75b97d9208..0c12e5aba9c 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -16,6 +16,8 @@
* limitations under the License.
*/
+import groovy.json.JsonOutput
+
apply from: project(":").file("build_rules.gradle")
applyJavaNature(enableFindbugs: false /* BEAM-925 */,
artifactId: "beam-runners-google-cloud-dataflow-java")
@@ -32,6 +34,7 @@ description = "Apache Beam :: Runners :: Google Cloud
Dataflow"
evaluationDependsOn(":sdks:java:io:google-cloud-platform")
evaluationDependsOn(":sdks:java:extensions:google-cloud-platform-core")
evaluationDependsOn(":model:fn-execution")
+evaluationDependsOn(":sdks:java:core")
processResources {
filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
@@ -47,6 +50,10 @@ test {
systemProperty "beamUseDummyRunner", "true"
}
+configurations {
+ validatesRunner
+}
+
dependencies {
compile library.java.guava
shadow project(path: ":model:pipeline", configuration: "shadow")
@@ -70,23 +77,60 @@ dependencies {
shadow library.java.jackson_annotations
shadow library.java.jackson_databind
shadow library.java.slf4j_api
- testCompile library.java.hamcrest_core
- testCompile library.java.junit
- testCompile
project(":sdks:java:io:google-cloud-platform").sourceSets.test.output
- testCompile project(path: ":sdks:java:core", configuration: "shadowTest")
- testCompile
project(":sdks:java:extensions:google-cloud-platform-core").sourceSets.test.output
- testCompile library.java.guava_testlib
- testCompile library.java.slf4j_jdk14
- testCompile library.java.mockito_core
- testCompile library.java.google_cloud_dataflow_java_proto_library_all
- testCompile library.java.datastore_v1_protos
- testCompile library.java.jackson_dataformat_yaml
+ shadowTest library.java.hamcrest_core
+ shadowTest library.java.junit
+ shadowTest
project(":sdks:java:io:google-cloud-platform").sourceSets.test.output
+ shadowTest project(path: ":sdks:java:core", configuration: "shadowTest")
+ shadowTest
project(":sdks:java:extensions:google-cloud-platform-core").sourceSets.test.output
+ shadowTest library.java.guava_testlib
+ shadowTest library.java.slf4j_jdk14
+ shadowTest library.java.mockito_core
+ shadowTest library.java.google_cloud_dataflow_java_proto_library_all
+ shadowTest library.java.datastore_v1_protos
+ shadowTest library.java.jackson_dataformat_yaml
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: project.path, configuration: "shadow")
}
test {
systemProperties = [ "beamUseDummyRunner" : "true" ]
}
+task validatesRunnerTest(type: Test) {
+ group = "Verification"
+ def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
+ def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-validates-runner-tests/'
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+ "--runner=TestDataflowRunner",
+ "--project=${dataflowProject}",
+ "--tempRoot=${dataflowTempRoot}",
+ ])
+
+ maxParallelForks 4
+ classpath = configurations.validatesRunner
+ testClassesDirs =
files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
+ useJUnit {
+ includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ excludeCategories 'org.apache.beam.sdk.testing.LargeKeys$Above10MB'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesAttemptedMetrics'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesDistributionMetrics'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesSetState'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesMapState'
+ excludeCategories
'org.apache.beam.sdk.testing.UsesSplittableParDoWithWindowedSideInputs'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedPCollections'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle'
+
+ }
+}
+
+task validatesRunner {
+ group = "Verification"
+ description "Validates Dataflow runner"
+ dependsOn validatesRunnerTest
+}
+
// Generates :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow
def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
def gcsBucket = project.findProperty('gcsBucket') ?:
'temp-storage-for-release-validation-tests/quickstart'
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 88405)
Time Spent: 3h 10m (was: 3h)
> Migrate ValidatesRunner Jenkins PostCommits to Gradle
> -----------------------------------------------------
>
> Key: BEAM-3250
> URL: https://issues.apache.org/jira/browse/BEAM-3250
> Project: Beam
> Issue Type: Sub-task
> Components: build-system, testing
> Reporter: Luke Cwik
> Assignee: Henning Rohde
> Priority: Major
> Time Spent: 3h 10m
> Remaining Estimate: 0h
>
> Update these targets to execute ValidatesRunner tests:
> https://github.com/apache/beam/search?l=Groovy&q=ValidatesRunner&type=&utf8=%E2%9C%93
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
