[
https://issues.apache.org/jira/browse/BEAM-4445?focusedWorklogId=116128&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-116128
]
ASF GitHub Bot logged work on BEAM-4445:
----------------------------------------
Author: ASF GitHub Bot
Created on: 26/Jun/18 20:03
Start Date: 26/Jun/18 20:03
Worklog Time Spent: 10m
Work Description: swegner closed pull request #5757: [BEAM-4445] Split
pre-commit tests into separate jobs based on trigger condition.
URL: https://github.com/apache/beam/pull/5757
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/JobBuilder.groovy
b/.test-infra/jenkins/PostcommitJobBuilder.groovy
similarity index 91%
rename from .test-infra/jenkins/JobBuilder.groovy
rename to .test-infra/jenkins/PostcommitJobBuilder.groovy
index 72ba4681db6..4ab6f718ffe 100644
--- a/.test-infra/jenkins/JobBuilder.groovy
+++ b/.test-infra/jenkins/PostcommitJobBuilder.groovy
@@ -24,12 +24,12 @@ import common_job_properties as cjp
* Purpose of this class is to define common strategies and reporting/building
paramereters
* for pre- and post- commit test jobs and unify them across the project.
*/
-class JobBuilder {
+class PostcommitJobBuilder {
private def scope;
private def jobDefinition;
private def job;
- private JobBuilder(scope, jobDefinition = {}) {
+ private PostcommitJobBuilder(scope, jobDefinition = {}) {
this.scope = scope
this.jobDefinition = jobDefinition
this.job = null
@@ -40,14 +40,14 @@ class JobBuilder {
githubUiHint,
scope,
jobDefinition = {}) {
- JobBuilder jb = new JobBuilder(scope, jobDefinition)
+ PostcommitJobBuilder jb = new PostcommitJobBuilder(scope, jobDefinition)
jb.defineAutoPostCommitJob(nameBase)
jb.defineGhprbTriggeredJob(nameBase + "_PR", triggerPhrase, githubUiHint,
false)
}
private void defineAutoPostCommitJob(name) {
def autoBuilds = scope.job(name) {
- cjp.setPostCommit(delegate)
+ cjp.setAutoJob delegate
}
autoBuilds.with(jobDefinition)
}
diff --git a/.test-infra/jenkins/PrecommitJobBuilder.groovy
b/.test-infra/jenkins/PrecommitJobBuilder.groovy
new file mode 100644
index 00000000000..f43d6ea6106
--- /dev/null
+++ b/.test-infra/jenkins/PrecommitJobBuilder.groovy
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+/** This class defines PrecommitJobBuilder.build() helper for defining
pre-comit jobs. */
+class PrecommitJobBuilder {
+ /** scope 'this' parameter from top-level script; used for binding Job DSL
methods. */
+ Object scope
+
+ /** Base name for each post-commit suite job, i.e. 'Go'. */
+ String nameBase
+
+ /** The Gradle task to execute. */
+ String gradleTask
+
+ /** Overall job timeout. */
+ int timeoutMins = 90
+
+ /** If defined, set of path expressions used to trigger the job on commit. */
+ List<String> triggerPathPatterns = []
+
+ /**
+ * Define a set of pre-commit jobs.
+ *
+ * @param additionalCustomization Job DSL closure with additional
customization to apply to the job.
+ */
+ void build(Closure additionalCustomization = {}) {
+ defineCronJob additionalCustomization
+ defineCommitJob additionalCustomization
+ definePhraseJob additionalCustomization
+ }
+
+ /** Create a pre-commit job which runs on a daily schedule. */
+ private void defineCronJob(Closure additionalCustomization) {
+ def job = createBaseJob 'Cron'
+ job.with {
+ description buildDescription('on a daily schedule.')
+ common_job_properties.setAutoJob delegate
+ }
+ job.with additionalCustomization
+ }
+
+ /** Create a pre-commit job which runs on every commit to a PR. */
+ private void defineCommitJob(Closure additionalCustomization) {
+ def job = createBaseJob 'Commit', true
+ def defaultPathTriggers = [
+ '^build.gradle$',
+ '^build_rules.gradle$',
+ '^gradle.properties$',
+ '^gradlew$',
+ '^gradle.bat$',
+ '^settings.gradle$'
+ ]
+ triggerPathPatterns.addAll defaultPathTriggers
+ job.with {
+ description buildDescription('for each commit push.')
+ concurrentBuild()
+ common_job_properties.setPullRequestBuildTrigger(delegate,
+ githubUiHint(),
+ '',
+ false,
+ triggerPathPatterns)
+ }
+ job.with additionalCustomization
+ }
+
+ private void definePhraseJob(Closure additionalCustomization) {
+ def job = createBaseJob 'Phrase'
+ job.with {
+ description buildDescription("on trigger phrase
'${buildTriggerPhrase()}'.")
+ concurrentBuild()
+ common_job_properties.setPullRequestBuildTrigger delegate,
githubUiHint(), buildTriggerPhrase()
+ }
+ job.with additionalCustomization
+ }
+
+ private Object createBaseJob(nameSuffix, usesRegionFilter = false) {
+ def allowRemotePoll = !usesRegionFilter
+ return scope.job("beam_PreCommit_${nameBase}_${nameSuffix}") {
+ common_job_properties.setTopLevelMainJobProperties(delegate,
+ 'master',
+ timeoutMins,
+ allowRemotePoll) // needed for included regions PR triggering; see
[JENKINS-23606]
+ steps {
+ gradle {
+ rootBuildScriptDir(common_job_properties.checkoutDir)
+ tasks(gradleTask)
+ common_job_properties.setGradleSwitches(delegate)
+ }
+ }
+ }
+ }
+
+ /** The magic phrase used to trigger the job when posted as a PR comment. */
+ private String buildTriggerPhrase() {
+ return "Run ${nameBase} PreCommit"
+ }
+
+ /** A human-readable description which will be used as the base of all suite
jobs. */
+ private buildDescription(String triggerDescription) {
+ return "Runs ${nameBase} PreCommit tests ${triggerDescription}"
+ }
+
+ private String githubUiHint() {
+ "${nameBase} (\"${buildTriggerPhrase()}\")"
+ }
+}
diff --git a/.test-infra/jenkins/common_job_properties.groovy
b/.test-infra/jenkins/common_job_properties.groovy
index 9249745075d..06c8a870f3a 100644
--- a/.test-infra/jenkins/common_job_properties.groovy
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -24,7 +24,7 @@ class common_job_properties {
static String checkoutDir = 'src'
- static void setSCM(def context, String repositoryName) {
+ static void setSCM(def context, String repositoryName, boolean
allowRemotePoll = true) {
context.scm {
git {
remote {
@@ -39,6 +39,9 @@ class common_job_properties {
extensions {
cleanAfterCheckout()
relativeTargetDirectory(checkoutDir)
+ if (!allowRemotePoll) {
+ disableRemotePoll()
+ }
}
}
}
@@ -51,7 +54,6 @@ class common_job_properties {
context,
'beam-site',
branch,
- 'beam',
30)
}
@@ -59,13 +61,13 @@ class common_job_properties {
static void setTopLevelMainJobProperties(def context,
String branch = 'master',
int timeout = 100,
- String jenkinsExecutorLabel =
'beam') {
+ boolean allowRemotePoll = true) {
setTopLevelJobProperties(
context,
'beam',
branch,
- jenkinsExecutorLabel,
- timeout)
+ timeout,
+ allowRemotePoll)
}
// Sets common top-level job properties. Accessed through one of the above
@@ -73,8 +75,9 @@ class common_job_properties {
private static void setTopLevelJobProperties(def context,
String repositoryName,
String defaultBranch,
- String jenkinsExecutorLabel,
- int defaultTimeout) {
+ int defaultTimeout,
+ boolean allowRemotePoll = true)
{
+ def jenkinsExecutorLabel = 'beam'
// GitHub project.
context.properties {
@@ -93,7 +96,7 @@ class common_job_properties {
}
// Source code management.
- setSCM(context, repositoryName)
+ setSCM(context, repositoryName, allowRemotePoll)
context.parameters {
// This is a recommended setup if you want to run the job manually. The
@@ -128,7 +131,7 @@ class common_job_properties {
String commitStatusContext,
String prTriggerPhrase = '',
boolean onlyTriggerPhraseToggle =
true,
- String successComment = '--none--') {
+ List<String> triggerPathPatterns =
[]) {
context.triggers {
githubPullRequest {
admins(['asfbot'])
@@ -147,17 +150,20 @@ class common_job_properties {
if (onlyTriggerPhraseToggle) {
onlyTriggerPhrase()
}
+ if (!triggerPathPatterns.isEmpty()) {
+ includedRegions(triggerPathPatterns.join('\n'))
+ }
extensions {
commitStatus {
// This is the name that will show up in the GitHub pull request UI
// for this Jenkins project. It has a limit of 255 characters.
- delegate.context(("Jenkins: " + commitStatusContext).take(255))
+ delegate.context commitStatusContext.take(255)
}
// Comment messages after build completes.
buildStatus {
- completedStatus('SUCCESS', successComment)
+ completedStatus('SUCCESS', '--none--')
completedStatus('FAILURE', '--none--')
completedStatus('ERROR', '--none--')
}
@@ -196,10 +202,9 @@ class common_job_properties {
// Sets common config for PreCommit jobs.
static void setPreCommit(context,
String commitStatusName,
- String prTriggerPhrase = '',
- String successComment = '--none--') {
+ String prTriggerPhrase = '') {
// Set pull request build trigger.
- setPullRequestBuildTrigger(context, commitStatusName, prTriggerPhrase,
false, successComment)
+ setPullRequestBuildTrigger(context, commitStatusName, prTriggerPhrase,
false)
}
// Enable triggering postcommit runs against pull requests. Users can
comment the trigger phrase
@@ -212,8 +217,7 @@ class common_job_properties {
context,
commitStatusName,
prTriggerPhrase,
- true,
- '--none--')
+ true)
}
// Sets this as a cron job, running on a schedule.
@@ -223,8 +227,8 @@ class common_job_properties {
}
}
- // Sets common config for PostCommit jobs.
- static void setPostCommit(context,
+ // Sets common config for jobs which run on a schedule / push
+ static void setAutoJob(context,
String buildSchedule = '0 */6 * * *',
boolean triggerEveryPush = true,
String notifyAddress = '[email protected]',
diff --git a/.test-infra/jenkins/job_Dependency_Check.groovy
b/.test-infra/jenkins/job_Dependency_Check.groovy
index bdd0bdd24d3..a3af7bb70a6 100644
--- a/.test-infra/jenkins/job_Dependency_Check.groovy
+++ b/.test-infra/jenkins/job_Dependency_Check.groovy
@@ -31,7 +31,7 @@ job('beam_Dependency_Check') {
'Run Dependency Check')
// This is a job that runs weekly.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'0 12 * * 1',
false)
diff --git a/.test-infra/jenkins/job_PerformanceTests_Dataflow.groovy
b/.test-infra/jenkins/job_PerformanceTests_Dataflow.groovy
index a18b4efe284..a2bf4483528 100644
--- a/.test-infra/jenkins/job_PerformanceTests_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_Dataflow.groovy
@@ -25,7 +25,7 @@ job('beam_PerformanceTests_Dataflow'){
// Run job in postcommit every 6 hours, don't trigger every push, and
// don't email individual committers.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'H */6 * * *',
false,
diff --git a/.test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT.groovy
b/.test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT.groovy
index 1ece77d1cbf..5d95be3f51b 100644
--- a/.test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT.groovy
@@ -112,7 +112,7 @@ private void
create_filebasedio_performance_test_job(testConfiguration) {
// Run job in postcommit every 6 hours, don't trigger every push, and
// don't email individual committers.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'H */6 * * *',
false,
diff --git
a/.test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT_HDFS.groovy
b/.test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT_HDFS.groovy
index 98a0e93893c..26e65202d43 100644
--- a/.test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT_HDFS.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT_HDFS.groovy
@@ -114,7 +114,7 @@ private void
create_filebasedio_performance_test_job(testConfiguration) {
// Run job in postcommit every 6 hours, don't trigger every push, and
// don't email individual committers.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'H */6 * * *',
false,
diff --git a/.test-infra/jenkins/job_PerformanceTests_HadoopInputFormat.groovy
b/.test-infra/jenkins/job_PerformanceTests_HadoopInputFormat.groovy
index 7d18a3124ce..384bbc20e63 100644
--- a/.test-infra/jenkins/job_PerformanceTests_HadoopInputFormat.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_HadoopInputFormat.groovy
@@ -26,7 +26,7 @@ job(jobName) {
// Run job in postcommit every 6 hours, don't trigger every push, and
// don't email individual committers.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'H */6 * * *',
false,
diff --git a/.test-infra/jenkins/job_PerformanceTests_JDBC.groovy
b/.test-infra/jenkins/job_PerformanceTests_JDBC.groovy
index 9b2e66803b5..81e7deaeebe 100644
--- a/.test-infra/jenkins/job_PerformanceTests_JDBC.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_JDBC.groovy
@@ -26,7 +26,7 @@ job(jobName) {
// Run job in postcommit every 6 hours, don't trigger every push, and
// don't email individual committers.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'H */6 * * *',
false,
diff --git a/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy
b/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy
index 42e6e3d91aa..eed28b5d81d 100644
--- a/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy
@@ -26,7 +26,7 @@ job(jobName) {
// Run job in postcommit every 6 hours, don't trigger every push, and
// don't email individual committers.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'H */6 * * *',
false,
diff --git a/.test-infra/jenkins/job_PerformanceTests_Python.groovy
b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
index 25a515ff907..e837029b6ce 100644
--- a/.test-infra/jenkins/job_PerformanceTests_Python.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
@@ -24,7 +24,7 @@ job('beam_PerformanceTests_Python'){
common_job_properties.setTopLevelMainJobProperties(delegate)
// Run job in postcommit every 6 hours, don't trigger every push.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'H */6 * * *',
false,
diff --git a/.test-infra/jenkins/job_PerformanceTests_Spark.groovy
b/.test-infra/jenkins/job_PerformanceTests_Spark.groovy
index 9af44226180..8c7689583a8 100644
--- a/.test-infra/jenkins/job_PerformanceTests_Spark.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_Spark.groovy
@@ -29,7 +29,7 @@ job('beam_PerformanceTests_Spark'){
// Run job in postcommit every 6 hours, don't trigger every push, and
// don't email individual committers.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'H */6 * * *',
false,
diff --git a/.test-infra/jenkins/job_PostCommit_Go_GradleBuild.groovy
b/.test-infra/jenkins/job_PostCommit_Go_GradleBuild.groovy
index c6f40531593..f7629554b77 100644
--- a/.test-infra/jenkins/job_PostCommit_Go_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Go_GradleBuild.groovy
@@ -17,12 +17,12 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This is the Go postcommit which runs a gradle build, and the current set
// of postcommit tests.
-JobBuilder.postCommitJob('beam_PostCommit_Go_GradleBuild', 'Run Go PostCommit',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go_GradleBuild', 'Run Go
PostCommit',
'./gradlew :goPostCommit', this) {
description('Runs Go PostCommit tests against master.')
diff --git a/.test-infra/jenkins/job_PostCommit_Java_GradleBuild.groovy
b/.test-infra/jenkins/job_PostCommit_Java_GradleBuild.groovy
index eb41d035a09..f0b88e2f2cb 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_GradleBuild.groovy
@@ -17,12 +17,12 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This job runs the Java postcommit tests, including the suite of integration
// tests.
-JobBuilder.postCommitJob('beam_PostCommit_Java_GradleBuild', 'Run Java
PostCommit',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_GradleBuild', 'Run
Java PostCommit',
'Java SDK Post Commit Tests', this) {
description('Runs PostCommit tests on the Java SDK.')
diff --git
a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Apex.groovy
b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Apex.groovy
index e9c9331ae76..ca439667912 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Apex.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Apex.groovy
@@ -17,10 +17,10 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This job runs the suite of ValidatesRunner tests against the Apex runner.
-JobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Apex_Gradle',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Apex_Gradle',
'Run Apex ValidatesRunner', 'Apache Apex Runner ValidatesRunner Tests',
this) {
description('Runs the ValidatesRunner suite on the Apex runner.')
previousNames('beam_PostCommit_Java_ValidatesRunner_Apex')
diff --git
a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy
b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy
index d61ee0df391..2ecbe331ac3 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy
@@ -17,12 +17,12 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This job runs the suite of ValidatesRunner tests against the Dataflow
// runner.
-JobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflow_Gradle',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflow_Gradle',
'Run Dataflow ValidatesRunner', 'Google Cloud Dataflow Runner
ValidatesRunner Tests', this) {
description('Runs the ValidatesRunner suite on the Dataflow runner.')
diff --git
a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy
b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy
index 4a935a0f493..ce21694f785 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy
@@ -17,10 +17,10 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This job runs the suite of ValidatesRunner tests against the Flink runner.
-JobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Flink_Gradle',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Flink_Gradle',
'Run Flink ValidatesRunner', 'Apache Flink Runner ValidatesRunner Tests',
this) {
description('Runs the ValidatesRunner suite on the Flink runner.')
diff --git
a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Gearpump.groovy
b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Gearpump.groovy
index a36250bf525..9ea2281e078 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Gearpump.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Gearpump.groovy
@@ -17,11 +17,11 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This job runs the suite of ValidatesRunner tests against the Gearpump
// runner.
-JobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Gearpump_Gradle',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Gearpump_Gradle',
'Run Gearpump ValidatesRunner', 'Apache Gearpump Runner ValidatesRunner
Tests',
this) {
description('Runs the ValidatesRunner suite on the Gearpump runner.')
diff --git
a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy
b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy
index 97b95e0b3c8..5d0e756a644 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy
@@ -17,10 +17,10 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This job runs the suite of ValidatesRunner tests against the Spark runner.
-JobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Spark_Gradle',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Spark_Gradle',
'Run Spark ValidatesRunner', 'Apache Spark Runner ValidatesRunner Tests',
this) {
description('Runs the ValidatesRunner suite on the Spark runner.')
previousNames('beam_PostCommit_Java_ValidatesRunner_Spark')
diff --git
a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
index 8887af46411..653b6ccede9 100644
---
a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
+++
b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
@@ -17,11 +17,11 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This job runs the suite of Python ValidatesContainer tests against the
// Dataflow runner.
-JobBuilder.postCommitJob('beam_PostCommit_Py_ValCont',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont',
'Run Python Dataflow ValidatesContainer', 'Google Cloud Dataflow Runner
Python ValidatesContainer Tests', this) {
description('Runs Python ValidatesContainer suite on the Dataflow runner.')
diff --git
a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
index b805c7102ab..29f497bfa75 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
@@ -17,11 +17,11 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This job runs the suite of Python ValidatesRunner tests against the
// Dataflow runner.
-JobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run Python
Dataflow ValidatesRunner',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run
Python Dataflow ValidatesRunner',
'Google Cloud Dataflow Runner Python ValidatesRunner Tests', this) {
description('Runs Python ValidatesRunner suite on the Dataflow runner.')
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Verify.groovy
b/.test-infra/jenkins/job_PostCommit_Python_Verify.groovy
index 7d65ba8be3b..ea0533f5063 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_Verify.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_Verify.groovy
@@ -17,10 +17,10 @@
*/
import common_job_properties
-import JobBuilder
+import PostcommitJobBuilder
// This job defines the Python postcommit tests.
-JobBuilder.postCommitJob('beam_PostCommit_Python_Verify', 'Run Python
PostCommit',
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Verify', 'Run
Python PostCommit',
'Python SDK PostCommit Tests', this) {
description('Runs postcommit tests on the Python SDK.')
diff --git a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
b/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
index 29c7ae11eff..1da2e5f4a71 100644
--- a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
@@ -39,7 +39,7 @@ job('beam_PostRelease_NightlySnapshot') {
}
// This is a post-commit job that runs once per day, not for every push.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'0 11 * * *',
false)
diff --git a/.test-infra/jenkins/job_PreCommit_Go_GradleBuild.groovy
b/.test-infra/jenkins/job_PreCommit_Go_GradleBuild.groovy
index 6a3bc15c74b..77a2ab22905 100644
--- a/.test-infra/jenkins/job_PreCommit_Go_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Go_GradleBuild.groovy
@@ -16,29 +16,18 @@
* limitations under the License.
*/
-import common_job_properties
+import PrecommitJobBuilder
-// This is the Go precommit which runs a gradle build, and the current set
-// of precommit tests.
-job('beam_PreCommit_Go_GradleBuild') {
- description('Runs Go PreCommit tests for the current GitHub Pull Request.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(
- delegate,
- 'master',
- 150)
-
- // Sets that this is a PreCommit job.
- common_job_properties.setPreCommit(delegate, './gradlew :goPreCommit', 'Run
Go PreCommit')
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':goPreCommit')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
-}
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+ scope: this,
+ nameBase: 'Go',
+ gradleTask: ':goPreCommit',
+ timeoutMins: 150,
+ triggerPathPatterns: [
+ '^model/.*$',
+ '^sdks/go/.*$',
+ '^runners/.*$',
+ '^release/.*$',
+ ]
+)
+builder.build()
diff --git a/.test-infra/jenkins/job_PreCommit_Java_GradleBuild.groovy
b/.test-infra/jenkins/job_PreCommit_Java_GradleBuild.groovy
index ce67d399ea6..dcb5fd54ce3 100644
--- a/.test-infra/jenkins/job_PreCommit_Java_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Java_GradleBuild.groovy
@@ -16,34 +16,22 @@
* limitations under the License.
*/
-import common_job_properties
+import PrecommitJobBuilder
-// This is the Java precommit which runs a Gradle build, and the current set
-// of precommit tests.
-job('beam_PreCommit_Java_GradleBuild') {
- description('Runs Java PreCommit tests for the current GitHub Pull Request.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(
- delegate,
- 'master',
- 90)
-
- // Publish all test results to Jenkins
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+ scope: this,
+ nameBase: 'Java',
+ gradleTask: ':javaPreCommit',
+ triggerPathPatterns: [
+ '^model/.*$',
+ '^sdks/java/.*$',
+ '^runners/.*$',
+ '^examples/java/.*$',
+ '^release/.*$',
+ ]
+)
+builder.build {
publishers {
archiveJunit('**/build/test-results/**/*.xml')
}
-
- // Sets that this is a PreCommit job.
- common_job_properties.setPreCommit(delegate, './gradlew :javaPreCommit',
'Run Java PreCommit')
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':javaPreCommit')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
}
diff --git a/.test-infra/jenkins/job_PreCommit_Python_GradleBuild.groovy
b/.test-infra/jenkins/job_PreCommit_Python_GradleBuild.groovy
index d03f04e7ed9..35b34f2c678 100644
--- a/.test-infra/jenkins/job_PreCommit_Python_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Python_GradleBuild.groovy
@@ -16,35 +16,23 @@
* limitations under the License.
*/
-import common_job_properties
-
-// This is the Python precommit which runs a Gradle build, and the current set
-// of precommit tests.
-job('beam_PreCommit_Python_GradleBuild') {
- description('Runs Python PreCommit tests for the current GitHub Pull
Request.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(
- delegate,
- 'master',
- 90)
+import PrecommitJobBuilder
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+ scope: this,
+ nameBase: 'Python',
+ gradleTask: ':pythonPreCommit',
+ triggerPathPatterns: [
+ '^model/.*$',
+ '^runners/.*$',
+ '^sdks/python/.*$',
+ '^release/.*$',
+ ]
+)
+builder.build {
// Publish all test results to Jenkins. Note that Nose documentation
// specifically mentions that it produces JUnit compatible test results.
publishers {
archiveJunit('**/nosetests.xml')
}
-
- // Sets that this is a PreCommit job.
- common_job_properties.setPreCommit(delegate, './gradlew :pythonPreCommit',
'Run Python PreCommit')
- steps {
- gradle {
- rootBuildScriptDir(common_job_properties.checkoutDir)
- tasks(':pythonPreCommit')
- common_job_properties.setGradleSwitches(delegate)
- }
- }
}
diff --git a/.test-infra/jenkins/job_Release_Gradle_NightlySnapshot.groovy
b/.test-infra/jenkins/job_Release_Gradle_NightlySnapshot.groovy
index f7d589732a1..175b64273c1 100644
--- a/.test-infra/jenkins/job_Release_Gradle_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_Release_Gradle_NightlySnapshot.groovy
@@ -30,7 +30,7 @@ job('beam_Release_Gradle_NightlySnapshot') {
common_job_properties.setTopLevelMainJobProperties(delegate)
// This is a post-commit job that runs once per day, not for every push.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'0 7 * * *',
false,
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Analysis.groovy
b/.test-infra/jenkins/job_beam_PerformanceTests_Analysis.groovy
index c23743fb840..2d9a967d1b0 100644
--- a/.test-infra/jenkins/job_beam_PerformanceTests_Analysis.groovy
+++ b/.test-infra/jenkins/job_beam_PerformanceTests_Analysis.groovy
@@ -54,7 +54,7 @@ job(testConfiguration.jobName) {
// Run job in postcommit every 24 hours, don't trigger every push, and
// don't email individual committers.
- common_job_properties.setPostCommit(
+ common_job_properties.setAutoJob(
delegate,
'30 */24 * * *',
false,
@@ -82,4 +82,4 @@ job(testConfiguration.jobName) {
// Launch performance tests analysis.
shell('.env/bin/python ' + common_job_properties.checkoutDir +
'/.test-infra/jenkins/verify_performance_test_results.py --bqtable \"'+
testConfiguration.bqTables + '\" ' + '--metric=\"run_time\" ' + '--mode=report
--send_notification')
}
-}
\ No newline at end of file
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 116128)
Time Spent: 3h 10m (was: 3h)
> Filter pre-commit triggering based on touched files
> ---------------------------------------------------
>
> Key: BEAM-4445
> URL: https://issues.apache.org/jira/browse/BEAM-4445
> Project: Beam
> Issue Type: Sub-task
> Components: build-system, testing
> Reporter: Scott Wegner
> Assignee: Scott Wegner
> Priority: Minor
> Labels: beam-site-automation-reliability
> Fix For: Not applicable
>
> Time Spent: 3h 10m
> Remaining Estimate: 0h
>
> This is discussed in the [Beam-Site Automation
> Reliability|https://s.apache.org/beam-site-automation] design, under
> "Pre-Commit Job Filtering"
> The proposal is to filter pre-commit job triggered on PR's based on which
> files are touched. The impact is that most PRs will only run one set of
> relevant tests, rather than all three. This will decrease test overhead and
> the impact of flaky tests.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
