Initial set of pipeline jobs. Signed-off-by: Jason Kuster <jasonkus...@google.com>
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4f7e0d65 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4f7e0d65 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4f7e0d65 Branch: refs/heads/master Commit: 4f7e0d65c514f022c0675dec50853ac3c7554be7 Parents: d6c6339 Author: Jason Kuster <jasonkus...@google.com> Authored: Wed Jun 28 16:22:52 2017 -0700 Committer: Thomas Groh <tg...@google.com> Committed: Thu Sep 21 19:21:38 2017 -0700 ---------------------------------------------------------------------- .test-infra/jenkins/PreCommit_Pipeline.groovy | 89 +++++++++ .../jenkins/common_job_properties.groovy | 185 ++++++++++++++----- .test-infra/jenkins/job_beam_Java_Build.groovy | 82 ++++++++ .../jenkins/job_beam_Java_CodeHealth.groovy | 39 ++++ .../job_beam_Java_IntegrationTest.groovy | 63 +++++++ .../jenkins/job_beam_Java_UnitTest.groovy | 49 +++++ .../jenkins/job_beam_PreCommit_Pipeline.groovy | 81 ++++++++ .../jenkins/job_beam_Python_UnitTest.groovy | 40 ++++ 8 files changed, 581 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/4f7e0d65/.test-infra/jenkins/PreCommit_Pipeline.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy new file mode 100644 index 0000000..20eaa56 --- /dev/null +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -0,0 +1,89 @@ +#!groovy +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import hudson.model.Result + +int NO_BUILD = -1 + +// These are args for the GitHub Pull Request Builder (ghprb) Plugin. Providing these arguments is +// necessary due to a bug in the ghprb plugin where environment variables are not correctly passed +// to jobs downstream of a Pipeline job. +// Tracked by https://github.com/jenkinsci/ghprb-plugin/issues/572. +List<Object> ghprbArgs = [ + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") +] + +// This argument is the commit at which to build. +List<Object> commitArg = [string(name: 'commit', value: "origin/pr/${ghprbPullId}/head")] + +int javaBuildNum = NO_BUILD + +// This (and the below) define "Stages" of a pipeline. These stages run serially, and inside can +// have "parallel" blocks which execute several work steps concurrently. This work is limited to +// simple operations -- more complicated operations need to be performed on an actual node. In this +// case we are using the pipeline to trigger downstream builds. +stage('Build') { + parallel ( + java: { + def javaBuild = build job: 'beam_Java_Build', parameters: commitArg + ghprbArgs + if(javaBuild.getResult() == Result.SUCCESS.toString()) { + javaBuildNum = javaBuild.getNumber() + } + }, + python_unit: { // Python doesn't have a build phase, so we include this here. + build job: 'beam_Python_UnitTest', parameters: commitArg + ghprbArgs + } + ) +} + +// This argument is provided to downstream jobs so they know from which build to pull artifacts. +javaBuildArg = [string(name: 'buildNum', value: "${javaBuildNum}")] +javaUnitPassed = false + +stage('Unit Test / Code Health') { + parallel ( + java_unit: { + if(javaBuildNum != NO_BUILD) { + def javaTest = build job: 'beam_Java_UnitTest', parameters: javaBuildArg + ghprbArgs + if(javaTest.getResult() == Result.SUCCESS.toString()) { + javaUnitPassed = true + } + } + }, + java_codehealth: { + if(javaBuildNum != NO_BUILD) { + build job: 'beam_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs + } + } + ) +} + +stage('Integration Test') { + parallel ( + // Not gated on codehealth because codehealth shouldn't affect whether tests provide useful + // signal. + java_integration: { + if(javaUnitPassed) { + build job: 'beam_Java_IntegrationTest', parameters: javaBuildArg + ghprbArgs + } + } + ) +} http://git-wip-us.apache.org/repos/asf/beam/blob/4f7e0d65/.test-infra/jenkins/common_job_properties.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 70534c6..43ed7cf 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -22,8 +22,27 @@ // http://groovy-lang.org/style-guide.html class common_job_properties { + static void setSCM(def context, String repositoryName) { + context.scm { + git { + remote { + // Double quotes here mean ${repositoryName} is interpolated. + github("apache/${repositoryName}") + // Single quotes here mean that ${ghprbPullId} is not interpolated and instead passed + // through to Jenkins where it refers to the environment variable. + refspec('+refs/heads/*:refs/remotes/origin/* ' + + '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*') + } + branch('${sha1}') + extensions { + cleanAfterCheckout() + } + } + } + } + // Sets common top-level job properties for website repository jobs. - static void setTopLevelWebsiteJobProperties(context, + static void setTopLevelWebsiteJobProperties(def context, String branch = 'asf-site') { setTopLevelJobProperties( context, @@ -34,7 +53,7 @@ class common_job_properties { } // Sets common top-level job properties for main repository jobs. - static void setTopLevelMainJobProperties(context, + static void setTopLevelMainJobProperties(def context, String branch = 'master', int timeout = 100, String jenkinsExecutorLabel = 'beam') { @@ -48,7 +67,7 @@ class common_job_properties { // Sets common top-level job properties. Accessed through one of the above // methods to protect jobs from internal details of param defaults. - private static void setTopLevelJobProperties(context, + private static void setTopLevelJobProperties(def context, String repositoryName, String defaultBranch, String jenkinsExecutorLabel, @@ -71,19 +90,7 @@ class common_job_properties { } // Source code management. - context.scm { - git { - remote { - url('https://github.com/apache/' + repositoryName + '.git') - refspec('+refs/heads/*:refs/remotes/origin/* ' + - '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*') - } - branch('${sha1}') - extensions { - cleanAfterCheckout() - } - } - } + setSCM(context, repositoryName) context.parameters { // This is a recommended setup if you want to run the job manually. The @@ -141,41 +148,19 @@ class common_job_properties { delegate.context("Jenkins: " + commitStatusContext) } - /* - This section is disabled, because of jenkinsci/ghprb-plugin#417 issue. - For the time being, an equivalent configure section below is added. - // Comment messages after build completes. buildStatus { completedStatus('SUCCESS', successComment) completedStatus('FAILURE', '--none--') completedStatus('ERROR', '--none--') } - */ } } } - - // Comment messages after build completes. - context.configure { - def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message(successComment) - result('SUCCESS') - } - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message('--none--') - result('ERROR') - } - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message('--none--') - result('FAILURE') - } - } } // Sets common config for Maven jobs. - static void setMavenConfig(context, mavenInstallation='Maven 3.3.3') { + static void setMavenConfig(context, String mavenInstallation='Maven 3.3.3') { context.mavenInstallation(mavenInstallation) context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true') context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS') @@ -236,10 +221,19 @@ class common_job_properties { } } + static def mapToArgString(LinkedHashMap<String, String> inputArgs) { + List argList = [] + inputArgs.each({ + // FYI: Replacement only works with double quotes. + key, value -> argList.add("--$key=$value") + }) + return argList.join(' ') + } + // Configures the argument list for performance tests, adding the standard // performance test job arguments. private static def genPerformanceArgs(def argMap) { - def standard_args = [ + LinkedHashMap<String, String> standardArgs = [ project: 'apache-beam-testing', dpb_log_level: 'INFO', maven_binary: '/home/jenkins/tools/maven/latest/bin/mvn', @@ -248,13 +242,8 @@ class common_job_properties { official: 'true' ] // Note: in case of key collision, keys present in ArgMap win. - def joined_args = standard_args.plus(argMap) - def argList = [] - joined_args.each({ - // FYI: Replacement only works with double quotes. - key, value -> argList.add("--$key=$value") - }) - return argList.join(' ') + LinkedHashMap<String, String> joinedArgs = standardArgs.plus(argMap) + return mapToArgString(joinedArgs) } // Adds the standard performance test job steps. @@ -273,4 +262,106 @@ class common_job_properties { shell("python PerfKitBenchmarker/pkb.py $pkbArgs") } } + + /** + * Sets properties for all jobs which are run by a pipeline top-level (maven) job. + * @param context The delegate from the top level of a MavenJob. + * @param jobTimeout How long (in minutes) to wait for the job to finish. + * @param descriptor A short string identifying the job, e.g. "Java Unit Test". + */ + static def setPipelineJobProperties(def context, int jobTimeout, String descriptor) { + context.parameters { + stringParam( + 'ghprbGhRepository', + 'N/A', + 'Repository name for use by ghprb plugin.') + stringParam( + 'ghprbActualCommit', + 'N/A', + 'Commit ID for use by ghprb plugin.') + stringParam( + 'ghprbPullId', + 'N/A', + 'PR # for use by ghprb plugin.') + + } + + // Set JDK version. + context.jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors as specified + context.label('beam') + + // Execute concurrent builds if necessary. + context.concurrentBuild() + + context.wrappers { + timeout { + absolute(jobTimeout) + abortBuild() + } + credentialsBinding { + string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") + } + downstreamCommitStatus { + delegate.context("Jenkins: ${descriptor}") + triggeredStatus("${descriptor} Pending") + startedStatus("Running ${descriptor}") + statusUrl() + completedStatus('SUCCESS', "${descriptor} Passed") + completedStatus('FAILURE', "${descriptor} Failed") + completedStatus('ERROR', "Error Executing ${descriptor}") + } + // Set SPARK_LOCAL_IP for spark tests. + environmentVariables { + env('SPARK_LOCAL_IP', '127.0.0.1') + } + } + + // Set Maven parameters. + setMavenConfig(context) + } + + /** + * Sets job properties common to pipeline jobs which are responsible for being the root of a + * build tree. Downstream jobs should pull artifacts from these jobs. + * @param context The delegate from the top level of a MavenJob. + */ + static def setPipelineBuildJobProperties(def context) { + context.properties { + githubProjectUrl('https://github.com/apache/beam/') + } + + context.parameters { + stringParam( + 'commit', + 'master', + 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') + } + + // Source code management. + setSCM(context, 'beam') + } + + /** + * Sets common job parameters for jobs which consume artifacts built for them by an upstream job. + * @param context The delegate from the top level of a MavenJob. + * @param jobName The job from which to copy artifacts. + */ + static def setPipelineDownstreamJobProperties(def context, String jobName) { + context.parameters { + stringParam( + 'buildNum', + 'N/A', + "Build number of ${jobName} to copy from.") + } + + context.preBuildSteps { + copyArtifacts(jobName) { + buildSelector { + buildNumber('${buildNum}') + } + } + } + } } http://git-wip-us.apache.org/repos/asf/beam/blob/4f7e0d65/.test-infra/jenkins/job_beam_Java_Build.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_Java_Build.groovy b/.test-infra/jenkins/job_beam_Java_Build.groovy new file mode 100644 index 0000000..7c6f4cf --- /dev/null +++ b/.test-infra/jenkins/job_beam_Java_Build.groovy @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java Jenkins job which builds artifacts for downstream jobs to consume. +mavenJob('beam_Java_Build') { + description('Builds Beam Java SDK and archives artifacts. Meant to be run as part of a pipeline.') + + // Set standard properties for a job which is part of a pipeline. + common_job_properties.setPipelineJobProperties(delegate, 15, "Java Build") + // Set standard properties for a pipeline job which needs to pull from GitHub instead of an + // upstream job. + common_job_properties.setPipelineBuildJobProperties(delegate) + + configure { project -> + // The CopyArtifact plugin doesn't support the job DSL so we have to configure it manually. + project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { + 'string' "beam_*" + } + // The Build Discarder also doesn't support the job DSL in the right way so we have to configure it manually. + // -1 indicates that a property is "infinite". + project / 'properties' / 'jenkins.model.BuildDiscarderProperty' / 'strategy'(class:'hudson.tasks.LogRotator') { + 'daysToKeep'(-1) + 'numToKeep'(-1) + 'artifactDaysToKeep'(1) + 'artifactNumToKeep'(-1) + } + } + + // Construct Maven goals for this job. + profiles = [ + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + args = [ + '-B', + '-e', + "-P${profiles.join(',')}", + 'clean', + 'install', + "-pl '!sdks/python,!sdks/java/javadoc'", + '-DskipTests', + '-Dcheckstyle.skip', + ] + goals(args.join(' ')) + + // This job publishes artifacts so that downstream jobs can use them. + publishers { + archiveArtifacts { + pattern('.repository/org/apache/beam/**/*') + pattern('.test-infra/**/*') + pattern('.github/**/*') + pattern('examples/**/*') + pattern('runners/**/*') + pattern('sdks/**/*') + pattern('target/**/*') + pattern('pom.xml') + exclude('examples/**/*.jar,runners/**/*.jar,sdks/**/*.jar,target/**/*.jar') + onlyIfSuccessful() + defaultExcludes() + } + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/4f7e0d65/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy new file mode 100644 index 0000000..e50bcd7 --- /dev/null +++ b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java Jenkins job which runs the Beam code health checks. +mavenJob('beam_Java_CodeHealth') { + description('Runs Java code health checks. Meant to be run as part of a pipeline.') + + // Set standard properties for a job which is part of a pipeline. + common_job_properties.setPipelineJobProperties(delegate, 15, "Java Code Health") + // This job runs downstream of the beam_Java_Build job and gets artifacts from that job. + common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build') + + args = [ + '-B', + '-e', + "-pl '!sdks/python'", + 'checkstyle:check', + 'findbugs:check', + 'org.apache.rat:apache-rat-plugin:check', + ] + goals(args.join(' ')) +} http://git-wip-us.apache.org/repos/asf/beam/blob/4f7e0d65/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy new file mode 100644 index 0000000..87470ef --- /dev/null +++ b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java Jenkins job which runs the set of precommit integration tests. +mavenJob('beam_Java_IntegrationTest') { + description('Runs Java Failsafe integration tests. Designed to be run as part of a pipeline.') + + // Set standard properties for a job which is part of a pipeline. + common_job_properties.setPipelineJobProperties(delegate, 25, "Java Integration Tests") + // Set standard properties for a job which pulls artifacts from an upstream job. + common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build') + + // Profiles to activate in order to ensure runners are available at test time. + profiles = [ + 'jenkins-precommit', + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + // In the case of the precommit integration tests, we are currently only running the integration + // tests in the examples directory. By directly invoking failsafe with an execution name (which we + // do in order to avoid building artifacts again) we are required to enumerate each execution we + // want to run, something which is feasible in this case. + examples_integration_executions = [ + 'apex-runner-integration-tests', + 'dataflow-runner-integration-tests', + 'dataflow-runner-integration-tests-streaming', + 'direct-runner-integration-tests', + 'flink-runner-integration-tests', + 'spark-runner-integration-tests', + ] + // Arguments to provide Maven. + args = [ + '-B', + '-e', + "-P${profiles.join(',')}", + "-pl examples/java", + ] + // This adds executions for each of the failsafe invocations listed above to the list of goals. + examples_integration_executions.each({ + value -> args.add("failsafe:integration-test@${value}") + }) + goals(args.join(' ')) +} http://git-wip-us.apache.org/repos/asf/beam/blob/4f7e0d65/.test-infra/jenkins/job_beam_Java_UnitTest.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy new file mode 100644 index 0000000..5a3d04e --- /dev/null +++ b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java Jenkins job which runs the current set of standard unit tests. +mavenJob('beam_Java_UnitTest') { + description('Runs Java Surefire unit tests. Designed to be run by a pipeline job.') + + // Set standard properties for a job which is part of a pipeline. + common_job_properties.setPipelineJobProperties(delegate, 20, "Java Unit Tests") + // Set standard properties for a job which pulls artifacts from an upstream job. + common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build') + + // Construct Maven goals for this job. + profiles = [ + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + args = [ + '-B', + '-e', + "-P${profiles.join(',')}", + 'surefire:test@default-test', + 'coveralls:report', // TODO: Will this work? Can't verify on my own Jenkins due to no coveralls. + "-pl '!sdks/python'", + '-DrepoToken=$COVERALLS_REPO_TOKEN', + '-DpullRequest=$ghprbPullId', + ] + goals(args.join(' ')) +} http://git-wip-us.apache.org/repos/asf/beam/blob/4f7e0d65/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy new file mode 100644 index 0000000..832712a --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy @@ -0,0 +1,81 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job owns the overall execution of the precommit pipeline. The actual pipeline code is in +// Precommit_Pipeline.groovy. +pipelineJob('beam_PreCommit_Pipeline') { + description('PreCommit Pipeline Job. Owns overall lifecycle of PreCommit tests.') + + properties { + githubProjectUrl('https://github.com/apache/beam/') + } + + parameters { + // Allow building at a specific commit. + stringParam( + 'commit', + 'master', + 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') + } + + wrappers { + // Set a timeout appropriate for the precommit tests. + timeout { + absolute(120) + abortBuild() + } + } + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + triggers { + githubPullRequest { + admins(['asfbot']) + useGitHubHooks() + orgWhitelist(['apache']) + allowMembersOfWhitelistedOrgsAsAdmin() + permitAll() + displayBuildErrorsOnDownstreamBuilds() + extensions { + commitStatus { + context("Jenkins: PreCommit Pipeline") + } + buildStatus { + completedStatus('SUCCESS', '--none--') + completedStatus('FAILURE', '--none--') + completedStatus('ERROR', '--none--') + } + } + } + } + + definition { + cpsScm { + // Source code management. + common_job_properties.setSCM(delegate, 'beam') + scriptPath('.test-infra/jenkins/PreCommit_Pipeline.groovy') + } + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/4f7e0d65/.test-infra/jenkins/job_beam_Python_UnitTest.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy new file mode 100644 index 0000000..299157a --- /dev/null +++ b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Python Jenkins job which runs a maven install, and the current set of precommit +// tests. +mavenJob('beam_Python_UnitTest') { + description('Runs Python unit tests on a specific commit. Designed to be run by a pipeline job.') + + // Set standard properties for a job which is part of a pipeline. + common_job_properties.setPipelineJobProperties(delegate, 25, "Python Unit Tests") + // Set standard properties for a pipeline job which needs to pull from GitHub instead of an + // upstream job. + common_job_properties.setPipelineBuildJobProperties(delegate) + + // Construct Maven goals for this job. + args = [ + '-B', + '-e', + 'clean install', + '-pl sdks/python', + ] + goals(args.join(' ')) +}
