Move travis/jenkins folders in a test-infra folder
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/66b20af9 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/66b20af9 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/66b20af9 Branch: refs/heads/master Commit: 66b20af9fed36ff419ca6da3de4b1b1a02e66464 Parents: 1788cef Author: Stephen Sisk <[email protected]> Authored: Fri Apr 7 15:57:33 2017 -0700 Committer: Thomas Groh <[email protected]> Committed: Wed Apr 12 13:52:28 2017 -0700 ---------------------------------------------------------------------- .jenkins/common_job_properties.groovy | 261 ------------------- .../job_beam_PerformanceTests_Dataflow.groovy | 43 --- .jenkins/job_beam_PerformanceTests_JDBC.groovy | 60 ----- .jenkins/job_beam_PerformanceTests_Spark.groovy | 44 ---- ...job_beam_PostCommit_Java_MavenInstall.groovy | 42 --- ..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 ---- ...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 ---- ...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 --- ...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 ---- ...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 ---- .../job_beam_PostCommit_Python_Verify.groovy | 55 ---- .../job_beam_PreCommit_Java_MavenInstall.groovy | 42 --- .../job_beam_PreCommit_Website_Stage.groovy | 80 ------ .jenkins/job_beam_PreCommit_Website_Test.groovy | 65 ----- .../job_beam_Release_NightlySnapshot.groovy | 45 ---- .jenkins/job_seed.groovy | 53 ---- .../jenkins/common_job_properties.groovy | 261 +++++++++++++++++++ .../job_beam_PerformanceTests_Dataflow.groovy | 43 +++ .../job_beam_PerformanceTests_JDBC.groovy | 60 +++++ .../job_beam_PerformanceTests_Spark.groovy | 44 ++++ ...job_beam_PostCommit_Java_MavenInstall.groovy | 42 +++ ..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 ++++ ...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 ++++ ...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 +++ ...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 ++++ ...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 ++++ .../job_beam_PostCommit_Python_Verify.groovy | 55 ++++ .../job_beam_PreCommit_Java_MavenInstall.groovy | 42 +++ .../job_beam_PreCommit_Website_Stage.groovy | 80 ++++++ .../job_beam_PreCommit_Website_Test.groovy | 65 +++++ .../job_beam_Release_NightlySnapshot.groovy | 45 ++++ .test-infra/jenkins/job_seed.groovy | 53 ++++ .test-infra/travis/README.md | 23 ++ .test-infra/travis/settings.xml | 33 +++ .test-infra/travis/test_wordcount.sh | 125 +++++++++ .travis.yml | 4 +- .travis/README.md | 23 -- .travis/settings.xml | 33 --- .travis/test_wordcount.sh | 125 --------- 39 files changed, 1202 insertions(+), 1202 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/common_job_properties.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/common_job_properties.groovy b/.jenkins/common_job_properties.groovy deleted file mode 100644 index ee10281..0000000 --- a/.jenkins/common_job_properties.groovy +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Contains functions that help build Jenkins projects. Functions typically set -// common properties that are shared among all Jenkins projects. -// Code in this directory should conform to the Groovy style guide. -// http://groovy-lang.org/style-guide.html -class common_job_properties { - - // Sets common top-level job properties for website repository jobs. - static void setTopLevelWebsiteJobProperties(context) { - setTopLevelJobProperties(context, 'beam-site', 'asf-site', 30) - } - - // Sets common top-level job properties for main repository jobs. - static void setTopLevelMainJobProperties(context, - String defaultBranch = 'master', - int defaultTimeout = 100) { - setTopLevelJobProperties(context, 'beam', defaultBranch, defaultTimeout) - } - - // Sets common top-level job properties. Accessed through one of the above - // methods to protect jobs from internal details of param defaults. - private static void setTopLevelJobProperties(context, - String repositoryName, - String defaultBranch, - int defaultTimeout) { - - // GitHub project. - context.properties { - githubProjectUrl('https://github.com/apache/' + repositoryName + '/') - } - - // Set JDK version. - context.jdk('JDK 1.8 (latest)') - - // Restrict this project to run only on Jenkins executors dedicated to the - // Apache Beam project. - context.label('beam') - - // Discard old builds. Build records are only kept up to this number of days. - context.logRotator { - daysToKeep(14) - } - - // Source code management. - context.scm { - git { - remote { - url('https://github.com/apache/' + repositoryName + '.git') - refspec('+refs/heads/*:refs/remotes/origin/* ' + - '+refs/pull/*:refs/remotes/origin/pr/*') - } - branch('${sha1}') - extensions { - cleanAfterCheckout() - pruneBranches() - } - } - } - - context.parameters { - // This is a recommended setup if you want to run the job manually. The - // ${sha1} parameter needs to be provided, and defaults to the main branch. - stringParam( - 'sha1', - defaultBranch, - 'Commit id or refname (eg: origin/pr/9/head) you want to build.') - } - - context.wrappers { - // Abort the build if it's stuck for more minutes than specified. - timeout { - absolute(defaultTimeout) - abortBuild() - } - - // Set SPARK_LOCAL_IP for spark tests. - environmentVariables { - env('SPARK_LOCAL_IP', '127.0.0.1') - } - credentialsBinding { - string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") - } - } - } - - // Sets the pull request build trigger. Accessed through precommit methods - // below to insulate callers from internal parameter defaults. - private static void setPullRequestBuildTrigger(context, - String commitStatusContext, - String successComment = '--none--', - String prTriggerPhrase = '') { - context.triggers { - githubPullRequest { - admins(['asfbot']) - useGitHubHooks() - orgWhitelist(['apache']) - allowMembersOfWhitelistedOrgsAsAdmin() - permitAll() - // prTriggerPhrase is the argument which gets set when we want to allow - // post-commit builds to run against pending pull requests. This block - // overrides the default trigger phrase with the new one. Setting this - // will disable automatic invocation of this build; the phrase will be - // required to start it. - if (prTriggerPhrase) { - triggerPhrase(prTriggerPhrase) - onlyTriggerPhrase() - } - - extensions { - commitStatus { - // This is the name that will show up in the GitHub pull request UI - // for this Jenkins project. - delegate.context("Jenkins: " + commitStatusContext) - } - - /* - This section is disabled, because of jenkinsci/ghprb-plugin#417 issue. - For the time being, an equivalent configure section below is added. - - // Comment messages after build completes. - buildStatus { - completedStatus('SUCCESS', successComment) - completedStatus('FAILURE', '--none--') - completedStatus('ERROR', '--none--') - } - */ - } - } - } - - // Comment messages after build completes. - context.configure { - def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message(successComment) - result('SUCCESS') - } - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message('--none--') - result('ERROR') - } - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message('--none--') - result('FAILURE') - } - } - } - - // Sets common config for Maven jobs. - static void setMavenConfig(context) { - context.mavenInstallation('Maven 3.3.3') - context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true') - context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS') - // The -XX:+TieredCompilation -XX:TieredStopAtLevel=1 JVM options enable - // tiered compilation to make the JVM startup times faster during the tests. - context.mavenOpts('-XX:+TieredCompilation') - context.mavenOpts('-XX:TieredStopAtLevel=1') - context.rootPOM('pom.xml') - // Use a repository local to the workspace for better isolation of jobs. - context.localRepository(LocalRepositoryLocation.LOCAL_TO_WORKSPACE) - // Disable archiving the built artifacts by default, as this is slow and flaky. - // We can usually recreate them easily, and we can also opt-in individual jobs - // to artifact archiving. - context.archivingDisabled(true) - } - - // Sets common config for PreCommit jobs. - static void setPreCommit(context, - String commitStatusName, - String successComment = '--none--') { - // Set pull request build trigger. - setPullRequestBuildTrigger(context, commitStatusName, successComment) - } - - // Enable triggering postcommit runs against pull requests. Users can comment the trigger phrase - // specified in the postcommit job and have the job run against their PR to run - // tests not in the presubmit suite for additional confidence. - static void enablePhraseTriggeringFromPullRequest(context, - String commitStatusName, - String prTriggerPhrase) { - setPullRequestBuildTrigger( - context, - commitStatusName, - '--none--', - prTriggerPhrase) - } - - // Sets common config for PostCommit jobs. - static void setPostCommit(context, - String buildSchedule = '0 */6 * * *', - boolean triggerEveryPush = true, - String notifyAddress = '[email protected]', - boolean emailIndividuals = true) { - // Set build triggers - context.triggers { - // By default runs every 6 hours. - cron(buildSchedule) - if (triggerEveryPush) { - githubPush() - } - } - - context.publishers { - // Notify an email address for each failed build (defaults to commits@). - mailer(notifyAddress, false, emailIndividuals) - } - } - - // Configures the argument list for performance tests, adding the standard - // performance test job arguments. - private static def genPerformanceArgs(def argMap) { - def standard_args = [ - project: 'apache-beam-testing', - dpb_log_level: 'INFO', - maven_binary: '/home/jenkins/tools/maven/latest/bin/mvn', - bigquery_table: 'beam_performance.pkb_results', - // Publishes results with official tag, for use in dashboards. - official: 'true' - ] - // Note: in case of key collision, keys present in ArgMap win. - def joined_args = standard_args.plus(argMap) - def argList = [] - joined_args.each({ - // FYI: Replacement only works with double quotes. - key, value -> argList.add("--$key=$value") - }) - return argList.join(' ') - } - - // Adds the standard performance test job steps. - static def buildPerformanceTest(def context, def argMap) { - def pkbArgs = genPerformanceArgs(argMap) - context.steps { - // Clean up environment. - shell('rm -rf PerfKitBenchmarker') - // Clone appropriate perfkit branch - shell('git clone https://github.com/GoogleCloudPlatform/PerfKitBenchmarker.git') - // Install job requirements. - shell('pip install --user -r PerfKitBenchmarker/requirements.txt') - // Launch performance test. - shell("python PerfKitBenchmarker/pkb.py $pkbArgs") - } - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PerformanceTests_Dataflow.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PerformanceTests_Dataflow.groovy b/.jenkins/job_beam_PerformanceTests_Dataflow.groovy deleted file mode 100644 index 51c73f3..0000000 --- a/.jenkins/job_beam_PerformanceTests_Dataflow.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job runs the Beam performance tests on PerfKit Benchmarker. -job('beam_PerformanceTests_Dataflow'){ - // Set default Beam job properties. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Run job in postcommit every 6 hours, don't trigger every push, and - // don't email individual committers. - common_job_properties.setPostCommit( - delegate, - '0 */6 * * *', - false, - '[email protected]', - false) - - def argMap = [ - benchmarks: 'dpb_wordcount_benchmark', - dpb_dataflow_staging_location: 'gs://temp-storage-for-perf-tests/staging', - dpb_wordcount_input: 'dataflow-samples/shakespeare/kinglear.txt', - config_override: 'dpb_wordcount_benchmark.dpb_service.service_type=dataflow' - ] - - common_job_properties.buildPerformanceTest(delegate, argMap) -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PerformanceTests_JDBC.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PerformanceTests_JDBC.groovy b/.jenkins/job_beam_PerformanceTests_JDBC.groovy deleted file mode 100644 index 8e581c2..0000000 --- a/.jenkins/job_beam_PerformanceTests_JDBC.groovy +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job runs the Beam performance tests on PerfKit Benchmarker. -job('beam_PerformanceTests_JDBC'){ - // Set default Beam job properties. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Run job in postcommit every 6 hours, don't trigger every push, and - // don't email individual committers. - common_job_properties.setPostCommit( - delegate, - '0 */6 * * *', - false, - '[email protected]', - false) - - def pipelineArgs = [ - tempRoot: 'gs://temp-storage-for-end-to-end-tests', - project: 'apache-beam-testing', - postgresServerName: '10.36.0.11', - postgresUsername: 'postgres', - postgresDatabaseName: 'postgres', - postgresPassword: 'uuinkks', - postgresSsl: 'false' - ] - def pipelineArgList = [] - pipelineArgs.each({ - key, value -> pipelineArgList.add("--$key=$value") - }) - def pipelineArgsJoined = pipelineArgList.join(',') - - def argMap = [ - benchmarks: 'beam_integration_benchmark', - beam_it_module: 'sdks/java/io/jdbc', - beam_it_args: pipelineArgsJoined, - beam_it_class: 'org.apache.beam.sdk.io.jdbc.JdbcIOIT', - // Profile is located in $BEAM_ROOT/sdks/java/io/pom.xml. - beam_it_profile: 'io-it' - ] - - common_job_properties.buildPerformanceTest(delegate, argMap) -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PerformanceTests_Spark.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PerformanceTests_Spark.groovy b/.jenkins/job_beam_PerformanceTests_Spark.groovy deleted file mode 100644 index ba719bf..0000000 --- a/.jenkins/job_beam_PerformanceTests_Spark.groovy +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job runs the Beam performance tests on PerfKit Benchmarker. -job('beam_PerformanceTests_Spark'){ - // Set default Beam job properties. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Run job in postcommit every 6 hours, don't trigger every push, and - // don't email individual committers. - common_job_properties.setPostCommit( - delegate, - '0 */6 * * *', - false, - '[email protected]', - false) - - def argMap = [ - benchmarks: 'dpb_wordcount_benchmark', - // There are currently problems uploading to Dataproc, so we use a file - // already present on the machines as input. - dpb_wordcount_input: '/etc/hosts', - config_override: 'dpb_wordcount_benchmark.dpb_service.service_type=dataproc' - ] - - common_job_properties.buildPerformanceTest(delegate, argMap) -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy deleted file mode 100644 index a288a84..0000000 --- a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job runs the Java postcommit tests, including the suite of integration -// tests. -mavenJob('beam_PostCommit_Java_MavenInstall') { - description('Runs postcommit tests on the Java SDK.') - - previousNames('beam_PostCommit_MavenVerify') - - // Execute concurrent builds if necessary. - concurrentBuild() - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Set maven parameters. - common_job_properties.setMavenConfig(delegate) - - // Sets that this is a PostCommit job. - common_job_properties.setPostCommit(delegate) - - // Maven goals for this job. - goals('-B -e -P release,dataflow-runner clean install coveralls:report -DrepoToken=$COVERALLS_REPO_TOKEN -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner" ]\'') -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy deleted file mode 100644 index c16a1e2..0000000 --- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job runs the suite of ValidatesRunner tests against the Apex runner. -mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') { - description('Runs the ValidatesRunner suite on the Apex runner.') - previousNames('beam_PostCommit_Java_RunnableOnService_Apex') - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Set maven parameters. - common_job_properties.setMavenConfig(delegate) - - // Sets that this is a PostCommit job. - common_job_properties.setPostCommit(delegate) - - // Allows triggering this build against pull requests. - common_job_properties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Apache Apex Runner ValidatesRunner Tests', - 'Run Apex ValidatesRunner') - - // Maven goals for this job. - goals('''clean verify --projects runners/apex \ - --also-make \ - --batch-mode \ - --errors \ - --activate-profiles validates-runner-tests \ - --activate-profiles local-validates-runner-tests''') -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy deleted file mode 100644 index 33235ff..0000000 --- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job runs the suite of ValidatesRunner tests against the Dataflow -// runner. -mavenJob('beam_PostCommit_Java_ValidatesRunner_Dataflow') { - description('Runs the ValidatesRunner suite on the Dataflow runner.') - previousNames('beam_PostCommit_Java_RunnableOnService_Dataflow') - - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 120) - - // Set maven parameters. - common_job_properties.setMavenConfig(delegate) - - // Sets that this is a PostCommit job. - common_job_properties.setPostCommit(delegate) - - // Allows triggering this build against pull requests. - common_job_properties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Google Cloud Dataflow Runner ValidatesRunner Tests', - 'Run Dataflow ValidatesRunner') - - // Maven goals for this job. - goals('-B -e clean verify -am -pl runners/google-cloud-dataflow-java -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner", "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-validates-runner-tests/" ]\'') -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy deleted file mode 100644 index 411106d..0000000 --- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job runs the suite of ValidatesRunner tests against the Flink runner. -mavenJob('beam_PostCommit_Java_ValidatesRunner_Flink') { - description('Runs the ValidatesRunner suite on the Flink runner.') - previousNames('beam_PostCommit_Java_RunnableOnService_Flink') - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Set maven parameters. - common_job_properties.setMavenConfig(delegate) - - // Sets that this is a PostCommit job. - common_job_properties.setPostCommit(delegate) - - // Allows triggering this build against pull requests. - common_job_properties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Apache Flink Runner ValidatesRunner Tests', - 'Run Flink ValidatesRunner') - - // Maven goals for this job. - goals('-B -e clean verify -am -pl runners/flink/runner -Plocal-validates-runner-tests -Pvalidates-runner-tests') -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy deleted file mode 100644 index 1348a19..0000000 --- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job runs the suite of ValidatesRunner tests against the Gearpump -// runner. -mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') { - description('Runs the ValidatesRunner suite on the Gearpump runner.') - - previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump') - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties( - delegate, - 'gearpump-runner') - - // Set maven parameters. - common_job_properties.setMavenConfig(delegate) - - // Sets that this is a PostCommit job. - // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties. - // In post-commit this job triggers only on SCM changes. - common_job_properties.setPostCommit(delegate, '0 5 31 2 *') - - // Allows triggering this build against pull requests. - common_job_properties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Apache Gearpump Runner ValidatesRunner Tests', - 'Run Gearpump ValidatesRunner') - - // Maven goals for this job. - goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner", "--streaming=false" ]\'') -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy deleted file mode 100644 index 9fbc219..0000000 --- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job runs the suite of ValidatesRunner tests against the Spark runner. -mavenJob('beam_PostCommit_Java_ValidatesRunner_Spark') { - description('Runs the ValidatesRunner suite on the Spark runner.') - - previousNames('beam_PostCommit_Java_RunnableOnService_Spark') - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Set maven parameters. - common_job_properties.setMavenConfig(delegate) - - // Sets that this is a PostCommit job. - common_job_properties.setPostCommit(delegate) - - // Allows triggering this build against pull requests. - common_job_properties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Apache Spark Runner ValidatesRunner Tests', - 'Run Spark ValidatesRunner') - - // Maven goals for this job. - goals('-B -e clean verify -am -pl runners/spark -Pvalidates-runner-tests -Plocal-validates-runner-tests -Dspark.ui.enabled=false') -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Python_Verify.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PostCommit_Python_Verify.groovy b/.jenkins/job_beam_PostCommit_Python_Verify.groovy deleted file mode 100644 index 28cf77e..0000000 --- a/.jenkins/job_beam_PostCommit_Python_Verify.groovy +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This job defines the Python postcommit tests. -job('beam_PostCommit_Python_Verify') { - description('Runs postcommit tests on the Python SDK.') - - previousNames('beam_PostCommit_PythonVerify') - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Sets that this is a PostCommit job. - common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *') - - // Allows triggering this build against pull requests. - common_job_properties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Python SDK PostCommit Tests', - 'Run Python PostCommit') - - // Allow the test to only run on particular nodes - // TODO(BEAM-1817): Remove once the tests can run on all nodes - parameters { - nodeParam('TEST_HOST') { - description('select test host as either beam1, 2 or 3') - defaultNodes(['beam3']) - allowedNodes(['beam1', 'beam2', 'beam3']) - trigger('multiSelectionDisallowed') - eligibility('IgnoreOfflineNodeEligibility') - } - } - - // Execute shell command to test Python SDK. - steps { - shell('bash sdks/python/run_postcommit.sh') - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy deleted file mode 100644 index 3718551..0000000 --- a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This is the Java precommit which runs a maven install, and the current set -// of precommit tests. -mavenJob('beam_PreCommit_Java_MavenInstall') { - description('Runs an install of the current GitHub Pull Request.') - - previousNames('beam_PreCommit_MavenVerify') - - // Execute concurrent builds if necessary. - concurrentBuild() - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Set Maven parameters. - common_job_properties.setMavenConfig(delegate) - - // Sets that this is a PreCommit job. - common_job_properties.setPreCommit(delegate, 'Maven clean install') - - // Maven goals for this job. - goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report') -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PreCommit_Website_Stage.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PreCommit_Website_Stage.groovy b/.jenkins/job_beam_PreCommit_Website_Stage.groovy deleted file mode 100644 index 7c64f11..0000000 --- a/.jenkins/job_beam_PreCommit_Website_Stage.groovy +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// Defines a job. -job('beam_PreCommit_Website_Stage') { - description('Stages the pull requests proposed for the Apache Beam ' + - 'website to a temporary location to ease reviews.') - - // Set common parameters. - common_job_properties.setTopLevelWebsiteJobProperties(delegate) - - // Set pull request build trigger. - common_job_properties.setPreCommit( - delegate, - 'Automatic staging of pull requests', - '\nJenkins built the site at commit id ${ghprbActualCommit} with ' + - 'Jekyll and staged it [here](http://apache-beam-website-pull-' + - 'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' + - 'Happy reviewing.\n\nNote that any previous site has been deleted. ' + - 'This staged site will be automatically deleted after its TTL ' + - 'expires. Push any commit to the pull request branch or re-trigger ' + - 'the build to get it staged again.') - - steps { - // Run the following shell script as a build step. - shell ''' - # Install RVM. - gpg --keyserver hkp://keys.gnupg.net --recv-keys \\ - 409B6B1796C275462A1703113804BB82D39DC0E3 - \\curl -sSL https://get.rvm.io | bash - source /home/jenkins/.rvm/scripts/rvm - - # Install Ruby. - RUBY_VERSION_NUM=2.3.0 - rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only - - # Install Bundler gem - PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH - GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH - gem install bundler --user-install - - # Install all needed gems. - bundle install --path ~/.gem/ - - # Remove current site if it exists. - GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/" - gsutil -m rm -r -f ${GCS_PATH} || true - - # Build the new site with the baseurl specified. - rm -fr ./content/ - bundle exec jekyll build --baseurl=/${ghprbPullId} - - # Install BeautifulSoup HTML Parser for python. - pip install --user beautifulsoup4 - - # Fix links on staged website. - python .jenkins/append_index_html_to_internal_links.py - - # Upload the new site. - gsutil -m cp -R ./content/* ${GCS_PATH} - '''.stripIndent().trim() - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PreCommit_Website_Test.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_PreCommit_Website_Test.groovy b/.jenkins/job_beam_PreCommit_Website_Test.groovy deleted file mode 100644 index 421b58a..0000000 --- a/.jenkins/job_beam_PreCommit_Website_Test.groovy +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// Defines a job. -job('beam_PreCommit_Website_Test') { - description('Runs tests on the pull requests proposed for the Apache Beam ' + - 'website.') - - // Set common parameters. - common_job_properties.setTopLevelWebsiteJobProperties(delegate) - - // Execute concurrent builds. Multiple builds of this project may be executed - // in parallel. This is safe because this build does not require exclusive - // access to any shared resources. - concurrentBuild() - - // Set pull request build trigger. - common_job_properties.setPreCommit( - delegate, - 'Test website (dead links, etc.)') - - steps { - // Run the following shell script as a build step. - shell ''' - # Install RVM. - gpg --keyserver hkp://keys.gnupg.net --recv-keys \\ - 409B6B1796C275462A1703113804BB82D39DC0E3 - \\curl -sSL https://get.rvm.io | bash - source /home/jenkins/.rvm/scripts/rvm - - # Install Ruby. - RUBY_VERSION_NUM=2.3.0 - rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only - - # Install Bundler gem - PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH - GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH - gem install bundler --user-install - - # Install all needed gems. - bundle install --path ~/.gem/ - - # Build the new site and test it. - rm -fr ./content/ - bundle exec rake test - '''.stripIndent().trim() - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_Release_NightlySnapshot.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_beam_Release_NightlySnapshot.groovy b/.jenkins/job_beam_Release_NightlySnapshot.groovy deleted file mode 100644 index f2c3ff0..0000000 --- a/.jenkins/job_beam_Release_NightlySnapshot.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This is the nightly snapshot build -- we use this to deploy a daily snapshot -// to https://repository.apache.org/content/groups/snapshots/org/apache/beam. -// Runs the postsubmit suite before deploying. -mavenJob('beam_Release_NightlySnapshot') { - description('Runs a mvn clean deploy of the nightly snapshot.') - - // Execute concurrent builds if necessary. - concurrentBuild() - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // Set maven paramaters. - common_job_properties.setMavenConfig(delegate) - - // This is a post-commit job that runs once per day, not for every push. - common_job_properties.setPostCommit( - delegate, - '0 7 * * *', - false, - '[email protected]') - - // Maven goals for this job. - goals('-B -e clean deploy -P release,dataflow-runner -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner" ]\'') -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_seed.groovy ---------------------------------------------------------------------- diff --git a/.jenkins/job_seed.groovy b/.jenkins/job_seed.groovy deleted file mode 100644 index 7e5272b..0000000 --- a/.jenkins/job_seed.groovy +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// Defines the seed job, which creates or updates all other Jenkins projects. -job('beam_SeedJob') { - description('Automatically configures all Apache Beam Jenkins projects based' + - ' on Jenkins DSL groovy files checked into the code repository.') - - previousNames('beam_SeedJob_Main') - - // Set common parameters. - common_job_properties.setTopLevelMainJobProperties(delegate) - - // This is a post-commit job that runs once per day, not for every push. - common_job_properties.setPostCommit( - delegate, - '0 6 * * *', - false, - '[email protected]') - - // Allows triggering this build against pull requests. - common_job_properties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Seed Job', - 'Run Seed Job') - - steps { - dsl { - // A list or a glob of other groovy files to process. - external('.jenkins/job_*.groovy') - - // If a job is removed from the script, disable it (rather than deleting). - removeAction('DISABLE') - } - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/common_job_properties.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy new file mode 100644 index 0000000..ee10281 --- /dev/null +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Contains functions that help build Jenkins projects. Functions typically set +// common properties that are shared among all Jenkins projects. +// Code in this directory should conform to the Groovy style guide. +// http://groovy-lang.org/style-guide.html +class common_job_properties { + + // Sets common top-level job properties for website repository jobs. + static void setTopLevelWebsiteJobProperties(context) { + setTopLevelJobProperties(context, 'beam-site', 'asf-site', 30) + } + + // Sets common top-level job properties for main repository jobs. + static void setTopLevelMainJobProperties(context, + String defaultBranch = 'master', + int defaultTimeout = 100) { + setTopLevelJobProperties(context, 'beam', defaultBranch, defaultTimeout) + } + + // Sets common top-level job properties. Accessed through one of the above + // methods to protect jobs from internal details of param defaults. + private static void setTopLevelJobProperties(context, + String repositoryName, + String defaultBranch, + int defaultTimeout) { + + // GitHub project. + context.properties { + githubProjectUrl('https://github.com/apache/' + repositoryName + '/') + } + + // Set JDK version. + context.jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors dedicated to the + // Apache Beam project. + context.label('beam') + + // Discard old builds. Build records are only kept up to this number of days. + context.logRotator { + daysToKeep(14) + } + + // Source code management. + context.scm { + git { + remote { + url('https://github.com/apache/' + repositoryName + '.git') + refspec('+refs/heads/*:refs/remotes/origin/* ' + + '+refs/pull/*:refs/remotes/origin/pr/*') + } + branch('${sha1}') + extensions { + cleanAfterCheckout() + pruneBranches() + } + } + } + + context.parameters { + // This is a recommended setup if you want to run the job manually. The + // ${sha1} parameter needs to be provided, and defaults to the main branch. + stringParam( + 'sha1', + defaultBranch, + 'Commit id or refname (eg: origin/pr/9/head) you want to build.') + } + + context.wrappers { + // Abort the build if it's stuck for more minutes than specified. + timeout { + absolute(defaultTimeout) + abortBuild() + } + + // Set SPARK_LOCAL_IP for spark tests. + environmentVariables { + env('SPARK_LOCAL_IP', '127.0.0.1') + } + credentialsBinding { + string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") + } + } + } + + // Sets the pull request build trigger. Accessed through precommit methods + // below to insulate callers from internal parameter defaults. + private static void setPullRequestBuildTrigger(context, + String commitStatusContext, + String successComment = '--none--', + String prTriggerPhrase = '') { + context.triggers { + githubPullRequest { + admins(['asfbot']) + useGitHubHooks() + orgWhitelist(['apache']) + allowMembersOfWhitelistedOrgsAsAdmin() + permitAll() + // prTriggerPhrase is the argument which gets set when we want to allow + // post-commit builds to run against pending pull requests. This block + // overrides the default trigger phrase with the new one. Setting this + // will disable automatic invocation of this build; the phrase will be + // required to start it. + if (prTriggerPhrase) { + triggerPhrase(prTriggerPhrase) + onlyTriggerPhrase() + } + + extensions { + commitStatus { + // This is the name that will show up in the GitHub pull request UI + // for this Jenkins project. + delegate.context("Jenkins: " + commitStatusContext) + } + + /* + This section is disabled, because of jenkinsci/ghprb-plugin#417 issue. + For the time being, an equivalent configure section below is added. + + // Comment messages after build completes. + buildStatus { + completedStatus('SUCCESS', successComment) + completedStatus('FAILURE', '--none--') + completedStatus('ERROR', '--none--') + } + */ + } + } + } + + // Comment messages after build completes. + context.configure { + def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages + messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { + message(successComment) + result('SUCCESS') + } + messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { + message('--none--') + result('ERROR') + } + messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { + message('--none--') + result('FAILURE') + } + } + } + + // Sets common config for Maven jobs. + static void setMavenConfig(context) { + context.mavenInstallation('Maven 3.3.3') + context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true') + context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS') + // The -XX:+TieredCompilation -XX:TieredStopAtLevel=1 JVM options enable + // tiered compilation to make the JVM startup times faster during the tests. + context.mavenOpts('-XX:+TieredCompilation') + context.mavenOpts('-XX:TieredStopAtLevel=1') + context.rootPOM('pom.xml') + // Use a repository local to the workspace for better isolation of jobs. + context.localRepository(LocalRepositoryLocation.LOCAL_TO_WORKSPACE) + // Disable archiving the built artifacts by default, as this is slow and flaky. + // We can usually recreate them easily, and we can also opt-in individual jobs + // to artifact archiving. + context.archivingDisabled(true) + } + + // Sets common config for PreCommit jobs. + static void setPreCommit(context, + String commitStatusName, + String successComment = '--none--') { + // Set pull request build trigger. + setPullRequestBuildTrigger(context, commitStatusName, successComment) + } + + // Enable triggering postcommit runs against pull requests. Users can comment the trigger phrase + // specified in the postcommit job and have the job run against their PR to run + // tests not in the presubmit suite for additional confidence. + static void enablePhraseTriggeringFromPullRequest(context, + String commitStatusName, + String prTriggerPhrase) { + setPullRequestBuildTrigger( + context, + commitStatusName, + '--none--', + prTriggerPhrase) + } + + // Sets common config for PostCommit jobs. + static void setPostCommit(context, + String buildSchedule = '0 */6 * * *', + boolean triggerEveryPush = true, + String notifyAddress = '[email protected]', + boolean emailIndividuals = true) { + // Set build triggers + context.triggers { + // By default runs every 6 hours. + cron(buildSchedule) + if (triggerEveryPush) { + githubPush() + } + } + + context.publishers { + // Notify an email address for each failed build (defaults to commits@). + mailer(notifyAddress, false, emailIndividuals) + } + } + + // Configures the argument list for performance tests, adding the standard + // performance test job arguments. + private static def genPerformanceArgs(def argMap) { + def standard_args = [ + project: 'apache-beam-testing', + dpb_log_level: 'INFO', + maven_binary: '/home/jenkins/tools/maven/latest/bin/mvn', + bigquery_table: 'beam_performance.pkb_results', + // Publishes results with official tag, for use in dashboards. + official: 'true' + ] + // Note: in case of key collision, keys present in ArgMap win. + def joined_args = standard_args.plus(argMap) + def argList = [] + joined_args.each({ + // FYI: Replacement only works with double quotes. + key, value -> argList.add("--$key=$value") + }) + return argList.join(' ') + } + + // Adds the standard performance test job steps. + static def buildPerformanceTest(def context, def argMap) { + def pkbArgs = genPerformanceArgs(argMap) + context.steps { + // Clean up environment. + shell('rm -rf PerfKitBenchmarker') + // Clone appropriate perfkit branch + shell('git clone https://github.com/GoogleCloudPlatform/PerfKitBenchmarker.git') + // Install job requirements. + shell('pip install --user -r PerfKitBenchmarker/requirements.txt') + // Launch performance test. + shell("python PerfKitBenchmarker/pkb.py $pkbArgs") + } + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy new file mode 100644 index 0000000..51c73f3 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the Beam performance tests on PerfKit Benchmarker. +job('beam_PerformanceTests_Dataflow'){ + // Set default Beam job properties. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Run job in postcommit every 6 hours, don't trigger every push, and + // don't email individual committers. + common_job_properties.setPostCommit( + delegate, + '0 */6 * * *', + false, + '[email protected]', + false) + + def argMap = [ + benchmarks: 'dpb_wordcount_benchmark', + dpb_dataflow_staging_location: 'gs://temp-storage-for-perf-tests/staging', + dpb_wordcount_input: 'dataflow-samples/shakespeare/kinglear.txt', + config_override: 'dpb_wordcount_benchmark.dpb_service.service_type=dataflow' + ] + + common_job_properties.buildPerformanceTest(delegate, argMap) +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy new file mode 100644 index 0000000..8e581c2 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the Beam performance tests on PerfKit Benchmarker. +job('beam_PerformanceTests_JDBC'){ + // Set default Beam job properties. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Run job in postcommit every 6 hours, don't trigger every push, and + // don't email individual committers. + common_job_properties.setPostCommit( + delegate, + '0 */6 * * *', + false, + '[email protected]', + false) + + def pipelineArgs = [ + tempRoot: 'gs://temp-storage-for-end-to-end-tests', + project: 'apache-beam-testing', + postgresServerName: '10.36.0.11', + postgresUsername: 'postgres', + postgresDatabaseName: 'postgres', + postgresPassword: 'uuinkks', + postgresSsl: 'false' + ] + def pipelineArgList = [] + pipelineArgs.each({ + key, value -> pipelineArgList.add("--$key=$value") + }) + def pipelineArgsJoined = pipelineArgList.join(',') + + def argMap = [ + benchmarks: 'beam_integration_benchmark', + beam_it_module: 'sdks/java/io/jdbc', + beam_it_args: pipelineArgsJoined, + beam_it_class: 'org.apache.beam.sdk.io.jdbc.JdbcIOIT', + // Profile is located in $BEAM_ROOT/sdks/java/io/pom.xml. + beam_it_profile: 'io-it' + ] + + common_job_properties.buildPerformanceTest(delegate, argMap) +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy new file mode 100644 index 0000000..ba719bf --- /dev/null +++ b/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the Beam performance tests on PerfKit Benchmarker. +job('beam_PerformanceTests_Spark'){ + // Set default Beam job properties. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Run job in postcommit every 6 hours, don't trigger every push, and + // don't email individual committers. + common_job_properties.setPostCommit( + delegate, + '0 */6 * * *', + false, + '[email protected]', + false) + + def argMap = [ + benchmarks: 'dpb_wordcount_benchmark', + // There are currently problems uploading to Dataproc, so we use a file + // already present on the machines as input. + dpb_wordcount_input: '/etc/hosts', + config_override: 'dpb_wordcount_benchmark.dpb_service.service_type=dataproc' + ] + + common_job_properties.buildPerformanceTest(delegate, argMap) +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy new file mode 100644 index 0000000..a288a84 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the Java postcommit tests, including the suite of integration +// tests. +mavenJob('beam_PostCommit_Java_MavenInstall') { + description('Runs postcommit tests on the Java SDK.') + + previousNames('beam_PostCommit_MavenVerify') + + // Execute concurrent builds if necessary. + concurrentBuild() + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Set maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Sets that this is a PostCommit job. + common_job_properties.setPostCommit(delegate) + + // Maven goals for this job. + goals('-B -e -P release,dataflow-runner clean install coveralls:report -DrepoToken=$COVERALLS_REPO_TOKEN -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner" ]\'') +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy new file mode 100644 index 0000000..c16a1e2 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the suite of ValidatesRunner tests against the Apex runner. +mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') { + description('Runs the ValidatesRunner suite on the Apex runner.') + previousNames('beam_PostCommit_Java_RunnableOnService_Apex') + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Set maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Sets that this is a PostCommit job. + common_job_properties.setPostCommit(delegate) + + // Allows triggering this build against pull requests. + common_job_properties.enablePhraseTriggeringFromPullRequest( + delegate, + 'Apache Apex Runner ValidatesRunner Tests', + 'Run Apex ValidatesRunner') + + // Maven goals for this job. + goals('''clean verify --projects runners/apex \ + --also-make \ + --batch-mode \ + --errors \ + --activate-profiles validates-runner-tests \ + --activate-profiles local-validates-runner-tests''') +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy new file mode 100644 index 0000000..33235ff --- /dev/null +++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the suite of ValidatesRunner tests against the Dataflow +// runner. +mavenJob('beam_PostCommit_Java_ValidatesRunner_Dataflow') { + description('Runs the ValidatesRunner suite on the Dataflow runner.') + previousNames('beam_PostCommit_Java_RunnableOnService_Dataflow') + + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 120) + + // Set maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Sets that this is a PostCommit job. + common_job_properties.setPostCommit(delegate) + + // Allows triggering this build against pull requests. + common_job_properties.enablePhraseTriggeringFromPullRequest( + delegate, + 'Google Cloud Dataflow Runner ValidatesRunner Tests', + 'Run Dataflow ValidatesRunner') + + // Maven goals for this job. + goals('-B -e clean verify -am -pl runners/google-cloud-dataflow-java -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner", "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-validates-runner-tests/" ]\'') +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy new file mode 100644 index 0000000..411106d --- /dev/null +++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the suite of ValidatesRunner tests against the Flink runner. +mavenJob('beam_PostCommit_Java_ValidatesRunner_Flink') { + description('Runs the ValidatesRunner suite on the Flink runner.') + previousNames('beam_PostCommit_Java_RunnableOnService_Flink') + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Set maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Sets that this is a PostCommit job. + common_job_properties.setPostCommit(delegate) + + // Allows triggering this build against pull requests. + common_job_properties.enablePhraseTriggeringFromPullRequest( + delegate, + 'Apache Flink Runner ValidatesRunner Tests', + 'Run Flink ValidatesRunner') + + // Maven goals for this job. + goals('-B -e clean verify -am -pl runners/flink/runner -Plocal-validates-runner-tests -Pvalidates-runner-tests') +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy new file mode 100644 index 0000000..1348a19 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the suite of ValidatesRunner tests against the Gearpump +// runner. +mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') { + description('Runs the ValidatesRunner suite on the Gearpump runner.') + + previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump') + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties( + delegate, + 'gearpump-runner') + + // Set maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Sets that this is a PostCommit job. + // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties. + // In post-commit this job triggers only on SCM changes. + common_job_properties.setPostCommit(delegate, '0 5 31 2 *') + + // Allows triggering this build against pull requests. + common_job_properties.enablePhraseTriggeringFromPullRequest( + delegate, + 'Apache Gearpump Runner ValidatesRunner Tests', + 'Run Gearpump ValidatesRunner') + + // Maven goals for this job. + goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner", "--streaming=false" ]\'') +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy new file mode 100644 index 0000000..9fbc219 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the suite of ValidatesRunner tests against the Spark runner. +mavenJob('beam_PostCommit_Java_ValidatesRunner_Spark') { + description('Runs the ValidatesRunner suite on the Spark runner.') + + previousNames('beam_PostCommit_Java_RunnableOnService_Spark') + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Set maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Sets that this is a PostCommit job. + common_job_properties.setPostCommit(delegate) + + // Allows triggering this build against pull requests. + common_job_properties.enablePhraseTriggeringFromPullRequest( + delegate, + 'Apache Spark Runner ValidatesRunner Tests', + 'Run Spark ValidatesRunner') + + // Maven goals for this job. + goals('-B -e clean verify -am -pl runners/spark -Pvalidates-runner-tests -Plocal-validates-runner-tests -Dspark.ui.enabled=false') +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy b/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy new file mode 100644 index 0000000..28cf77e --- /dev/null +++ b/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job defines the Python postcommit tests. +job('beam_PostCommit_Python_Verify') { + description('Runs postcommit tests on the Python SDK.') + + previousNames('beam_PostCommit_PythonVerify') + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Sets that this is a PostCommit job. + common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *') + + // Allows triggering this build against pull requests. + common_job_properties.enablePhraseTriggeringFromPullRequest( + delegate, + 'Python SDK PostCommit Tests', + 'Run Python PostCommit') + + // Allow the test to only run on particular nodes + // TODO(BEAM-1817): Remove once the tests can run on all nodes + parameters { + nodeParam('TEST_HOST') { + description('select test host as either beam1, 2 or 3') + defaultNodes(['beam3']) + allowedNodes(['beam1', 'beam2', 'beam3']) + trigger('multiSelectionDisallowed') + eligibility('IgnoreOfflineNodeEligibility') + } + } + + // Execute shell command to test Python SDK. + steps { + shell('bash sdks/python/run_postcommit.sh') + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy new file mode 100644 index 0000000..3718551 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java precommit which runs a maven install, and the current set +// of precommit tests. +mavenJob('beam_PreCommit_Java_MavenInstall') { + description('Runs an install of the current GitHub Pull Request.') + + previousNames('beam_PreCommit_MavenVerify') + + // Execute concurrent builds if necessary. + concurrentBuild() + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Sets that this is a PreCommit job. + common_job_properties.setPreCommit(delegate, 'Maven clean install') + + // Maven goals for this job. + goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report') +}
