This is an automated email from the ASF dual-hosted git repository. damccorm pushed a commit to branch users/damccorm/jenkinsPostcommits in repository https://gitbox.apache.org/repos/asf/beam.git
commit 1239304dce3aacfb81da9a8984eac1c9cb91ead9 Author: Danny McCormick <[email protected]> AuthorDate: Thu Jan 25 09:25:43 2024 -0500 Decommission all remaining jenkins jobs --- .test-infra/jenkins/job_00_seed.groovy | 115 --------------------- .test-infra/jenkins/job_Inventory.groovy | 83 --------------- ...mmit_CrossLanguageValidatesRunner_Direct.groovy | 51 --------- ...uageValidatesRunner_GoUsingJava_Dataflow.groovy | 44 -------- ...ValidatesRunner_PythonUsingJava_Dataflow.groovy | 49 --------- .../jenkins/job_PostCommit_Java_DataflowV2.groovy | 48 --------- .../job_PostCommit_Java_InfluxDBIO_IT.groovy | 55 ---------- ...ob_PostCommit_Java_ValidatesRunner_Flink.groovy | 45 -------- ...Commit_Java_ValidatesRunner_Flink_Java11.groovy | 61 ----------- .test-infra/jenkins/job_PostCommit_Python.groovy | 50 --------- ...tCommit_Python_CrossLanguage_IO_Dataflow.groovy | 55 ---------- .../job_PostCommit_Python_Examples_Direct.groovy | 43 -------- .../job_PostCommit_Python_Examples_Flink.groovy | 43 -------- ..._PostCommit_Python_ValidatesRunner_Flink.groovy | 38 ------- ..._PostCommit_Python_ValidatesRunner_Spark.groovy | 43 -------- .../jenkins/job_PostRelease_NightlySnapshot.groovy | 59 ----------- .test-infra/jenkins/job_seed_standalone.groovy | 115 --------------------- 17 files changed, 997 deletions(-) diff --git a/.test-infra/jenkins/job_00_seed.groovy b/.test-infra/jenkins/job_00_seed.groovy deleted file mode 100644 index 441b1eaf6cb..00000000000 --- a/.test-infra/jenkins/job_00_seed.groovy +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Defines the seed job, which creates or updates all other Jenkins projects. - -import Committers as committers - -job('beam_SeedJob') { - description('Automatically configures all Apache Beam Jenkins projects based' + - ' on Jenkins DSL groovy files checked into the code repository.') - - properties { - githubProjectUrl('https://github.com/apache/beam/') - } - - // Restrict to only run on Jenkins executors labeled 'beam' - label('beam') - - logRotator { - daysToKeep(30) - } - - scm { - git { - remote { - github('apache/beam') - - // ${ghprbPullId} is not interpolated by groovy, but passed through to Jenkins where it - // refers to the environment variable - refspec([ - '+refs/heads/*:refs/remotes/origin/*', - '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*'] - .join(' ')) - - // The variable ${sha1} is not interpolated by groovy, but a parameter of the Jenkins job - branch('${sha1}') - - extensions { - cleanAfterCheckout() - } - } - } - } - - parameters { - // Setup for running this job from a pull request - stringParam( - 'sha1', - 'master', - 'Commit id or refname (eg: origin/pr/4001/head) you want to build against.') - } - - wrappers { - timeout { - absolute(60) - abortBuild() - } - } - - triggers { - // Run every six hours, alternate job_seed_standalone has same freq - cron('H 0,6,12,18 * * *') - - githubPullRequest { - admins(['asfbot']) - useGitHubHooks() - userWhitelist(committers.GITHUB_USERNAMES) - - // Also run when manually kicked on a pull request - triggerPhrase('Run Seed Job') - onlyTriggerPhrase() - - extensions { - commitStatus { - context("Jenkins: Seed Job") - } - - buildStatus { - completedStatus('SUCCESS', '--none--') - completedStatus('FAILURE', '--none--') - completedStatus('ERROR', '--none--') - } - } - } - } - - publishers { - mailer('[email protected]', false, true) - } - - steps { - dsl { - // A list or a glob of other groovy files to process. - external('.test-infra/jenkins/job_*.groovy') - - // If a job is removed from the script, disable it (rather than deleting). - removeAction('DISABLE') - } - } -} diff --git a/.test-infra/jenkins/job_Inventory.groovy b/.test-infra/jenkins/job_Inventory.groovy deleted file mode 100644 index e115603d0d4..00000000000 --- a/.test-infra/jenkins/job_Inventory.groovy +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties - -import static PythonTestProperties.ALL_SUPPORTED_VERSIONS - -// These jobs list details about each beam runner, to clarify what software -// is on each machine. -def nums = 1..16 -nums.each { - def machine = "apache-beam-jenkins-${it}" - job("beam_Inventory_${machine}") { - description("Run inventory on ${machine}") - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Sets that this is a cron job. - commonJobProperties.setCronJob(delegate, '45 */8 * * *') - - // Allows triggering this build against pull requests. - commonJobProperties.enablePhraseTriggeringFromPullRequest( - delegate, - "Machine Inventory ${machine}", - "Run Inventory ${machine}") - - parameters { - nodeParam('TEST_HOST') { - description("Select test host ${machine}") - defaultNodes([machine]) - allowedNodes([machine]) - trigger('multiSelectionDisallowed') - eligibility('AllNodeEligibility') - } - stringParam { - name("tmp_unaccessed_for") - defaultValue("24") - description("Files from /tmp dir that were not accessed for last `tmp_unaccessed_for` hours will be deleted.") - trim(true) - } - } - - steps { - shell('ls /home/jenkins/tools') - shell('ls /home/jenkins/tools/*') - shell('python --version || echo "python not found"') - shell('python3 --version || echo "python3 not found"') - ALL_SUPPORTED_VERSIONS.each { version -> - shell("python${version} --version || echo \"python${version} not found\"") - } - shell('gcloud -v || echo "gcloud not found"') - shell('kubectl version || echo "kubectl not found"') - ALL_SUPPORTED_VERSIONS.each { version -> - def versionSuffix = version.replace('.', '') - shell("python${version} -m venv test${versionSuffix} && . ./test${versionSuffix}/bin/activate && python --version && deactivate || echo \"python ${version} not found\"") - } - shell('echo "Maven home $MAVEN_HOME"') - shell('env') - shell('docker system prune --all --filter until=24h --force') - shell('docker volume prune --force') - shell('echo "Current size of /tmp dir is \$(sudo du -sh /tmp)"') - shell('echo "Deleting files accessed later than \${tmp_unaccessed_for} hours ago"') - shell('sudo find /tmp -type f -amin +\$((60*\${tmp_unaccessed_for})) -print -delete') - shell('echo "Size of /tmp dir after cleanup is \$(sudo du -sh /tmp)"') - } - } -} diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy deleted file mode 100644 index 0bc3ca68401..00000000000 --- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS - -// This job runs the suite of ValidatesRunner tests against the Direct runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Direct', - 'Run XVR_Direct PostCommit', 'Direct CrossLanguageValidatesRunner Tests', this) { - description('Runs the CrossLanguageValidatesRunner suite on the Direct runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - // Gradle goals for this job. - steps { - CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion -> - shell("echo \"*** RUN CROSS-LANGUAGE DIRECT USING PYTHON ${pythonVersion} ***\"") - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner') - commonJobProperties.setGradleSwitches(delegate) - switches("-PpythonVersion=${pythonVersion}") - // only run non-python task (e.g. GoUsingJava) once - switches("-PskipNonPythonTask=${pythonVersion != CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS[0]}") - } - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_GoUsingJava_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_GoUsingJava_Dataflow.groovy deleted file mode 100644 index c13e1e34201..00000000000 --- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_GoUsingJava_Dataflow.groovy +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the suite of ValidatesRunner tests against the Dataflow runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_GoUsingJava_Dataflow', - 'Run XVR_GoUsingJava_Dataflow PostCommit', 'Dataflow CrossLanguageValidatesRunner GoUsingJava Tests', this) { - description('Runs the CrossLanguageValidatesRunner GoUsingJava suite on the Dataflow runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - // Gradle goals for this job. - steps { - shell("echo \"*** RUN CROSS-LANGUAGE DATAFLOW GO WITH JAVA EXTERNAL TRANSFORMS ***\"") - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJava_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJava_Dataflow.groovy deleted file mode 100644 index 03745685a29..00000000000 --- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJava_Dataflow.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS - -// This job runs the suite of ValidatesRunner tests against the Dataflow runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_PythonUsingJava_Dataflow', - 'Run XVR_PythonUsingJava_Dataflow PostCommit', 'Dataflow CrossLanguageValidatesRunner PythonUsingJava Tests', this) { - description('Runs the CrossLanguageValidatesRunner PythonUsingJava suite on the Dataflow runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/pytest*.xml') - } - - // Gradle goals for this job. - steps { - CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion -> - shell("echo \"*** RUN CROSS-LANGUAGE DATAFLOW PYTHON WITH JAVA EXTERNAL TRANSFORMS USING PYTHON ${pythonVersion} ***\"") - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingJava') - commonJobProperties.setGradleSwitches(delegate) - switches("-PpythonVersion=${pythonVersion}") - } - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy b/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy deleted file mode 100644 index 03d82d48974..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - - -// This job runs the Java postcommit tests, including the suite of integration -// tests. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_DataflowV2', 'Run PostCommit_Java_DataflowV2', - 'Dataflow V2 Java Post Commit Tests', this) { - - description('Dataflow V2 Java Post Commit Tests') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":runners:google-cloud-dataflow-java:postCommitRunnerV2") - commonJobProperties.setGradleSwitches(delegate) - // Specify maven home on Jenkins, needed by Maven archetype integration tests. - switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Java_InfluxDBIO_IT.groovy b/.test-infra/jenkins/job_PostCommit_Java_InfluxDBIO_IT.groovy deleted file mode 100644 index cb74cbf3228..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Java_InfluxDBIO_IT.groovy +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import CommonJobProperties as common -import PostcommitJobBuilder -import Kubernetes - -String jobName = "beam_PostCommit_Java_InfluxDbIO_IT" - -PostcommitJobBuilder.postCommitJob(jobName, 'Run Java InfluxDbIO_IT', 'Java InfluxDbIO Integration Test', this) { - description('Runs the Java InfluxDbIO Integration Test.') - previousNames(/beam_PerformanceTests_InfluxDbIO_IT/) - // Set common parameters. - common.setTopLevelMainJobProperties(delegate) - - // Deploy InfluxDb cluster - String namespace = common.getKubernetesNamespace(jobName) - String kubeconfigPath = common.getKubeconfigLocationForNamespace(namespace) - Kubernetes k8s = Kubernetes.create(delegate, kubeconfigPath, namespace) - - k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/influxdb/influxdb.yml")) - String influxDBHostName = "LOAD_BALANCER_IP" - k8s.loadBalancerIP("influxdb-load-balancer-service", influxDBHostName) - Map pipelineOptions = [ - influxDBURL : "http://\$${influxDBHostName}:8086";, - influxDBUserName : "superadmin", - influxDBPassword : "supersecretpassword", - databaseName : "db1" - ] - - steps { - gradle { - rootBuildScriptDir(common.checkoutDir) - common.setGradleSwitches(delegate) - switches("--info") - switches("-DintegrationTestPipelineOptions=\'${common.joinPipelineOptions(pipelineOptions)}\'") - switches("-DintegrationTestRunner=direct") - tasks(":sdks:java:io:influxdb:integrationTest --tests org.apache.beam.sdk.io.influxdb.InfluxDbIOIT") - } - } -} diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy deleted file mode 100644 index 6f24f9fe6a4..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import CommonTestProperties -import PostcommitJobBuilder - -// This job runs the suite of ValidatesRunner tests against the Flink runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Flink', - 'Run Flink ValidatesRunner', 'Apache Flink Runner ValidatesRunner Tests', this) { - description('Runs the ValidatesRunner suite on the Flink runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - previousNames(/beam_PostCommit_Java_ValidatesRunner_Flink_Gradle/) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:validatesRunner") - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink_Java11.groovy deleted file mode 100644 index 9178caebd81..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink_Java11.groovy +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import CommonTestProperties -import PostcommitJobBuilder - - -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Flink_Java11', - 'Run Flink ValidatesRunner Java 11', 'Apache Flink Runner ValidatesRunner Tests On Java 11', this) { - - description('Runs the ValidatesRunner suite on the Flink runner with Java 11.') - - def JAVA_11_HOME = '/usr/lib/jvm/java-11-openjdk-amd64' - def JAVA_8_HOME = '/usr/lib/jvm/java-8-openjdk-amd64' - - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 270) - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:jar") - tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:testJar") - switches("-Dorg.gradle.java.home=${JAVA_8_HOME}") - } - - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:validatesRunner") - switches('-x shadowJar') - switches('-x shadowTestJar') - switches('-x compileJava') - switches('-x compileTestJava') - switches('-x jar') - switches('-x testJar') - switches('-x classes') - switches('-x testClasses') - switches("-Dorg.gradle.java.home=${JAVA_11_HOME}") - - commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Python.groovy b/.test-infra/jenkins/job_PostCommit_Python.groovy deleted file mode 100644 index 6e8f5e81b07..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Python.groovy +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -import static PythonTestProperties.ALL_SUPPORTED_VERSIONS - -// This job defines the Python postcommit tests. -ALL_SUPPORTED_VERSIONS.each { pythonVersion -> - def versionSuffix = pythonVersion.replace('.', '') - PostcommitJobBuilder.postCommitJob("beam_PostCommit_Python${versionSuffix}", - "Run Python ${pythonVersion} PostCommit", - "Python${versionSuffix}_PC(\"Run Python ${pythonVersion} PostCommit\")", this) { - description("Runs Python postcommit tests using Python ${pythonVersion}.") - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240) - - publishers { - archiveJunit('**/pytest*.xml') - } - - // Execute shell command to test Python SDK. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":python${versionSuffix}PostCommit") - commonJobProperties.setGradleSwitches(delegate) - switches("-PuseWheelDistribution") - } - } - } -} - diff --git a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_IO_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_IO_Dataflow.groovy deleted file mode 100644 index 9a37e76cbde..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_IO_Dataflow.groovy +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - - -import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS - - -// This job runs end-to-end cross language GCP IO tests with DataflowRunner. -// Collects tests with the @pytest.mark.uses_io_java_expansion_service decorator -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Xlang_IO_Dataflow', - 'Run Python_Xlang_IO_Dataflow PostCommit', 'Python_Xlang_IO_Dataflow (\"Run Python_Xlang_IO_Dataflow PostCommit\")', this) { - description('Runs end-to-end cross language non-GCP IO tests on the Dataflow runner.') - - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180) - - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/pytest*.xml') - } - - - // Gradle goals for this job. - - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":sdks:python:test-suites:dataflow:ioCrossLanguagePostCommit") - commonJobProperties.setGradleSwitches(delegate) - switches("-PuseWheelDistribution") - switches("-PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094") - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy deleted file mode 100644 index e420b046271..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the Python examples tests with DirectRunner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Direct', - 'Run Python Examples_Direct', 'Python Direct Runner Examples', this) { - - description('Runs the Python Examples with DirectRunner') - - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/pytest*.xml') - } - - // Execute shell command to run examples. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":sdks:python:test-suites:direct:examplesPostCommit") - commonJobProperties.setGradleSwitches(delegate) - } - } - } \ No newline at end of file diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy deleted file mode 100644 index e9150ccae01..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the Python examples tests with FlinkRunner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Flink', - 'Run Python Examples_Flink', 'Python Flink Runner Examples', this) { - - description('Runs the Python Examples with Flink Runner') - - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120, true, 'beam', true, 40) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/pytest*.xml') - } - - // Execute shell command to run examples. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":sdks:python:test-suites:portable:flinkExamplesPostCommit") - commonJobProperties.setGradleSwitches(delegate) - } - } - } \ No newline at end of file diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Flink.groovy deleted file mode 100644 index 5f0aa8f13b9..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Flink.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the suite of Python ValidatesRunner tests against the Flink runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_VR_Flink', - 'Run Python Flink ValidatesRunner', 'Run Python Flink ValidatesRunner', this) { - description('Runs the Python ValidatesRunner suite on the Flink runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:portable:flinkValidatesRunner') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy deleted file mode 100644 index ce71871da7e..00000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the suite of Python ValidatesRunner tests against the Spark runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_VR_Spark', - 'Run Python Spark ValidatesRunner', 'Python Spark ValidatesRunner Tests', this) { - description('Runs the Python ValidatesRunner suite on the Spark runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':pythonSparkPostCommit') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy b/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy deleted file mode 100644 index 45290152f83..00000000000 --- a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties - -// This verifies the nightly snapshot build. -// From https://repository.apache.org/content/groups/snapshots/org/apache/beam. -job('beam_PostRelease_NightlySnapshot') { - description('Runs post release verification of the nightly snapshot.') - - // Set common parameters. Timeout is longer, to avoid [BEAM-5774]. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 200) - - parameters { - stringParam('snapshot_version', - '', - 'Version of the repository snapshot to install') - stringParam('snapshot_url', - '', - 'Repository URL to install from') - } - - // This is a post-commit job that runs once per day, not for every push. - commonJobProperties.setAutoJob( - delegate, - '@daily') - - - // Allows triggering this build against pull requests. - commonJobProperties.enablePhraseTriggeringFromPullRequest( - delegate, - './gradlew :release:runJavaExamplesValidationTask', - 'Run Dataflow PostRelease') - - steps { - // Run a quickstart from https://beam.apache.org/get-started/quickstart-java - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':release:runJavaExamplesValidationTask') - commonJobProperties.setGradleSwitches(delegate) - switches('-Pver=$snapshot_version -Prepourl=$snapshot_url') - } - } -} diff --git a/.test-infra/jenkins/job_seed_standalone.groovy b/.test-infra/jenkins/job_seed_standalone.groovy deleted file mode 100644 index 6f88448da4c..00000000000 --- a/.test-infra/jenkins/job_seed_standalone.groovy +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Defines the seed job, which creates or updates all other Jenkins projects. - -import Committers as committers - -job('beam_SeedJob_Standalone') { - description('Automatically configures all Apache Beam Jenkins projects based' + - ' on Jenkins DSL groovy files checked into the code repository.') - - properties { - githubProjectUrl('https://github.com/apache/beam/') - } - - // Restrict to only run on Jenkins executors labeled 'beam' - label('beam') - - logRotator { - daysToKeep(30) - } - - scm { - git { - remote { - github('apache/beam') - - // ${ghprbPullId} is not interpolated by groovy, but passed through to Jenkins where it - // refers to the environment variable - refspec([ - '+refs/heads/*:refs/remotes/origin/*', - '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*'] - .join(' ')) - - // The variable ${sha1} is not interpolated by groovy, but a parameter of the Jenkins job - branch('${sha1}') - - extensions { - cleanAfterCheckout() - } - } - } - } - - parameters { - // Setup for running this job from a pull request - stringParam( - 'sha1', - 'master', - 'Commit id or refname (eg: origin/pr/4001/head) you want to build against.') - } - - wrappers { - timeout { - absolute(60) - abortBuild() - } - } - - triggers { - // Run every six hours, alternate job_00_seed has same freq - cron('H 3,9,15,21 * * *') - - githubPullRequest { - admins(['asfbot']) - useGitHubHooks() - userWhitelist(committers.GITHUB_USERNAMES) - - // Also run when manually kicked on a pull request - triggerPhrase('Run Standalone Seed Job') - onlyTriggerPhrase() - - extensions { - commitStatus { - context("Jenkins: Standalone Seed Job") - } - - buildStatus { - completedStatus('SUCCESS', '--none--') - completedStatus('FAILURE', '--none--') - completedStatus('ERROR', '--none--') - } - } - } - } - - publishers { - mailer('[email protected]', false, true) - } - - steps { - dsl { - // A list or a glob of other groovy files to process. - external('.test-infra/jenkins/job_*.groovy') - - // If a job is removed from the script, disable it (rather than deleting). - removeAction('DISABLE') - } - } -}
