[
https://issues.apache.org/jira/browse/BEAM-3989?focusedWorklogId=86944&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-86944
]
ASF GitHub Bot logged work on BEAM-3989:
----------------------------------------
Author: ASF GitHub Bot
Created on: 03/Apr/18 05:10
Start Date: 03/Apr/18 05:10
Worklog Time Spent: 10m
Work Description: lukecwik closed pull request #4999: [BEAM-3989] Delete
Maven pipeline jobs consistently failing
URL: https://github.com/apache/beam/pull/4999
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/common_job_properties.groovy
b/.test-infra/jenkins/common_job_properties.groovy
index 118fe5068e4..c247ac94e87 100644
--- a/.test-infra/jenkins/common_job_properties.groovy
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -312,108 +312,6 @@ class common_job_properties {
}
}
- /**
- * Sets properties for all jobs which are run by a pipeline top-level
(maven) job.
- * @param context The delegate from the top level of a MavenJob.
- * @param jobTimeout How long (in minutes) to wait for the job to finish.
- * @param descriptor A short string identifying the job, e.g. "Java Unit
Test".
- */
- static def setPipelineJobProperties(def context, int jobTimeout, String
descriptor) {
- context.parameters {
- stringParam(
- 'ghprbGhRepository',
- 'N/A',
- 'Repository name for use by ghprb plugin.')
- stringParam(
- 'ghprbActualCommit',
- 'N/A',
- 'Commit ID for use by ghprb plugin.')
- stringParam(
- 'ghprbPullId',
- 'N/A',
- 'PR # for use by ghprb plugin.')
-
- }
-
- // Set JDK version.
- context.jdk('JDK 1.8 (latest)')
-
- // Restrict this project to run only on Jenkins executors as specified
- context.label('beam')
-
- // Execute concurrent builds if necessary.
- context.concurrentBuild()
-
- context.wrappers {
- timeout {
- absolute(jobTimeout)
- abortBuild()
- }
- credentialsBinding {
- string("COVERALLS_REPO_TOKEN", "beam-coveralls-token")
- }
- downstreamCommitStatus {
- delegate.context("Jenkins: ${descriptor}")
- triggeredStatus("${descriptor} Pending")
- startedStatus("Running ${descriptor}")
- statusUrl()
- completedStatus('SUCCESS', "${descriptor} Passed")
- completedStatus('FAILURE', "${descriptor} Failed")
- completedStatus('ERROR', "Error Executing ${descriptor}")
- }
- // Set SPARK_LOCAL_IP for spark tests.
- environmentVariables {
- env('SPARK_LOCAL_IP', '127.0.0.1')
- }
- }
-
- // Set Maven parameters.
- setMavenConfig(context)
- }
-
- /**
- * Sets job properties common to pipeline jobs which are responsible for
being the root of a
- * build tree. Downstream jobs should pull artifacts from these jobs.
- * @param context The delegate from the top level of a MavenJob.
- */
- static def setPipelineBuildJobProperties(def context) {
- context.properties {
- githubProjectUrl('https://github.com/apache/beam/')
- }
-
- context.parameters {
- stringParam(
- 'sha1',
- 'master',
- 'Commit id or refname (e.g. origin/pr/9/head) you want to
build.')
- }
-
- // Source code management.
- setSCM(context, 'beam')
- }
-
- /**
- * Sets common job parameters for jobs which consume artifacts built for
them by an upstream job.
- * @param context The delegate from the top level of a MavenJob.
- * @param jobName The job from which to copy artifacts.
- */
- static def setPipelineDownstreamJobProperties(def context, String jobName) {
- context.parameters {
- stringParam(
- 'buildNum',
- 'N/A',
- "Build number of ${jobName} to copy from.")
- }
-
- context.preBuildSteps {
- copyArtifacts(jobName) {
- buildSelector {
- buildNumber('${buildNum}')
- }
- }
- }
- }
-
/**
* Transforms pipeline options to a string of format like below:
* ["--pipelineOption=123", "--pipelineOption2=abc", ...]
diff --git a/.test-infra/jenkins/job_beam_Java_Build.groovy
b/.test-infra/jenkins/job_beam_Java_Build.groovy
deleted file mode 100644
index 87aa98df302..00000000000
--- a/.test-infra/jenkins/job_beam_Java_Build.groovy
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This is the Java Jenkins job which builds artifacts for downstream jobs to
consume.
-mavenJob('beam_Java_Build') {
- description('Builds Beam Java SDK and archives artifacts. Meant to be run as
part of a pipeline.')
-
- // Set standard properties for a job which is part of a pipeline.
- common_job_properties.setPipelineJobProperties(delegate, 30, "Java Build")
- // Set standard properties for a pipeline job which needs to pull from
GitHub instead of an
- // upstream job.
- common_job_properties.setPipelineBuildJobProperties(delegate)
-
- configure { project ->
- // The CopyArtifact plugin doesn't support the job DSL so we have to
configure it manually.
- project / 'properties' /
'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' /
'projectNameList' {
- 'string' "beam_*"
- }
- // The Build Discarder also doesn't support the job DSL in the right way
so we have to configure it manually.
- // -1 indicates that a property is "infinite".
- project / 'properties' / 'jenkins.model.BuildDiscarderProperty' /
'strategy'(class:'hudson.tasks.LogRotator') {
- 'daysToKeep'(-1)
- 'numToKeep'(-1)
- 'artifactDaysToKeep'(1)
- 'artifactNumToKeep'(-1)
- }
- }
-
- // Construct Maven goals for this job.
- args = [
- '-B',
- '-e',
- 'clean',
- 'install',
- "-pl '!sdks/python,!sdks/java/javadoc'",
- '-DskipTests',
- '-Dcheckstyle.skip',
- ]
- goals(args.join(' '))
-
- // This job publishes artifacts so that downstream jobs can use them.
- publishers {
- archiveArtifacts {
- pattern('.repository/org/apache/beam/**/*')
- pattern('.test-infra/**/*')
- pattern('.github/**/*')
- pattern('examples/**/*')
- pattern('runners/**/*')
- pattern('sdks/**/*')
- pattern('target/**/*')
- pattern('pom.xml')
-
exclude('examples/**/*.jar,runners/**/*.jar,sdks/**/*.jar,target/**/*.jar')
- onlyIfSuccessful()
- defaultExcludes()
- }
- }
-}
diff --git a/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy
b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy
deleted file mode 100644
index 41a45369ebd..00000000000
--- a/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This is the Java Jenkins job which runs the Beam code health checks.
-mavenJob('beam_Java_CodeHealth') {
- description('Runs Java code health checks. Meant to be run as part of a
pipeline.')
-
- // Set standard properties for a job which is part of a pipeline.
- common_job_properties.setPipelineJobProperties(delegate, 30, "Java Code
Health")
- // This job runs downstream of the beam_Java_Build job and gets artifacts
from that job.
- common_job_properties.setPipelineDownstreamJobProperties(delegate,
'beam_Java_Build')
-
- args = [
- '-B',
- '-e',
- "-pl '!sdks/python'",
- 'checkstyle:check',
- 'findbugs:check',
- 'org.apache.rat:apache-rat-plugin:check',
- ]
- goals(args.join(' '))
-}
diff --git a/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy
b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy
deleted file mode 100644
index 56daf731d84..00000000000
--- a/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This is the Java Jenkins job which runs the set of precommit integration
tests.
-mavenJob('beam_Java_IntegrationTest') {
- description('Runs Java Failsafe integration tests. Designed to be run as
part of a pipeline.')
-
- // Set standard properties for a job which is part of a pipeline.
- common_job_properties.setPipelineJobProperties(delegate, 30, "Java
Integration Tests")
- // Set standard properties for a job which pulls artifacts from an upstream
job.
- common_job_properties.setPipelineDownstreamJobProperties(delegate,
'beam_Java_Build')
-
- // Profiles to activate in order to ensure runners are available at test
time.
- profiles = [
- 'jenkins-precommit',
- 'direct-runner',
- 'dataflow-runner',
- 'spark-runner',
- 'flink-runner',
- 'apex-runner'
- ]
- // In the case of the precommit integration tests, we are currently only
running the integration
- // tests in the examples directory. By directly invoking failsafe with an
execution name (which we
- // do in order to avoid building artifacts again) we are required to
enumerate each execution we
- // want to run, something which is feasible in this case.
- examples_integration_executions = [
- 'apex-runner-integration-tests',
- 'dataflow-runner-integration-tests',
- 'dataflow-runner-integration-tests-streaming',
- 'direct-runner-integration-tests',
- 'flink-runner-integration-tests',
- 'spark-runner-integration-tests',
- ]
- // Arguments to provide Maven.
- args = [
- '-B',
- '-e',
- "-P${profiles.join(',')}",
- "-pl examples/java",
- ]
- // This adds executions for each of the failsafe invocations listed above to
the list of goals.
- examples_integration_executions.each({
- value -> args.add("failsafe:integration-test@${value}")
- })
- goals(args.join(' '))
-}
diff --git a/.test-infra/jenkins/job_beam_Java_UnitTest.groovy
b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy
deleted file mode 100644
index e558eead746..00000000000
--- a/.test-infra/jenkins/job_beam_Java_UnitTest.groovy
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This is the Java Jenkins job which runs the current set of standard unit
tests.
-mavenJob('beam_Java_UnitTest') {
- description('Runs Java Surefire unit tests. Designed to be run by a pipeline
job.')
-
- // Set standard properties for a job which is part of a pipeline.
- common_job_properties.setPipelineJobProperties(delegate, 30, "Java Unit
Tests")
- // Set standard properties for a job which pulls artifacts from an upstream
job.
- common_job_properties.setPipelineDownstreamJobProperties(delegate,
'beam_Java_Build')
-
- // Construct Maven goals for this job.
- args = [
- '-B',
- '-e',
- 'surefire:test@default-test',
- "-pl '!sdks/python'",
- '-DrepoToken=$COVERALLS_REPO_TOKEN',
- '-DpullRequest=$ghprbPullId',
- ]
- goals(args.join(' '))
-}
diff --git a/.test-infra/jenkins/job_beam_Python_UnitTest.groovy
b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy
deleted file mode 100644
index 89701d4474b..00000000000
--- a/.test-infra/jenkins/job_beam_Python_UnitTest.groovy
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This is the Python Jenkins job which runs a maven install, and the current
set of precommit
-// tests.
-mavenJob('beam_Python_UnitTest') {
- description('Runs Python unit tests on a specific commit. Designed to be run
by a pipeline job.')
-
- // Set standard properties for a job which is part of a pipeline.
- common_job_properties.setPipelineJobProperties(delegate, 35, "Python Unit
Tests")
- // Set standard properties for a pipeline job which needs to pull from
GitHub instead of an
- // upstream job.
- common_job_properties.setPipelineBuildJobProperties(delegate)
-
- // Construct Maven goals for this job.
- args = [
- '-B',
- '-e',
- 'clean install',
- '-pl sdks/python',
- ]
- goals(args.join(' '))
-}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 86944)
Time Spent: 40m (was: 0.5h)
> Maven pipeline jobs consistently failing
> ----------------------------------------
>
> Key: BEAM-3989
> URL: https://issues.apache.org/jira/browse/BEAM-3989
> Project: Beam
> Issue Type: Sub-task
> Components: build-system
> Reporter: Alan Myrvold
> Assignee: Alan Myrvold
> Priority: Major
> Time Spent: 40m
> Remaining Estimate: 0h
>
> These jobs are failing or disabled:
> * beam_Java_Build
> * beam_Java_CodeHealth
> * beam_Java_IntegrationTest
> * beam_Java_UnitTest
> * beam_Python_UnitTest
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
