This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new d8030c5 Refactor Jenkinsfiles (#13344)
d8030c5 is described below
commit d8030c576bb29dc8c1418052a9c6a66c8cd2f5e1
Author: Marco de Abreu <[email protected]>
AuthorDate: Wed Nov 21 12:07:17 2018 +0100
Refactor Jenkinsfiles (#13344)
* Add custom context for GitHub
* Refactor
* Split files
* Extract steps into separate file
* Revert main Jenkinsfile changes
* Resolve typos
* Fix typo around wildcards for examples
* Make debug output non blocking
* Skip website publishing in dev
* Revert Jenkinsfile and add deprecation message
* Remove unmatching nodes
* Improve documentation of assign_node_labels
* Remove unused code
---
Jenkinsfile | 26 +
ci/Jenkinsfile_utils.groovy | 105 ++-
Jenkinsfile => ci/jenkins/Jenkins_steps.groovy | 1045 ++++++++++++++----------
ci/jenkins/Jenkinsfile_centos_cpu | 53 ++
ci/jenkins/Jenkinsfile_centos_gpu | 51 ++
ci/jenkins/Jenkinsfile_clang | 51 ++
ci/jenkins/Jenkinsfile_edge | 56 ++
ci/jenkins/Jenkinsfile_miscellaneous | 54 ++
ci/jenkins/Jenkinsfile_sanity | 48 ++
ci/jenkins/Jenkinsfile_unix_cpu | 68 ++
ci/jenkins/Jenkinsfile_unix_gpu | 73 ++
ci/jenkins/Jenkinsfile_website | 52 ++
ci/jenkins/Jenkinsfile_windows_cpu | 52 ++
ci/jenkins/Jenkinsfile_windows_gpu | 54 ++
14 files changed, 1332 insertions(+), 456 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index fca8539..a7e3dbd 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -20,6 +20,32 @@
// Jenkins pipeline
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+/***
+ * _____ _
+ * | __ \| |
+ * | |__) | | ___ __ _ ___ ___
+ * | ___/| |/ _ \/ _` / __|/ _ \
+ * | | | || | __/ (_| \__ \ __/
+ * |_|_| ||_|\___|\__,_|___/\___|
+ * / _` |/ _ \
+ * | (_| | (_) |_
+ * \__,_|\___/| |
+ * _ __ ___ | |_
+ * | '_ \ / _ \| __| _ _ __
+ * | | | | (_) | |_ | (_)/ _|
+ * |_|_|_|\___/_\__| __| |_| |_ _ _
+ * | '_ ` _ \ / _ \ / _` | | _| | | |
+ * | | | | | | (_) | (_| | | | | |_| |
+ * |_| |_| |_|\___/ \__,_|_|_| \__, |
+ * __/ |
+ * |___/
+ *
+ * This file is about to be deprecated! See
https://github.com/apache/incubator-mxnet/pull/13344
+ * for more details
+ */
+
+
// mxnet libraries
mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/tvm/nnvm/lib/libnnvm.a'
diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy
index 43f296a..8794ad3 100644
--- a/ci/Jenkinsfile_utils.groovy
+++ b/ci/Jenkinsfile_utils.groovy
@@ -65,8 +65,10 @@ def init_git_win() {
// pack libraries for later use
def pack_lib(name, libs, include_gcov_data = false) {
sh """
+set +e
echo "Packing ${libs} into ${name}"
echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+return 0
"""
stash includes: libs, name: name
@@ -82,8 +84,10 @@ def unpack_and_init(name, libs, include_gcov_data = false) {
init_git()
unstash name
sh """
+set +e
echo "Unpacked ${libs} from ${name}"
echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+return 0
"""
if (include_gcov_data) {
// Restore GCNO files that are required for GCOV to operate during runtime
@@ -91,23 +95,32 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
}
}
+def get_jenkins_master_url() {
+ return env.BUILD_URL.split('/')[2].split(':')[0]
+}
+
+def get_git_commit_hash() {
+ lastCommitMessage = sh (script: "git log -1 --pretty=%B", returnStdout: true)
+ lastCommitMessage = lastCommitMessage.trim()
+ if (lastCommitMessage.startsWith("Merge commit '") &&
lastCommitMessage.endsWith("' into HEAD")) {
+ // Merge commit applied by Jenkins, skip that commit
+ git_commit_hash = sh (script: "git rev-parse @~", returnStdout: true)
+ } else {
+ git_commit_hash = sh (script: "git rev-parse @", returnStdout: true)
+ }
+ return git_commit_hash
+}
+
def publish_test_coverage() {
// CodeCovs auto detection has trouble with our CIs PR validation due the
merging strategy
- lastCommitMessage = sh (script: "git log -1 --pretty=%B", returnStdout:
true)
- lastCommitMessage = lastCommitMessage.trim()
- if (lastCommitMessage.startsWith("Merge commit '") &&
lastCommitMessage.endsWith("' into HEAD")) {
- // Merge commit applied by Jenkins, skip that commit
- GIT_COMMIT_HASH = sh (script: "git rev-parse @~", returnStdout: true)
- } else {
- GIT_COMMIT_HASH = sh (script: "git rev-parse @", returnStdout: true)
- }
+ git_commit_hash = get_git_commit_hash()
if (env.CHANGE_ID) {
// PR execution
- codecovArgs = "-B ${env.CHANGE_TARGET} -C ${GIT_COMMIT_HASH} -P
${env.CHANGE_ID}"
+ codecovArgs = "-B ${env.CHANGE_TARGET} -C ${git_commit_hash} -P
${env.CHANGE_ID}"
} else {
// Branch execution
- codecovArgs = "-B ${env.BRANCH_NAME} -C ${GIT_COMMIT_HASH}"
+ codecovArgs = "-B ${env.BRANCH_NAME} -C ${git_commit_hash}"
}
// To make sure we never fail because test coverage reporting is not
available
@@ -144,9 +157,78 @@ def docker_run(platform, function_name, use_nvidia,
shared_mem = '500m') {
sh command
}
+// Allow publishing to GitHub with a custom context (the status shown under a
PR)
+// Credit to https://plugins.jenkins.io/github
+def get_repo_url() {
+ checkout scm
+ sh "git config --get remote.origin.url > .git/remote-url"
+ return readFile(".git/remote-url").trim()
+}
+def update_github_commit_status(state, message) {
+ node(NODE_LINUX_CPU) {
+ // NOTE: https://issues.jenkins-ci.org/browse/JENKINS-39482
+ //The GitHubCommitStatusSetter requires that the Git Server is defined
under
+ //*Manage Jenkins > Configure System > GitHub > GitHub Servers*.
+ //Otherwise the GitHubCommitStatusSetter is not able to resolve the
repository name
+ //properly and you would see an empty list of repos:
+ //[Set GitHub commit status (universal)] PENDING on repos [] (sha:xxxxxxx)
with context:test/mycontext
+ //See
https://cwiki.apache.org/confluence/display/MXNET/Troubleshooting#Troubleshooting-GitHubcommit/PRstatusdoesnotgetpublished
+ repoUrl = get_repo_url()
+ commitSha = get_git_commit_hash()
+ context = get_github_context()
+
+ step([
+ $class: 'GitHubCommitStatusSetter',
+ reposSource: [$class: "ManuallyEnteredRepositorySource", url: repoUrl],
+ contextSource: [$class: "ManuallyEnteredCommitContextSource", context:
context],
+ commitShaSource: [$class: "ManuallyEnteredShaSource", sha: commitSha],
+ statusBackrefSource: [$class: "ManuallyEnteredBackrefSource", backref:
"${env.RUN_DISPLAY_URL}"],
+ errorHandlers: [[$class: 'ShallowAnyErrorHandler']],
+ statusResultSource: [
+ $class: 'ConditionalStatusResultSource',
+ results: [[$class: "AnyBuildResult", message: message, state: state]]
+ ]
+ ])
+ }
+}
+
+def get_github_context() {
+ // Since we use multi-branch pipelines, Jenkins appends the branch name to
the job name
+ if (JOB_NAME.contains('/')) {
+ short_job_name = JOB_NAME.split('/')[0]
+ } else {
+ short_job_name = JOB_NAME
+ }
+
+ return "ci/jenkins/${short_job_name}"
+}
+
+def parallel_stage(stage_name, steps) {
+ // Allow to pass an array of steps that will be executed in parallel in a
stage
+ new_map = [:]
+
+ for (def step in steps) {
+ new_map = new_map << step
+ }
+
+ stage(stage_name) {
+ parallel new_map
+ }
+}
def assign_node_labels(args) {
+ // This function allows to assign instance labels to the generalized
placeholders.
+ // This serves two purposes:
+ // 1. Allow generalized placeholders (e.g. NODE_WINDOWS_CPU) in the job
definition
+ // in order to abstract away the underlying node label. This allows to
schedule a job
+ // onto a different node for testing or security reasons. This could be,
for example,
+ // when you want to test a new set of slaves on separate labels or when a
job should
+ // only be run on restricted slaves
+ // 2. Restrict the allowed job types within a Jenkinsfile. For example, a
UNIX-CPU-only
+ // Jenkinsfile should not allowed access to Windows or GPU instances.
This prevents
+ // users from just copy&pasting something into an existing Jenkinsfile
without
+ // knowing about the limitations.
NODE_LINUX_CPU = args.linux_cpu
NODE_LINUX_GPU = args.linux_gpu
NODE_LINUX_GPU_P3 = args.linux_gpu_p3
@@ -164,15 +246,18 @@ def main_wrapper(args) {
// assign any caught errors here
err = null
try {
+ update_github_commit_status('PENDING', 'Job has been enqueued')
args['core_logic']()
// set build status to success at the end
currentBuild.result = "SUCCESS"
+ update_github_commit_status('SUCCESS', 'Job succeeded')
} catch (caughtError) {
node(NODE_LINUX_CPU) {
sh "echo caught ${caughtError}"
err = caughtError
currentBuild.result = "FAILURE"
+ update_github_commit_status('FAILURE', 'Job failed')
}
} finally {
node(NODE_LINUX_CPU) {
diff --git a/Jenkinsfile b/ci/jenkins/Jenkins_steps.groovy
similarity index 81%
copy from Jenkinsfile
copy to ci/jenkins/Jenkins_steps.groovy
index fca8539..f48a267 100644
--- a/Jenkinsfile
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -17,8 +17,10 @@
// specific language governing permissions and limitations
// under the License.
//
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+// This file contains the steps that will be used in the
+// Jenkins pipelines
+
+utils = load('ci/Jenkinsfile_utils.groovy')
// mxnet libraries
mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/tvm/nnvm/lib/libnnvm.a'
@@ -35,12 +37,8 @@ mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a,
build/3rdparty/dmlc-c
mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a,
build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests,
build/3rdparty/openmp/runtime/src/libomp.so,
build/3rdparty/mkldnn/src/libmkldnn.so.0'
mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so,
lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/tvm/nnvm/lib/libnnvm.a'
mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0,
lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
-mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a,
3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a,
3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a,
build/cpp-package/example/lenet, build/cpp-package/example/alexnet,
build/cpp-package/example/googlenet,
build/cpp-package/example/lenet_with_mxdataiter,
build/cpp-package/example/resnet, build/cpp-package/example/mlp,
build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, buil [...]
-mx_lib_cpp_examples_cpu = 'build/libmxnet.so,
build/cpp-package/example/mlp_cpu'
-
-// timeout in minutes
-max_time = 120
-
+mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a,
3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a,
3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a,
build/cpp-package/example/*'
+mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*'
// Python unittest for CPU
// Python 2
@@ -93,126 +91,178 @@ def python3_gpu_ut_nocudnn(docker_container_name) {
}
}
-def deploy_docs() {
- parallel 'Docs': {
- node(NODE_LINUX_CPU) {
- ws('workspace/docs') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.init_git()
- utils.docker_run('ubuntu_cpu', 'deploy_docs', false)
- sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
+//------------------------------------------------------------------------------------------
+
+def compile_unix_cpu_openblas() {
+ return ['CPU: Openblas': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/build-cpu-openblas') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)
+ utils.pack_lib('cpu', mx_dist_lib, true)
+ }
}
}
- }
- },
- 'Julia docs': {
- node(NODE_LINUX_CPU) {
- ws('workspace/julia-docs') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('cpu', mx_lib)
- utils.docker_run('ubuntu_cpu', 'deploy_jl_docs', false)
+ }]
+}
+
+def compile_unix_openblas_debug_cpu() {
+ return ['CPU: Openblas, debug': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/build-cpu-openblas') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_debug',
false)
+ utils.pack_lib('cpu_debug', mx_cmake_lib_debug, true)
+ }
}
}
- }
- }
+ }]
}
-node('mxnetlinux-cpu') {
- // Loading the utilities requires a node context unfortunately
- checkout scm
- utils = load('ci/Jenkinsfile_utils.groovy')
+def compile_unix_mkldnn_cpu() {
+ return ['CPU: MKLDNN': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/build-mkldnn-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn', false)
+ utils.pack_lib('mkldnn_cpu', mx_mkldnn_lib, true)
+ }
+ }
+ }
+ }]
}
-utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu:
'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu:
'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')
-utils.main_wrapper(
-core_logic: {
- stage('Sanity Check') {
- parallel 'Lint': {
+def compile_unix_mkldnn_gpu() {
+ return ['GPU: MKLDNN': {
node(NODE_LINUX_CPU) {
- ws('workspace/sanity-lint') {
- utils.init_git()
- utils.docker_run('ubuntu_cpu', 'sanity_check', false)
+ ws('workspace/build-mkldnn-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn',
false)
+ utils.pack_lib('mkldnn_gpu', mx_mkldnn_lib, true)
+ }
}
}
- },
- 'RAT License': {
+ }]
+}
+
+def compile_unix_mkldnn_nocudnn_gpu() {
+ return ['GPU: MKLDNN_CUDNNOFF': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/build-mkldnn-gpu-nocudnn') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.docker_run('ubuntu_build_cuda',
'build_ubuntu_gpu_mkldnn_nocudnn', false)
+ utils.pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib, true)
+ }
+ }
+ }
+ }]
+}
+
+def compile_unix_full_gpu() {
+ return ['GPU: CUDA9.1+cuDNN7': {
node(NODE_LINUX_CPU) {
- ws('workspace/sanity-rat') {
- utils.init_git()
- utils.docker_run('ubuntu_rat', 'nightly_test_rat_check', false)
+ ws('workspace/build-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.docker_run('ubuntu_build_cuda',
'build_ubuntu_gpu_cuda91_cudnn7', false)
+ utils.pack_lib('gpu', mx_lib_cpp_examples, true)
+ }
}
}
- }
- }
+ }]
+}
- stage('Build') {
- parallel 'CPU: CentOS 7': {
+def compile_unix_cmake_mkldnn_gpu() {
+ return ['GPU: CMake MKLDNN': {
node(NODE_LINUX_CPU) {
- ws('workspace/build-centos7-cpu') {
+ ws('workspace/build-cmake-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('centos7_cpu', 'build_centos7_cpu', false)
- utils.pack_lib('centos7_cpu', mx_dist_lib, true)
+ utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn',
false)
+ utils.pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib, true)
}
}
}
- },
- 'CPU: CentOS 7 MKLDNN': {
+ }]
+}
+
+def compile_unix_cmake_gpu() {
+ return ['GPU: CMake': {
node(NODE_LINUX_CPU) {
- ws('workspace/build-centos7-mkldnn') {
+ ws('workspace/build-cmake-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('centos7_cpu', 'build_centos7_mkldnn', false)
- utils.pack_lib('centos7_mkldnn', mx_lib, true)
+ utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false)
+ utils.pack_lib('cmake_gpu', mx_cmake_lib, true)
}
}
}
- },
- 'GPU: CentOS 7': {
+ }]
+}
+
+def compile_unix_tensorrt_gpu() {
+ return ['TensorRT': {
node(NODE_LINUX_CPU) {
- ws('workspace/build-centos7-gpu') {
+ ws('workspace/build-tensorrt') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('centos7_gpu', 'build_centos7_gpu', false)
- utils.pack_lib('centos7_gpu', mx_lib, true)
+ utils.docker_run('ubuntu_gpu_tensorrt',
'build_ubuntu_gpu_tensorrt', false)
+ utils.pack_lib('tensorrt', mx_tensorrt_lib, true)
}
}
}
- },
- 'CPU: Openblas': {
+ }]
+}
+
+def compile_centos7_cpu() {
+ return ['CPU: CentOS 7': {
node(NODE_LINUX_CPU) {
- ws('workspace/build-cpu-openblas') {
+ ws('workspace/build-centos7-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)
- utils.pack_lib('cpu', mx_dist_lib, true)
+ utils.docker_run('centos7_cpu', 'build_centos7_cpu', false)
+ utils.pack_lib('centos7_cpu', mx_dist_lib, true)
}
}
}
- },
- 'CPU: ASAN': {
+ }]
+}
+
+def compile_centos7_cpu_mkldnn() {
+ return ['CPU: CentOS 7 MKLDNN': {
node(NODE_LINUX_CPU) {
- ws('workspace/build-cpu-asan') {
+ ws('workspace/build-centos7-mkldnn') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_asan',
false)
- utils.pack_lib('cpu_asan', mx_lib_cpp_examples_cpu)
+ utils.docker_run('centos7_cpu', 'build_centos7_mkldnn', false)
+ utils.pack_lib('centos7_mkldnn', mx_lib, true)
}
}
}
- },
- 'CPU: Openblas, debug': {
+ }]
+}
+
+def compile_centos7_gpu() {
+ return ['GPU: CentOS 7': {
node(NODE_LINUX_CPU) {
- ws('workspace/build-cpu-openblas') {
+ ws('workspace/build-centos7-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_debug',
false)
- utils.pack_lib('cpu_debug', mx_cmake_lib_debug, true)
+ utils.docker_run('centos7_gpu', 'build_centos7_gpu', false)
+ utils.pack_lib('centos7_gpu', mx_lib, true)
}
}
}
- },
- 'CPU: Clang 3.9': {
+ }]
+}
+
+def compile_unix_clang_3_9_cpu() {
+ return ['CPU: Clang 3.9': {
node(NODE_LINUX_CPU) {
ws('workspace/build-cpu-clang39') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -221,8 +271,11 @@ core_logic: {
}
}
}
- },
- 'CPU: Clang 6': {
+ }]
+}
+
+def compile_unix_clang_6_cpu() {
+ return ['CPU: Clang 6': {
node(NODE_LINUX_CPU) {
ws('workspace/build-cpu-clang60') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -231,8 +284,11 @@ core_logic: {
}
}
}
- },
- 'CPU: Clang Tidy': {
+ }]
+}
+
+def compile_unix_clang_tidy_cpu() {
+ return ['CPU: Clang Tidy': {
node(NODE_LINUX_CPU) {
ws('workspace/build-cpu-clang60_tidy') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -241,8 +297,11 @@ core_logic: {
}
}
}
- },
- 'CPU: Clang 3.9 MKLDNN': {
+ }]
+}
+
+def compile_unix_clang_3_9_mkldnn_cpu() {
+ return ['CPU: Clang 3.9 MKLDNN': {
node(NODE_LINUX_CPU) {
ws('workspace/build-cpu-mkldnn-clang39') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -252,8 +311,11 @@ core_logic: {
}
}
}
- },
- 'CPU: Clang 6 MKLDNN': {
+ }]
+}
+
+def compile_unix_clang_6_mkldnn_cpu() {
+ return ['CPU: Clang 6 MKLDNN': {
node(NODE_LINUX_CPU) {
ws('workspace/build-cpu-mkldnn-clang60') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -263,106 +325,130 @@ core_logic: {
}
}
}
- },
- 'CPU: MKLDNN': {
+ }]
+}
+
+def compile_armv8_jetson_gpu() {
+ return ['NVidia Jetson / ARMv8':{
node(NODE_LINUX_CPU) {
- ws('workspace/build-mkldnn-cpu') {
+ ws('workspace/build-jetson-armv8') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn', false)
- utils.pack_lib('mkldnn_cpu', mx_mkldnn_lib, true)
+ utils.docker_run('jetson', 'build_jetson', false)
}
}
}
- },
- 'GPU: MKLDNN': {
+ }]
+}
+
+def compile_armv7_cpu() {
+ return ['ARMv7':{
node(NODE_LINUX_CPU) {
- ws('workspace/build-mkldnn-gpu') {
+ ws('workspace/build-ARMv7') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn',
false)
- utils.pack_lib('mkldnn_gpu', mx_mkldnn_lib, true)
+ utils.docker_run('armv7', 'build_armv7', false)
+ utils.pack_lib('armv7', mx_pip)
}
}
}
- },
- 'GPU: MKLDNN_CUDNNOFF': {
- node(NODE_LINUX_CPU) {
- ws('workspace/build-mkldnn-gpu-nocudnn') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.init_git()
- utils.docker_run('ubuntu_build_cuda',
'build_ubuntu_gpu_mkldnn_nocudnn', false)
- utils.pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib, true)
- }
- }
- }
- },
- 'GPU: CUDA9.1+cuDNN7': {
+ }]
+}
+
+def compile_armv6_cpu() {
+ return ['ARMv6':{
node(NODE_LINUX_CPU) {
- ws('workspace/build-gpu') {
+ ws('workspace/build-ARMv6') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_build_cuda',
'build_ubuntu_gpu_cuda91_cudnn7', false)
- utils.pack_lib('gpu', mx_lib_cpp_examples, true)
+ utils.docker_run('armv6', 'build_armv6', false)
}
}
}
- },
- 'Amalgamation MIN': {
+ }]
+}
+
+def compile_armv8_cpu() {
+ return ['ARMv8':{
node(NODE_LINUX_CPU) {
- ws('workspace/amalgamationmin') {
+ ws('workspace/build-ARMv8') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation_min',
false)
+ utils.docker_run('armv8', 'build_armv8', false)
}
}
}
- },
- 'Amalgamation': {
+ }]
+}
+
+def compile_armv8_android_cpu() {
+ return ['Android / ARMv8':{
node(NODE_LINUX_CPU) {
- ws('workspace/amalgamation') {
+ ws('workspace/android64') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation', false)
+ utils.docker_run('android_armv8', 'build_android_armv8', false)
}
}
}
- },
+ }]
+}
- 'GPU: CMake MKLDNN': {
+def compile_armv7_android_cpu() {
+ return ['Android / ARMv7':{
node(NODE_LINUX_CPU) {
- ws('workspace/build-cmake-mkldnn-gpu') {
+ ws('workspace/androidv7') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn',
false)
- utils.pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib, true)
+ utils.docker_run('android_armv7', 'build_android_armv7', false)
}
}
}
- },
- 'GPU: CMake': {
+ }]
+}
+
+def compile_unix_asan_cpu() {
+ return ['CPU: ASAN': {
node(NODE_LINUX_CPU) {
- ws('workspace/build-cmake-gpu') {
+ ws('workspace/build-cpu-asan') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false)
- utils.pack_lib('cmake_gpu', mx_cmake_lib, true)
+ utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_asan',
false)
+ utils.pack_lib('cpu_asan', mx_lib_cpp_examples_cpu)
}
}
}
- },
- 'TensorRT': {
+ }]
+}
+
+def compile_unix_amalgamation_min() {
+ return ['Amalgamation MIN': {
node(NODE_LINUX_CPU) {
- ws('workspace/build-tensorrt') {
+ ws('workspace/amalgamationmin') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_gpu_tensorrt',
'build_ubuntu_gpu_tensorrt', false)
- utils.pack_lib('tensorrt', mx_tensorrt_lib, true)
+ utils.docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation_min',
false)
+ }
+ }
+ }
+ }]
+}
+
+def compile_unix_amalgamation() {
+ return ['Amalgamation': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/amalgamation') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation', false)
}
}
}
- },
- 'Build CPU windows':{
+ }]
+}
+
+def compile_windows_cpu() {
+ return ['Build CPU windows':{
node(NODE_WINDOWS_CPU) {
timeout(time: max_time, unit: 'MINUTES') {
ws('workspace/build-cpu') {
@@ -374,9 +460,11 @@ core_logic: {
}
}
}
- },
+ }]
+}
- 'Build GPU windows':{
+def compile_windows_gpu() {
+ return ['Build GPU windows':{
node(NODE_WINDOWS_CPU) {
timeout(time: max_time, unit: 'MINUTES') {
ws('workspace/build-gpu') {
@@ -388,8 +476,11 @@ core_logic: {
}
}
}
- },
- 'Build GPU MKLDNN windows':{
+ }]
+}
+
+def compile_windows_gpu_mkldnn() {
+ return ['Build GPU MKLDNN windows':{
node(NODE_WINDOWS_CPU) {
timeout(time: max_time, unit: 'MINUTES') {
ws('workspace/build-gpu') {
@@ -401,73 +492,11 @@ core_logic: {
}
}
}
- },
- 'NVidia Jetson / ARMv8':{
- node(NODE_LINUX_CPU) {
- ws('workspace/build-jetson-armv8') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.init_git()
- utils.docker_run('jetson', 'build_jetson', false)
- }
- }
- }
- },
- 'ARMv7':{
- node(NODE_LINUX_CPU) {
- ws('workspace/build-ARMv7') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.init_git()
- utils.docker_run('armv7', 'build_armv7', false)
- utils.pack_lib('armv7', mx_pip)
- }
- }
- }
- },
- 'ARMv6':{
- node(NODE_LINUX_CPU) {
- ws('workspace/build-ARMv6') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.init_git()
- utils.docker_run('armv6', 'build_armv6', false)
- }
- }
- }
- },
- 'ARMv8':{
- node(NODE_LINUX_CPU) {
- ws('workspace/build-ARMv8') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.init_git()
- utils.docker_run('armv8', 'build_armv8', false)
- }
- }
- }
- },
- 'Android / ARMv8':{
- node(NODE_LINUX_CPU) {
- ws('workspace/android64') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.init_git()
- utils.docker_run('android_armv8', 'build_android_armv8', false)
- }
- }
- }
- },
- 'Android / ARMv7':{
- node(NODE_LINUX_CPU) {
- ws('workspace/androidv7') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.init_git()
- utils.docker_run('android_armv7', 'build_android_armv7', false)
- }
- }
- }
- }
-
- } // End of stage('Build')
+ }]
+}
- stage('Tests') {
- parallel 'Python2: CPU': {
+def test_unix_python2_cpu() {
+ return ['Python2: CPU': {
node(NODE_LINUX_CPU) {
ws('workspace/ut-python2-cpu') {
try {
@@ -481,56 +510,78 @@ core_logic: {
}
}
}
- },
- 'Python3: CPU': {
- node(NODE_LINUX_CPU) {
- ws('workspace/ut-python3-cpu') {
- try {
- utils.unpack_and_init('cpu', mx_lib, true)
- python3_ut('ubuntu_cpu')
+ }]
+}
+
+def test_unix_python2_gpu() {
+ return ['Python2: GPU': {
+ node(NODE_LINUX_GPU) {
+ ws('workspace/ut-python2-gpu') {
+ try {
+ utils.unpack_and_init('gpu', mx_lib, true)
+ python2_gpu_ut('ubuntu_gpu')
utils.publish_test_coverage()
} finally {
- utils.collect_test_results_unix('nosetests_unittest.xml',
'nosetests_python3_cpu_unittest.xml')
- utils.collect_test_results_unix('nosetests_quantization.xml',
'nosetests_python3_cpu_quantization.xml')
+ utils.collect_test_results_unix('nosetests_gpu.xml',
'nosetests_python2_gpu.xml')
}
}
}
- },
- 'CPU ASAN': {
- node(NODE_LINUX_CPU) {
- ws('workspace/ut-python3-cpu-asan') {
- utils.unpack_and_init('cpu_asan', mx_lib_cpp_examples_cpu)
- utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_asan',
false)
+ }]
+}
+
+def test_unix_python2_quantize_gpu() {
+ return ['Python2: Quantize GPU': {
+ node(NODE_LINUX_GPU_P3) {
+ ws('workspace/ut-python2-quantize-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ try {
+ utils.unpack_and_init('gpu', mx_lib, true)
+ utils.docker_run('ubuntu_gpu',
'unittest_ubuntu_python2_quantization_gpu', true)
+ utils.publish_test_coverage()
+ } finally {
+
utils.collect_test_results_unix('nosetests_quantization_gpu.xml',
'nosetests_python2_quantize_gpu.xml')
+ }
+ }
}
}
- },
- 'Python3: CPU debug': {
- node(NODE_LINUX_CPU) {
- ws('workspace/ut-python3-cpu-debug') {
+ }]
+}
+
+def test_unix_python2_mkldnn_gpu() {
+ return ['Python2: MKLDNN-GPU': {
+ node(NODE_LINUX_GPU) {
+ ws('workspace/ut-python2-mkldnn-gpu') {
try {
- utils.unpack_and_init('cpu_debug', mx_cmake_lib_debug, true)
- python3_ut('ubuntu_cpu')
+ utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib, true)
+ python2_gpu_ut('ubuntu_gpu')
+ utils.publish_test_coverage()
} finally {
- utils.collect_test_results_unix('nosetests_unittest.xml',
'nosetests_python3_cpu_debug_unittest.xml')
- utils.collect_test_results_unix('nosetests_quantization.xml',
'nosetests_python3_cpu_debug_quantization.xml')
+ utils.collect_test_results_unix('nosetests_gpu.xml',
'nosetests_python2_mkldnn_gpu.xml')
}
}
}
- },
- 'Python2: GPU': {
- node(NODE_LINUX_GPU) {
- ws('workspace/ut-python2-gpu') {
+ }]
+}
+
+def test_unix_python3_cpu() {
+ return ['Python3: CPU': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/ut-python3-cpu') {
try {
- utils.unpack_and_init('gpu', mx_lib, true)
- python2_gpu_ut('ubuntu_gpu')
+ utils.unpack_and_init('cpu', mx_lib, true)
+ python3_ut('ubuntu_cpu')
utils.publish_test_coverage()
} finally {
- utils.collect_test_results_unix('nosetests_gpu.xml',
'nosetests_python2_gpu.xml')
+ utils.collect_test_results_unix('nosetests_unittest.xml',
'nosetests_python3_cpu_unittest.xml')
+ utils.collect_test_results_unix('nosetests_quantization.xml',
'nosetests_python3_cpu_quantization.xml')
}
}
}
- },
- 'Python3: GPU': {
+ }]
+}
+
+def test_unix_python3_gpu() {
+ return ['Python3: GPU': {
node(NODE_LINUX_GPU) {
ws('workspace/ut-python3-gpu') {
try {
@@ -542,23 +593,11 @@ core_logic: {
}
}
}
- },
- 'Python2: Quantize GPU': {
- node(NODE_LINUX_GPU_P3) {
- ws('workspace/ut-python2-quantize-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu',
'unittest_ubuntu_python2_quantization_gpu', true)
- utils.publish_test_coverage()
- } finally {
-
utils.collect_test_results_unix('nosetests_quantization_gpu.xml',
'nosetests_python2_quantize_gpu.xml')
- }
- }
- }
- }
- },
- 'Python3: Quantize GPU': {
+ }]
+}
+
+def test_unix_python3_quantize_gpu() {
+ return ['Python3: Quantize GPU': {
node(NODE_LINUX_GPU_P3) {
ws('workspace/ut-python3-quantize-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -572,8 +611,27 @@ core_logic: {
}
}
}
- },
- 'Python2: MKLDNN-CPU': {
+ }]
+}
+
+def test_unix_python3_debug_cpu() {
+ return ['Python3: CPU debug': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/ut-python3-cpu-debug') {
+ try {
+ utils.unpack_and_init('cpu_debug', mx_cmake_lib_debug, true)
+ python3_ut('ubuntu_cpu')
+ } finally {
+ utils.collect_test_results_unix('nosetests_unittest.xml',
'nosetests_python3_cpu_debug_unittest.xml')
+ utils.collect_test_results_unix('nosetests_quantization.xml',
'nosetests_python3_cpu_debug_quantization.xml')
+ }
+ }
+ }
+ }]
+}
+
+def test_unix_python2_mkldnn_cpu() {
+ return ['Python2: MKLDNN-CPU': {
node(NODE_LINUX_CPU) {
ws('workspace/ut-python2-mkldnn-cpu') {
try {
@@ -587,21 +645,11 @@ core_logic: {
}
}
}
- },
- 'Python2: MKLDNN-GPU': {
- node(NODE_LINUX_GPU) {
- ws('workspace/ut-python2-mkldnn-gpu') {
- try {
- utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib, true)
- python2_gpu_ut('ubuntu_gpu')
- utils.publish_test_coverage()
- } finally {
- utils.collect_test_results_unix('nosetests_gpu.xml',
'nosetests_python2_mkldnn_gpu.xml')
- }
- }
- }
- },
- 'Python3: MKLDNN-CPU': {
+ }]
+}
+
+def test_unix_python3_mkldnn_cpu() {
+ return ['Python3: MKLDNN-CPU': {
node(NODE_LINUX_CPU) {
ws('workspace/ut-python3-mkldnn-cpu') {
try {
@@ -614,8 +662,11 @@ core_logic: {
}
}
}
- },
- 'Python3: MKLDNN-GPU': {
+ }]
+}
+
+def test_unix_python3_mkldnn_gpu() {
+ return ['Python3: MKLDNN-GPU': {
node(NODE_LINUX_GPU) {
ws('workspace/ut-python3-mkldnn-gpu') {
try {
@@ -627,8 +678,11 @@ core_logic: {
}
}
}
- },
- 'Python3: MKLDNN-GPU-NOCUDNN': {
+ }]
+}
+
+def test_unix_python3_mkldnn_nocudnn_gpu() {
+ return ['Python3: MKLDNN-GPU-NOCUDNN': {
node(NODE_LINUX_GPU) {
ws('workspace/ut-python3-mkldnn-gpu-nocudnn') {
try {
@@ -640,54 +694,72 @@ core_logic: {
}
}
}
- },
- 'Python3: CentOS 7 CPU': {
- node(NODE_LINUX_CPU) {
- ws('workspace/build-centos7-cpu') {
+ }]
+}
+
+def test_unix_python3_tensorrt_gpu() {
+ return ['Python3: TensorRT GPU': {
+ node(NODE_LINUX_GPU_P3) {
+ ws('workspace/build-tensorrt') {
timeout(time: max_time, unit: 'MINUTES') {
try {
- utils.unpack_and_init('centos7_cpu', mx_lib, true)
- utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false)
+ utils.unpack_and_init('tensorrt', mx_tensorrt_lib, true)
+ utils.docker_run('ubuntu_gpu_tensorrt',
'unittest_ubuntu_tensorrt_gpu', true)
utils.publish_test_coverage()
} finally {
- utils.collect_test_results_unix('nosetests_unittest.xml',
'nosetests_python3_centos7_cpu_unittest.xml')
- utils.collect_test_results_unix('nosetests_train.xml',
'nosetests_python3_centos7_cpu_train.xml')
+ utils.collect_test_results_unix('nosetests_tensorrt.xml',
'nosetests_python3_tensorrt_gpu.xml')
}
}
}
}
- },
- 'Python3: CentOS 7 GPU': {
+ }]
+}
+
+def test_unix_python3_integration_gpu() {
+ return ['Python Integration GPU': {
node(NODE_LINUX_GPU) {
- ws('workspace/build-centos7-gpu') {
+ ws('workspace/it-python-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
- try {
- utils.unpack_and_init('centos7_gpu', mx_lib, true)
- utils.docker_run('centos7_gpu', 'unittest_centos7_gpu', true)
- utils.publish_test_coverage()
- } finally {
- utils.collect_test_results_unix('nosetests_gpu.xml',
'nosetests_python3_centos7_gpu.xml')
- }
+ utils.unpack_and_init('gpu', mx_lib, true)
+ utils.docker_run('ubuntu_gpu',
'integrationtest_ubuntu_gpu_python', true)
+ utils.publish_test_coverage()
}
}
}
- },
- 'Python3: TensorRT GPU': {
- node(NODE_LINUX_GPU_P3) {
- ws('workspace/build-tensorrt') {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- utils.unpack_and_init('tensorrt', mx_tensorrt_lib, true)
- utils.docker_run('ubuntu_gpu_tensorrt',
'unittest_ubuntu_tensorrt_gpu', true)
- utils.publish_test_coverage()
- } finally {
- utils.collect_test_results_unix('nosetests_tensorrt.xml',
'nosetests_python3_tensorrt_gpu.xml')
+ }]
+}
+
+def test_unix_caffe_gpu() {
+ return ['Caffe GPU': {
+ node(NODE_LINUX_GPU) {
+ ws('workspace/it-caffe') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.unpack_lib('gpu', mx_lib)
+ utils.docker_run('ubuntu_gpu',
'integrationtest_ubuntu_gpu_caffe', true)
+ utils.publish_test_coverage()
+ }
}
+ }
+ }]
+}
+
+def test_unix_cpp_package_gpu() {
+ return ['cpp-package GPU': {
+ node(NODE_LINUX_GPU) {
+ ws('workspace/it-cpp-package') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.unpack_and_init('gpu', mx_lib_cpp_examples, true)
+ utils.docker_run('ubuntu_gpu',
'integrationtest_ubuntu_gpu_cpp_package', true)
+ utils.publish_test_coverage()
}
}
}
- },
- 'Scala: CPU': {
+ }]
+}
+
+def test_unix_scala_cpu() {
+ return ['Scala: CPU': {
node(NODE_LINUX_CPU) {
ws('workspace/ut-scala-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -697,19 +769,25 @@ core_logic: {
}
}
}
- },
- 'Scala: CentOS CPU': {
- node(NODE_LINUX_CPU) {
- ws('workspace/ut-scala-centos7-cpu') {
+ }]
+}
+
+def test_unix_scala_gpu() {
+ return ['Scala: GPU': {
+ node(NODE_LINUX_GPU) {
+ ws('workspace/ut-scala-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('centos7_cpu', mx_dist_lib, true)
- utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala',
false)
+ utils.unpack_and_init('gpu', mx_dist_lib, true)
+ utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala',
true)
utils.publish_test_coverage()
}
}
}
- },
- 'Clojure: CPU': {
+ }]
+}
+
+def test_unix_clojure_cpu() {
+ return ['Clojure: CPU': {
node(NODE_LINUX_CPU) {
ws('workspace/ut-clojure-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -719,8 +797,11 @@ core_logic: {
}
}
}
- },
- 'Perl: CPU': {
+ }]
+}
+
+def test_unix_r_cpu() {
+ return ['Perl: CPU': {
node(NODE_LINUX_CPU) {
ws('workspace/ut-perl-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -730,104 +811,177 @@ core_logic: {
}
}
}
- },
- 'Perl: GPU': {
+ }]
+}
+
+def test_unix_cpp_gpu() {
+ return ['Cpp: GPU': {
node(NODE_LINUX_GPU) {
- ws('workspace/ut-perl-gpu') {
+ ws('workspace/ut-cpp-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true)
+ utils.unpack_and_init('cmake_gpu', mx_cmake_lib, true)
+ utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
utils.publish_test_coverage()
}
}
}
- },
- 'Cpp: GPU': {
+ }]
+}
+
+def test_unix_cpp_mkldnn_gpu() {
+ return ['Cpp: MKLDNN+GPU': {
node(NODE_LINUX_GPU) {
- ws('workspace/ut-cpp-gpu') {
+ ws('workspace/ut-cpp-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('cmake_gpu', mx_cmake_lib, true)
+ utils.unpack_and_init('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib,
true)
utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
utils.publish_test_coverage()
}
}
}
- },
- 'Cpp: MKLDNN+GPU': {
+ }]
+}
+
+def test_unix_r_gpu() {
+ return ['Perl: GPU': {
node(NODE_LINUX_GPU) {
- ws('workspace/ut-cpp-mkldnn-gpu') {
+ ws('workspace/ut-perl-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib,
true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
+ utils.unpack_and_init('gpu', mx_lib, true)
+ utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true)
utils.publish_test_coverage()
}
}
}
- },
- 'R: CPU': {
+ }]
+}
+
+def test_unix_julia_cpu() {
+ return ['Julia 0.6: CPU': {
node(NODE_LINUX_CPU) {
- ws('workspace/ut-r-cpu') {
+ ws('workspace/ut-julia06-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.unpack_and_init('cpu', mx_lib)
+ utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia06',
false)
+ }
+ }
+ }
+ }]
+}
+
+def test_unix_onnx_cpu() {
+ return ['Onnx CPU': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/it-onnx-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cpu', mx_lib, true)
- utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false)
+ utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx',
false)
+ utils.publish_test_coverage()
+ }
+ }
+ }
+ }]
+}
+
+def test_unix_distributed_kvstore_cpu() {
+ return ['dist-kvstore tests CPU': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/it-dist-kvstore') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.unpack_and_init('cpu', mx_lib, true)
+ utils.docker_run('ubuntu_cpu',
'integrationtest_ubuntu_cpu_dist_kvstore', false)
utils.publish_test_coverage()
}
}
}
- },
- 'R: GPU': {
+ }]
+}
+
+def test_unix_distributed_kvstore_gpu() {
+ return ['dist-kvstore tests GPU': {
node(NODE_LINUX_GPU) {
- ws('workspace/ut-r-gpu') {
+ ws('workspace/it-dist-kvstore') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true)
+ utils.docker_run('ubuntu_gpu',
'integrationtest_ubuntu_gpu_dist_kvstore', true)
utils.publish_test_coverage()
}
}
}
- },
- 'Julia 0.6: CPU': {
+ }]
+}
+
+def test_centos7_python3_cpu() {
+ return ['Python3: CentOS 7 CPU': {
node(NODE_LINUX_CPU) {
- ws('workspace/ut-julia06-cpu') {
+ ws('workspace/build-centos7-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('cpu', mx_lib)
- utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia06',
false)
+ try {
+ utils.unpack_and_init('centos7_cpu', mx_lib, true)
+ utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false)
+ utils.publish_test_coverage()
+ } finally {
+ utils.collect_test_results_unix('nosetests_unittest.xml',
'nosetests_python3_centos7_cpu_unittest.xml')
+ utils.collect_test_results_unix('nosetests_train.xml',
'nosetests_python3_centos7_cpu_train.xml')
+ }
}
}
}
- },
+ }]
+}
- 'Python 2: CPU Win':{
- node(NODE_WINDOWS_CPU) {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-cpu') {
+def test_centos7_python3_gpu() {
+ return ['Python3: CentOS 7 GPU': {
+ node(NODE_LINUX_GPU) {
+ ws('workspace/build-centos7-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
try {
- utils.init_git_win()
- unstash 'windows_package_cpu'
- powershell 'ci/windows/test_py2_cpu.ps1'
+ utils.unpack_and_init('centos7_gpu', mx_lib, true)
+ utils.docker_run('centos7_gpu', 'unittest_centos7_gpu', true)
+ utils.publish_test_coverage()
} finally {
- utils.collect_test_results_windows('nosetests_unittest.xml',
'nosetests_unittest_windows_python2_cpu.xml')
+ utils.collect_test_results_unix('nosetests_gpu.xml',
'nosetests_python3_centos7_gpu.xml')
}
}
}
}
- },
- 'Python 3: CPU Win': {
+ }]
+}
+
+def test_centos7_scala_cpu() {
+ return ['Scala: CentOS CPU': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/ut-scala-centos7-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.unpack_and_init('centos7_cpu', mx_dist_lib, true)
+ utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala',
false)
+ utils.publish_test_coverage()
+ }
+ }
+ }
+ }]
+}
+
+def test_windows_python2_cpu() {
+ return ['Python 2: CPU Win':{
node(NODE_WINDOWS_CPU) {
timeout(time: max_time, unit: 'MINUTES') {
ws('workspace/ut-python-cpu') {
try {
utils.init_git_win()
unstash 'windows_package_cpu'
- powershell 'ci/windows/test_py3_cpu.ps1'
+ powershell 'ci/windows/test_py2_cpu.ps1'
} finally {
- utils.collect_test_results_windows('nosetests_unittest.xml',
'nosetests_unittest_windows_python3_cpu.xml')
+ utils.collect_test_results_windows('nosetests_unittest.xml',
'nosetests_unittest_windows_python2_cpu.xml')
}
}
}
}
- },
- 'Python 2: GPU Win':{
+ }]
+}
+
+def test_windows_python2_gpu() {
+ return ['Python 2: GPU Win':{
node(NODE_WINDOWS_GPU) {
timeout(time: max_time, unit: 'MINUTES') {
ws('workspace/ut-python-gpu') {
@@ -842,8 +996,11 @@ core_logic: {
}
}
}
- },
- 'Python 3: GPU Win':{
+ }]
+}
+
+def test_windows_python3_gpu() {
+ return ['Python 3: GPU Win':{
node(NODE_WINDOWS_GPU) {
timeout(time: max_time, unit: 'MINUTES') {
ws('workspace/ut-python-gpu') {
@@ -858,8 +1015,11 @@ core_logic: {
}
}
}
- },
- 'Python 3: MKLDNN-GPU Win':{
+ }]
+}
+
+def test_windows_python3_gpu_mkldnn() {
+ return ['Python 3: MKLDNN-GPU Win':{
node(NODE_WINDOWS_GPU) {
timeout(time: max_time, unit: 'MINUTES') {
ws('workspace/ut-python-gpu') {
@@ -874,111 +1034,104 @@ core_logic: {
}
}
}
- },
- 'Onnx CPU': {
- node(NODE_LINUX_CPU) {
- ws('workspace/it-onnx-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('cpu', mx_lib, true)
- utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx',
false)
- utils.publish_test_coverage()
+ }]
+}
+
+def test_windows_python3_cpu() {
+ return ['Python 3: CPU Win': {
+ node(NODE_WINDOWS_CPU) {
+ timeout(time: max_time, unit: 'MINUTES') {
+ ws('workspace/ut-python-cpu') {
+ try {
+ utils.init_git_win()
+ unstash 'windows_package_cpu'
+ powershell 'ci/windows/test_py3_cpu.ps1'
+ } finally {
+ utils.collect_test_results_windows('nosetests_unittest.xml',
'nosetests_unittest_windows_python3_cpu.xml')
+ }
}
}
}
- },
- 'Python GPU': {
- node(NODE_LINUX_GPU) {
- ws('workspace/it-python-gpu') {
+ }]
+}
+
+def test_qemu_armv7_cpu() {
+ return ['ARMv7 QEMU': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/ut-armv7-qemu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu',
'integrationtest_ubuntu_gpu_python', true)
- utils.publish_test_coverage()
+ utils.unpack_and_init('armv7', mx_pip)
+ sh "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} -p
test.arm_qemu ./runtime_functions.py run_ut_py3_qemu"
}
}
}
- },
- 'cpp-package GPU': {
- node(NODE_LINUX_GPU) {
- ws('workspace/it-cpp-package') {
+ }]
+}
+
+def docs_website() {
+ return ['Docs': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/docs') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('gpu', mx_lib_cpp_examples, true)
- utils.docker_run('ubuntu_gpu',
'integrationtest_ubuntu_gpu_cpp_package', true)
- utils.publish_test_coverage()
+ utils.init_git()
+ utils.docker_run('ubuntu_cpu', 'deploy_docs', false)
+
+ master_url = utils.get_jenkins_master_url()
+ if ( master_url == 'jenkins.mxnet-ci.amazon-ml.com') {
+ sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME}
${env.BUILD_NUMBER}"
+ } else {
+ print "Skipping staging documentation publishing since we are
not running in prod. Host: {$master_url}"
+ }
}
}
}
- },
- // Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407
- // 'Caffe GPU': {
- // node(NODE_LINUX_GPU) {
- // ws('workspace/it-caffe') {
- // timeout(time: max_time, unit: 'MINUTES') {
- // utils.init_git()
- // utils.unpack_lib('gpu', mx_lib)
- // utils.docker_run('ubuntu_gpu',
'integrationtest_ubuntu_gpu_caffe', true)
- // utils.publish_test_coverage()
- // }
- // }
- // }
- // },
- 'dist-kvstore tests GPU': {
- node(NODE_LINUX_GPU) {
- ws('workspace/it-dist-kvstore') {
+ }]
+}
+
+def docs_julia() {
+ return ['Julia docs': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/julia-docs') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu',
'integrationtest_ubuntu_gpu_dist_kvstore', true)
- utils.publish_test_coverage()
+ utils.unpack_and_init('cpu', mx_lib)
+ utils.docker_run('ubuntu_cpu', 'deploy_jl_docs', false)
}
}
}
- },
- /* Disabled due to master build failure:
- *
http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/
- * https://github.com/apache/incubator-mxnet/issues/11801
+ }]
+}
- 'dist-kvstore tests CPU': {
+def misc_asan_cpu() {
+ return ['CPU ASAN': {
node(NODE_LINUX_CPU) {
- ws('workspace/it-dist-kvstore') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('cpu', mx_lib, true)
- utils.docker_run('ubuntu_cpu',
'integrationtest_ubuntu_cpu_dist_kvstore', false)
- utils.publish_test_coverage()
- }
+ ws('workspace/ut-python3-cpu-asan') {
+ utils.unpack_and_init('cpu_asan', mx_lib_cpp_examples_cpu)
+ utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_asan',
false)
}
}
- }, */
- 'Scala: GPU': {
- node(NODE_LINUX_GPU) {
- ws('workspace/ut-scala-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('gpu', mx_dist_lib, true)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala',
true)
- utils.publish_test_coverage()
- }
+ }]
+}
+
+def sanity_lint() {
+ return ['Lint': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/sanity-lint') {
+ utils.init_git()
+ utils.docker_run('ubuntu_cpu', 'sanity_check', false)
}
}
- },
- 'ARMv7 QEMU': {
+ }]
+}
+
+def sanity_rat_license() {
+ return ['RAT License': {
node(NODE_LINUX_CPU) {
- ws('workspace/ut-armv7-qemu') {
- timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('armv7', mx_pip)
- sh "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} -p
test.arm_qemu ./runtime_functions.py run_ut_py3_qemu"
- }
+ ws('workspace/sanity-rat') {
+ utils.init_git()
+ utils.docker_run('ubuntu_rat', 'nightly_test_rat_check', false)
}
}
- }
- }
-
- stage('Deploy') {
- deploy_docs()
- }
+ }]
}
-,
-failure_handler: {
- // Only send email if master or release branches failed
- if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
- emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
- }
-}
-)
+
+return this
diff --git a/ci/jenkins/Jenkinsfile_centos_cpu
b/ci/jenkins/Jenkinsfile_centos_cpu
new file mode 100644
index 0000000..6d704bf
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_centos_cpu
@@ -0,0 +1,53 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_centos7_cpu(),
+ custom_steps.compile_centos7_cpu_mkldnn()
+ ])
+
+ utils.parallel_stage('Tests', [
+ custom_steps.test_centos7_python3_cpu(),
+ custom_steps.test_centos7_scala_cpu()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_centos_gpu
b/ci/jenkins/Jenkinsfile_centos_gpu
new file mode 100644
index 0000000..d99177e
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_centos_gpu
@@ -0,0 +1,51 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu:
'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_centos7_gpu()
+ ])
+
+ utils.parallel_stage('Tests', [
+ custom_steps.test_centos7_python3_gpu()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_clang b/ci/jenkins/Jenkinsfile_clang
new file mode 100644
index 0000000..2fa9b83
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_clang
@@ -0,0 +1,51 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu:
'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_unix_clang_3_9_cpu(),
+ custom_steps.compile_unix_clang_6_cpu(),
+ custom_steps.compile_unix_clang_tidy_cpu(),
+ custom_steps.compile_unix_clang_3_9_mkldnn_cpu(),
+ custom_steps.compile_unix_clang_6_mkldnn_cpu()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_edge b/ci/jenkins/Jenkinsfile_edge
new file mode 100644
index 0000000..64324cc
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_edge
@@ -0,0 +1,56 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu:
'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_armv8_jetson_gpu(),
+ custom_steps.compile_armv7_cpu(),
+ custom_steps.compile_armv6_cpu(),
+ custom_steps.compile_armv8_cpu(),
+ custom_steps.compile_armv8_android_cpu(),
+ custom_steps.compile_armv7_android_cpu()
+ ])
+
+ utils.parallel_stage('Tests', [
+ custom_steps.test_qemu_armv7_cpu()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_miscellaneous
b/ci/jenkins/Jenkinsfile_miscellaneous
new file mode 100644
index 0000000..367cd6f
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_miscellaneous
@@ -0,0 +1,54 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu:
'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu:
'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_unix_asan_cpu(),
+ custom_steps.compile_unix_amalgamation_min(),
+ custom_steps.compile_unix_amalgamation()
+ ])
+
+ utils.parallel_stage('Tests', [
+ custom_steps.misc_asan_cpu()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_sanity b/ci/jenkins/Jenkinsfile_sanity
new file mode 100644
index 0000000..cb74b03
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_sanity
@@ -0,0 +1,48 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu:
'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu:
'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Sanity Check', [
+ custom_steps.sanity_lint(),
+ custom_steps.sanity_rat_license()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
new file mode 100644
index 0000000..65da416
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -0,0 +1,68 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_unix_cpu_openblas(),
+ custom_steps.compile_unix_openblas_debug_cpu(),
+ custom_steps.compile_unix_mkldnn_cpu()
+ ])
+
+ utils.parallel_stage('Tests', [
+ custom_steps.test_unix_python2_cpu(),
+ custom_steps.test_unix_python3_cpu(),
+ custom_steps.test_unix_python3_debug_cpu(),
+ custom_steps.test_unix_python2_mkldnn_cpu(),
+ custom_steps.test_unix_python3_mkldnn_cpu(),
+ custom_steps.test_unix_scala_cpu(),
+ custom_steps.test_unix_clojure_cpu(),
+ custom_steps.test_unix_r_cpu(),
+ custom_steps.test_unix_julia_cpu(),
+ custom_steps.test_unix_onnx_cpu(),
+
+ /* Disabled due to master build failure:
+ *
http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/
+ * https://github.com/apache/incubator-mxnet/issues/11801
+ custom_steps.test_unix_distributed_kvstore_cpu()
+ */
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_unix_gpu b/ci/jenkins/Jenkinsfile_unix_gpu
new file mode 100644
index 0000000..f02458d
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_unix_gpu
@@ -0,0 +1,73 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu:
'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_unix_mkldnn_gpu(),
+ custom_steps.compile_unix_mkldnn_nocudnn_gpu(),
+ custom_steps.compile_unix_full_gpu(),
+ custom_steps.compile_unix_cmake_mkldnn_gpu(),
+ custom_steps.compile_unix_cmake_gpu(),
+ custom_steps.compile_unix_tensorrt_gpu(),
+ ])
+
+ utils.parallel_stage('Tests', [
+ custom_steps.test_unix_python2_gpu(),
+ custom_steps.test_unix_python3_gpu(),
+ custom_steps.test_unix_python2_quantize_gpu(),
+ custom_steps.test_unix_python3_quantize_gpu(),
+ custom_steps.test_unix_python2_mkldnn_gpu(),
+ custom_steps.test_unix_python3_mkldnn_gpu(),
+ custom_steps.test_unix_python3_mkldnn_nocudnn_gpu(),
+ custom_steps.test_unix_python3_tensorrt_gpu(),
+ custom_steps.test_unix_r_gpu(),
+ custom_steps.test_unix_cpp_gpu(),
+ custom_steps.test_unix_cpp_mkldnn_gpu(),
+ custom_steps.test_unix_python3_integration_gpu(),
+ custom_steps.test_unix_cpp_package_gpu(),
+ custom_steps.test_unix_scala_gpu(),
+ custom_steps.test_unix_distributed_kvstore_gpu()
+
+ // Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407
+ //custom_steps.test_unix_caffe_gpu()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_website b/ci/jenkins/Jenkinsfile_website
new file mode 100644
index 0000000..a2c151e
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_website
@@ -0,0 +1,52 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_unix_cpu_openblas()
+ ])
+
+ utils.parallel_stage('Deploy', [
+ custom_steps.docs_website(),
+ custom_steps.docs_julia()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_windows_cpu
b/ci/jenkins/Jenkinsfile_windows_cpu
new file mode 100644
index 0000000..f7f5442
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_windows_cpu
@@ -0,0 +1,52 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', windows_cpu:
'mxnetwindows-cpu')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_windows_cpu()
+ ])
+
+ utils.parallel_stage('Tests', [
+ custom_steps.test_windows_python2_cpu(),
+ custom_steps.test_windows_python3_cpu()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
diff --git a/ci/jenkins/Jenkinsfile_windows_gpu
b/ci/jenkins/Jenkinsfile_windows_gpu
new file mode 100644
index 0000000..906b6c7
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_windows_gpu
@@ -0,0 +1,54 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', windows_cpu:
'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_windows_gpu(),
+ custom_steps.compile_windows_gpu_mkldnn()
+ ])
+
+ utils.parallel_stage('Tests', [
+ custom_steps.test_windows_python2_gpu(),
+ custom_steps.test_windows_python3_gpu(),
+ custom_steps.test_windows_python3_gpu_mkldnn()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" ||
env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)
