marcoabreu closed pull request #13607: Add Intel MKL blas to Jenkins URL: https://github.com/apache/incubator-mxnet/pull/13607
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu index 7c7e2240ee6..2df9f5887f5 100644 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -54,6 +54,9 @@ RUN /work/ubuntu_clang.sh COPY install/ubuntu_gcc8.sh /work/ RUN /work/ubuntu_gcc8.sh +COPY install/ubuntu_mkl.sh /work/ +RUN /work/ubuntu_mkl.sh + COPY install/ubuntu_mklml.sh /work/ RUN /work/ubuntu_mklml.sh diff --git a/ci/docker/install/ubuntu_mkl.sh b/ci/docker/install/ubuntu_mkl.sh new file mode 100755 index 00000000000..36fc7b07ffd --- /dev/null +++ b/ci/docker/install/ubuntu_mkl.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex + +apt-get update || true +# Install Intel Math Kernel Library (latest major release) +# https://software.intel.com/en-us/articles/installing-intel-free-libs-and-python-apt-repo +wget -O - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | apt-key add - && \ + sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ + apt-get update && \ + apt-get install -y intel-mkl-2019.1-053 diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 1fc10bf0e08..6dd5bb6f239 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -326,6 +326,20 @@ build_ubuntu_cpu_openblas() { -j$(nproc) } +build_ubuntu_cpu_mkl() { + set -ex + export CC="ccache gcc" + export CXX="ccache g++" + make \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=mkl \ + USE_INTEL_PATH=/opt/intel \ + USE_DIST_KVSTORE=1 \ + -j$(nproc) +} + build_ubuntu_cpu_cmake_debug() { set -ex pushd . @@ -480,6 +494,20 @@ build_ubuntu_cpu_mkldnn() { -j$(nproc) } +build_ubuntu_cpu_mkldnn_mkl() { + set -ex + + build_ccache_wrappers + + make \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=mkl \ + USE_MKLDNN=1 \ + -j$(nproc) +} + build_ubuntu_gpu() { build_ubuntu_gpu_cuda91_cudnn7 } diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index f48a2673730..74bde1eee21 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -121,6 +121,20 @@ def compile_unix_openblas_debug_cpu() { }] } +def compile_unix_mkl_cpu() { + return ['CPU: MKL': { + node(NODE_LINUX_CPU) { + ws('workspace/build-cpu-mkl') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false) + utils.pack_lib('cpu_mkl', mx_dist_lib, true) + } + } + } + }] +} + def compile_unix_mkldnn_cpu() { return ['CPU: MKLDNN': { node(NODE_LINUX_CPU) { @@ -135,6 +149,20 @@ def compile_unix_mkldnn_cpu() { }] } +def compile_unix_mkldnn_mkl_cpu() { + return ['CPU: MKLDNN_MKL': { + node(NODE_LINUX_CPU) { + ws('workspace/build-mkldnn-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn_mkl', false) + utils.pack_lib('mkldnn_mkl_cpu', mx_mkldnn_lib, true) + } + } + } + }] +} + def compile_unix_mkldnn_gpu() { return ['GPU: MKLDNN': { node(NODE_LINUX_CPU) { @@ -580,6 +608,23 @@ def test_unix_python3_cpu() { }] } +def test_unix_python3_mkl_cpu() { + return ['Python3: MKL-CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-python3-cpu') { + try { + utils.unpack_and_init('cpu_mkl', mx_lib, true) + python3_ut('ubuntu_cpu') + utils.publish_test_coverage() + } finally { + utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml') + utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_quantization.xml') + } + } + } + }] +} + def test_unix_python3_gpu() { return ['Python3: GPU': { node(NODE_LINUX_GPU) { @@ -665,6 +710,23 @@ def test_unix_python3_mkldnn_cpu() { }] } +def test_unix_python3_mkldnn_mkl_cpu() { + return ['Python3: MKLDNN-MKL-CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-python3-mkldnn-mkl-cpu') { + try { + utils.unpack_and_init('mkldnn_mkl_cpu', mx_mkldnn_lib, true) + python3_ut_mkldnn('ubuntu_cpu') + utils.publish_test_coverage() + } finally { + utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_mkldnn_cpu_unittest.xml') + utils.collect_test_results_unix('nosetests_mkl.xml', 'nosetests_python3_mkldnn_cpu_mkl.xml') + } + } + } + }] +} + def test_unix_python3_mkldnn_gpu() { return ['Python3: MKLDNN-GPU': { node(NODE_LINUX_GPU) { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 9c9a4150377..e581bcf65dc 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -36,15 +36,19 @@ core_logic: { utils.parallel_stage('Build', [ custom_steps.compile_unix_cpu_openblas(), custom_steps.compile_unix_openblas_debug_cpu(), - custom_steps.compile_unix_mkldnn_cpu() + custom_steps.compile_unix_mkl_cpu(), + custom_steps.compile_unix_mkldnn_cpu(), + custom_steps.compile_unix_mkldnn_mkl_cpu() ]) utils.parallel_stage('Tests', [ custom_steps.test_unix_python2_cpu(), custom_steps.test_unix_python3_cpu(), custom_steps.test_unix_python3_debug_cpu(), + custom_steps.test_unix_python3_mkl_cpu(), custom_steps.test_unix_python2_mkldnn_cpu(), custom_steps.test_unix_python3_mkldnn_cpu(), + custom_steps.test_unix_python3_mkldnn_mkl_cpu(), custom_steps.test_unix_scala_cpu(), custom_steps.test_unix_clojure_cpu(), custom_steps.test_unix_r_cpu(), ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
