This is an automated email from the ASF dual-hosted git repository. marcoabreu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push: new bb105a1 [MXNET-405] Add 2 new pipelines to the Official CI and run nightly tests. (#10827) bb105a1 is described below commit bb105a1f8d977978af29867372ef0605c1c3cdbe Author: mbaijal <30911248+mbai...@users.noreply.github.com> AuthorDate: Wed Jun 20 19:17:45 2018 -0700 [MXNET-405] Add 2 new pipelines to the Official CI and run nightly tests. (#10827) * Migrate Nightlies PR1 * Run all tests * Fix RAT and Comment out KVstore for now * Comment out the chmod and test * remove chmod from rat and CW * One more check * changes * run install on cpu instead of gpu * Fix failing pip * Fix Pip * typo-fix * Run Pip on cpu instance * Add a README * Remove pip-test * add pip test back * Add BLC to jenkinsfile format * add permissions * include the git add, commit, push for broken link checker * push not working, comment * Review Comments 1: remove g++-5 and parallize make * Review Comments 2: change dockerfile name, update mx_lib, remove commented line * Review Comments 3: Separate out the dockerfiles for nightly tests * Review Comments 4: Move apache rat install to nightly install script * Review Comments 4: copy the rat folder into work * correct path * Change path Again * check tests * path * Uncomment KVStore * Fix rat and js tests * Move to docker_run and comment KVstore again * Adding docker_run for binaries Jenkinsfile * Permission and other fixes * fix RAT regex, add set -e to install funcs * Delete BLC, change chmod to 755 * JS: parallel make * Add links to github issues * change the instance type to cpu * Comment out pip installs to test build from source on cpu * typo * Add sudo for pip install * Add sudo to all pip install commands coz they cause CI to fail, linux only * Run compilation warnings on cpu instance * Change file perms and some cleanup * Fix the dockerfiles * Dockerfile updates * Merge JS dockerfile into nightly and remove export command * Pin repo versions and run cpu install from souce in cpu docker * Add yes to install in js script * remove sudo from index.md * Add sudo to test script instead * add sudo to test script --- .../docker/Dockerfile.build.ubuntu_base_cpu | 50 +++--- .../docker/Dockerfile.build.ubuntu_base_gpu | 50 +++--- ci/docker/Dockerfile.build.ubuntu_nightly_cpu | 61 +++++++ ci/docker/Dockerfile.build.ubuntu_nightly_gpu | 64 +++++++ .../docker/install/ubuntu_emscripten.sh | 47 +++--- .../docker/install/ubuntu_nightly_tests.sh | 35 ++-- .../docker/install/ubuntu_runas_sudo.sh | 30 +--- ci/docker/runtime_functions.sh | 56 +++++++ tests/jenkins/run_test_installation_docs.sh | 132 ++++++++++----- tests/nightly/Jenkinsfile | 185 +++++++++++++++++++++ tests/nightly/JenkinsfileForBinaries | 121 ++++++++++++++ tests/nightly/README.md | 49 ++++-- tests/nightly/apache_rat_license_check/README.md | 2 +- .../apache_rat_license_check/license_check.sh | 51 ++++++ .../{.rat-excludes => rat-excludes} | 2 + .../compilation_warnings/compilation_warnings.sh | 4 +- .../nightly/compilation_warnings/process_output.py | 0 tests/nightly/test_image_classification.sh | 35 +--- 18 files changed, 754 insertions(+), 220 deletions(-) diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh b/ci/docker/Dockerfile.build.ubuntu_base_cpu old mode 100644 new mode 100755 similarity index 54% copy from tests/nightly/compilation_warnings/compilation_warnings.sh copy to ci/docker/Dockerfile.build.ubuntu_base_cpu index a6c4863..b6515c7 --- a/tests/nightly/compilation_warnings/compilation_warnings.sh +++ b/ci/docker/Dockerfile.build.ubuntu_base_cpu @@ -1,5 +1,4 @@ -#!/bin/bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,30 +15,25 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04 +# This should run in an empty docker with ubuntu and cuda. + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +RUN apt-get update && apt-get -y install sudo + +ARG USER_ID=os.getuid() + +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY install/ubuntu_runas_sudo.sh /work/ +RUN /work/ubuntu_runas_sudo.sh + +COPY runtime_functions.sh /work/ -set -e -runme() { - cmd=$* - echo "$cmd" - $cmd - ret=$? - if [[ ${ret} != 0 ]]; then - echo " " - echo "ERROR: Return value non-zero for: $cmd" - echo " " - exit 1 - fi -} - -sudo add-apt-repository ppa:ubuntu-toolchain-r/test -sudo apt-get update -sudo apt-get -y install time g++-5 -runme make clean >/dev/null -runme mkdir build -echo "Starting make" -cp make/config.mk . -sed -i -e 's/gcc/gcc-5/g' config.mk -sed -i -e 's/g++/g++-5/g' config.mk -runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt -echo "Finished make. Now processing output" -python tests/nightly/compilation_warnings/process_output.py build/compile_output.txt +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh b/ci/docker/Dockerfile.build.ubuntu_base_gpu old mode 100644 new mode 100755 similarity index 54% copy from tests/nightly/compilation_warnings/compilation_warnings.sh copy to ci/docker/Dockerfile.build.ubuntu_base_gpu index a6c4863..bb0aa1d --- a/tests/nightly/compilation_warnings/compilation_warnings.sh +++ b/ci/docker/Dockerfile.build.ubuntu_base_gpu @@ -1,5 +1,4 @@ -#!/bin/bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,30 +15,25 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04 +# This should run in an empty docker with ubuntu and cuda. + +FROM nvidia/cuda:9.1-cudnn7-devel + +WORKDIR /work/deps + +RUN apt-get update && apt-get -y install sudo + +ARG USER_ID=os.getuid() + +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY install/ubuntu_runas_sudo.sh /work/ +RUN /work/ubuntu_runas_sudo.sh + +COPY runtime_functions.sh /work/ -set -e -runme() { - cmd=$* - echo "$cmd" - $cmd - ret=$? - if [[ ${ret} != 0 ]]; then - echo " " - echo "ERROR: Return value non-zero for: $cmd" - echo " " - exit 1 - fi -} - -sudo add-apt-repository ppa:ubuntu-toolchain-r/test -sudo apt-get update -sudo apt-get -y install time g++-5 -runme make clean >/dev/null -runme mkdir build -echo "Starting make" -cp make/config.mk . -sed -i -e 's/gcc/gcc-5/g' config.mk -sed -i -e 's/g++/g++-5/g' config.mk -runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt -echo "Finished make. Now processing output" -python tests/nightly/compilation_warnings/process_output.py build/compile_output.txt +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu new file mode 100755 index 0000000..fc02a4f --- /dev/null +++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu @@ -0,0 +1,61 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_core.sh /work/ +RUN /work/ubuntu_core.sh +COPY install/ubuntu_ccache.sh /work/ +RUN /work/ubuntu_ccache.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh +COPY install/ubuntu_r.sh /work/ +RUN /work/ubuntu_r.sh +COPY install/ubuntu_perl.sh /work/ +RUN /work/ubuntu_perl.sh +COPY install/ubuntu_clang.sh /work/ +RUN /work/ubuntu_clang.sh +COPY install/ubuntu_mklml.sh /work/ +RUN /work/ubuntu_mklml.sh +COPY install/ubuntu_caffe.sh /work/ +RUN /work/ubuntu_caffe.sh +COPY install/ubuntu_onnx.sh /work/ +RUN /work/ubuntu_onnx.sh +COPY install/ubuntu_docs.sh /work/ +RUN /work/ubuntu_docs.sh +COPY install/ubuntu_nightly_tests.sh /work/ +RUN /work/ubuntu_nightly_tests.sh +COPY install/ubuntu_emscripten.sh /work/ +RUN /work/ubuntu_emscripten.sh + +ARG USER_ID=0 + +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib +ENV LLVM=/work/deps/emscripten-fastcomp/build/bin \ No newline at end of file diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu new file mode 100755 index 0000000..c55b942 --- /dev/null +++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu @@ -0,0 +1,64 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for CPU + +FROM nvidia/cuda:9.1-cudnn7-devel + +WORKDIR /work/deps + +COPY install/ubuntu_core.sh /work/ +RUN /work/ubuntu_core.sh +COPY install/ubuntu_ccache.sh /work/ +RUN /work/ubuntu_ccache.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh +COPY install/ubuntu_r.sh /work/ +RUN /work/ubuntu_r.sh +COPY install/ubuntu_perl.sh /work/ +RUN /work/ubuntu_perl.sh +COPY install/ubuntu_clang.sh /work/ +RUN /work/ubuntu_clang.sh +COPY install/ubuntu_mklml.sh /work/ +RUN /work/ubuntu_mklml.sh +COPY install/ubuntu_tvm.sh /work/ +RUN /work/ubuntu_tvm.sh +COPY install/ubuntu_llvm.sh /work/ +RUN /work/ubuntu_llvm.sh +COPY install/ubuntu_caffe.sh /work/ +RUN /work/ubuntu_caffe.sh +COPY install/ubuntu_onnx.sh /work/ +RUN /work/ubuntu_onnx.sh +COPY install/ubuntu_docs.sh /work/ +RUN /work/ubuntu_docs.sh +COPY install/ubuntu_tutorials.sh /work/ +RUN /work/ubuntu_tutorials.sh +COPY install/ubuntu_nightly_tests.sh /work/ +RUN /work/ubuntu_nightly_tests.sh + +ARG USER_ID=0 + +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh b/ci/docker/install/ubuntu_emscripten.sh old mode 100644 new mode 100755 similarity index 50% copy from tests/nightly/compilation_warnings/compilation_warnings.sh copy to ci/docker/install/ubuntu_emscripten.sh index a6c4863..459d145 --- a/tests/nightly/compilation_warnings/compilation_warnings.sh +++ b/ci/docker/install/ubuntu_emscripten.sh @@ -17,29 +17,24 @@ # specific language governing permissions and limitations # under the License. -set -e -runme() { - cmd=$* - echo "$cmd" - $cmd - ret=$? - if [[ ${ret} != 0 ]]; then - echo " " - echo "ERROR: Return value non-zero for: $cmd" - echo " " - exit 1 - fi -} - -sudo add-apt-repository ppa:ubuntu-toolchain-r/test -sudo apt-get update -sudo apt-get -y install time g++-5 -runme make clean >/dev/null -runme mkdir build -echo "Starting make" -cp make/config.mk . -sed -i -e 's/gcc/gcc-5/g' config.mk -sed -i -e 's/g++/g++-5/g' config.mk -runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt -echo "Finished make. Now processing output" -python tests/nightly/compilation_warnings/process_output.py build/compile_output.txt +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +#The script has been copied as is from a dockerfile that existed on previous MXNet versions (0.11) +#Written By: Ly + +set -ex + +apt-get -y install nodejs + +git clone -b 1.38.6 https://github.com/kripken/emscripten.git +git clone -b 1.38.6 https://github.com/kripken/emscripten-fastcomp +cd emscripten-fastcomp +git clone -b 1.38.6 https://github.com/kripken/emscripten-fastcomp-clang tools/clang +mkdir build && cd build + +cmake .. -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="X86;JSBackend" \ +-DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF -DCLANG_INCLUDE_EXAMPLES=OFF \ +-DCLANG_INCLUDE_TESTS=OFF && make -j$(nproc) + +chmod -R 777 /work/deps/emscripten-fastcomp/ \ No newline at end of file diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh b/ci/docker/install/ubuntu_nightly_tests.sh old mode 100644 new mode 100755 similarity index 55% copy from tests/nightly/compilation_warnings/compilation_warnings.sh copy to ci/docker/install/ubuntu_nightly_tests.sh index a6c4863..abc2134 --- a/tests/nightly/compilation_warnings/compilation_warnings.sh +++ b/ci/docker/install/ubuntu_nightly_tests.sh @@ -17,29 +17,14 @@ # specific language governing permissions and limitations # under the License. -set -e -runme() { - cmd=$* - echo "$cmd" - $cmd - ret=$? - if [[ ${ret} != 0 ]]; then - echo " " - echo "ERROR: Return value non-zero for: $cmd" - echo " " - exit 1 - fi -} +#Install steps for the nightly tests -sudo add-apt-repository ppa:ubuntu-toolchain-r/test -sudo apt-get update -sudo apt-get -y install time g++-5 -runme make clean >/dev/null -runme mkdir build -echo "Starting make" -cp make/config.mk . -sed -i -e 's/gcc/gcc-5/g' config.mk -sed -i -e 's/g++/g++-5/g' config.mk -runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt -echo "Finished make. Now processing output" -python tests/nightly/compilation_warnings/process_output.py build/compile_output.txt +set -ex + +#Install for Compilation warning Nightly Test +add-apt-repository ppa:ubuntu-toolchain-r/test +apt-get update +apt-get -y install time + +# Install for RAT License Check Nightly Test +apt-get install -y subversion maven -y #>/dev/null diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh b/ci/docker/install/ubuntu_runas_sudo.sh old mode 100644 new mode 100755 similarity index 55% copy from tests/nightly/compilation_warnings/compilation_warnings.sh copy to ci/docker/install/ubuntu_runas_sudo.sh index a6c4863..40ec60e --- a/tests/nightly/compilation_warnings/compilation_warnings.sh +++ b/ci/docker/install/ubuntu_runas_sudo.sh @@ -17,29 +17,9 @@ # specific language governing permissions and limitations # under the License. -set -e -runme() { - cmd=$* - echo "$cmd" - $cmd - ret=$? - if [[ ${ret} != 0 ]]; then - echo " " - echo "ERROR: Return value non-zero for: $cmd" - echo " " - exit 1 - fi -} +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image -sudo add-apt-repository ppa:ubuntu-toolchain-r/test -sudo apt-get update -sudo apt-get -y install time g++-5 -runme make clean >/dev/null -runme mkdir build -echo "Starting make" -cp make/config.mk . -sed -i -e 's/gcc/gcc-5/g' config.mk -sed -i -e 's/g++/g++-5/g' config.mk -runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt -echo "Finished make. Now processing output" -python tests/nightly/compilation_warnings/process_output.py build/compile_output.txt +set -ex + +echo "jenkins_slave ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \ No newline at end of file diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 6cefeea..7af0401 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -774,6 +774,62 @@ build_docs() { popd } + +# Functions that run the nightly Tests: + +#Runs Apache RAT Check on MXNet Source for License Headers +nightly_test_rat_check() { + set -ex + ./tests/nightly/apache_rat_license_check/license_check.sh +} + +#Checks MXNet for Compilation Warnings +nightly_test_compilation_warning() { + set -ex + export PYTHONPATH=./python/ + ./tests/nightly/compilation_warnings/compilation_warnings.sh +} + +#Checks the MXNet Installation Guide - currently checks pip, build from source and virtual env on cpu and gpu +nightly_test_installation() { + set -ex + # The run_test_installation_docs.sh expects the path to index.md and the first and last line numbers of the index.md file + # First execute the test script and then call the method specified by the Jenkinsfile - ${1} + source ./tests/jenkins/run_test_installation_docs.sh docs/install/index.md 1 1686; ${1} +} + +#Runs a simple MNIST training example +nightly_test_image_classification() { + set -ex + ./tests/nightly/test_image_classification.sh +} + +#Single Node KVStore Test +nightly_test_KVStore_singleNode() { + set -ex + export PYTHONPATH=./python/ + python tests/nightly/test_kvstore.py +} + +#Tests Amalgamation Build with 5 different sets of flags +nightly_test_amalgamation() { + set -ex + # Amalgamation can not be run with -j nproc + make -C amalgamation/ clean + make -C amalgamation/ ${1} ${2} +} + +#Tests Amalgamation Build for Javascript +nightly_test_javascript() { + set -ex + export LLVM=/work/deps/emscripten-fastcomp/build/bin + # This part is needed to run emcc correctly + cd /work/deps/emscripten + ./emcc + touch ~/.emscripten + make -C /work/mxnet/amalgamation libmxnet_predict.js MIN=1 EMCC=/work/deps/emscripten/emcc +} + # Deploy deploy_docs() { diff --git a/tests/jenkins/run_test_installation_docs.sh b/tests/jenkins/run_test_installation_docs.sh index 812317b..d25f1d5 100755 --- a/tests/jenkins/run_test_installation_docs.sh +++ b/tests/jenkins/run_test_installation_docs.sh @@ -262,29 +262,52 @@ LINUX_PYTHON_CPU_END_LINENO=$(grep -n "END - Linux Python CPU Installation Instr set_instruction_set ${LINUX_PYTHON_CPU_START_LINENO} ${LINUX_PYTHON_CPU_END_LINENO} -echo -echo "### Testing Virtualenv ###" -echo "${virtualenv_commands}" -echo -docker run --rm ubuntu:14.04 bash -c "${virtualenv_commands}" +ubuntu_python_cpu_virtualenv() +{ + set -e + echo + echo "### Testing Virtualenv ###" + echo "${virtualenv_commands}" #> "$filewithcommands" + echo + eval ${virtualenv_commands} + echo "ubuntu_python_cpu_virtualenv: MXNet Installed Successfully" +} -echo -echo "### Testing Pip ###" -echo "${pip_commands}" -echo -docker run --rm ubuntu:14.04 bash -c "${pip_commands}" +ubuntu_python_cpu_pip() +{ + set -e + echo + echo "### Testing Pip ###" + pip_commands="sudo ${pip_commands}" + echo "${pip_commands}" + echo + eval ${pip_commands} + echo "ubuntu_python_cpu_pip: MXNet Installed Successfully" +} -echo -echo "### Testing Docker ###" -echo "${docker_commands}" -echo -eval ${docker_commands} +ubuntu_python_cpu_docker() +{ + set -e + echo + echo "### Testing Docker ###" + echo "${docker_commands}" + echo + eval ${docker_commands} + echo "ubuntu_python_cpu_docker: MXNet Installed Successfully" -echo -echo "### Testing Build From Source ###" -echo "${buildfromsource_commands}" -echo -docker run --rm ubuntu:14.04 bash -c "${buildfromsource_commands}" +} + +ubuntu_python_cpu_source() +{ + set -e + echo + echo "### Testing Build From Source ###" + echo "${buildfromsource_commands}" + echo + eval ${buildfromsource_commands} + echo "ubuntu_python_cpu_source: MXNet Installed Successfully" + +} #########################LINUX-PYTHON-GPU########################### @@ -299,28 +322,55 @@ LINUX_PYTHON_GPU_END_LINENO=$(grep -n "END - Linux Python GPU Installation Instr set_instruction_set ${LINUX_PYTHON_GPU_START_LINENO} ${LINUX_PYTHON_GPU_END_LINENO} -# mxnet/base-cuda9 is a simple Docker Image with 'nvidia/cuda:9.0-cudnn7-devel' and 'apt-get install sudo'. +ubuntu_python_gpu_virtualenv() +{ + set -e + echo + echo "### Testing Virtualenv ###" + echo "${virtualenv_commands}" + echo + eval ${virtualenv_commands} + echo "ubuntu_python_gpu_virtualenv: MXNet Installed Successfully" -echo -echo "### Testing Virtualenv ###" -echo "${virtualenv_commands}" -echo -nvidia-docker run --rm mxnet/base-cuda9 bash -c "${virtualenv_commands}" +} -echo -echo "### Testing Pip ###" -echo "${pip_commands}" -echo -nvidia-docker run --rm mxnet/base-cuda9 bash -c "${pip_commands}" +ubuntu_python_gpu_pip() +{ + set -e + echo + echo "### Testing Pip ###" + echo "${pip_commands}" + echo + eval ${pip_commands} + echo "ubuntu_python_gpu_pip: MXNet Installed Successfully" -echo -echo "### Testing Docker ###" -echo "${docker_commands}" -echo -eval ${docker_commands} +} -echo -echo "### Testing Build From Source ###" -echo "${buildfromsource_commands}" -echo -nvidia-docker run --rm mxnet/base-cuda9 bash -c "${buildfromsource_commands}" +ubuntu_python_gpu_docker() +{ + set -e + echo + echo "### Testing Docker ###" + echo "${docker_commands}" + echo + eval ${docker_commands} + echo "ubuntu_python_gpu_docker: MXNet Installed Successfully" + +} + +ubuntu_python_gpu_source() +{ + set -e + echo + echo "### Testing Build From Source ###" + echo "${buildfromsource_commands}" + echo + eval ${buildfromsource_commands} + echo "ubuntu_python_gpu_source: MXNet Installed Successfully" + +} + +func_virtual_commands() +{ + echo ${virtualenv_commands} +} diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile new file mode 100755 index 0000000..6805bc4 --- /dev/null +++ b/tests/nightly/Jenkinsfile @@ -0,0 +1,185 @@ +// -*- mode: groovy -*- +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +//This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile + +err = null +mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/nnvm/lib/libnnvm.a' + +// pack libraries for later use +def pack_lib(name, libs=mx_lib) { + sh """ +echo "Packing ${libs} into ${name}" +echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +""" + stash includes: libs, name: name +} + +// unpack libraries saved before +def unpack_lib(name, libs=mx_lib) { + unstash name + sh """ +echo "Unpacked ${libs} from ${name}" +echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +""" +} + +def init_git() { + deleteDir() + retry(5) { + try { + timeout(time: 15, unit: 'MINUTES') { + checkout scm + sh 'git submodule update --init --recursive' + sh 'git clean -d -f' + } + } catch (exc) { + deleteDir() + error "Failed to fetch source codes with ${exc}" + sleep 2 + } + } +} + +def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { + def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" + command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') + command = command.replaceAll('%PLATFORM%', platform) + command = command.replaceAll('%FUNCTION_NAME%', function_name) + command = command.replaceAll('%SHARED_MEM%', shared_mem) + + sh command +} + +try { + stage('NightlyTests'){ + parallel 'RATCheck: CPU': { + node('mxnetlinux-cpu') { + ws('workspace/nt-RATTest') { + init_git() + docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false) + } + } + }, + 'CompilationWarnings: CPU': { + node('mxnetlinux-cpu') { + ws('workspace/nt-compilationTest') { + init_git() + docker_run('ubuntu_nightly_cpu', 'nightly_test_compilation_warning', false) + } + } + }, + 'InstallationGuide: CPU': { + node('mxnetlinux-cpu') { + ws('workspace/nt-Installation-cpu') { + init_git() + docker_run('ubuntu_base_cpu', 'nightly_test_installation ubuntu_python_cpu_virtualenv', false) + docker_run('ubuntu_base_cpu', 'nightly_test_installation ubuntu_python_cpu_pip', false) + //Docker Install Test is currently disabled and tracked here: https://github.com/apache/incubator-mxnet/issues/11288 + //docker_run('ubuntu_base_cpu', 'nightly_test_installation ubuntu_python_cpu_docker', false) + docker_run('ubuntu_base_cpu', 'nightly_test_installation ubuntu_python_cpu_source', false) + } + } + }, + 'InstallationGuide: GPU': { + node('mxnetlinux-gpu') { + ws('workspace/nt-Installation-gpu') { + init_git() + docker_run('ubuntu_base_gpu', 'nightly_test_installation ubuntu_python_gpu_virtualenv', true) + docker_run('ubuntu_base_gpu', 'nightly_test_installation ubuntu_python_gpu_pip', true) + //Docker Install Test is currently disabled and tracked here: https://github.com/apache/incubator-mxnet/issues/11288 + //docker_run('ubuntu_base_gpu', 'nightly_test_installation ubuntu_python_gpu_docker', true) + docker_run('ubuntu_base_gpu', 'nightly_test_installation ubuntu_python_gpu_source', true) + } + } + }, + 'PipTest: GPU': { + node('mxnetlinux-gpu') { + ws('workspace/nt-pipTest') { + init_git() + } + } + }, + 'Amalgamation-atlas: CPU': { + node('mxnetlinux-cpu') { + ws('workspace/nt-amalgamation1') { + init_git() + docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas', false) + } + } + }, + 'Amalgamation-atlas-min: CPU': { + node('mxnetlinux-cpu') { + ws('workspace/nt-amalgamation2') { + init_git() + docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas MIN=1', false) + } + } + }, + 'Amalgamation-atlas-mkl: CPU': { + node('mxnetlinux-cpu') { + ws('workspace/nt-amalgamation3') { + init_git() + docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas MSHADOW_USE_MKL=1', false) + } + } + }, + 'Amalgamation-atlas-cuda: CPU': { + node('mxnetlinux-cpu') { + ws('workspace/nt-amalgamation4') { + init_git() + docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas MSHADOW_USE_CUDA=1', false) + } + } + }, + 'Amalgamation-atlas-openmp: CPU': { + node('mxnetlinux-cpu') { + ws('workspace/nt-amalgamation5') { + init_git() + docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas DISABLE_OPENMP=0', false) + } + } + }, + 'MXNetJS: CPU': { + node('mxnetlinux-cpu') { + ws('workspace/nt-mxnetjs') { + init_git() + docker_run('ubuntu_nightly_cpu', 'nightly_test_javascript', false) + } + } + } + } +} catch (caughtError) { + node("mxnetlinux-cpu") { + sh "echo caught ${caughtError}" + err = caughtError + currentBuild.result = "FAILURE" + } +} finally { + node("mxnetlinux-cpu") { + // Only send email if nightly test failed + if (currentBuild.result == "FAILURE") { + emailext body: 'Build for MXNet branch has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED] build ${BUILD_NUMBER}', to: 'meghnabaijal2...@gmail.com, meghnabai...@gmail.com' + } + // Remember to rethrow so the build is marked as failing + if (err) { + throw err + } + } +} diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries new file mode 100755 index 0000000..7c727d6 --- /dev/null +++ b/tests/nightly/JenkinsfileForBinaries @@ -0,0 +1,121 @@ +// -*- mode: groovy -*- +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile + +err = null +mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' + +// pack libraries for later use +def pack_lib(name, libs=mx_lib) { + sh """ +echo "Packing ${libs} into ${name}" +echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +""" + stash includes: libs, name: name +} + +// unpack libraries saved before +def unpack_lib(name, libs=mx_lib) { + unstash name + sh """ +echo "Unpacked ${libs} from ${name}" +echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +""" +} + +def init_git() { + deleteDir() + retry(5) { + try { + timeout(time: 15, unit: 'MINUTES') { + checkout scm + sh 'git submodule update --init --recursive' + sh 'git clean -d -f' + } + } catch (exc) { + deleteDir() + error "Failed to fetch source codes with ${exc}" + sleep 2 + } + } +} + +def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { + def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" + command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') + command = command.replaceAll('%PLATFORM%', platform) + command = command.replaceAll('%FUNCTION_NAME%', function_name) + command = command.replaceAll('%SHARED_MEM%', shared_mem) + + sh command +} + +try { + stage('Build') { + parallel 'GPU: CUDA9.1+cuDNN7': { + node('mxnetlinux-cpu') { + ws('workspace/build-gpu') { + init_git() + //sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7" + docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false) + pack_lib('gpu', mx_lib) + } + } + } + } + + stage('NightlyTests'){ + parallel 'ImageClassification: GPU': { + node('mxnetlinux-gpu') { + ws('workspace/nt-ImageClassificationTest') { + init_git() + unpack_lib('gpu', mx_lib) + docker_run('ubuntu_nightly_gpu', 'nightly_test_image_classification', true) + } + } + }, + 'KVStore_SingleNode: GPU': { + node('mxnetlinux-gpu-p3') { + ws('workspace/nt-KVStoreTest') { + init_git() + unpack_lib('gpu', mx_lib) + //https://github.com/apache/incubator-mxnet/issues/11289 + //docker_run('ubuntu_nightly_gpu', 'nightly_test_KVStore_singleNode', true) + } + } + } + } +} catch (caughtError) { + node("mxnetlinux-cpu") { + sh "echo caught ${caughtError}" + err = caughtError + currentBuild.result = "FAILURE" + } +} finally { + node("mxnetlinux-cpu") { + // Only send email if master failed + if (currentBuild.result == "FAILURE") { + emailext body: 'Build for MXNet Nightly Test is broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED] Build ${BUILD_NUMBER}', to: 'meghnabaijal2...@gmail.com' + } + // Remember to rethrow so the build is marked as failing + if (err) { + throw err + } + } +} diff --git a/tests/nightly/README.md b/tests/nightly/README.md old mode 100644 new mode 100755 index e1dd148..fa1771a --- a/tests/nightly/README.md +++ b/tests/nightly/README.md @@ -1,18 +1,43 @@ -# Nightly build for mxnet +# Nightly Tests for MXNet -This fold contains scripts to test some heavy jobs, often training with multiple -GPUs, to ensure everything is right. Normally it runs everyday. -The current build server is equipped with Intel i7-4790 and 4 Nvidia GTX -970 Tis. The build status is available at [ci.dmlc.ml](http://ci.dmlc.ml). We -thank [Dave Andersen](http://www.cs.cmu.edu/~dga) for providing the build machine. +These are some longer running tests that are scheduled to run every night. -## How to use +### Description +There are two Jenkins pipelines that run these tests - +1. Tests on the Source code: http://jenkins.mxnet-ci.amazon-ml.com/view/Nightly%20Tests/job/NightlyTests_onSource/ +2. Tests on Built Binaries: http://jenkins.mxnet-ci.amazon-ml.com/view/Nightly%20Tests/job/NightlyTests_onBinaries/ -### Run locallly +### Adding a new Nightly Test +Add your test script to the mxnet repo (preferably)in the tests/nightly folder) +Make sure to describe in a readme or comments the purpose of the test. -Run `tests/nightly/test_all.sh 4` if there are 4 GPUs. +#### Setting up the Docker Container +1. Your test must run on the CI slaves only within an official docker container available at ci/docker +2. Make sure your script itself does not install anything on the slaves. All dependencies must be added to the dockerfile. +3. For most tests you should be able to find an existing dockerfile that contains all the necessary dependencies - select based on the required os, cpu vs gpu, necessary dependencies etc +4. However, If there is no dockerfile which installs the required dependencies do the following - + a. Add your dependencies to a new install script in ci/docker/install + b. Now either run this install script within an existing dockerfile or create a new dockerfile. -### Run on Jenkins +#### Running the test from the Jenkinsfile +If the test runs on the MXNet source, modify tests/nightly/Jenkinsfile - +1. Add a function to ci/docker/runtimefunctions.sh which calls your test script. +2. In the Jenkinsfile, add a new test similar to existing tests under the `NightlyTests` stage. Make sure to call the right runtime function -First merge codes into the master branch, then go to -http://ci.dmlc.ml/job/mxnet/, click **Build Now**. +If the test runs on MXNet binaries modify tests/nightly/JenkinsfileForBinaries - +1. Add a function to ci/docker/runtimefunctions.sh which calls your test script. +2. In the Jenkinsfile, if needed, add a new build similar to existing tests under the `Build` stage. +3. Add a new test similar to existing tests under the `NightlyTests` stage. Make sure to call the right runtime function + +### Currently Running Tests + +#### Tests on Source +1. Amalgamation Tests +2. Compilation Warnings +3. Installation Guide +4. MXNet Javascript Test +5. Apache RAT check + +#### Tests on Binaries +1. Image Classification Test +2. Single Node KVStore diff --git a/tests/nightly/apache_rat_license_check/README.md b/tests/nightly/apache_rat_license_check/README.md old mode 100644 new mode 100755 index b040882..477db37 --- a/tests/nightly/apache_rat_license_check/README.md +++ b/tests/nightly/apache_rat_license_check/README.md @@ -7,7 +7,7 @@ This file lists all the files, directories and file formats that are excluded fr If you think something is wrong, feel free to change! ### Nightly test script for license check -Coming soon... +The license check script called by the Jenkinsfile ### How to run the RAT check locally The following commands can be used to run a Apache RAT check locally - diff --git a/tests/nightly/apache_rat_license_check/license_check.sh b/tests/nightly/apache_rat_license_check/license_check.sh new file mode 100755 index 0000000..84accf9 --- /dev/null +++ b/tests/nightly/apache_rat_license_check/license_check.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +#mvn and svn are installed in the docker container via the nghtly test script + +echo "download RAT" +svn co http://svn.apache.org/repos/asf/creadur/rat/trunk/ #>/dev/null + +echo "cd into directory" +cd trunk + +echo "mvn install" +mvn -Dmaven.test.skip=true install #>/dev/null + +echo "build success, cd into target" +cd apache-rat/target + + +echo "-------Run Apache RAT check on MXNet-------" + +#Command has been run twice, once for the logs and once to store in the variable to parse. +java -jar apache-rat-0.13-SNAPSHOT.jar -E /work/mxnet/tests/nightly/apache_rat_license_check/rat-excludes -d /work/mxnet +OUTPUT="$(java -jar apache-rat-0.13-SNAPSHOT.jar -E /work/mxnet/tests/nightly/apache_rat_license_check/rat-excludes -d /work/mxnet)" +SOURCE="^0 Unknown Licenses" + + +echo "-------Process The Output-------" + +if [[ "$OUTPUT" =~ "$SOURCE" ]]; then + echo "SUCCESS: There are no files with an Unknown License."; +else + echo "ERROR: RAT Check detected files with unknown licenses. Please fix and run test again!"; + exit 1 +fi \ No newline at end of file diff --git a/tests/nightly/apache_rat_license_check/.rat-excludes b/tests/nightly/apache_rat_license_check/rat-excludes old mode 100644 new mode 100755 similarity index 94% rename from tests/nightly/apache_rat_license_check/.rat-excludes rename to tests/nightly/apache_rat_license_check/rat-excludes index 885b131..ddcaec1 --- a/tests/nightly/apache_rat_license_check/.rat-excludes +++ b/tests/nightly/apache_rat_license_check/rat-excludes @@ -53,3 +53,5 @@ dataset.cPickle rcnn/* image-classification/* +rat-excludes +apache-rat-tasks/* diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh b/tests/nightly/compilation_warnings/compilation_warnings.sh old mode 100644 new mode 100755 index a6c4863..0e39424 --- a/tests/nightly/compilation_warnings/compilation_warnings.sh +++ b/tests/nightly/compilation_warnings/compilation_warnings.sh @@ -31,9 +31,7 @@ runme() { fi } -sudo add-apt-repository ppa:ubuntu-toolchain-r/test -sudo apt-get update -sudo apt-get -y install time g++-5 +# g++-5 is installed in ci/docker/install/ubuntu_nightly_tests.sh runme make clean >/dev/null runme mkdir build echo "Starting make" diff --git a/tests/nightly/compilation_warnings/process_output.py b/tests/nightly/compilation_warnings/process_output.py old mode 100644 new mode 100755 diff --git a/tests/nightly/test_image_classification.sh b/tests/nightly/test_image_classification.sh index b4b4dca..1e4508f 100755 --- a/tests/nightly/test_image_classification.sh +++ b/tests/nightly/test_image_classification.sh @@ -18,17 +18,6 @@ # under the License. -# setup -export LD_LIBRARY_PATH=`pwd`/`dirname $0`/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH -export PYTHONPATH=`pwd`/`dirname $0`/python -# bc is required by sh2ju.sh -apt-get install bc -cd `pwd`/`dirname $0` -. sh2ju.sh - -## clean last build log -juLogClean - if [ -f $(which nvidia-smi) ]; then if [ $# -eq 1 ]; then num_gpus=$1 @@ -41,34 +30,17 @@ else device_arg="" fi -# build -build() { - make -C ../.. clean - make -C ../.. -j8 - return $? -} - -cp ../../make/config.mk ../.. -cat >>../../config.mk <<EOF -USE_CUDA=1 -USE_CUDA_PATH=/usr/local/cuda -USE_CUDNN=1 -USE_DIST_KVSTORE=1 -EOF - -juLog -name=Build -error=Error build - # check if the final evaluation accuracy exceed the threshold check_val() { expected=$1 pass="Final validation >= $expected, PASS" fail="Final validation < $expected, FAIL" - python ../../tools/parse_log.py log --format none | tail -n1 | \ + python tools/parse_log.py log --format none | tail -n1 | \ awk "{ if (\$3~/^[.0-9]+$/ && \$3 > $expected) print \"$pass\"; else print \"$fail\"}" rm -f log } -example_dir=../../example/image-classification +example_dir=example/image-classification # python: lenet + mnist test_lenet() { optimizers="adam sgd adagrad" @@ -89,6 +61,7 @@ test_lenet() { check_val $desired_accuracy done } -juLog -name=Python.Lenet.Mnist -error=FAIL test_lenet + +test_lenet exit $errors