This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new bb105a1 [MXNET-405] Add 2 new pipelines to the Official CI and run
nightly tests. (#10827)
bb105a1 is described below
commit bb105a1f8d977978af29867372ef0605c1c3cdbe
Author: mbaijal <[email protected]>
AuthorDate: Wed Jun 20 19:17:45 2018 -0700
[MXNET-405] Add 2 new pipelines to the Official CI and run nightly tests.
(#10827)
* Migrate Nightlies PR1
* Run all tests
* Fix RAT and Comment out KVstore for now
* Comment out the chmod and test
* remove chmod from rat and CW
* One more check
* changes
* run install on cpu instead of gpu
* Fix failing pip
* Fix Pip
* typo-fix
* Run Pip on cpu instance
* Add a README
* Remove pip-test
* add pip test back
* Add BLC to jenkinsfile format
* add permissions
* include the git add, commit, push for broken link checker
* push not working, comment
* Review Comments 1: remove g++-5 and parallize make
* Review Comments 2: change dockerfile name, update mx_lib, remove
commented line
* Review Comments 3: Separate out the dockerfiles for nightly tests
* Review Comments 4: Move apache rat install to nightly install script
* Review Comments 4: copy the rat folder into work
* correct path
* Change path Again
* check tests
* path
* Uncomment KVStore
* Fix rat and js tests
* Move to docker_run and comment KVstore again
* Adding docker_run for binaries Jenkinsfile
* Permission and other fixes
* fix RAT regex, add set -e to install funcs
* Delete BLC, change chmod to 755
* JS: parallel make
* Add links to github issues
* change the instance type to cpu
* Comment out pip installs to test build from source on cpu
* typo
* Add sudo for pip install
* Add sudo to all pip install commands coz they cause CI to fail, linux only
* Run compilation warnings on cpu instance
* Change file perms and some cleanup
* Fix the dockerfiles
* Dockerfile updates
* Merge JS dockerfile into nightly and remove export command
* Pin repo versions and run cpu install from souce in cpu docker
* Add yes to install in js script
* remove sudo from index.md
* Add sudo to test script instead
* add sudo to test script
---
.../docker/Dockerfile.build.ubuntu_base_cpu | 50 +++---
.../docker/Dockerfile.build.ubuntu_base_gpu | 50 +++---
ci/docker/Dockerfile.build.ubuntu_nightly_cpu | 61 +++++++
ci/docker/Dockerfile.build.ubuntu_nightly_gpu | 64 +++++++
.../docker/install/ubuntu_emscripten.sh | 47 +++---
.../docker/install/ubuntu_nightly_tests.sh | 35 ++--
.../docker/install/ubuntu_runas_sudo.sh | 30 +---
ci/docker/runtime_functions.sh | 56 +++++++
tests/jenkins/run_test_installation_docs.sh | 132 ++++++++++-----
tests/nightly/Jenkinsfile | 185 +++++++++++++++++++++
tests/nightly/JenkinsfileForBinaries | 121 ++++++++++++++
tests/nightly/README.md | 49 ++++--
tests/nightly/apache_rat_license_check/README.md | 2 +-
.../apache_rat_license_check/license_check.sh | 51 ++++++
.../{.rat-excludes => rat-excludes} | 2 +
.../compilation_warnings/compilation_warnings.sh | 4 +-
.../nightly/compilation_warnings/process_output.py | 0
tests/nightly/test_image_classification.sh | 35 +---
18 files changed, 754 insertions(+), 220 deletions(-)
diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh
b/ci/docker/Dockerfile.build.ubuntu_base_cpu
old mode 100644
new mode 100755
similarity index 54%
copy from tests/nightly/compilation_warnings/compilation_warnings.sh
copy to ci/docker/Dockerfile.build.ubuntu_base_cpu
index a6c4863..b6515c7
--- a/tests/nightly/compilation_warnings/compilation_warnings.sh
+++ b/ci/docker/Dockerfile.build.ubuntu_base_cpu
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,30 +15,25 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04
+# This should run in an empty docker with ubuntu and cuda.
+
+FROM ubuntu:16.04
+
+WORKDIR /work/deps
+
+RUN apt-get update && apt-get -y install sudo
+
+ARG USER_ID=os.getuid()
+
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY install/ubuntu_runas_sudo.sh /work/
+RUN /work/ubuntu_runas_sudo.sh
+
+COPY runtime_functions.sh /work/
-set -e
-runme() {
- cmd=$*
- echo "$cmd"
- $cmd
- ret=$?
- if [[ ${ret} != 0 ]]; then
- echo " "
- echo "ERROR: Return value non-zero for: $cmd"
- echo " "
- exit 1
- fi
-}
-
-sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-sudo apt-get update
-sudo apt-get -y install time g++-5
-runme make clean >/dev/null
-runme mkdir build
-echo "Starting make"
-cp make/config.mk .
-sed -i -e 's/gcc/gcc-5/g' config.mk
-sed -i -e 's/g++/g++-5/g' config.mk
-runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt
-echo "Finished make. Now processing output"
-python tests/nightly/compilation_warnings/process_output.py
build/compile_output.txt
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh
b/ci/docker/Dockerfile.build.ubuntu_base_gpu
old mode 100644
new mode 100755
similarity index 54%
copy from tests/nightly/compilation_warnings/compilation_warnings.sh
copy to ci/docker/Dockerfile.build.ubuntu_base_gpu
index a6c4863..bb0aa1d
--- a/tests/nightly/compilation_warnings/compilation_warnings.sh
+++ b/ci/docker/Dockerfile.build.ubuntu_base_gpu
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,30 +15,25 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04
+# This should run in an empty docker with ubuntu and cuda.
+
+FROM nvidia/cuda:9.1-cudnn7-devel
+
+WORKDIR /work/deps
+
+RUN apt-get update && apt-get -y install sudo
+
+ARG USER_ID=os.getuid()
+
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY install/ubuntu_runas_sudo.sh /work/
+RUN /work/ubuntu_runas_sudo.sh
+
+COPY runtime_functions.sh /work/
-set -e
-runme() {
- cmd=$*
- echo "$cmd"
- $cmd
- ret=$?
- if [[ ${ret} != 0 ]]; then
- echo " "
- echo "ERROR: Return value non-zero for: $cmd"
- echo " "
- exit 1
- fi
-}
-
-sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-sudo apt-get update
-sudo apt-get -y install time g++-5
-runme make clean >/dev/null
-runme mkdir build
-echo "Starting make"
-cp make/config.mk .
-sed -i -e 's/gcc/gcc-5/g' config.mk
-sed -i -e 's/g++/g++-5/g' config.mk
-runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt
-echo "Finished make. Now processing output"
-python tests/nightly/compilation_warnings/process_output.py
build/compile_output.txt
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
new file mode 100755
index 0000000..fc02a4f
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
@@ -0,0 +1,61 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+
+FROM ubuntu:16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+COPY install/ubuntu_r.sh /work/
+RUN /work/ubuntu_r.sh
+COPY install/ubuntu_perl.sh /work/
+RUN /work/ubuntu_perl.sh
+COPY install/ubuntu_clang.sh /work/
+RUN /work/ubuntu_clang.sh
+COPY install/ubuntu_mklml.sh /work/
+RUN /work/ubuntu_mklml.sh
+COPY install/ubuntu_caffe.sh /work/
+RUN /work/ubuntu_caffe.sh
+COPY install/ubuntu_onnx.sh /work/
+RUN /work/ubuntu_onnx.sh
+COPY install/ubuntu_docs.sh /work/
+RUN /work/ubuntu_docs.sh
+COPY install/ubuntu_nightly_tests.sh /work/
+RUN /work/ubuntu_nightly_tests.sh
+COPY install/ubuntu_emscripten.sh /work/
+RUN /work/ubuntu_emscripten.sh
+
+ARG USER_ID=0
+
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
+ENV LLVM=/work/deps/emscripten-fastcomp/build/bin
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
new file mode 100755
index 0000000..c55b942
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
@@ -0,0 +1,64 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 16.04 for CPU
+
+FROM nvidia/cuda:9.1-cudnn7-devel
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+COPY install/ubuntu_r.sh /work/
+RUN /work/ubuntu_r.sh
+COPY install/ubuntu_perl.sh /work/
+RUN /work/ubuntu_perl.sh
+COPY install/ubuntu_clang.sh /work/
+RUN /work/ubuntu_clang.sh
+COPY install/ubuntu_mklml.sh /work/
+RUN /work/ubuntu_mklml.sh
+COPY install/ubuntu_tvm.sh /work/
+RUN /work/ubuntu_tvm.sh
+COPY install/ubuntu_llvm.sh /work/
+RUN /work/ubuntu_llvm.sh
+COPY install/ubuntu_caffe.sh /work/
+RUN /work/ubuntu_caffe.sh
+COPY install/ubuntu_onnx.sh /work/
+RUN /work/ubuntu_onnx.sh
+COPY install/ubuntu_docs.sh /work/
+RUN /work/ubuntu_docs.sh
+COPY install/ubuntu_tutorials.sh /work/
+RUN /work/ubuntu_tutorials.sh
+COPY install/ubuntu_nightly_tests.sh /work/
+RUN /work/ubuntu_nightly_tests.sh
+
+ARG USER_ID=0
+
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh
b/ci/docker/install/ubuntu_emscripten.sh
old mode 100644
new mode 100755
similarity index 50%
copy from tests/nightly/compilation_warnings/compilation_warnings.sh
copy to ci/docker/install/ubuntu_emscripten.sh
index a6c4863..459d145
--- a/tests/nightly/compilation_warnings/compilation_warnings.sh
+++ b/ci/docker/install/ubuntu_emscripten.sh
@@ -17,29 +17,24 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-runme() {
- cmd=$*
- echo "$cmd"
- $cmd
- ret=$?
- if [[ ${ret} != 0 ]]; then
- echo " "
- echo "ERROR: Return value non-zero for: $cmd"
- echo " "
- exit 1
- fi
-}
-
-sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-sudo apt-get update
-sudo apt-get -y install time g++-5
-runme make clean >/dev/null
-runme mkdir build
-echo "Starting make"
-cp make/config.mk .
-sed -i -e 's/gcc/gcc-5/g' config.mk
-sed -i -e 's/g++/g++-5/g' config.mk
-runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt
-echo "Finished make. Now processing output"
-python tests/nightly/compilation_warnings/process_output.py
build/compile_output.txt
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+#The script has been copied as is from a dockerfile that existed on previous
MXNet versions (0.11)
+#Written By: Ly
+
+set -ex
+
+apt-get -y install nodejs
+
+git clone -b 1.38.6 https://github.com/kripken/emscripten.git
+git clone -b 1.38.6 https://github.com/kripken/emscripten-fastcomp
+cd emscripten-fastcomp
+git clone -b 1.38.6 https://github.com/kripken/emscripten-fastcomp-clang
tools/clang
+mkdir build && cd build
+
+cmake .. -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="X86;JSBackend" \
+-DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF
-DCLANG_INCLUDE_EXAMPLES=OFF \
+-DCLANG_INCLUDE_TESTS=OFF && make -j$(nproc)
+
+chmod -R 777 /work/deps/emscripten-fastcomp/
\ No newline at end of file
diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh
b/ci/docker/install/ubuntu_nightly_tests.sh
old mode 100644
new mode 100755
similarity index 55%
copy from tests/nightly/compilation_warnings/compilation_warnings.sh
copy to ci/docker/install/ubuntu_nightly_tests.sh
index a6c4863..abc2134
--- a/tests/nightly/compilation_warnings/compilation_warnings.sh
+++ b/ci/docker/install/ubuntu_nightly_tests.sh
@@ -17,29 +17,14 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-runme() {
- cmd=$*
- echo "$cmd"
- $cmd
- ret=$?
- if [[ ${ret} != 0 ]]; then
- echo " "
- echo "ERROR: Return value non-zero for: $cmd"
- echo " "
- exit 1
- fi
-}
+#Install steps for the nightly tests
-sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-sudo apt-get update
-sudo apt-get -y install time g++-5
-runme make clean >/dev/null
-runme mkdir build
-echo "Starting make"
-cp make/config.mk .
-sed -i -e 's/gcc/gcc-5/g' config.mk
-sed -i -e 's/g++/g++-5/g' config.mk
-runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt
-echo "Finished make. Now processing output"
-python tests/nightly/compilation_warnings/process_output.py
build/compile_output.txt
+set -ex
+
+#Install for Compilation warning Nightly Test
+add-apt-repository ppa:ubuntu-toolchain-r/test
+apt-get update
+apt-get -y install time
+
+# Install for RAT License Check Nightly Test
+apt-get install -y subversion maven -y #>/dev/null
diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh
b/ci/docker/install/ubuntu_runas_sudo.sh
old mode 100644
new mode 100755
similarity index 55%
copy from tests/nightly/compilation_warnings/compilation_warnings.sh
copy to ci/docker/install/ubuntu_runas_sudo.sh
index a6c4863..40ec60e
--- a/tests/nightly/compilation_warnings/compilation_warnings.sh
+++ b/ci/docker/install/ubuntu_runas_sudo.sh
@@ -17,29 +17,9 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-runme() {
- cmd=$*
- echo "$cmd"
- $cmd
- ret=$?
- if [[ ${ret} != 0 ]]; then
- echo " "
- echo "ERROR: Return value non-zero for: $cmd"
- echo " "
- exit 1
- fi
-}
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-sudo apt-get update
-sudo apt-get -y install time g++-5
-runme make clean >/dev/null
-runme mkdir build
-echo "Starting make"
-cp make/config.mk .
-sed -i -e 's/gcc/gcc-5/g' config.mk
-sed -i -e 's/g++/g++-5/g' config.mk
-runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt
-echo "Finished make. Now processing output"
-python tests/nightly/compilation_warnings/process_output.py
build/compile_output.txt
+set -ex
+
+echo "jenkins_slave ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
\ No newline at end of file
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 6cefeea..7af0401 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -774,6 +774,62 @@ build_docs() {
popd
}
+
+# Functions that run the nightly Tests:
+
+#Runs Apache RAT Check on MXNet Source for License Headers
+nightly_test_rat_check() {
+ set -ex
+ ./tests/nightly/apache_rat_license_check/license_check.sh
+}
+
+#Checks MXNet for Compilation Warnings
+nightly_test_compilation_warning() {
+ set -ex
+ export PYTHONPATH=./python/
+ ./tests/nightly/compilation_warnings/compilation_warnings.sh
+}
+
+#Checks the MXNet Installation Guide - currently checks pip, build from source
and virtual env on cpu and gpu
+nightly_test_installation() {
+ set -ex
+ # The run_test_installation_docs.sh expects the path to index.md and the
first and last line numbers of the index.md file
+ # First execute the test script and then call the method specified by the
Jenkinsfile - ${1}
+ source ./tests/jenkins/run_test_installation_docs.sh docs/install/index.md
1 1686; ${1}
+}
+
+#Runs a simple MNIST training example
+nightly_test_image_classification() {
+ set -ex
+ ./tests/nightly/test_image_classification.sh
+}
+
+#Single Node KVStore Test
+nightly_test_KVStore_singleNode() {
+ set -ex
+ export PYTHONPATH=./python/
+ python tests/nightly/test_kvstore.py
+}
+
+#Tests Amalgamation Build with 5 different sets of flags
+nightly_test_amalgamation() {
+ set -ex
+ # Amalgamation can not be run with -j nproc
+ make -C amalgamation/ clean
+ make -C amalgamation/ ${1} ${2}
+}
+
+#Tests Amalgamation Build for Javascript
+nightly_test_javascript() {
+ set -ex
+ export LLVM=/work/deps/emscripten-fastcomp/build/bin
+ # This part is needed to run emcc correctly
+ cd /work/deps/emscripten
+ ./emcc
+ touch ~/.emscripten
+ make -C /work/mxnet/amalgamation libmxnet_predict.js MIN=1
EMCC=/work/deps/emscripten/emcc
+}
+
# Deploy
deploy_docs() {
diff --git a/tests/jenkins/run_test_installation_docs.sh
b/tests/jenkins/run_test_installation_docs.sh
index 812317b..d25f1d5 100755
--- a/tests/jenkins/run_test_installation_docs.sh
+++ b/tests/jenkins/run_test_installation_docs.sh
@@ -262,29 +262,52 @@ LINUX_PYTHON_CPU_END_LINENO=$(grep -n "END - Linux Python
CPU Installation Instr
set_instruction_set ${LINUX_PYTHON_CPU_START_LINENO}
${LINUX_PYTHON_CPU_END_LINENO}
-echo
-echo "### Testing Virtualenv ###"
-echo "${virtualenv_commands}"
-echo
-docker run --rm ubuntu:14.04 bash -c "${virtualenv_commands}"
+ubuntu_python_cpu_virtualenv()
+{
+ set -e
+ echo
+ echo "### Testing Virtualenv ###"
+ echo "${virtualenv_commands}" #> "$filewithcommands"
+ echo
+ eval ${virtualenv_commands}
+ echo "ubuntu_python_cpu_virtualenv: MXNet Installed Successfully"
+}
-echo
-echo "### Testing Pip ###"
-echo "${pip_commands}"
-echo
-docker run --rm ubuntu:14.04 bash -c "${pip_commands}"
+ubuntu_python_cpu_pip()
+{
+ set -e
+ echo
+ echo "### Testing Pip ###"
+ pip_commands="sudo ${pip_commands}"
+ echo "${pip_commands}"
+ echo
+ eval ${pip_commands}
+ echo "ubuntu_python_cpu_pip: MXNet Installed Successfully"
+}
-echo
-echo "### Testing Docker ###"
-echo "${docker_commands}"
-echo
-eval ${docker_commands}
+ubuntu_python_cpu_docker()
+{
+ set -e
+ echo
+ echo "### Testing Docker ###"
+ echo "${docker_commands}"
+ echo
+ eval ${docker_commands}
+ echo "ubuntu_python_cpu_docker: MXNet Installed Successfully"
-echo
-echo "### Testing Build From Source ###"
-echo "${buildfromsource_commands}"
-echo
-docker run --rm ubuntu:14.04 bash -c "${buildfromsource_commands}"
+}
+
+ubuntu_python_cpu_source()
+{
+ set -e
+ echo
+ echo "### Testing Build From Source ###"
+ echo "${buildfromsource_commands}"
+ echo
+ eval ${buildfromsource_commands}
+ echo "ubuntu_python_cpu_source: MXNet Installed Successfully"
+
+}
#########################LINUX-PYTHON-GPU###########################
@@ -299,28 +322,55 @@ LINUX_PYTHON_GPU_END_LINENO=$(grep -n "END - Linux Python
GPU Installation Instr
set_instruction_set ${LINUX_PYTHON_GPU_START_LINENO}
${LINUX_PYTHON_GPU_END_LINENO}
-# mxnet/base-cuda9 is a simple Docker Image with
'nvidia/cuda:9.0-cudnn7-devel' and 'apt-get install sudo'.
+ubuntu_python_gpu_virtualenv()
+{
+ set -e
+ echo
+ echo "### Testing Virtualenv ###"
+ echo "${virtualenv_commands}"
+ echo
+ eval ${virtualenv_commands}
+ echo "ubuntu_python_gpu_virtualenv: MXNet Installed Successfully"
-echo
-echo "### Testing Virtualenv ###"
-echo "${virtualenv_commands}"
-echo
-nvidia-docker run --rm mxnet/base-cuda9 bash -c "${virtualenv_commands}"
+}
-echo
-echo "### Testing Pip ###"
-echo "${pip_commands}"
-echo
-nvidia-docker run --rm mxnet/base-cuda9 bash -c "${pip_commands}"
+ubuntu_python_gpu_pip()
+{
+ set -e
+ echo
+ echo "### Testing Pip ###"
+ echo "${pip_commands}"
+ echo
+ eval ${pip_commands}
+ echo "ubuntu_python_gpu_pip: MXNet Installed Successfully"
-echo
-echo "### Testing Docker ###"
-echo "${docker_commands}"
-echo
-eval ${docker_commands}
+}
-echo
-echo "### Testing Build From Source ###"
-echo "${buildfromsource_commands}"
-echo
-nvidia-docker run --rm mxnet/base-cuda9 bash -c "${buildfromsource_commands}"
+ubuntu_python_gpu_docker()
+{
+ set -e
+ echo
+ echo "### Testing Docker ###"
+ echo "${docker_commands}"
+ echo
+ eval ${docker_commands}
+ echo "ubuntu_python_gpu_docker: MXNet Installed Successfully"
+
+}
+
+ubuntu_python_gpu_source()
+{
+ set -e
+ echo
+ echo "### Testing Build From Source ###"
+ echo "${buildfromsource_commands}"
+ echo
+ eval ${buildfromsource_commands}
+ echo "ubuntu_python_gpu_source: MXNet Installed Successfully"
+
+}
+
+func_virtual_commands()
+{
+ echo ${virtualenv_commands}
+}
diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile
new file mode 100755
index 0000000..6805bc4
--- /dev/null
+++ b/tests/nightly/Jenkinsfile
@@ -0,0 +1,185 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+ try {
+ timeout(time: 15, unit: 'MINUTES') {
+ checkout scm
+ sh 'git submodule update --init --recursive'
+ sh 'git clean -d -f'
+ }
+ } catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+ }
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('NightlyTests'){
+ parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/nt-RATTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
+ }
+ }
+ },
+ 'CompilationWarnings: CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/nt-compilationTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_compilation_warning',
false)
+ }
+ }
+ },
+ 'InstallationGuide: CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/nt-Installation-cpu') {
+ init_git()
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_virtualenv', false)
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_pip', false)
+ //Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_docker', false)
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_source', false)
+ }
+ }
+ },
+ 'InstallationGuide: GPU': {
+ node('mxnetlinux-gpu') {
+ ws('workspace/nt-Installation-gpu') {
+ init_git()
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_virtualenv', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_pip', true)
+ //Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_docker', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_source', true)
+ }
+ }
+ },
+ 'PipTest: GPU': {
+ node('mxnetlinux-gpu') {
+ ws('workspace/nt-pipTest') {
+ init_git()
+ }
+ }
+ },
+ 'Amalgamation-atlas: CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/nt-amalgamation1') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation
USE_BLAS=atlas', false)
+ }
+ }
+ },
+ 'Amalgamation-atlas-min: CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/nt-amalgamation2') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation
USE_BLAS=atlas MIN=1', false)
+ }
+ }
+ },
+ 'Amalgamation-atlas-mkl: CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/nt-amalgamation3') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation
USE_BLAS=atlas MSHADOW_USE_MKL=1', false)
+ }
+ }
+ },
+ 'Amalgamation-atlas-cuda: CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/nt-amalgamation4') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation
USE_BLAS=atlas MSHADOW_USE_CUDA=1', false)
+ }
+ }
+ },
+ 'Amalgamation-atlas-openmp: CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/nt-amalgamation5') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation
USE_BLAS=atlas DISABLE_OPENMP=0', false)
+ }
+ }
+ },
+ 'MXNetJS: CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/nt-mxnetjs') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_javascript', false)
+ }
+ }
+ }
+ }
+} catch (caughtError) {
+ node("mxnetlinux-cpu") {
+ sh "echo caught ${caughtError}"
+ err = caughtError
+ currentBuild.result = "FAILURE"
+ }
+} finally {
+ node("mxnetlinux-cpu") {
+ // Only send email if nightly test failed
+ if (currentBuild.result == "FAILURE") {
+ emailext body: 'Build for MXNet branch has broken. Please view the build
at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED] build
${BUILD_NUMBER}', to: '[email protected], [email protected]'
+ }
+ // Remember to rethrow so the build is marked as failing
+ if (err) {
+ throw err
+ }
+ }
+}
diff --git a/tests/nightly/JenkinsfileForBinaries
b/tests/nightly/JenkinsfileForBinaries
new file mode 100755
index 0000000..7c727d6
--- /dev/null
+++ b/tests/nightly/JenkinsfileForBinaries
@@ -0,0 +1,121 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/tvm/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+ try {
+ timeout(time: 15, unit: 'MINUTES') {
+ checkout scm
+ sh 'git submodule update --init --recursive'
+ sh 'git clean -d -f'
+ }
+ } catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+ }
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('Build') {
+ parallel 'GPU: CUDA9.1+cuDNN7': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-gpu') {
+ init_git()
+ //sh "ci/build.py --platform ubuntu_build_cuda
/work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7"
+ docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7',
false)
+ pack_lib('gpu', mx_lib)
+ }
+ }
+ }
+ }
+
+ stage('NightlyTests'){
+ parallel 'ImageClassification: GPU': {
+ node('mxnetlinux-gpu') {
+ ws('workspace/nt-ImageClassificationTest') {
+ init_git()
+ unpack_lib('gpu', mx_lib)
+ docker_run('ubuntu_nightly_gpu',
'nightly_test_image_classification', true)
+ }
+ }
+ },
+ 'KVStore_SingleNode: GPU': {
+ node('mxnetlinux-gpu-p3') {
+ ws('workspace/nt-KVStoreTest') {
+ init_git()
+ unpack_lib('gpu', mx_lib)
+ //https://github.com/apache/incubator-mxnet/issues/11289
+ //docker_run('ubuntu_nightly_gpu',
'nightly_test_KVStore_singleNode', true)
+ }
+ }
+ }
+ }
+} catch (caughtError) {
+ node("mxnetlinux-cpu") {
+ sh "echo caught ${caughtError}"
+ err = caughtError
+ currentBuild.result = "FAILURE"
+ }
+} finally {
+ node("mxnetlinux-cpu") {
+ // Only send email if master failed
+ if (currentBuild.result == "FAILURE") {
+ emailext body: 'Build for MXNet Nightly Test is broken. Please view the
build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED]
Build ${BUILD_NUMBER}', to: '[email protected]'
+ }
+ // Remember to rethrow so the build is marked as failing
+ if (err) {
+ throw err
+ }
+ }
+}
diff --git a/tests/nightly/README.md b/tests/nightly/README.md
old mode 100644
new mode 100755
index e1dd148..fa1771a
--- a/tests/nightly/README.md
+++ b/tests/nightly/README.md
@@ -1,18 +1,43 @@
-# Nightly build for mxnet
+# Nightly Tests for MXNet
-This fold contains scripts to test some heavy jobs, often training with
multiple
-GPUs, to ensure everything is right. Normally it runs everyday.
-The current build server is equipped with Intel i7-4790 and 4 Nvidia GTX
-970 Tis. The build status is available at [ci.dmlc.ml](http://ci.dmlc.ml). We
-thank [Dave Andersen](http://www.cs.cmu.edu/~dga) for providing the build
machine.
+These are some longer running tests that are scheduled to run every night.
-## How to use
+### Description
+There are two Jenkins pipelines that run these tests -
+1. Tests on the Source code:
http://jenkins.mxnet-ci.amazon-ml.com/view/Nightly%20Tests/job/NightlyTests_onSource/
+2. Tests on Built Binaries:
http://jenkins.mxnet-ci.amazon-ml.com/view/Nightly%20Tests/job/NightlyTests_onBinaries/
-### Run locallly
+### Adding a new Nightly Test
+Add your test script to the mxnet repo (preferably)in the tests/nightly folder)
+Make sure to describe in a readme or comments the purpose of the test.
-Run `tests/nightly/test_all.sh 4` if there are 4 GPUs.
+#### Setting up the Docker Container
+1. Your test must run on the CI slaves only within an official docker
container available at ci/docker
+2. Make sure your script itself does not install anything on the slaves. All
dependencies must be added to the dockerfile.
+3. For most tests you should be able to find an existing dockerfile that
contains all the necessary dependencies - select based on the required os, cpu
vs gpu, necessary dependencies etc
+4. However, If there is no dockerfile which installs the required dependencies
do the following -
+ a. Add your dependencies to a new install script in ci/docker/install
+ b. Now either run this install script within an existing dockerfile or
create a new dockerfile.
-### Run on Jenkins
+#### Running the test from the Jenkinsfile
+If the test runs on the MXNet source, modify tests/nightly/Jenkinsfile -
+1. Add a function to ci/docker/runtimefunctions.sh which calls your test
script.
+2. In the Jenkinsfile, add a new test similar to existing tests under the
`NightlyTests` stage. Make sure to call the right runtime function
-First merge codes into the master branch, then go to
-http://ci.dmlc.ml/job/mxnet/, click **Build Now**.
+If the test runs on MXNet binaries modify tests/nightly/JenkinsfileForBinaries
-
+1. Add a function to ci/docker/runtimefunctions.sh which calls your test
script.
+2. In the Jenkinsfile, if needed, add a new build similar to existing tests
under the `Build` stage.
+3. Add a new test similar to existing tests under the `NightlyTests` stage.
Make sure to call the right runtime function
+
+### Currently Running Tests
+
+#### Tests on Source
+1. Amalgamation Tests
+2. Compilation Warnings
+3. Installation Guide
+4. MXNet Javascript Test
+5. Apache RAT check
+
+#### Tests on Binaries
+1. Image Classification Test
+2. Single Node KVStore
diff --git a/tests/nightly/apache_rat_license_check/README.md
b/tests/nightly/apache_rat_license_check/README.md
old mode 100644
new mode 100755
index b040882..477db37
--- a/tests/nightly/apache_rat_license_check/README.md
+++ b/tests/nightly/apache_rat_license_check/README.md
@@ -7,7 +7,7 @@ This file lists all the files, directories and file formats
that are excluded fr
If you think something is wrong, feel free to change!
### Nightly test script for license check
-Coming soon...
+The license check script called by the Jenkinsfile
### How to run the RAT check locally
The following commands can be used to run a Apache RAT check locally -
diff --git a/tests/nightly/apache_rat_license_check/license_check.sh
b/tests/nightly/apache_rat_license_check/license_check.sh
new file mode 100755
index 0000000..84accf9
--- /dev/null
+++ b/tests/nightly/apache_rat_license_check/license_check.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+#mvn and svn are installed in the docker container via the nghtly test script
+
+echo "download RAT"
+svn co http://svn.apache.org/repos/asf/creadur/rat/trunk/ #>/dev/null
+
+echo "cd into directory"
+cd trunk
+
+echo "mvn install"
+mvn -Dmaven.test.skip=true install #>/dev/null
+
+echo "build success, cd into target"
+cd apache-rat/target
+
+
+echo "-------Run Apache RAT check on MXNet-------"
+
+#Command has been run twice, once for the logs and once to store in the
variable to parse.
+java -jar apache-rat-0.13-SNAPSHOT.jar -E
/work/mxnet/tests/nightly/apache_rat_license_check/rat-excludes -d /work/mxnet
+OUTPUT="$(java -jar apache-rat-0.13-SNAPSHOT.jar -E
/work/mxnet/tests/nightly/apache_rat_license_check/rat-excludes -d /work/mxnet)"
+SOURCE="^0 Unknown Licenses"
+
+
+echo "-------Process The Output-------"
+
+if [[ "$OUTPUT" =~ "$SOURCE" ]]; then
+ echo "SUCCESS: There are no files with an Unknown License.";
+else
+ echo "ERROR: RAT Check detected files with unknown licenses. Please fix
and run test again!";
+ exit 1
+fi
\ No newline at end of file
diff --git a/tests/nightly/apache_rat_license_check/.rat-excludes
b/tests/nightly/apache_rat_license_check/rat-excludes
old mode 100644
new mode 100755
similarity index 94%
rename from tests/nightly/apache_rat_license_check/.rat-excludes
rename to tests/nightly/apache_rat_license_check/rat-excludes
index 885b131..ddcaec1
--- a/tests/nightly/apache_rat_license_check/.rat-excludes
+++ b/tests/nightly/apache_rat_license_check/rat-excludes
@@ -53,3 +53,5 @@ dataset.cPickle
rcnn/*
image-classification/*
+rat-excludes
+apache-rat-tasks/*
diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh
b/tests/nightly/compilation_warnings/compilation_warnings.sh
old mode 100644
new mode 100755
index a6c4863..0e39424
--- a/tests/nightly/compilation_warnings/compilation_warnings.sh
+++ b/tests/nightly/compilation_warnings/compilation_warnings.sh
@@ -31,9 +31,7 @@ runme() {
fi
}
-sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-sudo apt-get update
-sudo apt-get -y install time g++-5
+# g++-5 is installed in ci/docker/install/ubuntu_nightly_tests.sh
runme make clean >/dev/null
runme mkdir build
echo "Starting make"
diff --git a/tests/nightly/compilation_warnings/process_output.py
b/tests/nightly/compilation_warnings/process_output.py
old mode 100644
new mode 100755
diff --git a/tests/nightly/test_image_classification.sh
b/tests/nightly/test_image_classification.sh
index b4b4dca..1e4508f 100755
--- a/tests/nightly/test_image_classification.sh
+++ b/tests/nightly/test_image_classification.sh
@@ -18,17 +18,6 @@
# under the License.
-# setup
-export LD_LIBRARY_PATH=`pwd`/`dirname
$0`/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
-export PYTHONPATH=`pwd`/`dirname $0`/python
-# bc is required by sh2ju.sh
-apt-get install bc
-cd `pwd`/`dirname $0`
-. sh2ju.sh
-
-## clean last build log
-juLogClean
-
if [ -f $(which nvidia-smi) ]; then
if [ $# -eq 1 ]; then
num_gpus=$1
@@ -41,34 +30,17 @@ else
device_arg=""
fi
-# build
-build() {
- make -C ../.. clean
- make -C ../.. -j8
- return $?
-}
-
-cp ../../make/config.mk ../..
-cat >>../../config.mk <<EOF
-USE_CUDA=1
-USE_CUDA_PATH=/usr/local/cuda
-USE_CUDNN=1
-USE_DIST_KVSTORE=1
-EOF
-
-juLog -name=Build -error=Error build
-
# check if the final evaluation accuracy exceed the threshold
check_val() {
expected=$1
pass="Final validation >= $expected, PASS"
fail="Final validation < $expected, FAIL"
- python ../../tools/parse_log.py log --format none | tail -n1 | \
+ python tools/parse_log.py log --format none | tail -n1 | \
awk "{ if (\$3~/^[.0-9]+$/ && \$3 > $expected) print \"$pass\"; else
print \"$fail\"}"
rm -f log
}
-example_dir=../../example/image-classification
+example_dir=example/image-classification
# python: lenet + mnist
test_lenet() {
optimizers="adam sgd adagrad"
@@ -89,6 +61,7 @@ test_lenet() {
check_val $desired_accuracy
done
}
-juLog -name=Python.Lenet.Mnist -error=FAIL test_lenet
+
+test_lenet
exit $errors
