This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch v1.4.x
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/v1.4.x by this push:
new 44265aa v1.4.x: Backport of Windows CI Fixes (#14420)
44265aa is described below
commit 44265aa8a6ae29c0771aa7bcb2cc4c61848a7ffc
Author: perdasilva <[email protected]>
AuthorDate: Thu Mar 14 10:44:38 2019 +0100
v1.4.x: Backport of Windows CI Fixes (#14420)
* Backports Windows CI pipeline fixes from master
* Updates Jenkins file to use python 3 when calling build_windows.py
* Installs qemu pip requirements from qemu requirements file (#14355)
* remove a white space
---
Jenkinsfile | 32 +++++++++++++-------------------
ci/build_windows.py | 6 +++---
ci/docker/qemu/runtime_functions.py | 4 ++--
ci/docker/qemu/vmcontrol.py | 1 +
ci/jenkins/Jenkins_steps.groovy | 32 +++++++++++++-------------------
ci/jenkins/Jenkinsfile_windows_cpu | 4 ++--
ci/qemu/README.md | 4 ++++
ci/qemu/test_requirements.txt | 3 +++
ci/windows/test_py2_cpu.ps1 | 8 +++++---
ci/windows/test_py2_gpu.ps1 | 12 +++++++-----
ci/windows/test_py3_cpu.ps1 | 8 +++++---
ci/windows/test_py3_gpu.ps1 | 12 +++++++-----
tests/requirements.txt | 4 ++++
13 files changed, 69 insertions(+), 61 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index a9051db..03ad7d8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -390,13 +390,11 @@ core_logic: {
},
'Build CPU windows':{
node(NODE_WINDOWS_CPU) {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-cpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas',
'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
- utils.init_git_win()
- powershell 'python ci/build_windows.py -f WIN_CPU'
- stash includes: 'windows_package.7z', name: 'windows_package_cpu'
- }
+ ws('workspace/build-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git_win()
+ powershell 'py -3 ci/build_windows.py -f WIN_CPU'
+ stash includes: 'windows_package.7z', name: 'windows_package_cpu'
}
}
}
@@ -404,26 +402,22 @@ core_logic: {
'Build GPU windows':{
node(NODE_WINDOWS_CPU) {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-gpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas',
'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
+ ws('workspace/build-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
- powershell 'python ci/build_windows.py -f WIN_GPU'
+ powershell 'py -3 ci/build_windows.py -f WIN_GPU'
stash includes: 'windows_package.7z', name: 'windows_package_gpu'
- }
}
}
}
},
'Build GPU MKLDNN windows':{
node(NODE_WINDOWS_CPU) {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-gpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas',
'OpenCV_DIR=C:\\mxnet\\opencv_vc14',
'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) {
- utils.init_git_win()
- powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN'
- stash includes: 'windows_package.7z', name:
'windows_package_gpu_mkldnn'
- }
+ ws('workspace/build-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git_win()
+ powershell 'py -3 ci/build_windows.py -f WIN_GPU_MKLDNN'
+ stash includes: 'windows_package.7z', name:
'windows_package_gpu_mkldnn'
}
}
}
diff --git a/ci/build_windows.py b/ci/build_windows.py
index 56769f7..427e50f 100755
--- a/ci/build_windows.py
+++ b/ci/build_windows.py
@@ -216,11 +216,11 @@ def main():
if system == 'Windows':
logging.info("Detected Windows platform")
if 'OpenBLAS_HOME' not in os.environ:
- os.environ["OpenBLAS_HOME"] = "C:\\mxnet\\openblas"
+ os.environ["OpenBLAS_HOME"] = "C:\\Program Files\\OpenBLAS-v0.2.19"
if 'OpenCV_DIR' not in os.environ:
- os.environ["OpenCV_DIR"] = "C:\\mxnet\\opencv_vc14"
+ os.environ["OpenCV_DIR"] = "C:\\Program
Files\\OpenCV-v3.4.1\\build"
if 'CUDA_PATH' not in os.environ:
- os.environ["CUDA_PATH"] = "C:\\CUDA\\v8.0"
+ os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing
Toolkit\\CUDA\\v9.2"
windows_build(args)
elif system == 'Linux' or system == 'Darwin':
diff --git a/ci/docker/qemu/runtime_functions.py
b/ci/docker/qemu/runtime_functions.py
index 8b8e5ac..9c137f6 100755
--- a/ci/docker/qemu/runtime_functions.py
+++ b/ci/docker/qemu/runtime_functions.py
@@ -77,8 +77,8 @@ def run_ut_python3_qemu_internal():
logging.info("=== NOW Running inside QEMU ===")
logging.info("PIP Installing %s", pkg)
check_call(['sudo', 'pip3', 'install', pkg])
- logging.info("PIP Installing mxnet/tests/requirements.txt")
- check_call(['sudo', 'pip3', 'install', '-r',
'mxnet/tests/requirements.txt'])
+ logging.info("PIP Installing mxnet/test_requirements.txt")
+ check_call(['sudo', 'pip3', 'install', '-r',
'mxnet/test_requirements.txt'])
logging.info("Running tests in mxnet/tests/python/unittest/")
check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file',
'nosetests_unittest.xml', '--verbose',
'mxnet/tests/python/unittest/test_engine.py'])
# Example to run a single unit test:
diff --git a/ci/docker/qemu/vmcontrol.py b/ci/docker/qemu/vmcontrol.py
index a7e8c0f..a5b025b 100644
--- a/ci/docker/qemu/vmcontrol.py
+++ b/ci/docker/qemu/vmcontrol.py
@@ -187,6 +187,7 @@ def qemu_provision(ssh_port=QEMU_SSH_PORT):
qemu_rsync(ssh_port, '/work/runtime_functions.py','')
qemu_rsync(ssh_port, '/work/vmcontrol.py','')
qemu_rsync(ssh_port, 'mxnet/tests', 'mxnet')
+ qemu_rsync(ssh_port, 'mxnet/ci/qemu/test_requirements.txt',
'mxnet/test_requirements.txt')
logging.info("Provisioning completed successfully.")
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 81b573a..73c6029 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -450,13 +450,11 @@ def compile_unix_amalgamation() {
def compile_windows_cpu() {
return ['Build CPU windows':{
node(NODE_WINDOWS_CPU) {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-cpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas',
'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
- utils.init_git_win()
- powershell 'python ci/build_windows.py -f WIN_CPU'
- stash includes: 'windows_package.7z', name: 'windows_package_cpu'
- }
+ ws('workspace/build-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git_win()
+ powershell 'py -3 ci/build_windows.py -f WIN_CPU'
+ stash includes: 'windows_package.7z', name: 'windows_package_cpu'
}
}
}
@@ -466,13 +464,11 @@ def compile_windows_cpu() {
def compile_windows_gpu() {
return ['Build GPU windows':{
node(NODE_WINDOWS_CPU) {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-gpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas',
'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
+ ws('workspace/build-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
- powershell 'python ci/build_windows.py -f WIN_GPU'
+ powershell 'py -3 ci/build_windows.py -f WIN_GPU'
stash includes: 'windows_package.7z', name: 'windows_package_gpu'
- }
}
}
}
@@ -482,13 +478,11 @@ def compile_windows_gpu() {
def compile_windows_gpu_mkldnn() {
return ['Build GPU MKLDNN windows':{
node(NODE_WINDOWS_CPU) {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-gpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas',
'OpenCV_DIR=C:\\mxnet\\opencv_vc14',
'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) {
- utils.init_git_win()
- powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN'
- stash includes: 'windows_package.7z', name:
'windows_package_gpu_mkldnn'
- }
+ ws('workspace/build-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git_win()
+ powershell 'py -3 ci/build_windows.py -f WIN_GPU_MKLDNN'
+ stash includes: 'windows_package.7z', name:
'windows_package_gpu_mkldnn'
}
}
}
diff --git a/ci/jenkins/Jenkinsfile_windows_cpu
b/ci/jenkins/Jenkinsfile_windows_cpu
index 9e70df3..67a0611 100644
--- a/ci/jenkins/Jenkinsfile_windows_cpu
+++ b/ci/jenkins/Jenkinsfile_windows_cpu
@@ -35,12 +35,12 @@ utils.main_wrapper(
core_logic: {
utils.parallel_stage('Build', [
custom_steps.compile_windows_cpu()
- ])
+ ])
utils.parallel_stage('Tests', [
custom_steps.test_windows_python2_cpu(),
custom_steps.test_windows_python3_cpu()
- ])
+ ])
}
,
failure_handler: {
diff --git a/ci/qemu/README.md b/ci/qemu/README.md
index 498f8b7..4beca4a 100644
--- a/ci/qemu/README.md
+++ b/ci/qemu/README.md
@@ -86,3 +86,7 @@ pip3 install -r mxnet_requirements.txt
To access qemu control console from tmux: `ctrl-a a c`
+
+# CI and Testing
+
+Formally,
[runtime_functions.py](https://github.com/apache/incubator-mxnet/blob/master/ci/docker/qemu/runtime_functions.py)
would
[run](https://github.com/apache/incubator-mxnet/blob/8beea18e3d9835f90b59d3f9de8f9945ac819423/ci/docker/qemu/runtime_functions.py#L81)
*pip install -r
[mxnet/tests/requirements.txt](https://github.com/apache/incubator-mxnet/blob/master/tests/requirements.txt)*.
If the requirements change, there can be an unfortunate side-effect that there
are no wheel files fo [...]
diff --git a/ci/qemu/test_requirements.txt b/ci/qemu/test_requirements.txt
new file mode 100644
index 0000000..77037d8
--- /dev/null
+++ b/ci/qemu/test_requirements.txt
@@ -0,0 +1,3 @@
+mock
+nose
+nose-timer
\ No newline at end of file
diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1
index 702a2db..4cef0ea 100644
--- a/ci/windows/test_py2_cpu.ps1
+++ b/ci/windows/test_py2_cpu.ps1
@@ -19,8 +19,10 @@
$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll
$env:PYTHONPATH=join-path $pwd.Path windows_package\python
$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
-c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
-c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_unittest.xml tests\python\unittest
+$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')
+
+C:\Python27\Scripts\pip install -r tests\requirements.txt
+C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml
tests\python\unittest
if (! $?) { Throw ("Error running unittest") }
-c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_train.xml tests\python\train
+C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml
tests\python\train
if (! $?) { Throw ("Error running train tests") }
diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1
index 0cf2717..a113ee0 100644
--- a/ci/windows/test_py2_gpu.ps1
+++ b/ci/windows/test_py2_gpu.ps1
@@ -19,12 +19,14 @@
$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll
$env:PYTHONPATH=join-path $pwd.Path windows_package\python
$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
-c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
-c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_unittest.xml tests\python\unittest
+$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')
+
+C:\Python27\Scripts\pip install -r tests\requirements.txt
+C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml
tests\python\unittest
if (! $?) { Throw ("Error running unittest") }
-c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_operator.xml tests\python\gpu\test_operator_gpu.py
+C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml
tests\python\gpu\test_operator_gpu.py
if (! $?) { Throw ("Error running tests") }
-c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_forward.xml tests\python\gpu\test_forward.py
+C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml
tests\python\gpu\test_forward.py
if (! $?) { Throw ("Error running tests") }
-c:\Anaconda3\envs\py2\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error tests\python\train
+C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error tests\python\train
if (! $?) { Throw ("Error running tests") }
diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1
index a7774a6..9d838c2 100644
--- a/ci/windows/test_py3_cpu.ps1
+++ b/ci/windows/test_py3_cpu.ps1
@@ -19,8 +19,10 @@
$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll
$env:PYTHONPATH=join-path $pwd.Path windows_package\python
$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
-c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
-c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_unittest.xml tests\python\unittest
+$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')
+
+C:\Python37\Scripts\pip install -r tests\requirements.txt
+C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml
tests\python\unittest
if (! $?) { Throw ("Error running unittest") }
-c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_train.xml tests\python\train
+C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml
tests\python\train
if (! $?) { Throw ("Error running train tests") }
diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1
index f9955ef..5492538 100644
--- a/ci/windows/test_py3_gpu.ps1
+++ b/ci/windows/test_py3_gpu.ps1
@@ -19,12 +19,14 @@
$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll
$env:PYTHONPATH=join-path $pwd.Path windows_package\python
$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
-c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
-c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_unittest.xml tests\python\unittest
+$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')
+
+C:\Python37\Scripts\pip install -r tests\requirements.txt
+C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml
tests\python\unittest
if (! $?) { Throw ("Error running unittest") }
-c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_operator.xml tests\python\gpu\test_operator_gpu.py
+C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml
tests\python\gpu\test_operator_gpu.py
if (! $?) { Throw ("Error running tests") }
-c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_forward.xml tests\python\gpu\test_forward.py
+C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml
tests\python\gpu\test_forward.py
if (! $?) { Throw ("Error running tests") }
-c:\Anaconda3\envs\py3\python.exe -m nose -v --with-timer --timer-ok 1
--timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file
nosetests_train.xml tests\python\train
+C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15
--timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml
tests\python\train
if (! $?) { Throw ("Error running tests") }
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 3ca696b..e16b764 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,4 +1,8 @@
# Requirements for tests, those are installed before running on the virtualenv
+# Requirements for tests run within the qemu requirement see
ci/qemu/test_requirements.txt
mock
nose
nose-timer
+ipython
+numpy
+scipy
