This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new a1cc2ba Improve dev_menu usability, local build and virtualenv
(#13529)
a1cc2ba is described below
commit a1cc2ba54acd98bf157c68a99ba4508ef103ee79
Author: Pedro Larroy <pedro.larroy.li...@gmail.com>
AuthorDate: Fri Dec 14 00:20:18 2018 +0100
Improve dev_menu usability, local build and virtualenv (#13529)
* Improve dev_menu, add build command and virtualenv creation with local
builds for easy testing
* Update dev_menu.py
Co-Authored-By: larroy <pedro.larroy.li...@gmail.com>
* Cuda off by default, use ccache
* address CR
---
.gitignore | 2 ++
cmake/cmake_options.yml | 5 +++-
dev_menu.py | 74 ++++++++++++++++++++++++++++++++++++++++++++-----
tests/requirements.txt | 1 +
4 files changed, 74 insertions(+), 8 deletions(-)
diff --git a/.gitignore b/.gitignore
index c8a8136..c7530ab 100644
--- a/.gitignore
+++ b/.gitignore
@@ -177,3 +177,5 @@ cxx
*.gcno
coverage.xml
+# Local CMake build config
+cmake_options.yml
diff --git a/cmake/cmake_options.yml b/cmake/cmake_options.yml
index 01446f7..a4323fe 100644
--- a/cmake/cmake_options.yml
+++ b/cmake/cmake_options.yml
@@ -16,7 +16,7 @@
# under the License.
--- # CMake configuration
-USE_CUDA: "ON" # Build with CUDA support
+USE_CUDA: "OFF" # Build with CUDA support
USE_OLDCMAKECUDA: "OFF" # Build with old cmake cuda
USE_NCCL: "OFF" # Use NVidia NCCL with CUDA
USE_OPENCV: "ON" # Build with OpenCV support
@@ -48,3 +48,6 @@ USE_TENSORRT: "OFF" # Enable infeference optimization with
TensorRT.
USE_ASAN: "OFF" # Enable Clang/GCC ASAN sanitizers.
ENABLE_TESTCOVERAGE: "OFF" # Enable compilation with test coverage metric
output
CMAKE_BUILD_TYPE: "Debug"
+CMAKE_CUDA_COMPILER_LAUNCHER: "ccache"
+CMAKE_C_COMPILER_LAUNCHER: "ccache"
+CMAKE_CXX_COMPILER_LAUNCHER: "ccache"
diff --git a/dev_menu.py b/dev_menu.py
index 0fd78cb..ebffe14 100755
--- a/dev_menu.py
+++ b/dev_menu.py
@@ -20,6 +20,7 @@
# -*- coding: utf-8 -*-
"""Tool to ease working with the build system and reproducing test results"""
+import argparse
import os
import sys
from subprocess import check_call
@@ -29,6 +30,11 @@ from typing import List
from collections import OrderedDict
import logging
import yaml
+import shutil
+
+DEFAULT_PYENV=os.environ.get('DEFAULT_PYENV','py3_venv')
+DEFAULT_PYTHON=os.environ.get('DEFAULT_PYTHON','python3')
+DEFAULT_CMAKE_OPTIONS=os.environ.get('DEFAULT_CMAKE_OPTIONS','cmake_options.yml')
class Confirm(object):
def __init__(self, cmds):
@@ -46,7 +52,7 @@ class Confirm(object):
resp = input("Please answer yes or no: ")
class CMake(object):
- def __init__(self, cmake_options_yaml='cmake_options.yml',
cmake_options_yaml_default='cmake/cmake_options.yml'):
+ def __init__(self, cmake_options_yaml=DEFAULT_CMAKE_OPTIONS,
cmake_options_yaml_default='cmake/cmake_options.yml'):
if os.path.exists(cmake_options_yaml):
self.cmake_options_yaml = cmake_options_yaml
else:
@@ -87,10 +93,29 @@ class CMake(object):
logging.info('Now building')
check_call(shlex.split(build_cmd))
-
+def create_virtualenv(venv_exe, pyexe, venv) -> None:
+ logging.info("Creating virtualenv in %s with python %s", venv, pyexe)
+ if not (venv_exe and pyexe and venv):
+ logging.warn("Skipping creation of virtualenv")
+ return
+ check_call([venv_exe, '-p', pyexe, venv])
+ activate_this_py = os.path.join(venv, 'bin', 'activate_this.py')
+ # Activate virtualenv in this interpreter
+ exec(open(activate_this_py).read(), dict(__file__=activate_this_py))
+ check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e',
'python'])
+ check_call(['pip', 'install', '-r', 'tests/requirements.txt'])
+
+def create_virtualenv_default():
+ create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV)
+ logging.info("You can use the virtualenv by executing 'source
%s/bin/activate'", DEFAULT_PYENV)
COMMANDS = OrderedDict([
- ('[Docker] sanity_check',
+ ('[Local build] CMake/Ninja build (using cmake_options.yaml (cp
cmake/cmake_options.yml .) and edit) (creates {} virtualenv in
"{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV),
+ [
+ CMake(),
+ create_virtualenv_default,
+ ]),
+ ('[Docker] sanity_check. Check for linting and code formatting.',
"ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
sanity_check"),
('[Docker] Python3 CPU unittests',
[
@@ -117,8 +142,6 @@ COMMANDS = OrderedDict([
"ci/build.py -p armv7",
"ci/build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu"
]),
- ('[Local] CMake build (using cmake/cmake_options.yaml)',
- CMake()),
('Clean (RESET HARD) repository (Warning! erases local changes / DATA
LOSS)',
Confirm("ci/docker/runtime_functions.sh clean_repo"))
])
@@ -128,6 +151,7 @@ def clip(x, mini, maxi):
@retry((ValueError, RuntimeError), 3, delay_s = 0)
def show_menu(items: List[str], header=None) -> int:
+ print('\n-- MXNet dev menu --\n')
def hr():
print(''.join(['-']*30))
if header:
@@ -156,11 +180,47 @@ def handle_commands(cmds) -> None:
else:
raise RuntimeError("handle_commands(cmds): argument should be str or
List[str] but is %s", type(cmds))
-def main():
- logging.getLogger().setLevel(logging.INFO)
+def use_menu_ui(args) -> None:
command_list = list(COMMANDS.keys())
choice = show_menu(command_list, 'Available actions')
handle_commands(COMMANDS[command_list[choice]])
+
+def build(args) -> None:
+ """Build using CMake"""
+ venv_exe = shutil.which('virtualenv')
+ pyexe = shutil.which(args.pyexe)
+ if not venv_exe:
+ logging.warn("virtualenv wasn't found in path, it's recommended to
install virutalenv to manage python environments")
+ if not pyexe:
+ logging.warn("Python executable %s not found in path", args.pyexe)
+ if args.cmake_options:
+ cmake = CMake(args.cmake_options)
+ else:
+ cmake = CMake()
+ cmake()
+ create_virtualenv(venv_exe, pyexe, args.venv)
+
+def main():
+ logging.getLogger().setLevel(logging.INFO)
+ parser = argparse.ArgumentParser(description="""Utility for compiling and
testing MXNet easily""")
+ parser.set_defaults(command='use_menu_ui')
+
+ subparsers = parser.add_subparsers(help='sub-command help')
+ build_parser = subparsers.add_parser('build', help='build with the
specified flags from file')
+ build_parser.add_argument('cmake_options', nargs='?',
+ help='File containing CMake options in YAML')
+ build_parser.add_argument('-v', '--venv',
+ type=str,
+ default=DEFAULT_PYENV,
+ help='virtualenv dir')
+ build_parser.add_argument('-p', '--pyexe',
+ type=str,
+ default=DEFAULT_PYTHON,
+ help='python executable')
+
+ build_parser.set_defaults(command='build')
+ args = parser.parse_args()
+ globals()[args.command](args)
return 0
if __name__ == '__main__':
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 3ca696b..f64f7ff 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,3 +2,4 @@
mock
nose
nose-timer
+ipython
