marcoabreu closed pull request #12563: Speed up CI windows builds
URL: https://github.com/apache/incubator-mxnet/pull/12563
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/build_windows.py b/ci/build_windows.py
index 39cb85f45d3..56769f7cdaf 100755
--- a/ci/build_windows.py
+++ b/ci/build_windows.py
@@ -20,31 +20,26 @@
"""User friendly / multi platform builder script"""
-import subprocess
+import argparse
+import datetime
+import glob
import logging
import os
-import tempfile
-import sys
-from distutils import spawn
-import logging
-from subprocess import check_call
import platform
-import argparse
-from util import *
-import json
-from enum import Enum
-import time
-import datetime
import shutil
-import glob
+import sys
+import time
from distutils.dir_util import copy_tree
+from enum import Enum
+from subprocess import check_call
+
+from util import *
+
+KNOWN_VCVARS = {
+ 'VS 2015': r'C:\Program Files (x86)\Microsoft Visual Studio
14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat',
+ 'VS 2017': r'C:\Program Files (x86)\Microsoft Visual
Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat'
+}
-KNOWN_VCVARS = [
- # VS 2015
- r'C:\Program Files (x86)\Microsoft Visual Studio
14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat'
- # VS 2017
- , r'c:\Program Files (x86)\Microsoft Visual
Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat'
-]
class BuildFlavour(Enum):
WIN_CPU = 'WIN_CPU'
@@ -52,110 +47,96 @@ class BuildFlavour(Enum):
WIN_GPU = 'WIN_GPU'
WIN_GPU_MKLDNN = 'WIN_GPU_MKLDNN'
+
CMAKE_FLAGS = {
- 'WIN_CPU': '-DUSE_CUDA=0 \
- -DUSE_CUDNN=0 \
- -DUSE_NVRTC=0 \
- -DUSE_OPENCV=1 \
- -DUSE_OPENMP=1 \
- -DUSE_PROFILER=1 \
- -DUSE_BLAS=open \
- -DUSE_LAPACK=1 \
- -DUSE_DIST_KVSTORE=0 \
- -DBUILD_CPP_EXAMPLES=1 \
- -DUSE_MKL_IF_AVAILABLE=0'
-
- ,'WIN_CPU_MKLDNN': '-DUSE_CUDA=0 \
- -DUSE_CUDNN=0 \
- -DUSE_NVRTC=0 \
- -DUSE_OPENCV=1 \
- -DUSE_OPENMP=1 \
- -DUSE_PROFILER=1 \
- -DUSE_BLAS=open \
- -DUSE_LAPACK=1 \
- -DUSE_DIST_KVSTORE=0 \
- -DUSE_MKL_IF_AVAILABLE=1'
-
- ,'WIN_GPU': '-DUSE_CUDA=1 \
- -DUSE_CUDNN=1 \
- -DUSE_NVRTC=1 \
- -DUSE_OPENCV=1 \
- -DUSE_OPENMP=1 \
- -DUSE_PROFILER=1 \
- -DUSE_BLAS=open \
- -DUSE_LAPACK=1 \
- -DUSE_DIST_KVSTORE=0 \
- -DCUDA_ARCH_NAME=Manual \
- -DCUDA_ARCH_BIN=52 \
- -DCUDA_ARCH_PTX=52 \
- -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" \
- -DUSE_MKL_IF_AVAILABLE=0 \
- -DCMAKE_BUILD_TYPE=Release'
-
- ,'WIN_GPU_MKLDNN': '-DUSE_CUDA=1 \
- -DUSE_CUDNN=1 \
- -DUSE_NVRTC=1 \
- -DUSE_OPENCV=1 \
- -DUSE_OPENMP=1 \
- -DUSE_PROFILER=1 \
- -DUSE_BLAS=open \
- -DUSE_LAPACK=1 \
- -DUSE_DIST_KVSTORE=0 \
- -DCUDA_ARCH_NAME=Manual \
- -DCUDA_ARCH_BIN=52 \
- -DCUDA_ARCH_PTX=52 \
- -DUSE_MKLDNN=1 \
- -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 \
- /DNDEBUG" \
- -DCMAKE_BUILD_TYPE=Release'
+ 'WIN_CPU': ('-DUSE_CUDA=0 '
+ '-DUSE_CUDNN=0 '
+ '-DUSE_NVRTC=0 '
+ '-DUSE_OPENCV=1 '
+ '-DUSE_OPENMP=1 '
+ '-DUSE_PROFILER=1 '
+ '-DUSE_BLAS=open '
+ '-DUSE_LAPACK=1 '
+ '-DUSE_DIST_KVSTORE=0 '
+ '-DBUILD_CPP_EXAMPLES=1 '
+ '-DUSE_MKL_IF_AVAILABLE=0 '
+ '-DCMAKE_BUILD_TYPE=Release')
+
+ , 'WIN_CPU_MKLDNN': ('-DUSE_CUDA=0 '
+ '-DUSE_CUDNN=0 '
+ '-DUSE_NVRTC=0 '
+ '-DUSE_OPENCV=1 '
+ '-DUSE_OPENMP=1 '
+ '-DUSE_PROFILER=1 '
+ '-DUSE_BLAS=open '
+ '-DUSE_LAPACK=1 '
+ '-DUSE_DIST_KVSTORE=0 '
+ '-DUSE_MKL_IF_AVAILABLE=1 '
+ '-DCMAKE_BUILD_TYPE=Release')
+
+ , 'WIN_GPU': ('-DUSE_CUDA=1 '
+ '-DUSE_CUDNN=1 '
+ '-DUSE_NVRTC=1 '
+ '-DUSE_OPENCV=1 '
+ '-DUSE_OPENMP=1 '
+ '-DUSE_PROFILER=1 '
+ '-DUSE_BLAS=open '
+ '-DUSE_LAPACK=1 '
+ '-DUSE_DIST_KVSTORE=0 '
+ '-DCUDA_ARCH_NAME=Manual '
+ '-DCUDA_ARCH_BIN=52 '
+ '-DCUDA_ARCH_PTX=52 '
+ '-DCMAKE_CXX_FLAGS="/FS /MD /O2 /Ob2" '
+ '-DUSE_MKL_IF_AVAILABLE=0 '
+ '-DCMAKE_BUILD_TYPE=Release')
+
+ , 'WIN_GPU_MKLDNN': ('-DUSE_CUDA=1 '
+ '-DUSE_CUDNN=1 '
+ '-DUSE_NVRTC=1 '
+ '-DUSE_OPENCV=1 '
+ '-DUSE_OPENMP=1 '
+ '-DUSE_PROFILER=1 '
+ '-DUSE_BLAS=open '
+ '-DUSE_LAPACK=1 '
+ '-DUSE_DIST_KVSTORE=0 '
+ '-DCUDA_ARCH_NAME=Manual '
+ '-DCUDA_ARCH_BIN=52 '
+ '-DCUDA_ARCH_PTX=52 '
+ '-DUSE_MKLDNN=1 '
+ '-DCMAKE_CXX_FLAGS="/FS /MD /O2 /Ob2" '
+ '-DCMAKE_BUILD_TYPE=Release')
}
-def get_vcvars_environment(architecture, vcvars):
- """
- Returns a dictionary containing the environment variables set up by vcvars
- """
- result = None
- python = sys.executable
-
- vcvars_list = [vcvars]
- vcvars_list.extend(KNOWN_VCVARS)
- for vcvars in vcvars_list:
- if os.path.isfile(vcvars):
- process = subprocess.Popen('("%s" %s>nul) && "%s" -c "import os;
import json; print(json.dumps(dict(os.environ)))"' % (vcvars, architecture,
python), stdout=subprocess.PIPE, shell=True)
- stdout, stderr = process.communicate()
- exitcode = process.wait()
- if exitcode == 0:
- logging.info("Using build environment from: %s", vcvars)
- return(json.loads(stdout.strip()))
- else:
- raise RuntimeError('Failed cloning environment from vcvars
file: %s stdout: %s stderr: %s', vcvars, stdout, stderr)
- raise RuntimeError('Couldn\'t find vcvars batch file: %s', vcvars)
-
-
def windows_build(args):
- vcvars_env = get_vcvars_environment(args.arch, args.vcvars)
- logging.debug("vcvars environment: %s", vcvars_env)
- os.environ.update(vcvars_env)
+ logging.info("Using vcvars environment:\n{}".format(args.vcvars))
path = args.output
os.makedirs(path, exist_ok=True)
+
mxnet_root = get_mxnet_root()
- logging.info("Found mxnet root: {}".format(mxnet_root))
+ logging.info("Found MXNet root: {}".format(mxnet_root))
+
with remember_cwd():
os.chdir(path)
- logging.info("Generating project with CMake")
- check_call("cmake -G \"Visual Studio 14 2015 Win64\" {}
{}".format(CMAKE_FLAGS[args.flavour], mxnet_root), shell=True)
- logging.info("Building with visual studio")
+ cmd = "\"{}\" && cmake -G \"NMake Makefiles JOM\" {}
{}".format(args.vcvars,
+
CMAKE_FLAGS[args.flavour],
+
mxnet_root)
+ logging.info("Generating project with CMake:\n{}".format(cmd))
+ check_call(cmd, shell=True)
+
+ cmd = "\"{}\" && jom".format(args.vcvars)
+ logging.info("Building with jom:\n{}".format(cmd))
+
t0 = int(time.time())
- check_call(
- ["msbuild", "mxnet.sln",
"/p:configuration=release;platform=x64;BuildInParallel=true", "/maxcpucount",
- "/v:minimal"])
- logging.info("Build flavour: %s complete in directory: \"%s\"",
args.flavour, os.path.abspath(path))
- logging.info("Build took %s" ,
datetime.timedelta(seconds=int(time.time()-t0)))
+ check_call(cmd, shell=True)
+
+ logging.info("Build flavour: {} complete in directory:
\"{}\"".format(args.flavour, os.path.abspath(path)))
+ logging.info("Build took
{}".format(datetime.timedelta(seconds=int(time.time() - t0))))
windows_package(args)
+
def windows_package(args):
pkgfile = 'windows_package.7z'
pkgdir = os.path.abspath('windows_package')
@@ -200,6 +181,7 @@ def nix_build(args):
-GNinja ..", shell=True)
check_call("ninja", shell=True)
+
def main():
logging.getLogger().setLevel(logging.INFO)
logging.basicConfig(format='%(asctime)-15s %(message)s')
@@ -213,7 +195,7 @@ def main():
parser.add_argument("--vcvars",
help="vcvars batch file location, typically inside vs studio install
dir",
- default=r'c:\Program Files (x86)\Microsoft Visual
Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat',
+ default=KNOWN_VCVARS['VS 2015'],
type=str)
parser.add_argument("--arch",
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
