lanking520 closed pull request #13648: scripts for building libmxnet binary and wheel URL: https://github.com/apache/incubator-mxnet/pull/13648
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/tools/build/build_lib.sh b/tools/build/build_lib.sh new file mode 100755 index 00000000000..032fcb92045 --- /dev/null +++ b/tools/build/build_lib.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the libraries of mxnet. +make_config=config/pip_${PLATFORM}_${VARIANT}.mk +if [[ ! -f $make_config ]]; then + >&2 echo "Couldn't find make config $make_config for the current settings." + exit 1 +fi + +git clone --recursive https://github.com/apache/incubator-mxnet mxnet-build + +>&2 echo "Now building mxnet modules..." +cp $make_config mxnet-build/config.mk + +cd mxnet-build + +make DEPS_PATH=$DEPS_PATH DMLCCORE +make DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a +make DEPS_PATH=$DEPS_PATH PSLITE + +if [[ $VARIANT == *mkl ]]; then + MKLDNN_LICENSE='license.txt' + if [[ $PLATFORM == 'linux' ]]; then + IOMP_LIBFILE='libiomp5.so' + MKLML_LIBFILE='libmklml_intel.so' + MKLDNN_LIBFILE='libmkldnn.so.0' + else + IOMP_LIBFILE='libiomp5.dylib' + MKLML_LIBFILE='libmklml.dylib' + MKLDNN_LIBFILE='libmkldnn.0.dylib' + fi + make DEPS_PATH=$DEPS_PATH mkldnn + cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE +fi + +if [[ $VARIANT == *mkl ]]; then + >&2 echo "Copying MKL license." + rm lib/libmkldnn.{so,dylib} + rm lib/libmkldnn.0.*.dylib + rm lib/libmkldnn.so.0.* +fi + +>&2 echo "Now building mxnet..." +make DEPS_PATH=$DEPS_PATH || exit 1; + +if [[ $PLATFORM == 'linux' ]]; then + cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3 + cp -L /usr/lib/x86_64-linux-gnu/libquadmath.so.0 lib/libquadmath.so.0 +fi + +# Print the linked objects on libmxnet.so +>&2 echo "Checking linked objects on libmxnet.so..." +if [[ ! -z $(command -v readelf) ]]; then + readelf -d lib/libmxnet.so + strip --strip-unneeded lib/libmxnet.so +elif [[ ! -z $(command -v otool) ]]; then + otool -L lib/libmxnet.so + strip -u -r -x lib/libmxnet.so +else + >&2 echo "Not available" +fi + +cd ../ diff --git a/tools/pip_package/MANIFEST.in b/tools/build/build_wheel.sh old mode 100644 new mode 100755 similarity index 69% rename from tools/pip_package/MANIFEST.in rename to tools/build/build_wheel.sh index 5c6a72377e9..a79634117c2 --- a/tools/pip_package/MANIFEST.in +++ b/tools/build/build_wheel.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,6 +17,15 @@ # specific language governing permissions and limitations # under the License. -include README -recursive-include * *.py -recursive-include * *.so +# This script builds the wheel for binary distribution and performs sanity check. + +cd mxnet-build +echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH +cd - + +# Make wheel for testing +python setup.py bdist_wheel + +wheel_name=$(ls -t dist | head -n 1) +pip install -U --user --force-reinstall dist/$wheel_name +python sanity_test.py diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh new file mode 100755 index 00000000000..d678fddcc02 --- /dev/null +++ b/tools/dependencies/make_shared_dependencies.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This is a convenience script for calling the build scripts of all dependency libraries. +# Environment variables should be set beforehand. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + + +if [[ ! $PLATFORM == 'darwin' ]]; then + source $DIR/openblas.sh +fi +source $DIR/libz.sh +source $DIR/libturbojpeg.sh +source $DIR/libpng.sh +source $DIR/libtiff.sh +source $DIR/openssl.sh +source $DIR/curl.sh +source $DIR/eigen.sh +source $DIR/opencv.sh +source $DIR/protobuf.sh +source $DIR/cityhash.sh +source $DIR/zmq.sh +source $DIR/lz4.sh diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh index 98ff115f176..99d0ecb71c3 100755 --- a/tools/dependencies/opencv.sh +++ b/tools/dependencies/opencv.sh @@ -20,6 +20,7 @@ # This script builds the static library of opencv that can be used as dependency of mxnet. # It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in $DEPS_PATH. OPENCV_VERSION=3.4.2 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" if [[ $PLATFORM == 'linux' ]]; then OPENCV_LAPACK_OPTIONS=" \ -D OpenBLAS_HOME=$DEPS_PATH \ @@ -181,7 +182,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. if [[ $PLATFORM == 'linux' ]]; then - cp $DEPS_PATH/../patch/opencv_lapack.h ./ + cp $DIR/patch/opencv_lapack.h ./ fi make make install diff --git a/tools/dependencies/patch/opencv_lapack.h b/tools/dependencies/patch/opencv_lapack.h new file mode 100644 index 00000000000..97af9d67ea3 --- /dev/null +++ b/tools/dependencies/patch/opencv_lapack.h @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +extern "C" { +#include "cblas.h" +#include "lapacke.h" +} diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh index dfa3d71f375..1564701042a 100755 --- a/tools/dependencies/protobuf.sh +++ b/tools/dependencies/protobuf.sh @@ -39,3 +39,5 @@ if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then make install cd - fi + +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib diff --git a/tools/pip/MANIFEST.in b/tools/pip/MANIFEST.in new file mode 100644 index 00000000000..5e072064193 --- /dev/null +++ b/tools/pip/MANIFEST.in @@ -0,0 +1,11 @@ +include README +include mxnet/COMMIT_HASH +recursive-include mxnet/tools * +recursive-include mxnet *.py +recursive-include mxnet *.so +recursive-include mxnet *.so.* +recursive-include mxnet *.dylib +recursive-include mxnet *_LICENSE +recursive-include mxnet *.h +recursive-include mxnet *.cuh +recursive-include dmlc_tracker *.py diff --git a/tools/pip/sanity_test.py b/tools/pip/sanity_test.py new file mode 100644 index 00000000000..dc51e479906 --- /dev/null +++ b/tools/pip/sanity_test.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +"""Sanity test.""" +from __future__ import print_function +import sys +from base64 import b64decode + +try: + import mxnet as mx + mx.img.imdecode(b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==')).asnumpy() + print('Test succeeded') +except: + import traceback + print('Test failed') + traceback.print_exc() + sys.exit(1) diff --git a/tools/pip/setup.py b/tools/pip/setup.py new file mode 100644 index 00000000000..d5db6d87fc1 --- /dev/null +++ b/tools/pip/setup.py @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +# pylint: disable=invalid-name, exec-used +"""Setup mxnet package for pip.""" +from __future__ import absolute_import +from datetime import datetime +import os +import sys +import shutil +import platform + +if platform.system() == 'Linux': + sys.argv.append('--universal') + sys.argv.append('--plat-name=manylinux1_x86_64') + +from setuptools import setup, find_packages +from setuptools.dist import Distribution + +# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py +# Will be invoked which introduces dependences +CURRENT_DIR = os.path.dirname(__file__) +libinfo_py = os.path.join(CURRENT_DIR, 'mxnet-build/python/mxnet/libinfo.py') +libinfo = {'__file__': libinfo_py} +exec(compile(open(libinfo_py, "rb").read(), libinfo_py, 'exec'), libinfo, libinfo) + +LIB_PATH = libinfo['find_lib_path']() +__version__ = libinfo['__version__'] +if 'TRAVIS_TAG' not in os.environ or not os.environ['TRAVIS_TAG'].strip(): + __version__ += 'b{0}'.format(datetime.today().strftime('%Y%m%d')) +elif 'TRAVIS_TAG' in os.environ and os.environ['TRAVIS_TAG'].startswith('patch-'): + __version__ = os.environ['TRAVIS_TAG'].split('-')[1] + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return platform.system() == 'Darwin' + + +DEPENDENCIES = [ + 'numpy<1.15.0,>=1.8.2', + 'requests>=2.20.0', + 'graphviz<0.9.0,>=0.8.1' +] + +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True) +shutil.rmtree(os.path.join(CURRENT_DIR, 'dmlc_tracker'), ignore_errors=True) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/python/mxnet'), + os.path.join(CURRENT_DIR, 'mxnet')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dmlc-core/tracker/dmlc_tracker'), + os.path.join(CURRENT_DIR, 'dmlc_tracker')) +shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet')) + +# copy tools to mxnet package +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet/tools'), ignore_errors=True) +os.mkdir(os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/launch.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/im2rec.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/kill-mxnet.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/parse_log.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/diagnose.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/tools/caffe_converter'), os.path.join(CURRENT_DIR, 'mxnet/tools/caffe_converter')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/tools/bandwidth'), os.path.join(CURRENT_DIR, 'mxnet/tools/bandwidth')) + +# copy headers to mxnet package +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet/include'), ignore_errors=True) +os.mkdir(os.path.join(CURRENT_DIR, 'mxnet/include')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/include/mxnet'), + os.path.join(CURRENT_DIR, 'mxnet/include/mxnet')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dlpack/include/dlpack'), + os.path.join(CURRENT_DIR, 'mxnet/include/dlpack')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dmlc-core/include/dmlc'), + os.path.join(CURRENT_DIR, 'mxnet/include/dmlc')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/mshadow/mshadow'), + os.path.join(CURRENT_DIR, 'mxnet/include/mshadow')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/tvm/nnvm/include/nnvm'), + os.path.join(CURRENT_DIR, 'mxnet/include/nnvm')) + +package_name = 'mxnet' + +variant = os.environ['mxnet_variant'].upper() +if variant != 'CPU': + package_name = 'mxnet_{0}'.format(variant.lower()) + +with open('doc/PYPI_README.md') as readme_file: + long_description = readme_file.read() + +with open('doc/{0}_ADDITIONAL.md'.format(variant)) as variant_doc: + long_description = long_description + variant_doc.read() + +# pypi only supports rst, so use pandoc to convert +import pypandoc +if platform.system() == 'Darwin': + pypandoc.download_pandoc() +long_description = pypandoc.convert_text(long_description, 'rst', 'md') +short_description = 'MXNet is an ultra-scalable deep learning framework.' +libraries = [] +if variant == 'CPU': + libraries.append('openblas') +else: + if variant.startswith('CU92'): + libraries.append('CUDA-9.2') + elif variant.startswith('CU91'): + libraries.append('CUDA-9.1') + elif variant.startswith('CU90'): + libraries.append('CUDA-9.0') + elif variant.startswith('CU80'): + libraries.append('CUDA-8.0') + elif variant.startswith('CU75'): + libraries.append('CUDA-7.5') + if variant.endswith('MKL'): + libraries.append('MKLDNN') + +short_description += ' This version uses {0}.'.format(' and '.join(libraries)) + +package_data = {'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))], + 'dmlc_tracker': []} +if variant.endswith('MKL'): + if platform.system() == 'Darwin': + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmklml.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libiomp5.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmkldnn.0.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libmklml.dylib') + package_data['mxnet'].append('mxnet/libiomp5.dylib') + package_data['mxnet'].append('mxnet/libmkldnn.0.dylib') + else: + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmklml_intel.so'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libiomp5.so'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmkldnn.so.0'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libmklml_intel.so') + package_data['mxnet'].append('mxnet/libiomp5.so') + package_data['mxnet'].append('mxnet/libmkldnn.so.0') + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), '../MKLML_LICENSE'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/MKLML_LICENSE') +if platform.system() == 'Linux': + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libgfortran.so.3'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libgfortran.so.3') + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libquadmath.so.0'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libquadmath.so.0') + +from mxnet.base import _generate_op_module_signature +from mxnet.ndarray.register import _generate_ndarray_function_code +from mxnet.symbol.register import _generate_symbol_function_code +_generate_op_module_signature('mxnet', 'symbol', _generate_symbol_function_code) +_generate_op_module_signature('mxnet', 'ndarray', _generate_ndarray_function_code) + +setup(name=package_name, + version=__version__, + long_description=long_description, + description=short_description, + zip_safe=False, + packages=find_packages(), + package_data=package_data, + include_package_data=True, + install_requires=DEPENDENCIES, + distclass=BinaryDistribution, + license='Apache 2.0', + classifiers=[ # https://pypi.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: C++', + 'Programming Language :: Cython', + 'Programming Language :: Other', # R, Scala + 'Programming Language :: Perl', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + url='https://github.com/apache/incubator-mxnet') diff --git a/tools/pip_package/README.md b/tools/pip_package/README.md deleted file mode 100644 index f289c98b715..00000000000 --- a/tools/pip_package/README.md +++ /dev/null @@ -1,9 +0,0 @@ -MXNet Python Package -==================== -MXNet is a deep learning framework designed for both *efficiency* and *flexibility*. -It allows you to mix the flavours of deep learning programs together to maximize the efficiency and your productivity. - - -Installation ------------- -To install, check [Build Instruction](http://mxnet.io/get_started/setup.html) diff --git a/tools/pip_package/make_pip_package.sh b/tools/pip_package/make_pip_package.sh deleted file mode 100755 index 46b4938b078..00000000000 --- a/tools/pip_package/make_pip_package.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# Assuming the script is run at mxnet/tools/pip_package -# This script builds from scratch the dependencies of mxnet into static -# librareis and statically links them to produce a (mostly) standalone -# libmxnet.so, then packages it into the python wheel. -# It assumes the build environment to be a sandbox that doesn't have the .so -# objects for the dependencies, i.e. zlib, openblas, libjpeg, libpng, libtiff -# and opencv. - -# Install necessary build tools -if [ -n "$(command -v apt-get)" ]; then - sudo apt-get update; - sudo apt-get install -y build-essential git python-pip zip pkg-config cmake -elif [ -n "$(command -v yum)" ]; then - sudo yum install -y cmake - sudo yum groupinstall -y "Development Tools" - sudo yum install -y python27 python27-setuptools python27-tools python-pip -else - echo "Need a package manager to install build tools, e.g. apt/yum" - exit 1 -fi -sudo pip install -U pip setuptools wheel - -# Set up path as temporary working directory -DEPS_PATH=$PWD/../../deps -mkdir $DEPS_PATH - -# Dependencies can be updated here. Be sure to verify the download link before -# changing. The dependencies are: -ZLIB_VERSION=1.2.6 -OPENBLAS_VERSION=0.2.19 -JPEG_VERSION=8.4.0 -PNG_VERSION=1.5.10 -TIFF_VERSION=3.8.2 -OPENCV_VERSION=2.4.13 - -# Setup path to dependencies -export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$PKG_CONFIG_PATH -export CPATH=$DEPS_PATH/include:$CPATH - -# Position Independent code must be turned on for statically linking .a -export CC="gcc -fPIC" -export CXX="g++ -fPIC" - -# Download and build zlib -curl -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip -unzip $DEPS_PATH/zlib.zip -d $DEPS_PATH -mkdir $DEPS_PATH/zlib-$ZLIB_VERSION/build -cd $DEPS_PATH/zlib-$ZLIB_VERSION/build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ - -D BUILD_SHARED_LIBS=OFF .. -make -j$(nproc) -make install -cd - - -# download and build openblas -curl -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip -unzip $DEPS_PATH/openblas.zip -d $DEPS_PATH -cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION -make FC=gfortran -j $(($(nproc) + 1)) -make PREFIX=$DEPS_PATH install -cd - -ln -s $DEPS_PATH/lib/libopenblas_haswellp-r0.2.19.a $DEPS_PATH/lib/libcblas.a - -# download and build libjpeg -curl -L https://github.com/LuaDist/libjpeg/archive/$JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip -unzip $DEPS_PATH/libjpeg.zip -d $DEPS_PATH -cd $DEPS_PATH/libjpeg-$JPEG_VERSION -./configure --disable-shared --prefix=$DEPS_PATH -make -j$(nproc) -make test -make install -cd - - -# download and build libpng -curl -L https://github.com/LuaDist/libpng/archive/$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip -unzip $DEPS_PATH/libpng.zip -d $DEPS_PATH -mkdir $DEPS_PATH/libpng-$PNG_VERSION/build -cd $DEPS_PATH/libpng-$PNG_VERSION/build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ - -D PNG_CONFIGURE_LIBPNG=-fPIC \ - -D BUILD_SHARED_LIBS=OFF .. -make -j$(nproc) -make install -cd - - -# download and build libtiff -curl -L https://github.com/LuaDist/libtiff/archive/$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip -unzip $DEPS_PATH/libtiff.zip -d $DEPS_PATH -cd $DEPS_PATH/libtiff-$TIFF_VERSION -./configure --disable-shared --prefix=$DEPS_PATH -make -j$(nproc) -make install -cd - - -# download and build opencv since we need the static library -curl -L https://github.com/Itseez/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip -unzip $DEPS_PATH/opencv.zip -d $DEPS_PATH -mkdir $DEPS_PATH/opencv-$OPENCV_VERSION/build -cd $DEPS_PATH/opencv-$OPENCV_VERSION/build -cmake -D WITH_1394=OFF \ - -D WITH_AVFOUNDATION=OFF \ - -D WITH_CUDA=OFF \ - -D WITH_VTK=OFF \ - -D WITH_CUFFT=OFF \ - -D WITH_CUBLAS=OFF \ - -D WITH_NVCUVID=OFF \ - -D WITH_EIGEN=ON \ - -D WITH_VFW=OFF \ - -D WITH_FFMPEG=OFF \ - -D WITH_GSTREAMER=OFF \ - -D WITH_GTK=OFF \ - -D WITH_JASPER=OFF \ - -D WITH_JPEG=ON \ - -D WITH_PNG=ON \ - -D WITH_QUICKTIME=OFF \ - -D WITH_TBB=ON \ - -D WITH_TIFF=OFF \ - -D WITH_V4L=OFF \ - -D WITH_LIBV4L=OFF \ - -D WITH_DSHOW=OFF \ - -D WITH_MSMF=OFF \ - -D WITH_OPENCL=OFF \ - -D WITH_OPENCLAMDFFT=OFF \ - -D WITH_OPENCLAMDBLAS=OFF \ - -D BUILD_SHARED_LIBS=OFF \ - -D BUILD_opencv_apps=OFF \ - -D BUILD_opencv_gpu=OFF \ - -D BUILD_opencv_video=OFF \ - -D BUILD_opencv_contrib=OFF \ - -D BUILD_opencv_nonfree=OFF \ - -D BUILD_opencv_flann=OFF \ - -D BUILD_opencv_features2d=OFF \ - -D BUILD_opencv_calib3d=OFF \ - -D BUILD_opencv_objdetect=OFF \ - -D BUILD_opencv_ml=OFF \ - -D BUILD_opencv_photo=OFF \ - -D BUILD_DOCS=OFF \ - -D BUILD_PACKAGE=OFF \ - -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. -make -j $(nproc) -make install # user will always have access to home, so no sudo needed -cd - - -# Although .so building is explicitly turned off for most libraries, sometimes -# they still get created. So, remove them just to make sure they don't -# interfere, or otherwise we might get libmxnet.so that is not self-contained. -rm $DEPS_PATH/{lib,lib64}/*.{so,so.0} - -# Go to the parent path and build mxnet -cd ../../ -cp make/pip_$(uname | tr '[:upper:]' '[:lower:]')_cpu.mk config.mk -make -j $(nproc) - -# Generate wheel. The output is in the mxnet/tools/pip_package/dist path. -cd tools/pip_package -python setup.py bdist_wheel diff --git a/tools/pip_package/setup.py b/tools/pip_package/setup.py deleted file mode 100644 index e4bf48236bd..00000000000 --- a/tools/pip_package/setup.py +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# pylint: disable=invalid-name, exec-used -"""Setup mxnet package.""" -from __future__ import absolute_import -import os -import shutil - -from setuptools import setup, find_packages -from setuptools.dist import Distribution - -# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py -# Will be invoked which introduces dependences -CURRENT_DIR = os.path.dirname(__file__) -libinfo_py = os.path.join(CURRENT_DIR, '../../python/mxnet/libinfo.py') -libinfo = {'__file__': libinfo_py} -exec(compile(open(libinfo_py, "rb").read(), libinfo_py, 'exec'), libinfo, libinfo) - -LIB_PATH = libinfo['find_lib_path']() -__version__ = libinfo['__version__'] - -class BinaryDistribution(Distribution): - def has_ext_modules(self): - return True - - -DEPENDENCIES = [ - 'numpy', -] - -shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True) -shutil.copytree(os.path.join(CURRENT_DIR, '../../python/mxnet'), - os.path.join(CURRENT_DIR, 'mxnet')) -shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet')) - -setup(name='mxnet', - version=__version__, - description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), - zip_safe=False, - packages=find_packages(), - package_data={'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))]}, - include_package_data=True, - install_requires=DEPENDENCIES, - distclass=BinaryDistribution, - url='https://github.com/dmlc/mxnet') ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
