This is an automated email from the ASF dual-hosted git repository.

marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
     new 1b60478  CI scripts refinements. Separate Py2 and Py3 installs cripts. 
Fix perms. (#12125)
1b60478 is described below

commit 1b60478db2c66ef82f5b3ed48b12832d6da72e61
Author: Pedro Larroy <928489+lar...@users.noreply.github.com>
AuthorDate: Sat Aug 11 16:20:29 2018 +0200

    CI scripts refinements. Separate Py2 and Py3 installs cripts. Fix perms. 
(#12125)
---
 ci/build.py                                        | 97 +++++++++++++---------
 ci/docker/Dockerfile.build.android_armv7           |  0
 ci/docker/Dockerfile.build.android_armv8           |  0
 ci/docker/Dockerfile.build.armv6                   |  0
 ci/docker/Dockerfile.build.armv7                   |  0
 ci/docker/Dockerfile.build.armv8                   |  0
 ci/docker/Dockerfile.build.centos7_cpu             |  0
 ci/docker/Dockerfile.build.centos7_gpu             |  0
 ci/docker/Dockerfile.build.jetson                  |  0
 ci/docker/Dockerfile.build.ubuntu_base_cpu         |  0
 ci/docker/Dockerfile.build.ubuntu_base_gpu         |  0
 ci/docker/Dockerfile.build.ubuntu_blc              |  9 +-
 ci/docker/Dockerfile.build.ubuntu_build_cuda       | 14 +++-
 ci/docker/Dockerfile.build.ubuntu_cpu              |  7 +-
 ci/docker/Dockerfile.build.ubuntu_gpu              |  7 +-
 ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt     | 10 ++-
 ci/docker/Dockerfile.build.ubuntu_nightly_cpu      |  7 +-
 ci/docker/Dockerfile.build.ubuntu_nightly_gpu      |  7 +-
 ci/docker/Dockerfile.build.ubuntu_rat              |  0
 .../{ubuntu_python.sh => ubuntu_python2.sh}        |  6 +-
 .../{ubuntu_python.sh => ubuntu_python3.sh}        |  6 +-
 ci/docker/runtime_functions.sh                     |  1 +
 ci/test_docker_cache.py                            |  0
 23 files changed, 111 insertions(+), 60 deletions(-)

diff --git a/ci/build.py b/ci/build.py
index a9d6a63..0a1ad4c 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -43,6 +43,43 @@ from util import *
 
 CCACHE_MAXSIZE = '500G'
 
+
+
+def retry(ExceptionToCheck, tries=4, delay_s=1, backoff=2):
+    """Retry calling the decorated function using an exponential backoff.
+
+    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+
+    :param ExceptionToCheck: the exception to check. may be a tuple of
+        exceptions to check
+    :type ExceptionToCheck: Exception or tuple
+    :param tries: number of times to try (not retry) before giving up
+    :type tries: int
+    :param delay_s: initial delay between retries in seconds
+    :type delay_s: int
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+        each retry
+    :type backoff: int
+    """
+    import time
+    from functools import wraps
+    def decorated_retry(f):
+        @wraps(f)
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay_s
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except ExceptionToCheck as e:
+                    logging.warning("Exception: %s, Retrying in %d 
seconds...", str(e), mdelay)
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+        return f_retry  # true decorator
+    return decorated_retry
+
 def under_ci() -> bool:
     """:return: True if we run in Jenkins."""
     return 'JOB_NAME' in os.environ
@@ -77,9 +114,8 @@ def build_docker(platform: str, docker_binary: str, 
registry: str, num_retries:
     :param num_retries: Number of retries to build the docker image
     :return: Id of the top level image
     """
-
     tag = get_docker_tag(platform=platform, registry=registry)
-    logging.info("Building container tagged '%s' with %s", tag, docker_binary)
+    logging.info("Building docker container tagged '%s' with %s", tag, 
docker_binary)
     #
     # We add a user with the same group as the executing non-root user so 
files created in the
     # container match permissions of the local user. Same for the group.
@@ -91,40 +127,24 @@ def build_docker(platform: str, docker_binary: str, 
registry: str, num_retries:
     # docker pull see: docker_cache.load_docker_cache
     #
     # This doesn't work with multi head docker files.
-    # 
-
-    for i in range(num_retries):
-        logging.info('%d out of %d tries to build the docker image.', i + 1, 
num_retries)
-
-        cmd = [docker_binary, "build",
-               "-f", get_dockerfile(platform),
-               "--build-arg", "USER_ID={}".format(os.getuid()),
-               "--build-arg", "GROUP_ID={}".format(os.getgid()),
-               "--cache-from", tag,
-               "-t", tag,
-               "docker"]
+    #
+    cmd = [docker_binary, "build",
+           "-f", get_dockerfile(platform),
+           "--build-arg", "USER_ID={}".format(os.getuid()),
+           "--build-arg", "GROUP_ID={}".format(os.getgid()),
+           "--cache-from", tag,
+           "-t", tag,
+           "docker"]
+
+    @retry(subprocess.CalledProcessError, tries=num_retries)
+    def run_cmd():
         logging.info("Running command: '%s'", ' '.join(cmd))
-        try:
-            check_call(cmd)
-            # Docker build was successful. Call break to break out of the 
retry mechanism
-            break
-        except subprocess.CalledProcessError as e:
-            saved_exception = e
-            logging.error('Failed to build docker image')
-            # Building the docker image failed. Call continue to trigger the 
retry mechanism
-            continue
-    else:
-        # Num retries exceeded
-        logging.exception('Exception during build of docker image', 
saved_exception)
-        logging.fatal('Failed to build the docker image, aborting...')
-        sys.exit(1)
+        check_call(cmd)
 
+    run_cmd()
     # Get image id by reading the tag. It's guaranteed (except race condition) 
that the tag exists. Otherwise, the
     # check_call would have failed
-    image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
-    if not image_id:
-        raise FileNotFoundError('Unable to find docker image id matching with 
{}'.format(tag))
-    return image_id
+    return _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
 
 
 def _get_local_image_id(docker_binary, docker_tag):
@@ -136,6 +156,8 @@ def _get_local_image_id(docker_binary, docker_tag):
     cmd = [docker_binary, "images", "-q", docker_tag]
     image_id_b = subprocess.check_output(cmd)
     image_id = image_id_b.decode('utf-8').strip()
+    if not image_id:
+        raise RuntimeError('Unable to find docker image id matching with tag 
{}'.format(tag))
     return image_id
 
 
@@ -186,7 +208,7 @@ def container_run(platform: str,
                '-e', "CCACHE_LOGFILE=/tmp/ccache.log",  # a container-scoped 
log, useful for ccache verification.
                tag]
     runlist.extend(command)
-    cmd = '\\\n\t'.join(runlist)
+    cmd = ' \\\n\t'.join(runlist)
     ret = 0
     if not dry_run and not interactive:
         logging.info("Running %s in container %s", command, tag)
@@ -199,14 +221,14 @@ def container_run(platform: str,
         # -ti can't be after the tag, as is interpreted as a command so hook 
it up after the -u argument
         idx = into_cmd.index('-u') + 2
         into_cmd[idx:idx] = ['-ti']
-        cmd = '\\\n\t'.join(into_cmd)
+        cmd = ' \\\n\t'.join(into_cmd)
         logging.info("Executing:\n%s\n", cmd)
         docker_run_cmd = ' '.join(into_cmd)
         ret = call(into_cmd)
 
     if not dry_run and not interactive and ret != 0:
         logging.error("Running of command in container failed (%s):\n%s\n", 
ret, cmd)
-        logging.error("You can get into the container by adding the -i option")
+        logging.error("You can get into the container by adding the -i option 
to this script")
         raise subprocess.CalledProcessError(ret, cmd)
 
     return docker_run_cmd
@@ -303,7 +325,6 @@ def main() -> int:
     command = list(chain(*args.command))
     docker_binary = get_docker_binary(args.nvidiadocker)
     shared_memory_size = args.shared_memory_size
-    num_docker_build_retires = args.docker_build_retries
 
     if args.list:
         list_platforms()
@@ -312,7 +333,7 @@ def main() -> int:
         tag = get_docker_tag(platform=platform, registry=args.docker_registry)
         if use_cache():
             load_docker_cache(tag=tag, docker_registry=args.docker_registry)
-        build_docker(platform, docker_binary, registry=args.docker_registry, 
num_retries=num_docker_build_retires)
+        build_docker(platform, docker_binary, registry=args.docker_registry, 
num_retries=args.docker_build_retries)
         if args.build_only:
             logging.warning("Container was just built. Exiting due to 
build-only.")
             return 0
@@ -346,7 +367,7 @@ def main() -> int:
             tag = get_docker_tag(platform=platform, 
registry=args.docker_registry)
             if use_cache():
                 load_docker_cache(tag=tag, 
docker_registry=args.docker_registry)
-            build_docker(platform, docker_binary, args.docker_registry, 
num_retries=num_docker_build_retires)
+            build_docker(platform, docker_binary, args.docker_registry, 
num_retries=args.docker_build_retries)
             if args.build_only:
                 continue
             build_platform = "build_{}".format(platform)
diff --git a/ci/docker/Dockerfile.build.android_armv7 
b/ci/docker/Dockerfile.build.android_armv7
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.android_armv8 
b/ci/docker/Dockerfile.build.android_armv8
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.centos7_cpu 
b/ci/docker/Dockerfile.build.centos7_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.centos7_gpu 
b/ci/docker/Dockerfile.build.centos7_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.jetson 
b/ci/docker/Dockerfile.build.jetson
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu 
b/ci/docker/Dockerfile.build.ubuntu_base_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu 
b/ci/docker/Dockerfile.build.ubuntu_base_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_blc 
b/ci/docker/Dockerfile.build.ubuntu_blc
old mode 100755
new mode 100644
index 294740c..208cba2
--- a/ci/docker/Dockerfile.build.ubuntu_blc
+++ b/ci/docker/Dockerfile.build.ubuntu_blc
@@ -24,8 +24,13 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
 COPY install/ubuntu_npm_blc.sh /work/
 RUN /work/ubuntu_npm_blc.sh
 
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda 
b/ci/docker/Dockerfile.build.ubuntu_build_cuda
old mode 100755
new mode 100644
index 9ed0cbb..19e9265
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -27,20 +27,30 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
+
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/
 RUN /work/ubuntu_scala.sh
+
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh
+
 COPY install/ubuntu_perl.sh /work/
 RUN /work/ubuntu_perl.sh
+
 COPY install/ubuntu_clang.sh /work/
 RUN /work/ubuntu_clang.sh
+
 COPY install/ubuntu_mklml.sh /work/
 RUN /work/ubuntu_mklml.sh
 
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu 
b/ci/docker/Dockerfile.build.ubuntu_cpu
old mode 100755
new mode 100644
index 58a8e9a..76df3df
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu 
b/ci/docker/Dockerfile.build.ubuntu_gpu
old mode 100755
new mode 100644
index de38948..8b4cd1e
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt 
b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
old mode 100755
new mode 100644
index 255da31..3f0bbc6
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
@@ -24,10 +24,16 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
+
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
 COPY install/tensorrt.sh /work
 RUN /work/tensorrt.sh
 
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu 
b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
old mode 100755
new mode 100644
index 58ff33e..c803188
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu 
b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
old mode 100755
new mode 100644
index 017bade..c0e31e2
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_rat 
b/ci/docker/Dockerfile.build.ubuntu_rat
old mode 100755
new mode 100644
diff --git a/ci/docker/install/ubuntu_python.sh 
b/ci/docker/install/ubuntu_python2.sh
similarity index 84%
copy from ci/docker/install/ubuntu_python.sh
copy to ci/docker/install/ubuntu_python2.sh
index e71cac8..f0526e2 100755
--- a/ci/docker/install/ubuntu_python.sh
+++ b/ci/docker/install/ubuntu_python2.sh
@@ -22,12 +22,10 @@
 
 set -ex
 # install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev python3-dev virtualenv
+apt-get install -y python-dev virtualenv wget
 
 # the version of the pip shipped with ubuntu may be too lower, install a 
recent version here
 wget -nv https://bootstrap.pypa.io/get-pip.py
-python3 get-pip.py
 python2 get-pip.py
 
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' 
nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' 
nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' 
nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
diff --git a/ci/docker/install/ubuntu_python.sh 
b/ci/docker/install/ubuntu_python3.sh
similarity index 84%
rename from ci/docker/install/ubuntu_python.sh
rename to ci/docker/install/ubuntu_python3.sh
index e71cac8..1dad5a7 100755
--- a/ci/docker/install/ubuntu_python.sh
+++ b/ci/docker/install/ubuntu_python3.sh
@@ -22,12 +22,10 @@
 
 set -ex
 # install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev python3-dev virtualenv
+apt-get install -y python3-dev virtualenv wget
 
 # the version of the pip shipped with ubuntu may be too lower, install a 
recent version here
 wget -nv https://bootstrap.pypa.io/get-pip.py
 python3 get-pip.py
-python2 get-pip.py
 
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' 
nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' 
nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' 
nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 815eae9..2a813c8 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -1005,6 +1005,7 @@ broken_link_checker() {
     ./tests/nightly/broken_link_checker_test/broken_link_checker.sh
 }
 
+
 ##############################################################
 # MAIN
 #
diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py
old mode 100644
new mode 100755

Reply via email to