Revert "CI scripts refinements. Separate Py2 and Py3 installs cripts.…

ci/build.py

@@ -43,43 +43,6 @@
 
 CCACHE_MAXSIZE = '500G'
 
-
-
-def retry(ExceptionToCheck, tries=4, delay_s=1, backoff=2):
-    """Retry calling the decorated function using an exponential backoff.
-
-    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
-    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
-
-    :param ExceptionToCheck: the exception to check. may be a tuple of
-        exceptions to check
-    :type ExceptionToCheck: Exception or tuple
-    :param tries: number of times to try (not retry) before giving up
-    :type tries: int
-    :param delay_s: initial delay between retries in seconds
-    :type delay_s: int
-    :param backoff: backoff multiplier e.g. value of 2 will double the delay
-        each retry
-    :type backoff: int
-    """
-    import time
-    from functools import wraps
-    def decorated_retry(f):
-        @wraps(f)
-        def f_retry(*args, **kwargs):
-            mtries, mdelay = tries, delay_s
-            while mtries > 1:
-                try:
-                    return f(*args, **kwargs)
-                except ExceptionToCheck as e:
-                    logging.warning("Exception: %s, Retrying in %d seconds...", str(e), mdelay)
-                    time.sleep(mdelay)
-                    mtries -= 1
-                    mdelay *= backoff
-            return f(*args, **kwargs)
-        return f_retry  # true decorator
-    return decorated_retry
-
 def under_ci() -> bool:
     """:return: True if we run in Jenkins."""
     return 'JOB_NAME' in os.environ
@@ -114,8 +77,9 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
     :param num_retries: Number of retries to build the docker image
     :return: Id of the top level image
     """
+
     tag = get_docker_tag(platform=platform, registry=registry)
-    logging.info("Building docker container tagged '%s' with %s", tag, docker_binary)
+    logging.info("Building container tagged '%s' with %s", tag, docker_binary)
     #
     # We add a user with the same group as the executing non-root user so files created in the
     # container match permissions of the local user. Same for the group.
@@ -127,24 +91,40 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
     # docker pull see: docker_cache.load_docker_cache
     #
     # This doesn't work with multi head docker files.
-    #
-    cmd = [docker_binary, "build",
-           "-f", get_dockerfile(platform),
-           "--build-arg", "USER_ID={}".format(os.getuid()),
-           "--build-arg", "GROUP_ID={}".format(os.getgid()),
-           "--cache-from", tag,
-           "-t", tag,
-           "docker"]
-
-    @retry(subprocess.CalledProcessError, tries=num_retries)
-    def run_cmd():
+    # 
+
+    for i in range(num_retries):
+        logging.info('%d out of %d tries to build the docker image.', i + 1, num_retries)
+
+        cmd = [docker_binary, "build",
+               "-f", get_dockerfile(platform),
+               "--build-arg", "USER_ID={}".format(os.getuid()),
+               "--build-arg", "GROUP_ID={}".format(os.getgid()),
+               "--cache-from", tag,
+               "-t", tag,
+               "docker"]
         logging.info("Running command: '%s'", ' '.join(cmd))
-        check_call(cmd)
+        try:
+            check_call(cmd)
+            # Docker build was successful. Call break to break out of the retry mechanism
+            break
+        except subprocess.CalledProcessError as e:
+            saved_exception = e
+            logging.error('Failed to build docker image')
+            # Building the docker image failed. Call continue to trigger the retry mechanism
+            continue
+    else:
+        # Num retries exceeded
+        logging.exception('Exception during build of docker image', saved_exception)
+        logging.fatal('Failed to build the docker image, aborting...')
+        sys.exit(1)
 
-    run_cmd()
     # Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
     # check_call would have failed
-    return _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
+    image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
+    if not image_id:
+        raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
+    return image_id
 
 
 def _get_local_image_id(docker_binary, docker_tag):
@@ -156,8 +136,6 @@ def _get_local_image_id(docker_binary, docker_tag):
     cmd = [docker_binary, "images", "-q", docker_tag]
     image_id_b = subprocess.check_output(cmd)
     image_id = image_id_b.decode('utf-8').strip()
-    if not image_id:
-        raise RuntimeError('Unable to find docker image id matching with tag {}'.format(tag))
     return image_id
 
 
@@ -208,7 +186,7 @@ def container_run(platform: str,
                '-e', "CCACHE_LOGFILE=/tmp/ccache.log",  # a container-scoped log, useful for ccache verification.
                tag]
     runlist.extend(command)
-    cmd = ' \\\n\t'.join(runlist)
+    cmd = '\\\n\t'.join(runlist)
     ret = 0
     if not dry_run and not interactive:
         logging.info("Running %s in container %s", command, tag)
@@ -221,14 +199,14 @@ def container_run(platform: str,
         # -ti can't be after the tag, as is interpreted as a command so hook it up after the -u argument
         idx = into_cmd.index('-u') + 2
         into_cmd[idx:idx] = ['-ti']
-        cmd = ' \\\n\t'.join(into_cmd)
+        cmd = '\\\n\t'.join(into_cmd)
         logging.info("Executing:\n%s\n", cmd)
         docker_run_cmd = ' '.join(into_cmd)
         ret = call(into_cmd)
 
     if not dry_run and not interactive and ret != 0:
         logging.error("Running of command in container failed (%s):\n%s\n", ret, cmd)
-        logging.error("You can get into the container by adding the -i option to this script")
+        logging.error("You can get into the container by adding the -i option")
         raise subprocess.CalledProcessError(ret, cmd)
 
     return docker_run_cmd
@@ -325,6 +303,7 @@ def use_cache():
     command = list(chain(*args.command))
     docker_binary = get_docker_binary(args.nvidiadocker)
     shared_memory_size = args.shared_memory_size
+    num_docker_build_retires = args.docker_build_retries
 
     if args.list:
         list_platforms()
@@ -333,7 +312,7 @@ def use_cache():
         tag = get_docker_tag(platform=platform, registry=args.docker_registry)
         if use_cache():
             load_docker_cache(tag=tag, docker_registry=args.docker_registry)
-        build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=args.docker_build_retries)
+        build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=num_docker_build_retires)
         if args.build_only:
             logging.warning("Container was just built. Exiting due to build-only.")
             return 0
@@ -367,7 +346,7 @@ def use_cache():
             tag = get_docker_tag(platform=platform, registry=args.docker_registry)
             if use_cache():
                 load_docker_cache(tag=tag, docker_registry=args.docker_registry)
-            build_docker(platform, docker_binary, args.docker_registry, num_retries=args.docker_build_retries)
+            build_docker(platform, docker_binary, args.docker_registry, num_retries=num_docker_build_retires)
             if args.build_only:
                 continue
             build_platform = "build_{}".format(platform)

ci/docker/Dockerfile.build.ubuntu_blc

@@ -24,13 +24,8 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
-
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
-
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
 COPY install/ubuntu_npm_blc.sh /work/
 RUN /work/ubuntu_npm_blc.sh
 

ci/docker/Dockerfile.build.ubuntu_build_cuda

@@ -27,30 +27,20 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
-
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
-
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
-
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/
 RUN /work/ubuntu_scala.sh
-
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh
-
 COPY install/ubuntu_perl.sh /work/
 RUN /work/ubuntu_perl.sh
-
 COPY install/ubuntu_clang.sh /work/
 RUN /work/ubuntu_clang.sh
-
 COPY install/ubuntu_mklml.sh /work/
 RUN /work/ubuntu_mklml.sh
 

ci/docker/Dockerfile.build.ubuntu_cpu

@@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/

ci/docker/Dockerfile.build.ubuntu_gpu

@@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/

ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt

@@ -24,16 +24,10 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
-
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
-
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
-
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
 COPY install/tensorrt.sh /work
 RUN /work/tensorrt.sh
 

ci/docker/Dockerfile.build.ubuntu_nightly_cpu

@@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/

ci/docker/Dockerfile.build.ubuntu_nightly_gpu

@@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/

ci/docker/install/ubuntu_python2.sh → ci/docker/install/ubuntu_python.sh

@@ -22,10 +22,12 @@
 
 set -ex
 # install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev virtualenv wget
+apt-get install -y python-dev python3-dev virtualenv
 
 # the version of the pip shipped with ubuntu may be too lower, install a recent version here
 wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
 python2 get-pip.py
 
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3

ci/docker/runtime_functions.sh

@@ -994,7 +994,6 @@ broken_link_checker() {
     ./tests/nightly/broken_link_checker_test/broken_link_checker.sh
 }
 
-
 ##############################################################
 # MAIN
 #