From 1f178c7fe8a7b9e7ddc81431228a1706f715af7f Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Tue, 29 May 2018 07:42:44 +0000
Subject: [PATCH 01/14] [MXNET-472] Add ccache support to docker builds

---
 Makefile                                  |  9 ++-
 ci/build.py                               | 88 +++++++++++++++++------
 ci/docker/Dockerfile.build.armv6          |  7 +-
 ci/docker/Dockerfile.build.centos7.ccache | 42 +++++++++++
 ci/docker/Dockerfile.build.jetson         | 14 ++--
 ci/docker/Dockerfile.build.ubuntu.ccache  | 46 ++++++++++++
 ci/docker/runtime_functions.sh            |  5 +-
 make/config.mk                            |  6 ++
 make/crosscompile.jetson.mk               |  6 +-
 9 files changed, 192 insertions(+), 31 deletions(-)
 create mode 100644 ci/docker/Dockerfile.build.centos7.ccache
 create mode 100644 ci/docker/Dockerfile.build.ubuntu.ccache

diff --git a/Makefile b/Makefile
index 03212841fa31..2c8560c83d2f 100644
--- a/Makefile
+++ b/Makefile
@@ -79,7 +79,7 @@ include $(DMLC_CORE)/make/dmlc.mk
 
 # all tge possible warning tread
 WARNFLAGS= -Wall -Wsign-compare
-CFLAGS = -DMSHADOW_FORCE_STREAM $(WARNFLAGS)
+CFLAGS += -DMSHADOW_FORCE_STREAM $(WARNFLAGS)
 
 ifeq ($(DEV), 1)
 	CFLAGS += -g -Werror
@@ -92,8 +92,13 @@ ifeq ($(DEBUG), 1)
 else
 	CFLAGS += -O3 -DNDEBUG=1
 endif
+<<<<<<< HEAD
 CFLAGS += -I$(TPARTYDIR)/mshadow/ -I$(TPARTYDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(TPARTYDIR)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
 LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
+=======
+CFLAGS += -I$(TPARTYDIR)/mshadow/ -I$(TPARTYDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(NNVM_PATH)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
+LDFLAGS += -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
+>>>>>>> [MXNET-472] Add ccache support to docker builds
 ifeq ($(DEBUG), 1)
 	NVCCFLAGS += -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
 else
@@ -477,7 +482,7 @@ endif
 $(PS_PATH)/build/libps.a: PSLITE
 
 PSLITE:
-	$(MAKE) CXX=$(CXX) DEPS_PATH=$(DEPS_PATH) -C $(PS_PATH) ps
+	$(MAKE) CXX="$(CXX)" DEPS_PATH="$(DEPS_PATH)" -C $(PS_PATH) ps
 
 $(DMLC_CORE)/libdmlc.a: DMLCCORE
 
diff --git a/ci/build.py b/ci/build.py
index e52fa794bc9b..7ed1b9416b27 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -33,6 +33,7 @@
 import shutil
 import subprocess
 import sys
+import tempfile
 from copy import deepcopy
 from itertools import chain
 from subprocess import call, check_call
@@ -72,11 +73,11 @@ def build_docker(platform: str, docker_binary: str, registry: str) -> None:
     tag = get_docker_tag(platform=platform, registry=registry)
     logging.info("Building container tagged '%s' with %s", tag, docker_binary)
     cmd = [docker_binary, "build",
-        "-f", get_dockerfile(platform),
-        "--build-arg", "USER_ID={}".format(os.getuid()),
-        "--cache-from", tag,
-        "-t", tag,
-        "docker"]
+           "-f", get_dockerfile(platform),
+           "--build-arg", "USER_ID={}".format(os.getuid()),
+           "--cache-from", tag,
+           "-t", tag,
+           "docker"]
     logging.info("Running command: '%s'", ' '.join(cmd))
     check_call(cmd)
 
@@ -87,6 +88,31 @@ def build_docker(platform: str, docker_binary: str, registry: str) -> None:
         raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
     return image_id
 
+def build_ccache(platform: str, docker_binary: str) -> None:
+    """
+    Build a ccache container for the given platform
+    :param platform: Platform
+    :param docker_binary: docker binary to use (docker/nvidia-docker)
+    :return: Id of the top level image
+    """
+
+    tag = "ccache/build.{}".format(platform)
+    logging.info("Building ccache container tagged '%s' with %s", tag, docker_binary)
+    cmd = [docker_binary, "build",
+           "-f", get_dockerfile("{}.ccache".format(platform)),
+           "--rm=true", # We only need the binary
+           "--cache-from", tag,
+           "-t", tag,
+           "docker"]
+    logging.info("Running command: '%s'", ' '.join(cmd))
+    check_call(cmd)
+
+    # Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
+    # check_call would have failed
+    image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
+    if not image_id:
+        raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
+    return image_id
 
 def _get_local_image_id(docker_binary, docker_tag):
     """
@@ -115,11 +141,19 @@ def is_mxnet_root(path: str) -> bool:
 def buildir() -> str:
     return os.path.join(get_mxnet_root(), "build")
 
+def default_ccache_dir() -> str:
+    if 'CCACHE_DIR' in os.environ:
+        ccache_dir = os.path.realpath(os.environ['CCACHE_DIR'])
+        os.makedirs(ccache_dir, exist_ok=True)
+        return ccache_dirpython
+    # Share ccache across containers (should we have a separate dir per platform?)
+    return os.path.join(tempfile.gettempdir(), "ci_ccache")
 
 def container_run(platform: str,
                   docker_binary: str,
                   docker_registry: str,
                   shared_memory_size: str,
+                  local_ccache_dir: str,
                   command: List[str],
                   dry_run: bool = False,
                   into_container: bool = False) -> str:
@@ -128,12 +162,16 @@ def container_run(platform: str,
     local_build_folder = buildir()
     # We need to create it first, otherwise it will be created by the docker daemon with root only permissions
     os.makedirs(local_build_folder, exist_ok=True)
+    os.makedirs(local_ccache_dir, exist_ok=True)
+    logging.info("Using ccache directory: %s", local_ccache_dir)
     runlist = [docker_binary, 'run', '--rm', '-t',
-        '--shm-size={}'.format(shared_memory_size),
-        '-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
-        '-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
-        '-u', '{}:{}'.format(os.getuid(), os.getgid()),
-        tag]
+               '--shm-size={}'.format(shared_memory_size),
+               '-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
+               '-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
+               '-v', "{}:/work/ccache".format(local_ccache_dir),
+               '-u', '{}:{}'.format(os.getuid(), os.getgid()),
+               '-e', "CCACHE_DIR=/work/ccache", # this path is inside the container as /work/ccache is mounted
+               tag]
     runlist.extend(command)
     cmd = ' '.join(runlist)
     if not dry_run and not into_container:
@@ -180,12 +218,11 @@ def main() -> int:
     os.chdir(base)
 
     logging.getLogger().setLevel(logging.INFO)
-
     def script_name() -> str:
         return os.path.split(sys.argv[0])[1]
-
     logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
 
+
     parser = argparse.ArgumentParser(description="""Utility for building and testing MXNet on docker
     containers""",epilog="")
     parser.add_argument("-p", "--platform",
@@ -230,6 +267,11 @@ def script_name() -> str:
                         help="command to run in the container",
                         nargs='*', action='append', type=str)
 
+    parser.add_argument("--ccache-dir",
+                        default=default_ccache_dir(),
+                        help="Ccache directory",
+                        type=str)
+
     args = parser.parse_args()
     docker_registry = args.docker_registry
     command = list(chain(*args.command))
@@ -241,27 +283,33 @@ def script_name() -> str:
         list_platforms()
     elif args.platform:
         platform = args.platform
-        tag = get_docker_tag(platform=platform, registry=docker_registry)
-        load_docker_cache(tag=tag, docker_registry=args.docker_registry)
-        build_docker(platform, docker_binary, registry=docker_registry)
+        tag = get_docker_tag(platform)
+        if args.download_docker_cache:
+            import docker_cache
+            logging.info('Docker cache download is enabled')
+            docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag)
+        # prepare ccache
+        build_ccache("ubuntu", docker_binary)
+        build_ccache("centos7", docker_binary)
+        build_docker(platform, docker_binary)
         if args.build_only:
             logging.warning("Container was just built. Exiting due to build-only.")
             return 0
 
         if command:
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=command, docker_registry=docker_registry)
+                          command=command, docker_registry=docker_registry, ccache_dir=args.ccache_dir)
         elif args.print_docker_run:
             print(container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                                command=[], dry_run=True, docker_registry=docker_registry))
+                                command=[], dry_run=True, docker_registry=docker_registry, ccache_dir=args.ccache_dir))
         elif args.into_container:
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=[], dry_run=False, into_container=True, docker_registry=docker_registry)
+                          command=[], dry_run=False, into_container=True, docker_registry=docker_registry, ccache_dir=args.ccache_dir)
         else:
             cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
             logging.info("No command specified, trying default build: %s", ' '.join(cmd))
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=cmd, docker_registry=docker_registry)
+                          command=cmd, docker_registry=docker_registry, ccache_dir=args.ccache_dir)
 
     elif args.all:
         platforms = get_platforms()
@@ -277,7 +325,7 @@ def script_name() -> str:
             cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform]
             shutil.rmtree(buildir(), ignore_errors=True)
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=cmd, docker_registry=docker_registry)
+                          command=cmd, docker_registry=docker_registry, ccache_dir=args.ccache_dir)
             plat_buildir = os.path.join(get_mxnet_root(), build_platform)
             shutil.move(buildir(), plat_buildir)
             logging.info("Built files left in: %s", plat_buildir)
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index 20739dabe2eb..77465bdb18b7 100755
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -18,10 +18,15 @@
 #
 # Dockerfile to build MXNet for ARMv6
 
+FROM ccache/build.ubuntu as ccachebuilder
+
 FROM dockcross/linux-armv6
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 ENV ARCH armv6l
-ENV FC=/usr/bin/${CROSS_TRIPLE}-gfortran
 ENV HOSTCC gcc
 ENV TARGET ARMV6
 
diff --git a/ci/docker/Dockerfile.build.centos7.ccache b/ci/docker/Dockerfile.build.centos7.ccache
new file mode 100644
index 000000000000..aea47310ad81
--- /dev/null
+++ b/ci/docker/Dockerfile.build.centos7.ccache
@@ -0,0 +1,42 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build ccache for ubuntu based images
+
+FROM centos:7
+
+# Multipackage installation does not fail in yum
+RUN yum -y install epel-release && \
+    yum -y install git \
+    yum -y install ssh \
+    yum -y install autoconf \
+    yum -y install wget \
+    yum -y install make \
+    yum -y install google-perftools \
+    yum -y install asciidoc \
+    yum -y install gcc-c++-4.8.*
+
+WORKDIR /work/deps
+
+# build ccache
+RUN git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git && \
+    cd ccache && \
+    ./autogen.sh && \
+    ./configure && \
+    make -j$(nproc) && \
+    make install
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index c358edb1fb04..649b057cd77e 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -22,12 +22,15 @@
 
 FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder
 
-# Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
-# FROM dockcross/linux-arm64
-FROM mxnetci/dockcross-linux-arm64:05082018
+FROM ccache/build.ubuntu as ccachebuilder
+
+FROM dockcross/linux-arm64
+
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
 
 ENV ARCH aarch64
-ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
 ENV HOSTCC gcc
 ENV TARGET ARMV8
 
@@ -39,6 +42,9 @@ RUN git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git && \
     make -j$(nproc) && \
     PREFIX=${CROSS_ROOT} make install
 
+ENV OpenBLAS_HOME=${CROSS_ROOT}
+ENV OpenBLAS_DIR=${CROSS_ROOT}
+
 # Setup CUDA build env (including configuring and copying nvcc)
 COPY --from=cudabuilder /usr/local/cuda /usr/local/cuda
 ENV TARGET_ARCH aarch64
diff --git a/ci/docker/Dockerfile.build.ubuntu.ccache b/ci/docker/Dockerfile.build.ubuntu.ccache
new file mode 100644
index 000000000000..aa1f106615fa
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu.ccache
@@ -0,0 +1,46 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build ccache for ubuntu based images
+
+FROM ubuntu:16.04
+
+RUN apt update && \
+    apt install -y --no-install-recommends \
+        git \
+        ssh \
+        apt-transport-https \
+        build-essential \
+        ca-certificates \
+        autoconf \
+        google-perftools \
+        asciidoc \
+        libxslt1-dev \
+        docbook-xsl \
+        xsltproc \
+        libxml2-utils
+
+WORKDIR /work/deps
+
+# build ccache
+RUN git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git && \
+    cd ccache && \
+    ./autogen.sh && \
+    ./configure && \
+    make -j$(nproc) && \
+    make install
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index fa9de6112ff8..d08c5e0c121b 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -38,7 +38,9 @@ clean_repo() {
 build_jetson() {
     set -ex
     pushd .
-    mv make/crosscompile.jetson.mk make/config.mk
+
+    cp -f make/crosscompile.jetson.mk ./config.mk
+
     make -j$(nproc)
 
     export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
@@ -73,6 +75,7 @@ build_armv6() {
 
     cmake \
         -DCMAKE_TOOLCHAIN_FILE=$CROSS_ROOT/Toolchain.cmake \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=OFF \
         -DUSE_OPENCV=OFF \
         -DUSE_OPENMP=OFF \
diff --git a/make/config.mk b/make/config.mk
index dd67c33cc9e5..b65f77c605f7 100644
--- a/make/config.mk
+++ b/make/config.mk
@@ -37,9 +37,15 @@
 # choice of compiler
 #--------------------
 
+ifndef CC
 export CC = gcc
+endif
+ifndef CXX
 export CXX = g++
+endif
+ifndef NVCC
 export NVCC = nvcc
+endif
 
 # whether compile with options for MXNet developer
 DEV = 0
diff --git a/make/crosscompile.jetson.mk b/make/crosscompile.jetson.mk
index acc9c4a5a8a4..868fefe05ec2 100644
--- a/make/crosscompile.jetson.mk
+++ b/make/crosscompile.jetson.mk
@@ -57,10 +57,10 @@ DEBUG = 0
 USE_SIGNAL_HANDLER = 1
 
 # the additional link flags you want to add
-ADD_LDFLAGS =
+ADD_LDFLAGS = -L${CROSS_ROOT}/lib
 
 # the additional compile flags you want to add
-ADD_CFLAGS =
+ADD_CFLAGS = -I${CROSS_ROOT}/include
 
 #---------------------------------------------
 # matrix computation libraries for CPU/GPU
@@ -96,7 +96,7 @@ USE_LIBJPEG_TURBO = 0
 USE_LIBJPEG_TURBO_PATH = NONE
 
 # use openmp for parallelization
-USE_OPENMP = 1
+USE_OPENMP = 0
 
 # whether use MKL-DNN library
 USE_MKLDNN = 0

From 9ed19548a82f69aa7933b4c5b8e11b83e90dd56c Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Tue, 29 May 2018 16:24:22 +0000
Subject: [PATCH 02/14] Added ccache stages to all containers

---
 .gitignore                                   |  4 ++++
 ci/docker/Dockerfile.build.android_arm64     |  6 ++++++
 ci/docker/Dockerfile.build.android_armv7     |  6 ++++++
 ci/docker/Dockerfile.build.arm64             |  6 ++++++
 ci/docker/Dockerfile.build.armv7             |  6 ++++++
 ci/docker/Dockerfile.build.centos7_cpu       |  6 ++++++
 ci/docker/Dockerfile.build.centos7_gpu       |  6 ++++++
 ci/docker/Dockerfile.build.jetson            |  5 ++++-
 ci/docker/Dockerfile.build.ubuntu_build_cuda |  6 ++++++
 ci/docker/Dockerfile.build.ubuntu_cpu        |  6 ++++++
 ci/docker/Dockerfile.build.ubuntu_gpu        |  6 ++++++
 ci/docker/runtime_functions.sh               | 21 ++++++++++++++++++++
 12 files changed, 83 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index d585672ab7d9..416741a5e704 100644
--- a/.gitignore
+++ b/.gitignore
@@ -166,3 +166,7 @@ python/.eggs
 *DartConfiguration.tcl
 tests/Makefile
 tests/mxnet_unit_tests
+
+# generated wrappers for ccache
+cc
+cxx
diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64
index d7687514d7a6..12d0732bf0eb 100755
--- a/ci/docker/Dockerfile.build.android_arm64
+++ b/ci/docker/Dockerfile.build.android_arm64
@@ -18,9 +18,15 @@
 #
 # Dockerfile to build MXNet for Android ARM64/ARMv8
 
+FROM ccache/build.ubuntu as ccachebuilder
+
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 # The cross-compiling emulator
 RUN apt-get update && apt-get install -y \
   qemu-user \
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
index c22e000cad1e..8f4afa8929d5 100755
--- a/ci/docker/Dockerfile.build.android_armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -18,9 +18,15 @@
 #
 # Dockerfile to build MXNet for Android ARMv7
 
+FROM ccache/build.ubuntu as ccachebuilder
+
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 # The cross-compiling emulator
 RUN apt-get update && apt-get install -y \
   qemu-user \
diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64
index ec949600f739..7580172a2aab 100755
--- a/ci/docker/Dockerfile.build.arm64
+++ b/ci/docker/Dockerfile.build.arm64
@@ -18,10 +18,16 @@
 #
 # Dockerfile to build MXNet for ARM64/ARMv8
 
+FROM ccache/build.ubuntu as ccachebuilder
+
 # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
 #FROM dockcross/linux-arm64
 FROM mxnetci/dockcross-linux-arm64:05082018
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 ENV ARCH aarch64
 ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
 ENV HOSTCC gcc
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
index c24930635181..3ccc496558c5 100755
--- a/ci/docker/Dockerfile.build.armv7
+++ b/ci/docker/Dockerfile.build.armv7
@@ -18,8 +18,14 @@
 #
 # Dockerfile to build MXNet for Android ARMv7
 
+FROM ccache/build.ubuntu as ccachebuilder
+
 FROM dockcross/linux-armv7
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 ENV ARCH armv71
 ENV CC /usr/bin/arm-linux-gnueabihf-gcc
 ENV CXX /usr/bin/arm-linux-gnueabihf-g++
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
index 92314faf1214..0ad358531670 100755
--- a/ci/docker/Dockerfile.build.centos7_cpu
+++ b/ci/docker/Dockerfile.build.centos7_cpu
@@ -18,8 +18,14 @@
 #
 # Dockerfile to build and run MXNet on CentOS 7 for CPU
 
+FROM ccache/build.centos7 as ccachebuilder
+
 FROM centos:7
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
index 2d28170f11bb..68980a6d5b86 100755
--- a/ci/docker/Dockerfile.build.centos7_gpu
+++ b/ci/docker/Dockerfile.build.centos7_gpu
@@ -18,8 +18,14 @@
 #
 # Dockerfile to build and run MXNet on CentOS 7 for GPU
 
+FROM ccache/build.centos7 as ccachebuilder
+
 FROM nvidia/cuda:9.1-cudnn7-devel-centos7
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index 649b057cd77e..540134fe467d 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -24,7 +24,9 @@ FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder
 
 FROM ccache/build.ubuntu as ccachebuilder
 
-FROM dockcross/linux-arm64
+# Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
+# FROM dockcross/linux-arm64
+FROM mxnetci/dockcross-linux-arm64:05082018
 
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 ENV CCACHE_MAXSIZE 10G
@@ -32,6 +34,7 @@ ENV CCACHE_DIR /work/ccache
 
 ENV ARCH aarch64
 ENV HOSTCC gcc
+ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
 ENV TARGET ARMV8
 
 WORKDIR /work
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index 4d3c4664363e..36860efbadf5 100755
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -21,8 +21,14 @@
 # package generation, requiring the actual CUDA library to be
 # present
 
+FROM ccache/build.ubuntu as ccachebuilder
+
 FROM nvidia/cuda:9.1-cudnn7-devel
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
index 2dc7ef13f210..b54d9025f8b0 100755
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -18,8 +18,14 @@
 #
 # Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
 
+FROM ccache/build.ubuntu as ccachebuilder
+
 FROM ubuntu:16.04
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
index 10971724aaa8..6118750a4450 100755
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -18,8 +18,14 @@
 #
 # Dockerfile to run MXNet on Ubuntu 16.04 for CPU
 
+FROM ccache/build.ubuntu as ccachebuilder
+
 FROM nvidia/cuda:9.1-cudnn7-devel
 
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE 10G
+ENV CCACHE_DIR /work/ccache
+
 WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index d08c5e0c121b..bf48f15f84d7 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -31,6 +31,25 @@ clean_repo() {
     git submodule update --init --recursive
 }
 
+# this function is nessesary for cuda enabled make based builds, since nvcc needs just an exacutable for -ccbin
+build_ccache_wrappers() {
+    set -ex
+
+    rm -f cc
+    rm -f cxx
+
+    touch cc
+    touch cxx
+
+    echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CC} \"\$@\"\n" >> cc
+    echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CXX} \"\$@\"\n" >> cxx
+
+    chmod +x cc
+    chmod +x cxx
+
+    export CC=`pwd`/cc
+    export CXX=`pwd`/cxx
+}
 
 # Build commands: Every platform in docker/Dockerfile.build.<platform> should have a corresponding
 # function here with the same suffix:
@@ -39,6 +58,8 @@ build_jetson() {
     set -ex
     pushd .
 
+    build_ccache_wrappers
+
     cp -f make/crosscompile.jetson.mk ./config.mk
 
     make -j$(nproc)

From 88ec7a45bf98d256b448af375f80252270f0cf14 Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Wed, 30 May 2018 08:22:58 +0000
Subject: [PATCH 03/14] Refactored ccache installs in docker images

---
 ci/build.py                                   | 28 --------
 ci/docker/Dockerfile.build.arm64              |  5 +-
 ci/docker/Dockerfile.build.armv6              |  5 +-
 ci/docker/Dockerfile.build.armv7              |  5 +-
 ci/docker/Dockerfile.build.centos7_cpu        |  5 +-
 ci/docker/Dockerfile.build.centos7_gpu        |  5 +-
 ci/docker/Dockerfile.build.jetson             |  5 +-
 ci/docker/Dockerfile.build.ubuntu_build_cuda  |  5 +-
 ci/docker/Dockerfile.build.ubuntu_cpu         |  5 +-
 ci/docker/Dockerfile.build.ubuntu_gpu         |  5 +-
 .../centos7_install_ccache.sh}                | 57 +++++++++-------
 .../ubuntu_install_ccache.sh}                 | 68 +++++++++++--------
 12 files changed, 95 insertions(+), 103 deletions(-)
 rename ci/docker/{Dockerfile.build.centos7.ccache => install/centos7_install_ccache.sh} (55%)
 mode change 100644 => 100755
 rename ci/docker/{Dockerfile.build.ubuntu.ccache => install/ubuntu_install_ccache.sh} (55%)
 mode change 100644 => 100755

diff --git a/ci/build.py b/ci/build.py
index 7ed1b9416b27..8fe04ecd567e 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -88,32 +88,6 @@ def build_docker(platform: str, docker_binary: str, registry: str) -> None:
         raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
     return image_id
 
-def build_ccache(platform: str, docker_binary: str) -> None:
-    """
-    Build a ccache container for the given platform
-    :param platform: Platform
-    :param docker_binary: docker binary to use (docker/nvidia-docker)
-    :return: Id of the top level image
-    """
-
-    tag = "ccache/build.{}".format(platform)
-    logging.info("Building ccache container tagged '%s' with %s", tag, docker_binary)
-    cmd = [docker_binary, "build",
-           "-f", get_dockerfile("{}.ccache".format(platform)),
-           "--rm=true", # We only need the binary
-           "--cache-from", tag,
-           "-t", tag,
-           "docker"]
-    logging.info("Running command: '%s'", ' '.join(cmd))
-    check_call(cmd)
-
-    # Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
-    # check_call would have failed
-    image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
-    if not image_id:
-        raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
-    return image_id
-
 def _get_local_image_id(docker_binary, docker_tag):
     """
     Get the image id of the local docker layer with the passed tag
@@ -289,8 +263,6 @@ def script_name() -> str:
             logging.info('Docker cache download is enabled')
             docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag)
         # prepare ccache
-        build_ccache("ubuntu", docker_binary)
-        build_ccache("centos7", docker_binary)
         build_docker(platform, docker_binary)
         if args.build_only:
             logging.warning("Container was just built. Exiting due to build-only.")
diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64
index 7580172a2aab..babd6ead24b7 100755
--- a/ci/docker/Dockerfile.build.arm64
+++ b/ci/docker/Dockerfile.build.arm64
@@ -18,7 +18,10 @@
 #
 # Dockerfile to build MXNet for ARM64/ARMv8
 
-FROM ccache/build.ubuntu as ccachebuilder
+FROM ubuntu:16.04 as ccachebuilder
+
+COPY install/ubuntu_install_ccache.sh /work/
+RUN /work/ubuntu_install_ccache.sh
 
 # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
 #FROM dockcross/linux-arm64
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index 77465bdb18b7..af97bfb364a5 100755
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -18,7 +18,10 @@
 #
 # Dockerfile to build MXNet for ARMv6
 
-FROM ccache/build.ubuntu as ccachebuilder
+FROM ubuntu:16.04 as ccachebuilder
+
+COPY install/ubuntu_install_ccache.sh /work/
+RUN /work/ubuntu_install_ccache.sh
 
 FROM dockcross/linux-armv6
 
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
index 3ccc496558c5..baaa81a98151 100755
--- a/ci/docker/Dockerfile.build.armv7
+++ b/ci/docker/Dockerfile.build.armv7
@@ -18,7 +18,10 @@
 #
 # Dockerfile to build MXNet for Android ARMv7
 
-FROM ccache/build.ubuntu as ccachebuilder
+FROM ubuntu:16.04 as ccachebuilder
+
+COPY install/ubuntu_install_ccache.sh /work/
+RUN /work/ubuntu_install_ccache.sh
 
 FROM dockcross/linux-armv7
 
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
index 0ad358531670..86f724246fe1 100755
--- a/ci/docker/Dockerfile.build.centos7_cpu
+++ b/ci/docker/Dockerfile.build.centos7_cpu
@@ -18,11 +18,8 @@
 #
 # Dockerfile to build and run MXNet on CentOS 7 for CPU
 
-FROM ccache/build.centos7 as ccachebuilder
-
 FROM centos:7
 
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 ENV CCACHE_MAXSIZE 10G
 ENV CCACHE_DIR /work/ccache
 
@@ -30,6 +27,8 @@ WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
 RUN /work/centos7_core.sh
+COPY install/centos7_install_ccache.sh /work/
+RUN /work/centos7_install_ccache.sh
 COPY install/centos7_python.sh /work/
 RUN /work/centos7_python.sh
 COPY install/ubuntu_mklml.sh /work/
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
index 68980a6d5b86..45699c80a461 100755
--- a/ci/docker/Dockerfile.build.centos7_gpu
+++ b/ci/docker/Dockerfile.build.centos7_gpu
@@ -18,11 +18,8 @@
 #
 # Dockerfile to build and run MXNet on CentOS 7 for GPU
 
-FROM ccache/build.centos7 as ccachebuilder
-
 FROM nvidia/cuda:9.1-cudnn7-devel-centos7
 
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 ENV CCACHE_MAXSIZE 10G
 ENV CCACHE_DIR /work/ccache
 
@@ -30,6 +27,8 @@ WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
 RUN /work/centos7_core.sh
+COPY install/centos7_install_ccache.sh /work/
+RUN /work/centos7_install_ccache.sh
 COPY install/centos7_python.sh /work/
 RUN /work/centos7_python.sh
 
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index 540134fe467d..1cfab7e56f6c 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -22,7 +22,10 @@
 
 FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder
 
-FROM ccache/build.ubuntu as ccachebuilder
+FROM ubuntu:16.04 as ccachebuilder
+
+COPY install/ubuntu_install_ccache.sh /work/
+RUN /work/ubuntu_install_ccache.sh
 
 # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
 # FROM dockcross/linux-arm64
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index 36860efbadf5..c0661fa315bf 100755
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -21,11 +21,8 @@
 # package generation, requiring the actual CUDA library to be
 # present
 
-FROM ccache/build.ubuntu as ccachebuilder
-
 FROM nvidia/cuda:9.1-cudnn7-devel
 
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 ENV CCACHE_MAXSIZE 10G
 ENV CCACHE_DIR /work/ccache
 
@@ -33,6 +30,8 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
+COPY install/ubuntu_install_ccache.sh /work/
+RUN /work/ubuntu_install_ccache.sh
 COPY install/ubuntu_python.sh /work/
 RUN /work/ubuntu_python.sh
 COPY install/ubuntu_scala.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
index b54d9025f8b0..628ea4b77e86 100755
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -18,11 +18,8 @@
 #
 # Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
 
-FROM ccache/build.ubuntu as ccachebuilder
-
 FROM ubuntu:16.04
 
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 ENV CCACHE_MAXSIZE 10G
 ENV CCACHE_DIR /work/ccache
 
@@ -30,6 +27,8 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
+COPY install/ubuntu_install_ccache.sh /work/
+RUN /work/ubuntu_install_ccache.sh
 COPY install/ubuntu_python.sh /work/
 RUN /work/ubuntu_python.sh
 COPY install/ubuntu_scala.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
index 6118750a4450..f6dc5e1a761d 100755
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -18,11 +18,8 @@
 #
 # Dockerfile to run MXNet on Ubuntu 16.04 for CPU
 
-FROM ccache/build.ubuntu as ccachebuilder
-
 FROM nvidia/cuda:9.1-cudnn7-devel
 
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 ENV CCACHE_MAXSIZE 10G
 ENV CCACHE_DIR /work/ccache
 
@@ -30,6 +27,8 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
+COPY install/ubuntu_install_ccache.sh /work/
+RUN /work/ubuntu_install_ccache.sh
 COPY install/ubuntu_python.sh /work/
 RUN /work/ubuntu_python.sh
 COPY install/ubuntu_scala.sh /work/
diff --git a/ci/docker/Dockerfile.build.centos7.ccache b/ci/docker/install/centos7_install_ccache.sh
old mode 100644
new mode 100755
similarity index 55%
rename from ci/docker/Dockerfile.build.centos7.ccache
rename to ci/docker/install/centos7_install_ccache.sh
index aea47310ad81..3391a15b31c9
--- a/ci/docker/Dockerfile.build.centos7.ccache
+++ b/ci/docker/install/centos7_install_ccache.sh
@@ -1,4 +1,5 @@
-# -*- mode: dockerfile -*-
+#!/bin/bash
+
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,28 +16,32 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#
-# Dockerfile to build ccache for ubuntu based images
-
-FROM centos:7
-
-# Multipackage installation does not fail in yum
-RUN yum -y install epel-release && \
-    yum -y install git \
-    yum -y install ssh \
-    yum -y install autoconf \
-    yum -y install wget \
-    yum -y install make \
-    yum -y install google-perftools \
-    yum -y install asciidoc \
-    yum -y install gcc-c++-4.8.*
-
-WORKDIR /work/deps
-
-# build ccache
-RUN git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git && \
-    cd ccache && \
-    ./autogen.sh && \
-    ./configure && \
-    make -j$(nproc) && \
-    make install
+
+# Script to build ccache for centos7 based images
+
+set -ex
+
+pushd .
+
+yum -y install epel-release
+yum -y install git
+yum -y install ssh
+yum -y install autoconf
+yum -y install wget
+yum -y install make
+yum -y install google-perftools
+yum -y install asciidoc
+yum -y install gcc-c++-4.8.*
+
+cd /work
+
+git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git
+
+cd ccache
+
+./autogen.sh
+./configure
+make -j$(nproc)
+make install
+
+popd
diff --git a/ci/docker/Dockerfile.build.ubuntu.ccache b/ci/docker/install/ubuntu_install_ccache.sh
old mode 100644
new mode 100755
similarity index 55%
rename from ci/docker/Dockerfile.build.ubuntu.ccache
rename to ci/docker/install/ubuntu_install_ccache.sh
index aa1f106615fa..8565b5dae412
--- a/ci/docker/Dockerfile.build.ubuntu.ccache
+++ b/ci/docker/install/ubuntu_install_ccache.sh
@@ -1,4 +1,5 @@
-# -*- mode: dockerfile -*-
+#!/bin/bash
+
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,32 +16,39 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#
-# Dockerfile to build ccache for ubuntu based images
-
-FROM ubuntu:16.04
-
-RUN apt update && \
-    apt install -y --no-install-recommends \
-        git \
-        ssh \
-        apt-transport-https \
-        build-essential \
-        ca-certificates \
-        autoconf \
-        google-perftools \
-        asciidoc \
-        libxslt1-dev \
-        docbook-xsl \
-        xsltproc \
-        libxml2-utils
-
-WORKDIR /work/deps
-
-# build ccache
-RUN git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git && \
-    cd ccache && \
-    ./autogen.sh && \
-    ./configure && \
-    make -j$(nproc) && \
-    make install
+
+# Script to build ccache for ubuntu based images
+
+set -ex
+
+pushd .
+
+apt update
+apt install -y --no-install-recommends \
+    git \
+    ssh \
+    apt-transport-https \
+    build-essential \
+    ca-certificates \
+    autoconf \
+    google-perftools \
+    asciidoc \
+    libxslt1-dev \
+    docbook-xsl \
+    xsltproc \
+    libxml2-utils
+
+cd /work
+
+git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git
+
+cd ccache
+
+./autogen.sh
+./configure
+make -j$(nproc)
+make install
+
+popd
+
+rm -rf /work/ccache

From 4ac54a9c46d6bcf00c71a230c53b17a6f1b7e2e2 Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Wed, 30 May 2018 08:28:23 +0000
Subject: [PATCH 04/14] Reformatted build.py

---
 ci/build.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/ci/build.py b/ci/build.py
index 8fe04ecd567e..03c35b0fdb95 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -40,7 +40,7 @@
 from typing import *
 
 
-def get_platforms(path: Optional[str]="docker"):
+def get_platforms(path: Optional[str] = "docker"):
     """Get a list of architectures given our dockerfiles"""
     dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
     dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
@@ -88,6 +88,7 @@ def build_docker(platform: str, docker_binary: str, registry: str) -> None:
         raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
     return image_id
 
+
 def _get_local_image_id(docker_binary, docker_tag):
     """
     Get the image id of the local docker layer with the passed tag
@@ -102,8 +103,10 @@ def _get_local_image_id(docker_binary, docker_tag):
 
 def get_mxnet_root() -> str:
     curpath = os.path.abspath(os.path.dirname(__file__))
+
     def is_mxnet_root(path: str) -> bool:
         return os.path.exists(os.path.join(path, ".mxnet_root"))
+
     while not is_mxnet_root(curpath):
         parent = os.path.abspath(os.path.join(curpath, os.pardir))
         if parent == curpath:
@@ -115,6 +118,7 @@ def is_mxnet_root(path: str) -> bool:
 def buildir() -> str:
     return os.path.join(get_mxnet_root(), "build")
 
+
 def default_ccache_dir() -> str:
     if 'CCACHE_DIR' in os.environ:
         ccache_dir = os.path.realpath(os.environ['CCACHE_DIR'])
@@ -123,6 +127,7 @@ def default_ccache_dir() -> str:
     # Share ccache across containers (should we have a separate dir per platform?)
     return os.path.join(tempfile.gettempdir(), "ci_ccache")
 
+
 def container_run(platform: str,
                   docker_binary: str,
                   docker_registry: str,
@@ -140,11 +145,11 @@ def container_run(platform: str,
     logging.info("Using ccache directory: %s", local_ccache_dir)
     runlist = [docker_binary, 'run', '--rm', '-t',
                '--shm-size={}'.format(shared_memory_size),
-               '-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
-               '-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
+               '-v', "{}:/work/mxnet".format(mx_root),  # mount mxnet root
+               '-v', "{}:/work/build".format(local_build_folder),  # mount mxnet/build for storing build artifacts
                '-v', "{}:/work/ccache".format(local_ccache_dir),
                '-u', '{}:{}'.format(os.getuid(), os.getgid()),
-               '-e', "CCACHE_DIR=/work/ccache", # this path is inside the container as /work/ccache is mounted
+               '-e', "CCACHE_DIR=/work/ccache",  # this path is inside the container as /work/ccache is mounted
                tag]
     runlist.extend(command)
     cmd = ' '.join(runlist)
@@ -172,7 +177,6 @@ def container_run(platform: str,
 def list_platforms() -> str:
     print("\nSupported platforms:\n{}".format('\n'.join(get_platforms())))
 
-
 def load_docker_cache(tag, docker_registry) -> None:
     if docker_registry:
         try:
@@ -184,7 +188,6 @@ def load_docker_cache(tag, docker_registry) -> None:
     else:
         logging.info('Distributed docker cache disabled')
 
-
 def main() -> int:
     # We need to be in the same directory than the script so the commands in the dockerfiles work as
     # expected. But the script can be invoked from a different path
@@ -192,13 +195,14 @@ def main() -> int:
     os.chdir(base)
 
     logging.getLogger().setLevel(logging.INFO)
+
     def script_name() -> str:
         return os.path.split(sys.argv[0])[1]
-    logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
 
+    logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
 
     parser = argparse.ArgumentParser(description="""Utility for building and testing MXNet on docker
-    containers""",epilog="")
+    containers""", epilog="")
     parser.add_argument("-p", "--platform",
                         help="platform",
                         type=str)

From a609022b34f9a09c2eb72515ca0607349c70c9be Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Wed, 30 May 2018 08:33:30 +0000
Subject: [PATCH 05/14] Added ccache install to android docker builds

---
 ci/docker/Dockerfile.build.android_arm64 | 3 +++
 ci/docker/Dockerfile.build.android_armv7 | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64
index 12d0732bf0eb..5cdb25c93240 100755
--- a/ci/docker/Dockerfile.build.android_arm64
+++ b/ci/docker/Dockerfile.build.android_arm64
@@ -20,6 +20,9 @@
 
 FROM ccache/build.ubuntu as ccachebuilder
 
+COPY install/ubuntu_install_ccache.sh /work/
+RUN /work/ubuntu_install_ccache.sh
+
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
index 8f4afa8929d5..51d7edea2c53 100755
--- a/ci/docker/Dockerfile.build.android_armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -20,6 +20,9 @@
 
 FROM ccache/build.ubuntu as ccachebuilder
 
+COPY install/ubuntu_install_ccache.sh /work/
+RUN /work/ubuntu_install_ccache.sh
+
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 

From 59d0e64aecb3a6b9752bbb6e07733c5da106de83 Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Wed, 30 May 2018 08:45:28 +0000
Subject: [PATCH 06/14] Improved setting ccache directory and max cache size

---
 ci/docker/Dockerfile.build.android_arm64     | 2 --
 ci/docker/Dockerfile.build.android_armv7     | 2 --
 ci/docker/Dockerfile.build.arm64             | 2 --
 ci/docker/Dockerfile.build.armv6             | 2 --
 ci/docker/Dockerfile.build.armv7             | 2 --
 ci/docker/Dockerfile.build.centos7_cpu       | 3 ---
 ci/docker/Dockerfile.build.centos7_gpu       | 3 ---
 ci/docker/Dockerfile.build.jetson            | 2 --
 ci/docker/Dockerfile.build.ubuntu_build_cuda | 3 ---
 ci/docker/Dockerfile.build.ubuntu_cpu        | 3 ---
 ci/docker/Dockerfile.build.ubuntu_gpu        | 3 ---
 ci/docker/install/centos7_install_ccache.sh  | 5 +++++
 ci/docker/install/ubuntu_install_ccache.sh   | 3 +++
 13 files changed, 8 insertions(+), 27 deletions(-)

diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64
index 5cdb25c93240..f55ac4b1ff8d 100755
--- a/ci/docker/Dockerfile.build.android_arm64
+++ b/ci/docker/Dockerfile.build.android_arm64
@@ -27,8 +27,6 @@ FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
 
 # The cross-compiling emulator
 RUN apt-get update && apt-get install -y \
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
index 51d7edea2c53..30c4242a3dbe 100755
--- a/ci/docker/Dockerfile.build.android_armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -27,8 +27,6 @@ FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
 
 # The cross-compiling emulator
 RUN apt-get update && apt-get install -y \
diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64
index babd6ead24b7..5de4d22c988f 100755
--- a/ci/docker/Dockerfile.build.arm64
+++ b/ci/docker/Dockerfile.build.arm64
@@ -28,8 +28,6 @@ RUN /work/ubuntu_install_ccache.sh
 FROM mxnetci/dockcross-linux-arm64:05082018
 
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
 
 ENV ARCH aarch64
 ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index af97bfb364a5..42555ebee133 100755
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -26,8 +26,6 @@ RUN /work/ubuntu_install_ccache.sh
 FROM dockcross/linux-armv6
 
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
 
 ENV ARCH armv6l
 ENV HOSTCC gcc
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
index baaa81a98151..58f3c37ce008 100755
--- a/ci/docker/Dockerfile.build.armv7
+++ b/ci/docker/Dockerfile.build.armv7
@@ -26,8 +26,6 @@ RUN /work/ubuntu_install_ccache.sh
 FROM dockcross/linux-armv7
 
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
 
 ENV ARCH armv71
 ENV CC /usr/bin/arm-linux-gnueabihf-gcc
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
index 86f724246fe1..1777640273f8 100755
--- a/ci/docker/Dockerfile.build.centos7_cpu
+++ b/ci/docker/Dockerfile.build.centos7_cpu
@@ -20,9 +20,6 @@
 
 FROM centos:7
 
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
-
 WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
index 45699c80a461..cb2ba08c6aa7 100755
--- a/ci/docker/Dockerfile.build.centos7_gpu
+++ b/ci/docker/Dockerfile.build.centos7_gpu
@@ -20,9 +20,6 @@
 
 FROM nvidia/cuda:9.1-cudnn7-devel-centos7
 
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
-
 WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index 1cfab7e56f6c..7d28cf6f07ce 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -32,8 +32,6 @@ RUN /work/ubuntu_install_ccache.sh
 FROM mxnetci/dockcross-linux-arm64:05082018
 
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
 
 ENV ARCH aarch64
 ENV HOSTCC gcc
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index c0661fa315bf..46ed811400b8 100755
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -23,9 +23,6 @@
 
 FROM nvidia/cuda:9.1-cudnn7-devel
 
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
-
 WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
index 628ea4b77e86..24558adce687 100755
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -20,9 +20,6 @@
 
 FROM ubuntu:16.04
 
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
-
 WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
index f6dc5e1a761d..614d43b911bd 100755
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -20,9 +20,6 @@
 
 FROM nvidia/cuda:9.1-cudnn7-devel
 
-ENV CCACHE_MAXSIZE 10G
-ENV CCACHE_DIR /work/ccache
-
 WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
diff --git a/ci/docker/install/centos7_install_ccache.sh b/ci/docker/install/centos7_install_ccache.sh
index 3391a15b31c9..39a8cf56434a 100755
--- a/ci/docker/install/centos7_install_ccache.sh
+++ b/ci/docker/install/centos7_install_ccache.sh
@@ -45,3 +45,8 @@ make -j$(nproc)
 make install
 
 popd
+
+rm -rf /work/ccache
+
+export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:=10G}
+export CCACHE_DIR=${CCACHE_DIR:=/work/ccache}
diff --git a/ci/docker/install/ubuntu_install_ccache.sh b/ci/docker/install/ubuntu_install_ccache.sh
index 8565b5dae412..75db51fcbdbd 100755
--- a/ci/docker/install/ubuntu_install_ccache.sh
+++ b/ci/docker/install/ubuntu_install_ccache.sh
@@ -52,3 +52,6 @@ make install
 popd
 
 rm -rf /work/ccache
+
+export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:=10G}
+export CCACHE_DIR=${CCACHE_DIR:=/work/ccache}

From abc3a3d81b6245be4b396e122c0913c902712488 Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Wed, 30 May 2018 09:14:17 +0000
Subject: [PATCH 07/14] Added ccache to cmake based docker builds

---
 ci/docker/runtime_functions.sh | 66 ++++++++++++++++++++++++++--------
 1 file changed, 52 insertions(+), 14 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index bf48f15f84d7..621e83c522fe 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -31,7 +31,7 @@ clean_repo() {
     git submodule update --init --recursive
 }
 
-# this function is nessesary for cuda enabled make based builds, since nvcc needs just an exacutable for -ccbin
+# wrap compiler calls with ccache
 build_ccache_wrappers() {
     set -ex
 
@@ -41,6 +41,8 @@ build_ccache_wrappers() {
     touch cc
     touch cxx
 
+    # this function is nessesary for cuda enabled make based builds, since nvcc needs just an exacutable for -ccbin
+
     echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CC} \"\$@\"\n" >> cc
     echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CXX} \"\$@\"\n" >> cxx
 
@@ -97,6 +99,7 @@ build_armv6() {
     cmake \
         -DCMAKE_TOOLCHAIN_FILE=$CROSS_ROOT/Toolchain.cmake \
         -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=OFF \
         -DUSE_OPENCV=OFF \
         -DUSE_OPENMP=OFF \
@@ -119,7 +122,9 @@ build_armv7() {
     set -ex
     pushd .
     cd /work/build
-    cmake\
+    cmake \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=OFF\
         -DUSE_OPENCV=OFF\
         -DUSE_OPENMP=OFF\
@@ -137,7 +142,9 @@ build_armv7() {
 
 build_amzn_linux_cpu() {
     cd /work/build
-    cmake\
+    cmake \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=OFF\
         -DUSE_OPENCV=ON\
         -DUSE_OPENMP=ON\
@@ -152,7 +159,9 @@ build_amzn_linux_cpu() {
 }
 
 build_arm64() {
-    cmake\
+    cmake \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=OFF\
         -DUSE_OPENCV=OFF\
         -DUSE_OPENMP=OFF\
@@ -170,7 +179,9 @@ build_arm64() {
 build_android_arm64() {
     set -ex
     cd /work/build
-    cmake\
+    cmake \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=OFF\
         -DUSE_SSE=OFF\
         -DUSE_LAPACK=OFF\
@@ -189,6 +200,7 @@ build_android_arm64() {
 
 build_centos7_cpu() {
     set -ex
+    build_ccache_wrappers
     cd /work/mxnet
     make \
         DEV=1 \
@@ -201,6 +213,7 @@ build_centos7_cpu() {
 
 build_centos7_mkldnn() {
     set -ex
+    build_ccache_wrappers
     cd /work/mxnet
     make \
         DEV=1 \
@@ -213,6 +226,7 @@ build_centos7_mkldnn() {
 
 build_centos7_gpu() {
     set -ex
+    build_ccache_wrappers
     cd /work/mxnet
     make \
         DEV=1 \
@@ -228,6 +242,7 @@ build_centos7_gpu() {
 
 build_ubuntu_cpu_openblas() {
     set -ex
+    build_ccache_wrappers
     make \
         DEV=1                         \
         USE_CPP_PACKAGE=1             \
@@ -238,54 +253,71 @@ build_ubuntu_cpu_openblas() {
 
 build_ubuntu_cpu_clang39() {
     set -ex
+
+    export CXX=clang++-3.9
+    export CC=clang-3.9
+
+    build_ccache_wrappers
+
     make \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
         USE_OPENMP=0                  \
         USE_DIST_KVSTORE=1            \
-        CXX=clang++-3.9               \
-        CC=clang-3.9                  \
         -j$(nproc)
 }
 
 build_ubuntu_cpu_clang50() {
     set -ex
-    make \
+
+    export CXX=clang++-5.0
+    export CC=clang-5.0
+
+    build_ccache_wrappers
+
+    make  \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
         USE_OPENMP=1                  \
         USE_DIST_KVSTORE=1            \
-        CXX=clang++-5.0               \
-        CC=clang-5.0                  \
         -j$(nproc)
 }
 
 build_ubuntu_cpu_clang39_mkldnn() {
     set -ex
+
+    export CXX=clang++-3.9
+    export CC=clang-3.9
+
+    build_ccache_wrappers
+
     make \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
         USE_MKLDNN=1                  \
         USE_OPENMP=0                  \
-        CXX=clang++-3.9               \
-        CC=clang-3.9                  \
         -j$(nproc)
 }
 
 build_ubuntu_cpu_clang50_mkldnn() {
     set -ex
+
+    export CXX=clang++-5.0
+    export CC=clang-5.0
+
+    build_ccache_wrappers
+
     make \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
         USE_MKLDNN=1                  \
         USE_OPENMP=1                  \
-        CXX=clang++-5.0               \
-        CC=clang-5.0                  \
         -j$(nproc)
 }
 
 build_ubuntu_cpu_mkldnn() {
     set -ex
+    build_ccache_wrappers
     make  \
         DEV=1                         \
         USE_CPP_PACKAGE=1             \
@@ -296,6 +328,7 @@ build_ubuntu_cpu_mkldnn() {
 
 build_ubuntu_gpu_mkldnn() {
     set -ex
+    build_ccache_wrappers
     make  \
         DEV=1                         \
         USE_CPP_PACKAGE=1             \
@@ -309,6 +342,7 @@ build_ubuntu_gpu_mkldnn() {
 
 build_ubuntu_gpu_cuda91_cudnn7() {
     set -ex
+    build_ccache_wrappers
     make  \
         DEV=1                         \
         USE_BLAS=openblas             \
@@ -338,6 +372,8 @@ build_ubuntu_gpu_cmake_mkldnn() {
     set -ex
     cd /work/build
     cmake \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=1               \
         -DUSE_CUDNN=1              \
         -DUSE_MKLML_MKL=1          \
@@ -356,6 +392,8 @@ build_ubuntu_gpu_cmake() {
     set -ex
     cd /work/build
     cmake \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=1               \
         -DUSE_CUDNN=1              \
         -DUSE_MKLML_MKL=0          \

From 0c63b9eb29c0c710aea5c14adc59e64903849979 Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Wed, 30 May 2018 10:07:11 +0000
Subject: [PATCH 08/14] Removed unnessesary yum install from centos7 ccache
 build

---
 ci/docker/install/centos7_install_ccache.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/docker/install/centos7_install_ccache.sh b/ci/docker/install/centos7_install_ccache.sh
index 39a8cf56434a..4ad17e74b834 100755
--- a/ci/docker/install/centos7_install_ccache.sh
+++ b/ci/docker/install/centos7_install_ccache.sh
@@ -25,7 +25,6 @@ pushd .
 
 yum -y install epel-release
 yum -y install git
-yum -y install ssh
 yum -y install autoconf
 yum -y install wget
 yum -y install make

From 166da1e6071a35f7216d2b9ce4d264809b594549 Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Wed, 30 May 2018 11:48:07 +0000
Subject: [PATCH 09/14] Added default compilers for ccache for docker builds

---
 ci/docker/runtime_functions.sh | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 621e83c522fe..ad9d28765ab4 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -41,6 +41,16 @@ build_ccache_wrappers() {
     touch cc
     touch cxx
 
+    if [ -z ${CC+x} ]; then
+        echo "No \$CC set, defaulting to gcc";
+        export CC=gcc
+    fi
+
+    if [ -z ${CXX+x} ]; then
+        echo "No \$CXX set, defaulting to g++";
+        export CXX=g++
+    fi
+
     # this function is nessesary for cuda enabled make based builds, since nvcc needs just an exacutable for -ccbin
 
     echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CC} \"\$@\"\n" >> cc
@@ -343,7 +353,7 @@ build_ubuntu_gpu_mkldnn() {
 build_ubuntu_gpu_cuda91_cudnn7() {
     set -ex
     build_ccache_wrappers
-    make  \
+    make \
         DEV=1                         \
         USE_BLAS=openblas             \
         USE_CUDA=1                    \

From 13c26de6cbf2609e4754f7976d526d123d6c4870 Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Wed, 30 May 2018 11:56:09 +0000
Subject: [PATCH 10/14] Added README comment about ccache mapping into docker
 builds

---
 ci/README.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ci/README.md b/ci/README.md
index 1c59a3af7c84..ca46434a30f1 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -54,7 +54,7 @@ The artifacts are located in the build/ directory in the project root. In case
 
 ## Add a platform
 
-To add a platform, you should add the appropiate dockerfile in
+To add a platform, you should add the appropriate dockerfile in
 docker/Dockerfile.build.<platform> and add a shell function named
 build_<platform> to the file docker/runtime_functions.sh with build
 instructions for that platform.
@@ -63,3 +63,9 @@ instructions for that platform.
 Due to current limitations of the CMake build system creating artifacts in the
 source 3rdparty folder of the parent mxnet sources concurrent builds of
 different platforms is NOT SUPPORTED.
+
+## ccache
+For all builds a directory from the host system is mapped where ccache will store cached 
+compiled object files (defaults to /tmp/ci_ccache). This will speed up rebuilds 
+significantly. You can set this directory explicitly by setting CCACHE_DIR environment 
+variable. All ccache instances are currently set to be 10 Gigabytes max in size.

From bed2c31d6c3d0ac31882b28c5cb1c44ac93b4782 Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Wed, 30 May 2018 15:38:47 +0000
Subject: [PATCH 11/14] Reverted not working ccache configurations

---
 Makefile                                    |  9 ++------
 ci/docker/install/centos7_install_ccache.sh |  8 ++++---
 ci/docker/install/ubuntu_install_ccache.sh  |  8 ++++---
 ci/docker/runtime_functions.sh              | 25 ++++++++++++++-------
 4 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/Makefile b/Makefile
index 2c8560c83d2f..19a3bcd34d4e 100644
--- a/Makefile
+++ b/Makefile
@@ -79,7 +79,7 @@ include $(DMLC_CORE)/make/dmlc.mk
 
 # all tge possible warning tread
 WARNFLAGS= -Wall -Wsign-compare
-CFLAGS += -DMSHADOW_FORCE_STREAM $(WARNFLAGS)
+CFLAGS = -DMSHADOW_FORCE_STREAM $(WARNFLAGS)
 
 ifeq ($(DEV), 1)
 	CFLAGS += -g -Werror
@@ -92,13 +92,8 @@ ifeq ($(DEBUG), 1)
 else
 	CFLAGS += -O3 -DNDEBUG=1
 endif
-<<<<<<< HEAD
-CFLAGS += -I$(TPARTYDIR)/mshadow/ -I$(TPARTYDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(TPARTYDIR)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
-LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
-=======
 CFLAGS += -I$(TPARTYDIR)/mshadow/ -I$(TPARTYDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(NNVM_PATH)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
-LDFLAGS += -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
->>>>>>> [MXNET-472] Add ccache support to docker builds
+LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
 ifeq ($(DEBUG), 1)
 	NVCCFLAGS += -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
 else
diff --git a/ci/docker/install/centos7_install_ccache.sh b/ci/docker/install/centos7_install_ccache.sh
index 4ad17e74b834..3face0a93b52 100755
--- a/ci/docker/install/centos7_install_ccache.sh
+++ b/ci/docker/install/centos7_install_ccache.sh
@@ -32,7 +32,8 @@ yum -y install google-perftools
 yum -y install asciidoc
 yum -y install gcc-c++-4.8.*
 
-cd /work
+mkdir -p /work/deps
+cd /work/deps
 
 git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git
 
@@ -43,9 +44,10 @@ cd ccache
 make -j$(nproc)
 make install
 
-popd
+cd /work/deps
+rm -rf /work/deps/ccache
 
-rm -rf /work/ccache
+popd
 
 export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:=10G}
 export CCACHE_DIR=${CCACHE_DIR:=/work/ccache}
diff --git a/ci/docker/install/ubuntu_install_ccache.sh b/ci/docker/install/ubuntu_install_ccache.sh
index 75db51fcbdbd..3ceaaad8032e 100755
--- a/ci/docker/install/ubuntu_install_ccache.sh
+++ b/ci/docker/install/ubuntu_install_ccache.sh
@@ -38,7 +38,8 @@ apt install -y --no-install-recommends \
     xsltproc \
     libxml2-utils
 
-cd /work
+mkdir -p /work/deps
+cd /work/deps
 
 git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git
 
@@ -49,9 +50,10 @@ cd ccache
 make -j$(nproc)
 make install
 
-popd
+cd /work/deps
+rm -rf /work/deps/ccache
 
-rm -rf /work/ccache
+popd
 
 export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:=10G}
 export CCACHE_DIR=${CCACHE_DIR:=/work/ccache}
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index ad9d28765ab4..3585427f4d8b 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -47,11 +47,11 @@ build_ccache_wrappers() {
     fi
 
     if [ -z ${CXX+x} ]; then
-        echo "No \$CXX set, defaulting to g++";
-        export CXX=g++
+       echo "No \$CXX set, defaulting to g++";
+       export CXX=g++
     fi
 
-    # this function is nessesary for cuda enabled make based builds, since nvcc needs just an exacutable for -ccbin
+    # this function is nessesary for cuda enabled make based builds, since nvcc needs just an executable for -ccbin
 
     echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CC} \"\$@\"\n" >> cc
     echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CXX} \"\$@\"\n" >> cxx
@@ -210,8 +210,9 @@ build_android_arm64() {
 
 build_centos7_cpu() {
     set -ex
-    build_ccache_wrappers
     cd /work/mxnet
+    export CC="ccache gcc"
+    export CXX="ccache g++"
     make \
         DEV=1 \
         USE_LAPACK=1 \
@@ -223,8 +224,9 @@ build_centos7_cpu() {
 
 build_centos7_mkldnn() {
     set -ex
-    build_ccache_wrappers
     cd /work/mxnet
+    export CC="ccache gcc"
+    export CXX="ccache g++"
     make \
         DEV=1 \
         USE_LAPACK=1 \
@@ -236,8 +238,9 @@ build_centos7_mkldnn() {
 
 build_centos7_gpu() {
     set -ex
-    build_ccache_wrappers
     cd /work/mxnet
+    # unfortunately this build has problems in 3rdparty dependencies with ccache and make
+    # build_ccache_wrappers
     make \
         DEV=1 \
         USE_LAPACK=1 \
@@ -250,9 +253,14 @@ build_centos7_gpu() {
         -j$(nproc)
 }
 
+build_ubuntu_cpu() {
+    build_ubuntu_cpu_openblas
+}
+
 build_ubuntu_cpu_openblas() {
     set -ex
-    build_ccache_wrappers
+    export CC="ccache gcc"
+    export CXX="ccache g++"
     make \
         DEV=1                         \
         USE_CPP_PACKAGE=1             \
@@ -352,7 +360,8 @@ build_ubuntu_gpu_mkldnn() {
 
 build_ubuntu_gpu_cuda91_cudnn7() {
     set -ex
-    build_ccache_wrappers
+    # unfortunately this build has problems in 3rdparty dependencies with ccache and make
+    # build_ccache_wrappers
     make \
         DEV=1                         \
         USE_BLAS=openblas             \

From 4b3e09f34dcf216fba329a9b3036a94fba91bd4d Mon Sep 17 00:00:00 2001
From: Anton Chernov <mechernov@gmail.com>
Date: Thu, 31 May 2018 11:05:57 +0000
Subject: [PATCH 12/14] Added comments about ccache installation

---
 Makefile                                 |  2 +-
 ci/build.py                              | 23 ++++++++++-------------
 ci/docker/Dockerfile.build.android_arm64 |  1 +
 ci/docker/Dockerfile.build.android_armv7 |  1 +
 ci/docker/Dockerfile.build.arm64         |  1 +
 ci/docker/Dockerfile.build.armv6         |  1 +
 ci/docker/Dockerfile.build.armv7         |  1 +
 ci/docker/Dockerfile.build.jetson        |  1 +
 make/crosscompile.jetson.mk              |  2 +-
 9 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index 19a3bcd34d4e..ff4446ab80c7 100644
--- a/Makefile
+++ b/Makefile
@@ -92,7 +92,7 @@ ifeq ($(DEBUG), 1)
 else
 	CFLAGS += -O3 -DNDEBUG=1
 endif
-CFLAGS += -I$(TPARTYDIR)/mshadow/ -I$(TPARTYDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(NNVM_PATH)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
+CFLAGS += -I$(TPARTYDIR)/mshadow/ -I$(TPARTYDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(TPARTYDIR)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
 LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
 ifeq ($(DEBUG), 1)
 	NVCCFLAGS += -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
diff --git a/ci/build.py b/ci/build.py
index 03c35b0fdb95..ef80bc017549 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -124,7 +124,7 @@ def default_ccache_dir() -> str:
         ccache_dir = os.path.realpath(os.environ['CCACHE_DIR'])
         os.makedirs(ccache_dir, exist_ok=True)
         return ccache_dirpython
-    # Share ccache across containers (should we have a separate dir per platform?)
+    # Share ccache across containers
     return os.path.join(tempfile.gettempdir(), "ci_ccache")
 
 
@@ -261,31 +261,28 @@ def script_name() -> str:
         list_platforms()
     elif args.platform:
         platform = args.platform
-        tag = get_docker_tag(platform)
-        if args.download_docker_cache:
-            import docker_cache
-            logging.info('Docker cache download is enabled')
-            docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag)
-        # prepare ccache
-        build_docker(platform, docker_binary)
+        tag = get_docker_tag(platform=platform, registry=docker_registry)
+        load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+        build_docker(platform, docker_binary, registry=docker_registry)
         if args.build_only:
             logging.warning("Container was just built. Exiting due to build-only.")
             return 0
 
         if command:
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=command, docker_registry=docker_registry, ccache_dir=args.ccache_dir)
+                          command=command, docker_registry=docker_registry, local_ccache_dir=args.ccache_dir)
         elif args.print_docker_run:
             print(container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                                command=[], dry_run=True, docker_registry=docker_registry, ccache_dir=args.ccache_dir))
+                                command=[], dry_run=True, docker_registry=docker_registry, local_ccache_dir=args.ccache_dir))
         elif args.into_container:
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=[], dry_run=False, into_container=True, docker_registry=docker_registry, ccache_dir=args.ccache_dir)
+                          command=[], dry_run=False, into_container=True,
+                          docker_registry=docker_registry, local_ccache_dir=args.ccache_dir)
         else:
             cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
             logging.info("No command specified, trying default build: %s", ' '.join(cmd))
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=cmd, docker_registry=docker_registry, ccache_dir=args.ccache_dir)
+                          command=cmd, docker_registry=docker_registry, local_ccache_dir=args.ccache_dir)
 
     elif args.all:
         platforms = get_platforms()
@@ -301,7 +298,7 @@ def script_name() -> str:
             cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform]
             shutil.rmtree(buildir(), ignore_errors=True)
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=cmd, docker_registry=docker_registry, ccache_dir=args.ccache_dir)
+                          command=cmd, docker_registry=docker_registry, local_ccache_dir=args.ccache_dir)
             plat_buildir = os.path.join(get_mxnet_root(), build_platform)
             shutil.move(buildir(), plat_buildir)
             logging.info("Built files left in: %s", plat_buildir)
diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64
index f55ac4b1ff8d..fe9490f8298b 100755
--- a/ci/docker/Dockerfile.build.android_arm64
+++ b/ci/docker/Dockerfile.build.android_arm64
@@ -26,6 +26,7 @@ RUN /work/ubuntu_install_ccache.sh
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
+# extract ccache binary into latest context
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 
 # The cross-compiling emulator
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
index 30c4242a3dbe..bc2c0e897a52 100755
--- a/ci/docker/Dockerfile.build.android_armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -26,6 +26,7 @@ RUN /work/ubuntu_install_ccache.sh
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
+# extract ccache binary into latest context
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 
 # The cross-compiling emulator
diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64
index 5de4d22c988f..edf78856722a 100755
--- a/ci/docker/Dockerfile.build.arm64
+++ b/ci/docker/Dockerfile.build.arm64
@@ -27,6 +27,7 @@ RUN /work/ubuntu_install_ccache.sh
 #FROM dockcross/linux-arm64
 FROM mxnetci/dockcross-linux-arm64:05082018
 
+# extract ccache binary into latest context
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 
 ENV ARCH aarch64
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index 42555ebee133..1d000bb0962a 100755
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -25,6 +25,7 @@ RUN /work/ubuntu_install_ccache.sh
 
 FROM dockcross/linux-armv6
 
+# extract ccache binary into latest context
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 
 ENV ARCH armv6l
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
index 58f3c37ce008..7d55dc463272 100755
--- a/ci/docker/Dockerfile.build.armv7
+++ b/ci/docker/Dockerfile.build.armv7
@@ -25,6 +25,7 @@ RUN /work/ubuntu_install_ccache.sh
 
 FROM dockcross/linux-armv7
 
+# extract ccache binary into latest context
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 
 ENV ARCH armv71
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index 7d28cf6f07ce..c963457299bc 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -31,6 +31,7 @@ RUN /work/ubuntu_install_ccache.sh
 # FROM dockcross/linux-arm64
 FROM mxnetci/dockcross-linux-arm64:05082018
 
+# extract ccache binary into latest context
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 
 ENV ARCH aarch64
diff --git a/make/crosscompile.jetson.mk b/make/crosscompile.jetson.mk
index 868fefe05ec2..5bb4961bf01e 100644
--- a/make/crosscompile.jetson.mk
+++ b/make/crosscompile.jetson.mk
@@ -96,7 +96,7 @@ USE_LIBJPEG_TURBO = 0
 USE_LIBJPEG_TURBO_PATH = NONE
 
 # use openmp for parallelization
-USE_OPENMP = 0
+USE_OPENMP = 1
 
 # whether use MKL-DNN library
 USE_MKLDNN = 0

From e25c8c9810d9f88c44d2964ca911a860f70ed9a6 Mon Sep 17 00:00:00 2001
From: Marco de Abreu <marco.g.abreu+github@gmail.com>
Date: Tue, 5 Jun 2018 18:18:11 +0200
Subject: [PATCH 13/14] Move install scripts

---
 ci/build.py                                         |  2 ++
 ci/docker/Dockerfile.build.android_arm64            |  6 ++++--
 ci/docker/Dockerfile.build.android_armv7            |  6 ++++--
 ci/docker/Dockerfile.build.arm64                    |  6 ++++--
 ci/docker/Dockerfile.build.armv6                    |  6 ++++--
 ci/docker/Dockerfile.build.armv7                    |  6 ++++--
 ci/docker/Dockerfile.build.centos7_cpu              |  4 ++--
 ci/docker/Dockerfile.build.centos7_gpu              |  4 ++--
 ci/docker/Dockerfile.build.jetson                   |  8 +++++---
 ci/docker/Dockerfile.build.ubuntu_build_cuda        |  4 ++--
 ci/docker/Dockerfile.build.ubuntu_cpu               |  4 ++--
 ci/docker/Dockerfile.build.ubuntu_gpu               |  4 ++--
 ...{centos7_install_ccache.sh => centos7_ccache.sh} |  8 --------
 .../{ubuntu_install_ccache.sh => ubuntu_ccache.sh}  | 13 +------------
 ci/docker/install/ubuntu_r.sh                       |  2 +-
 15 files changed, 39 insertions(+), 44 deletions(-)
 rename ci/docker/install/{centos7_install_ccache.sh => centos7_ccache.sh} (82%)
 rename ci/docker/install/{ubuntu_install_ccache.sh => ubuntu_ccache.sh} (80%)

diff --git a/ci/build.py b/ci/build.py
index ef80bc017549..0ff14befc509 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -39,6 +39,7 @@
 from subprocess import call, check_call
 from typing import *
 
+CCACHE_MAXSIZE = '10G'
 
 def get_platforms(path: Optional[str] = "docker"):
     """Get a list of architectures given our dockerfiles"""
@@ -149,6 +150,7 @@ def container_run(platform: str,
                '-v', "{}:/work/build".format(local_build_folder),  # mount mxnet/build for storing build artifacts
                '-v', "{}:/work/ccache".format(local_ccache_dir),
                '-u', '{}:{}'.format(os.getuid(), os.getgid()),
+               '-e', 'CCACHE_MAXSIZE={}'.format(CCACHE_MAXSIZE),
                '-e', "CCACHE_DIR=/work/ccache",  # this path is inside the container as /work/ccache is mounted
                tag]
     runlist.extend(command)
diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64
index fe9490f8298b..44d369258182 100755
--- a/ci/docker/Dockerfile.build.android_arm64
+++ b/ci/docker/Dockerfile.build.android_arm64
@@ -20,8 +20,10 @@
 
 FROM ccache/build.ubuntu as ccachebuilder
 
-COPY install/ubuntu_install_ccache.sh /work/
-RUN /work/ubuntu_install_ccache.sh
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
 
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
index bc2c0e897a52..28dc3fd23903 100755
--- a/ci/docker/Dockerfile.build.android_armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -20,8 +20,10 @@
 
 FROM ccache/build.ubuntu as ccachebuilder
 
-COPY install/ubuntu_install_ccache.sh /work/
-RUN /work/ubuntu_install_ccache.sh
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
 
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64
index edf78856722a..a1f752bbf64a 100755
--- a/ci/docker/Dockerfile.build.arm64
+++ b/ci/docker/Dockerfile.build.arm64
@@ -20,8 +20,10 @@
 
 FROM ubuntu:16.04 as ccachebuilder
 
-COPY install/ubuntu_install_ccache.sh /work/
-RUN /work/ubuntu_install_ccache.sh
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
 
 # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
 #FROM dockcross/linux-arm64
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index 1d000bb0962a..c073992406fb 100755
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -20,8 +20,10 @@
 
 FROM ubuntu:16.04 as ccachebuilder
 
-COPY install/ubuntu_install_ccache.sh /work/
-RUN /work/ubuntu_install_ccache.sh
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
 
 FROM dockcross/linux-armv6
 
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
index 7d55dc463272..627486c0537d 100755
--- a/ci/docker/Dockerfile.build.armv7
+++ b/ci/docker/Dockerfile.build.armv7
@@ -20,8 +20,10 @@
 
 FROM ubuntu:16.04 as ccachebuilder
 
-COPY install/ubuntu_install_ccache.sh /work/
-RUN /work/ubuntu_install_ccache.sh
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
 
 FROM dockcross/linux-armv7
 
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
index 1777640273f8..076ef5df911a 100755
--- a/ci/docker/Dockerfile.build.centos7_cpu
+++ b/ci/docker/Dockerfile.build.centos7_cpu
@@ -24,8 +24,8 @@ WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
 RUN /work/centos7_core.sh
-COPY install/centos7_install_ccache.sh /work/
-RUN /work/centos7_install_ccache.sh
+COPY install/centos7_ccache.sh /work/
+RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/
 RUN /work/centos7_python.sh
 COPY install/ubuntu_mklml.sh /work/
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
index cb2ba08c6aa7..8bf2442731af 100755
--- a/ci/docker/Dockerfile.build.centos7_gpu
+++ b/ci/docker/Dockerfile.build.centos7_gpu
@@ -24,8 +24,8 @@ WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
 RUN /work/centos7_core.sh
-COPY install/centos7_install_ccache.sh /work/
-RUN /work/centos7_install_ccache.sh
+COPY install/centos7_ccache.sh /work/
+RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/
 RUN /work/centos7_python.sh
 
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index c963457299bc..8a8bb97aa156 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -24,8 +24,10 @@ FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder
 
 FROM ubuntu:16.04 as ccachebuilder
 
-COPY install/ubuntu_install_ccache.sh /work/
-RUN /work/ubuntu_install_ccache.sh
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
 
 # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
 # FROM dockcross/linux-arm64
@@ -35,8 +37,8 @@ FROM mxnetci/dockcross-linux-arm64:05082018
 COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
 
 ENV ARCH aarch64
-ENV HOSTCC gcc
 ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
+ENV HOSTCC gcc
 ENV TARGET ARMV8
 
 WORKDIR /work
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index 46ed811400b8..a87651a4f5a2 100755
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -27,8 +27,8 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
-COPY install/ubuntu_install_ccache.sh /work/
-RUN /work/ubuntu_install_ccache.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
 COPY install/ubuntu_python.sh /work/
 RUN /work/ubuntu_python.sh
 COPY install/ubuntu_scala.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
index 24558adce687..f82ee75c482e 100755
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -24,8 +24,8 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
-COPY install/ubuntu_install_ccache.sh /work/
-RUN /work/ubuntu_install_ccache.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
 COPY install/ubuntu_python.sh /work/
 RUN /work/ubuntu_python.sh
 COPY install/ubuntu_scala.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
index 614d43b911bd..5f4bfc5a87d9 100755
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -24,8 +24,8 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
-COPY install/ubuntu_install_ccache.sh /work/
-RUN /work/ubuntu_install_ccache.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
 COPY install/ubuntu_python.sh /work/
 RUN /work/ubuntu_python.sh
 COPY install/ubuntu_scala.sh /work/
diff --git a/ci/docker/install/centos7_install_ccache.sh b/ci/docker/install/centos7_ccache.sh
similarity index 82%
rename from ci/docker/install/centos7_install_ccache.sh
rename to ci/docker/install/centos7_ccache.sh
index 3face0a93b52..846a407001b7 100755
--- a/ci/docker/install/centos7_install_ccache.sh
+++ b/ci/docker/install/centos7_ccache.sh
@@ -23,14 +23,8 @@ set -ex
 
 pushd .
 
-yum -y install epel-release
-yum -y install git
 yum -y install autoconf
-yum -y install wget
-yum -y install make
-yum -y install google-perftools
 yum -y install asciidoc
-yum -y install gcc-c++-4.8.*
 
 mkdir -p /work/deps
 cd /work/deps
@@ -49,5 +43,3 @@ rm -rf /work/deps/ccache
 
 popd
 
-export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:=10G}
-export CCACHE_DIR=${CCACHE_DIR:=/work/ccache}
diff --git a/ci/docker/install/ubuntu_install_ccache.sh b/ci/docker/install/ubuntu_ccache.sh
similarity index 80%
rename from ci/docker/install/ubuntu_install_ccache.sh
rename to ci/docker/install/ubuntu_ccache.sh
index 3ceaaad8032e..09838d20e69e 100755
--- a/ci/docker/install/ubuntu_install_ccache.sh
+++ b/ci/docker/install/ubuntu_ccache.sh
@@ -25,18 +25,9 @@ pushd .
 
 apt update
 apt install -y --no-install-recommends \
-    git \
-    ssh \
-    apt-transport-https \
-    build-essential \
-    ca-certificates \
     autoconf \
-    google-perftools \
     asciidoc \
-    libxslt1-dev \
-    docbook-xsl \
-    xsltproc \
-    libxml2-utils
+    xsltproc
 
 mkdir -p /work/deps
 cd /work/deps
@@ -55,5 +46,3 @@ rm -rf /work/deps/ccache
 
 popd
 
-export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:=10G}
-export CCACHE_DIR=${CCACHE_DIR:=/work/ccache}
diff --git a/ci/docker/install/ubuntu_r.sh b/ci/docker/install/ubuntu_r.sh
index e04e94d64865..0c4a6f2e66f4 100755
--- a/ci/docker/install/ubuntu_r.sh
+++ b/ci/docker/install/ubuntu_r.sh
@@ -27,4 +27,4 @@ gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
 gpg -a --export E084DAB9 | apt-key add -
 
 apt-get update
-apt-get install -y r-base r-base-dev libxml2-dev libssl-dev libxt-dev
\ No newline at end of file
+apt-get install -y --alow-unauthenticated r-base r-base-dev libxml2-dev libssl-dev libxt-dev

From df60ac77a084e55d6b754a1543d06d3e42121a8e Mon Sep 17 00:00:00 2001
From: Marco de Abreu <marcoabreu@users.noreply.github.com>
Date: Tue, 5 Jun 2018 23:08:19 +0200
Subject: [PATCH 14/14] Update ubuntu_r.sh

---
 ci/docker/install/ubuntu_r.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/docker/install/ubuntu_r.sh b/ci/docker/install/ubuntu_r.sh
index 0c4a6f2e66f4..4bf1a897e085 100755
--- a/ci/docker/install/ubuntu_r.sh
+++ b/ci/docker/install/ubuntu_r.sh
@@ -27,4 +27,4 @@ gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
 gpg -a --export E084DAB9 | apt-key add -
 
 apt-get update
-apt-get install -y --alow-unauthenticated r-base r-base-dev libxml2-dev libssl-dev libxt-dev
+apt-get install -y --allow-unauthenticated r-base r-base-dev libxml2-dev libssl-dev libxt-dev