[WIP] Windows dev environment configuration, update install instructions from source in the docs.

ci/build_windows.py

@@ -37,9 +37,16 @@
 
 from util import *
 
+
+# Fix for broken PATH with newline inserted presumably by VS studio installation of SQL server or
+# other component which makes visual studio stop working.
+os.environ['PATH']=os.environ.get('PATH').replace('\n','')
+
 KNOWN_VCVARS = {
+    # https://gitlab.kitware.com/cmake/cmake/issues/18920
     'VS 2015': r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat',
-    'VS 2017': r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat'
+    'VS 2017': r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat',
+    'VS 2019': r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat'
 }
 
 
@@ -54,10 +61,14 @@ class BuildFlavour(Enum):
 
 CMAKE_FLAGS = {
     'WIN_CPU': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=OFF '
         '-DUSE_CUDNN=OFF '
         '-DENABLE_CUDA_RTC=OFF '
         '-DUSE_OPENCV=ON '
+        '-DOpenCV_RUNTIME=vc15 '
+        '-DOpenCV_ARCH=x64 '
         '-DUSE_OPENMP=ON '
         '-DUSE_BLAS=open '
         '-DUSE_LAPACK=ON '
@@ -67,10 +78,14 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_CPU_MKLDNN': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=OFF '
         '-DUSE_CUDNN=OFF '
         '-DENABLE_CUDA_RTC=OFF '
         '-DUSE_OPENCV=ON '
+        '-DOpenCV_RUNTIME=vc15 '
+        '-DOpenCV_ARCH=x64 '
         '-DUSE_OPENMP=ON '
         '-DUSE_BLAS=open '
         '-DUSE_LAPACK=ON '
@@ -80,10 +95,14 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_CPU_MKLDNN_MKL': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=OFF '
         '-DUSE_CUDNN=OFF '
         '-DENABLE_CUDA_RTC=OFF '
         '-DUSE_OPENCV=ON '
+        '-DOpenCV_RUNTIME=vc15 '
+        '-DOpenCV_ARCH=x64 '
         '-DUSE_OPENMP=ON '
         '-DUSE_BLAS=mkl '
         '-DUSE_LAPACK=ON '
@@ -93,10 +112,14 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_CPU_MKL': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=OFF '
         '-DUSE_CUDNN=OFF '
         '-DENABLE_CUDA_RTC=OFF '
         '-DUSE_OPENCV=ON '
+        '-DOpenCV_RUNTIME=vc15 '
+        '-DOpenCV_ARCH=x64 '
         '-DUSE_OPENMP=ON '
         '-DUSE_BLAS=mkl '
         '-DUSE_LAPACK=ON '
@@ -106,10 +129,14 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_GPU': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=ON '
         '-DUSE_CUDNN=ON '
         '-DENABLE_CUDA_RTC=ON '
         '-DUSE_OPENCV=ON  '
+        '-DOpenCV_RUNTIME=vc15 '
+        '-DOpenCV_ARCH=x64 '
         '-DUSE_OPENMP=ON '
         '-DUSE_BLAS=open '
         '-DUSE_LAPACK=ON '
@@ -120,10 +147,14 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_GPU_MKLDNN': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=ON '
         '-DUSE_CUDNN=ON '
         '-DENABLE_CUDA_RTC=ON '
         '-DUSE_OPENCV=ON '
+        '-DOpenCV_RUNTIME=vc15 '
+        '-DOpenCV_ARCH=x64 '
         '-DUSE_OPENMP=ON '
         '-DUSE_BLAS=open '
         '-DUSE_LAPACK=ON '
@@ -145,35 +176,23 @@ def windows_build(args):
     mxnet_root = get_mxnet_root()
     logging.info("Found MXNet root: {}".format(mxnet_root))
 
-    url = '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/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.zip'
-    with tempfile.TemporaryDirectory() as tmpdir:
-        cmake_file_path = download_file(url, tmpdir)
-        with zipfile.ZipFile(cmake_file_path, 'r') as zip_ref:
-            # Create $tmpdir\cmake-3.16.1-win64-x64\bin\cmake.exe
-            zip_ref.extractall(tmpdir)
-
-        with remember_cwd():
-            os.chdir(path)
-            cmd = "\"{}\" && {} -G \"NMake Makefiles JOM\" {} {}".format(
-                args.vcvars,
-                os.path.join(tmpdir, 'cmake-3.16.1-win64-x64', 'bin', 'cmake.exe'),
-                CMAKE_FLAGS[args.flavour], mxnet_root)
-            logging.info("Generating project with CMake:\n{}".format(cmd))
-            check_call(cmd, shell=True)
-
-            cmd = "\"{}\" && jom".format(args.vcvars)
-            logging.info("Building with jom:\n{}".format(cmd))
-
-            t0 = int(time.time())
-            check_call(cmd, shell=True)
-
-            logging.info(
-                "Build flavour: {} complete in directory: \"{}\"".format(
-                    args.flavour, os.path.abspath(path)))
-            logging.info("Build took {}".format(
-                datetime.timedelta(seconds=int(time.time() - t0))))
-    windows_package(args)
+    with remember_cwd():
+        os.chdir(path)
+        cmd = "\"{}\" && cmake -G Ninja {} {}".format(args.vcvars,
+                                                                        CMAKE_FLAGS[args.flavour],
+                                                                        mxnet_root)
+        logging.info("Generating project with CMake:\n{}".format(cmd))
+        check_call(cmd, shell=True)
+
+        cmd = "\"{}\" && ninja".format(args.vcvars)
+        logging.info("Building:\n{}".format(cmd))
 
+        t0 = int(time.time())
+        check_call(cmd, shell=True)
+
+        logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path)))
+        logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0))))
+    windows_package(args)
 
 def windows_package(args):
     pkgfile = 'windows_package.7z'
@@ -230,7 +249,7 @@ def main():
 
     parser.add_argument("--vcvars",
         help="vcvars batch file location, typically inside vs studio install dir",
-        default=KNOWN_VCVARS['VS 2015'],
+        default=KNOWN_VCVARS['VS 2017'],
         type=str)
 
     parser.add_argument("--arch",
@@ -250,10 +269,6 @@ def main():
     system = platform.system()
     if system == 'Windows':
         logging.info("Detected Windows platform")
-        if 'OpenBLAS_HOME' not in os.environ:
-            os.environ["OpenBLAS_HOME"] = "C:\\Program Files\\OpenBLAS-v0.2.19"
-        if 'OpenCV_DIR' not in os.environ:
-            os.environ["OpenCV_DIR"] = "C:\\Program Files\\OpenCV-v3.4.1\\build"
         if 'CUDA_PATH' not in os.environ:
             os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.2"
         if 'MKL_ROOT' not in os.environ:

ci/safe_docker_run.py

@@ -38,8 +38,9 @@
 
 from util import config_logging
 
-DOCKER_STOP_TIMEOUT_SECONDS = 3
+DOCKER_STOP_TIMEOUT_SECONDS = 10
 CONTAINER_WAIT_SECONDS = 600
+DOCKER_CLIENT_TIMEOUT = 600
 
 
 class SafeDockerClient:
@@ -54,7 +55,7 @@ def _trim_container_id(cid):
         return cid[:12]
 
     def __init__(self):
-        self._docker_client = docker.from_env()
+        self._docker_client = docker.from_env(timeout=DOCKER_CLIENT_TIMEOUT)
         self._containers = set()
         self._docker_stop_timeout = DOCKER_STOP_TIMEOUT_SECONDS
         self._container_wait_seconds = CONTAINER_WAIT_SECONDS

ci/windows_dev_env/requirements.txt

@@ -0,0 +1,4 @@
+psutil
+boto3
+python-jenkins
+progressbar2

ci/windows_dev_env/setup.ps1

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version Latest
+function Check-Call {
+    param (
+        [scriptblock]$ScriptBlock
+    )
+    Write-Host "Executing $ScriptBlock"
+    & @ScriptBlock
+    if (($lastexitcode -ne 0)) {
+	Write-Error "Execution failed with $lastexitcode"
+        exit $lastexitcode
+    }
+}
+Check-Call { setx PATH "$($env:path);c:\Program Files\CMake\bin;" /m }
+Set-ExecutionPolicy Bypass -Scope Process -Force
+Invoke-WebRequest -Uri https://chocolatey.org/install.ps1 -OutFile install.ps1
+./install.ps1
+#Check-Call { C:\ProgramData\chocolatey\choco install python2 -y --no-progress }
+Check-Call { C:\ProgramData\chocolatey\choco install python --version=3.7.0 --force -y --no-progress -r}
+Check-Call { C:\Python37\python -m pip install --upgrade pip  }
+Check-Call { C:\Python37\python -m pip install -r requirements.txt  }
+#Check-Call { C:\Python27\python -m pip install --upgrade pip  }
+#Check-Call { C:\Python27\python -m pip install -r requirements.txt  }
+
+Check-Call { C:\ProgramData\chocolatey\choco install git -y -r --no-progress }
+Check-Call { C:\ProgramData\chocolatey\choco install 7zip -y -r --no-progress }
+Check-Call { C:\ProgramData\chocolatey\choco install cmake -y -r --no-progress }
+Check-Call { C:\ProgramData\chocolatey\choco install ninja -y -r --no-progress }
+
+# Deps
+Check-Call { C:\Python37\python  windows_deps_headless_installer.py }
+
+# Other software
+#Check-Call { C:\ProgramData\chocolatey\choco install jom -y }
+#Check-Call { C:\ProgramData\chocolatey\choco install mingw -y -r --no-progress }
+Check-Call { C:\ProgramData\chocolatey\choco install javaruntime -y -r --no-progress }
+
+# update path after all software is installed
+refreshenv
+
+Write-Output "End"