diff --git a/.gitignore b/.gitignore index c8a813649bb8..7cdbc260e6b3 100644 --- a/.gitignore +++ b/.gitignore @@ -146,6 +146,7 @@ tools/pip_package/mxnet # temporary path for building dependencies when building wheel ./deps/ +./staticdeps/ bld ./tmp/* *.jar diff --git a/Makefile b/Makefile index 16ea59f3d585..4f8bedb7e44b 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,6 @@ ROOTDIR = $(CURDIR) TPARTYDIR = $(ROOTDIR)/3rdparty -SCALA_VERSION_PROFILE := scala-2.11 - ifeq ($(OS),Windows_NT) UNAME_S := Windows else @@ -397,18 +395,13 @@ PLUGIN_OBJ = PLUGIN_CUOBJ = include $(MXNET_PLUGINS) -ifeq ($(UNAME_S), Windows) - # TODO(yizhi) currently scala package does not support windows - SCALA_PKG_PROFILE := windows -else +ifneq ($(UNAME_S), Windows) ifeq ($(UNAME_S), Darwin) WHOLE_ARCH= -all_load NO_WHOLE_ARCH= -noall_load - SCALA_PKG_PROFILE := osx-x86_64 else WHOLE_ARCH= --whole-archive NO_WHOLE_ARCH= --no-whole-archive - SCALA_PKG_PROFILE := linux-x86_64 endif endif @@ -427,7 +420,6 @@ ifeq ($(USE_CUDA), 1) # Make sure to add stubs as fallback in order to be able to build # without full CUDA install (especially if run without nvidia-docker) LDFLAGS += -L/usr/local/cuda/lib64/stubs - SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu ifeq ($(USE_NCCL), 1) ifneq ($(USE_NCCL_PATH), NONE) CFLAGS += -I$(USE_NCCL_PATH)/include @@ -439,7 +431,6 @@ ifeq ($(USE_CUDA), 1) CFLAGS += -DMXNET_USE_NCCL=0 endif else - SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu CFLAGS += -DMXNET_USE_NCCL=0 endif @@ -454,6 +445,10 @@ else CFLAGS += -DMXNET_USE_LIBJPEG_TURBO=0 endif +ifeq ($(CI), 1) + MAVEN_ARGS := -B +endif + # For quick compile test, used smaller subset ALLX_DEP= $(ALL_DEP) @@ -600,63 +595,22 @@ rpkgtest: Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)' scalaclean: - (cd $(ROOTDIR)/scala-package; \ - mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE)) + (cd $(ROOTDIR)/scala-package && mvn clean) scalapkg: - (cd $(ROOTDIR)/scala-package; \ - mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ - -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dcurrent_libdir="$(ROOTDIR)/lib" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + (cd $(ROOTDIR)/scala-package && mvn install -DskipTests) + +scalainstall: + (cd $(ROOTDIR)/scala-package && mvn install) scalaunittest: - (cd $(ROOTDIR)/scala-package; \ - mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) + (cd $(ROOTDIR)/scala-package && mvn install) scalaintegrationtest: - (cd $(ROOTDIR)/scala-package; \ - mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) - -scalainstall: - (cd $(ROOTDIR)/scala-package; \ - mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \ - -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") - -scalarelease-dryrun: - (cd $(ROOTDIR)/scala-package; \ - mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \ - -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") - -scalarelease-prepare: - (cd $(ROOTDIR)/scala-package; \ - mvn release:clean release:prepare -DautoVersionSubmodules=true \ - -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") - -scalarelease-perform: - (cd $(ROOTDIR)/scala-package; \ - mvn release:perform -DautoVersionSubmodules=true \ - -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") - -scaladeploy: - (cd $(ROOTDIR)/scala-package; \ - mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ - -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + (cd $(ROOTDIR)/scala-package && mvn integration-test -DskipTests=false) jnilint: - 3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src + 3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src --exclude_path scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h rclean: $(RM) -r R-package/src/image_recordio.h R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \ diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index fb84e0045d53..e36d353d873c 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -64,7 +64,7 @@ def init_git_win() { // pack libraries for later use def pack_lib(name, libs, include_gcov_data = false) { - sh """ + sh returnStatus: true, script: """ set +e echo "Packing ${libs} into ${name}" echo ${libs} | sed -e 's/,/ /g' | xargs md5sum @@ -83,7 +83,7 @@ return 0 def unpack_and_init(name, libs, include_gcov_data = false) { init_git() unstash name - sh """ + sh returnStatus: true, script: """ set +e echo "Unpacked ${libs} from ${name}" echo ${libs} | sed -e 's/,/ /g' | xargs md5sum @@ -147,8 +147,9 @@ def collect_test_results_windows(original_file_name, new_file_name) { } -def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" +def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "") { + def command = "ci/build.py %ENV_VARS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" + command = command.replaceAll('%ENV_VARS%', env_vars.length() > 0 ? "-e ${env_vars}" : '') command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') command = command.replaceAll('%PLATFORM%', platform) command = command.replaceAll('%FUNCTION_NAME%', function_name) diff --git a/ci/build.py b/ci/build.py index acf277dd2c32..52d4623e682a 100755 --- a/ci/build.py +++ b/ci/build.py @@ -92,22 +92,24 @@ def get_dockerfiles_path(): def get_platforms(path: str = get_dockerfiles_path()) -> List[str]: """Get a list of architectures given our dockerfiles""" - dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*")) + dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*")) dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles)) - files = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles)) + files = list(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles)) platforms = list(map(lambda x: os.path.split(x)[1], sorted(files))) return platforms def get_docker_tag(platform: str, registry: str) -> str: """:return: docker tag to be used for the container""" + platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform) if not registry: registry = "mxnet_local" - return "{0}/build.{1}".format(registry, platform) + return "{0}/{1}".format(registry, platform) def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str: - return os.path.join(path, "Dockerfile.build.{0}".format(platform)) + platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform) + return os.path.join(path, "Dockerfile.{0}".format(platform)) def get_docker_binary(use_nvidia_docker: bool) -> str: @@ -215,20 +217,21 @@ def container_run(platform: str, local_ccache_dir: str, command: List[str], cleanup: Cleanup, + environment: Dict[str, str], dry_run: bool = False) -> int: """Run command in a container""" container_wait_s = 600 # # Environment setup # - environment = { + environment.update({ 'CCACHE_MAXSIZE': '500G', 'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared 'CCACHE_DIR': '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/work/ccache', # this path is inside the container as /work/ccache is # mounted 'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache # verification. - } + }) # These variables are passed to the container to the process tree killer can find runaway # process inside the container # https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller @@ -446,6 +449,10 @@ def main() -> int: parser.add_argument("--no-cache", action="store_true", help="passes --no-cache to docker build") + parser.add_argument("-e", "--environment", nargs="*", default=[], + help="Environment variables for the docker container. " + "Specify with a list containing either names or name=value") + parser.add_argument("command", help="command to run in the container", nargs='*', action='append', type=str) @@ -474,6 +481,9 @@ def signal_handler(signum, _): signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) + environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e])) + for e in args.environment]) + if args.list: print(list_platforms()) elif args.platform: @@ -493,13 +503,13 @@ def signal_handler(signum, _): ret = container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment) elif args.print_docker_run: command = [] ret = container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup, environment=environment) else: # With no commands, execute a build function for the target platform command = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)] @@ -507,7 +517,7 @@ def signal_handler(signum, _): ret = container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment) if ret != 0: logging.critical("Execution of %s failed with status: %d", command, ret) @@ -515,6 +525,7 @@ def signal_handler(signum, _): elif args.all: platforms = get_platforms() + platforms = [platform for platform in platforms if 'build.' in platform] logging.info("Building for all architectures: %s", platforms) logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: @@ -535,7 +546,7 @@ def signal_handler(signum, _): container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment) shutil.move(buildir(), plat_buildir) logging.info("Built files left in: %s", plat_buildir) diff --git a/ci/docker/Dockerfile.publish.test.centos7_cpu b/ci/docker/Dockerfile.publish.test.centos7_cpu new file mode 100644 index 000000000000..7d284452971b --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.centos7_cpu @@ -0,0 +1,38 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on CentOS 7 for CPU + +FROM centos:7 + +WORKDIR /work/deps + +COPY install/centos7_base.sh /work/ +RUN /work/centos7_base.sh + +COPY install/centos7_scala.sh /work/ +RUN /work/centos7_scala.sh + +ARG USER_ID=0 +COPY install/centos7_adduser.sh /work/ +RUN /work/centos7_adduser.sh + +ENV PYTHONPATH=./python/ +WORKDIR /work/mxnet + +COPY runtime_functions.sh /work/ diff --git a/ci/docker/Dockerfile.publish.test.centos7_gpu b/ci/docker/Dockerfile.publish.test.centos7_gpu new file mode 100644 index 000000000000..e7f584683109 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.centos7_gpu @@ -0,0 +1,38 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on CentOS 7 for CPU + +FROM nvidia/cuda:9.2-cudnn7-devel-centos7 + +WORKDIR /work/deps + +COPY install/centos7_base.sh /work/ +RUN /work/centos7_base.sh + +COPY install/centos7_scala.sh /work/ +RUN /work/centos7_scala.sh + +ARG USER_ID=0 +COPY install/centos7_adduser.sh /work/ +RUN /work/centos7_adduser.sh + +ENV PYTHONPATH=./python/ +WORKDIR /work/mxnet + +COPY runtime_functions.sh /work/ diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu new file mode 100644 index 000000000000..035837686554 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu new file mode 100644 index 000000000000..854dd68a63c1 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu @@ -0,0 +1,40 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 14.04 for GPU + +# Use CPU with setup_gpu script +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu new file mode 100644 index 000000000000..bbb7b6a0d7bd --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu new file mode 100644 index 000000000000..660461dc0cfa --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu new file mode 100644 index 000000000000..e3a8c193f234 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu @@ -0,0 +1,41 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU + +FROM ubuntu:18.04 + +WORKDIR /work/deps + +ENV DEBIAN_FRONTEND noninteractive + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu new file mode 100644 index 000000000000..99f7e0d3eff9 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu @@ -0,0 +1,41 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 18.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu18.04 + +WORKDIR /work/deps + +ENV DEBIAN_FRONTEND noninteractive + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1404_cpu b/ci/docker/Dockerfile.publish.ubuntu1404_cpu new file mode 100644 index 000000000000..04ce94f95eae --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1404_cpu @@ -0,0 +1,36 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_publish.sh /work/ +RUN /work/ubuntu_publish.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1404_gpu b/ci/docker/Dockerfile.publish.ubuntu1404_gpu new file mode 100644 index 000000000000..9855986a2891 --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1404_gpu @@ -0,0 +1,36 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 14.04 for GPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_publish.sh /work/ +RUN /work/ubuntu_publish.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1604_cpu b/ci/docker/Dockerfile.publish.ubuntu1604_cpu new file mode 100644 index 000000000000..df284be2b37e --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1604_cpu @@ -0,0 +1,42 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1604_gpu b/ci/docker/Dockerfile.publish.ubuntu1604_gpu new file mode 100644 index 000000000000..2a1f8594e752 --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1604_gpu @@ -0,0 +1,42 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_base.sh b/ci/docker/install/centos7_base.sh new file mode 100755 index 000000000000..3b84aeb57b06 --- /dev/null +++ b/ci/docker/install/centos7_base.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex + +# Multipackage installation does not fail in yum +yum -y install epel-release +yum -y install git +yum -y install wget +yum -y install make +yum -y install cmake +yum -y install unzip +yum -y install ninja-build +yum -y install gcc-gfortran diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_scala.sh index ea46de9b9311..5c43f011cbf1 100755 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/install/centos7_scala.sh @@ -23,9 +23,17 @@ set -ex yum install -y java-1.8.0-openjdk-devel +export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk +export PATH=$JAVA_HOME/bin:$PATH # Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +wget -q http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz tar xzf apache-maven-3.3.9-bin.tar.gz mkdir /usr/local/maven mv apache-maven-3.3.9/ /usr/local/maven/ alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 + +echo "export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk" >> /etc/profile.d/maven.sh +echo "export M3_HOME=/usr/local/src/apache-maven" >> /etc/profile.d/maven.sh +echo "export PATH=$M3_HOME/bin:$JAVA_HOME/bin:$PATH" >> /etc/profile.d/maven.sh +chmod +x /etc/profile.d/maven.sh +source /etc/profile.d/maven.sh diff --git a/ci/docker/install/ubuntu_base.sh b/ci/docker/install/ubuntu_base.sh new file mode 100755 index 000000000000..f36e53279f57 --- /dev/null +++ b/ci/docker/install/ubuntu_base.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex +apt-get update || true +apt-get install -y \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + git \ + ninja-build \ + libgfortran3 \ + software-properties-common \ + sudo \ + unzip \ + expect \ + gnupg \ + gnupg2 \ + gnupg-agent \ + wget diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 942b686c1602..c0b5e6a039bf 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -30,7 +30,6 @@ apt-get install -y \ apt-transport-https \ build-essential \ ca-certificates \ - cmake \ curl \ git \ libatlas-base-dev \ @@ -46,11 +45,9 @@ apt-get install -y \ unzip \ wget -# Note: we specify an exact cmake version to work around a cmake 3.10 CUDA 10 issue. -# Reference: https://github.com/clab/dynet/issues/1457 -mkdir /opt/cmake && cd /opt/cmake -wget -nv https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh -sh cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license -ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -rm cmake-3.12.4-Linux-x86_64.sh -cmake --version +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + apt-get install -y cmake3 +else + apt-get install -y cmake +fi diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh new file mode 100644 index 000000000000..1ad6ab947842 --- /dev/null +++ b/ci/docker/install/ubuntu_publish.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Build on Ubuntu 14.04 LTS for LINUX CPU/GPU +apt-get update +apt-get install -y software-properties-common +add-apt-repository ppa:ubuntu-toolchain-r/test -y +add-apt-repository ppa:openjdk-r/ppa -y # Java lib +apt-get update +apt-get install -y git \ + cmake3 \ + libcurl4-openssl-dev \ + unzip \ + gcc-4.8 \ + g++-4.8 \ + gfortran \ + gfortran-4.8 \ + binutils \ + nasm \ + libtool \ + curl \ + wget \ + sudo \ + gnupg \ + gnupg2 \ + gnupg-agent \ + pandoc \ + python3-pip \ + automake \ + pkg-config \ + openjdk-8-jdk +curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +tar xzf apache-maven-3.3.9-bin.tar.gz +mkdir /usr/local/maven +mv apache-maven-3.3.9/ /usr/local/maven/ +update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +update-ca-certificates -f + +apt-get install -y python python3 + +# the version of the pip shipped with ubuntu may be too lower, install a recent version here +wget -nv https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py +python2 get-pip.py + +apt-get remove -y python3-urllib3 + +pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 6ecb8d801186..8b23d614b490 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -24,13 +24,28 @@ set -ex cd "$(dirname "$0")" # install libraries for mxnet's scala package on ubuntu echo 'Installing Scala...' -apt-get update || true -apt-get install -y software-properties-common -apt-get update || true -apt-get install -y openjdk-8-jdk -apt-get install -y openjdk-8-jre +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + add-apt-repository -y ppa:openjdk-r/ppa +fi + +# All Ubuntu apt-get update || true apt-get install -y \ - maven \ + openjdk-8-jdk \ + openjdk-8-jre \ + software-properties-common \ scala + +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz + tar xzf apache-maven-3.3.9-bin.tar.gz + mkdir /usr/local/maven + mv apache-maven-3.3.9/ /usr/local/maven/ + update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 + update-ca-certificates -f +else + apt-get install -y maven +fi diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index b4564e0b41ef..c35cf8c22d89 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -36,6 +36,11 @@ clean_repo() { git submodule update --init --recursive } +scala_prepare() { + # Clean up maven logs + export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" +} + build_ccache_wrappers() { set -ex @@ -273,7 +278,6 @@ build_amzn_linux_cpu() { ninja -v } - build_centos7_mkldnn() { set -ex cd /work/mxnet @@ -759,21 +763,24 @@ unittest_ubuntu_python3_quantization_gpu() { unittest_ubuntu_cpu_scala() { set -ex - make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 - make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + scala_prepare + make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 + make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 } unittest_centos7_cpu_scala() { set -ex cd /work/mxnet + scala_prepare make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 } unittest_ubuntu_cpu_clojure() { set -ex - make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 - make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + scala_prepare + make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 + make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 ./contrib/clojure-package/ci-test.sh } @@ -917,8 +924,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() { integrationtest_ubuntu_gpu_scala() { set -ex - make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 - make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + scala_prepare + make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1 + make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 } integrationtest_ubuntu_gpu_dist_kvstore() { @@ -982,7 +990,6 @@ build_docs() { popd } - # Functions that run the nightly Tests: #Runs Apache RAT Check on MXNet Source for License Headers @@ -1171,6 +1178,40 @@ deploy_jl_docs() { # ... } +build_scala_static_mkl() { + set -ex + pushd . + scala_prepare + export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu + export mxnet_variant=mkl + ./ci/publish/scala/build.sh + popd +} + +publish_scala_build() { + set -ex + pushd . + scala_prepare + ./ci/publish/scala/build.sh + popd +} + +publish_scala_test() { + set -ex + pushd . + scala_prepare + ./ci/publish/scala/test.sh + popd +} + +publish_scala_deploy() { + set -ex + pushd . + scala_prepare + ./ci/publish/scala/deploy.sh + popd +} + # broken_link_checker broken_link_checker() { diff --git a/ci/docker_cache.py b/ci/docker_cache.py index fe1882a567aa..24a806c7dc97 100755 --- a/ci/docker_cache.py +++ b/ci/docker_cache.py @@ -211,8 +211,7 @@ def _get_dockerhub_credentials(): # pragma: no cover logging.exception("The request was invalid due to:") elif client_error.response['Error']['Code'] == 'InvalidParameterException': logging.exception("The request had invalid params:") - else: - raise + raise else: secret = get_secret_value_response['SecretString'] secret_dict = json.loads(secret) diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 73c60290d32b..c0b038244636 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -489,6 +489,19 @@ def compile_windows_gpu_mkldnn() { }] } +def test_static_scala_cpu() { + return ['Static build CPU 14.04 Scala' : { + node(NODE_LINUX_CPU) { + ws('workspace/ut-publish-scala-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run("publish.ubuntu1404_cpu", 'build_scala_static_mkl', false) + } + } + } + }] +} + def test_unix_python2_cpu() { return ['Python2: CPU': { node(NODE_LINUX_CPU) { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 9c9a41503772..e93dcf576bcd 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -50,7 +50,8 @@ core_logic: { custom_steps.test_unix_r_cpu(), custom_steps.test_unix_julia_cpu(), custom_steps.test_unix_onnx_cpu(), - + custom_steps.test_unix_cpp_cpu(), + custom_steps.test_static_scala_cpu(), /* Disabled due to master build failure: * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/ * https://github.com/apache/incubator-mxnet/issues/11801 diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile new file mode 100644 index 000000000000..2b91f4f749d4 --- /dev/null +++ b/ci/publish/Jenkinsfile @@ -0,0 +1,110 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +//mxnet libraries +mx_scala_pub = 'lib/**, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/**/target/**, scala-package/*/target/repo/**' + +// timeout in minutes +max_time = 120 + +node('restricted-utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') +} +utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') + +// CPU and GPU. OSX nodes are not currently supported by Jenkins +def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3] +def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu'] +def scalaVariantMap = ['cpu': 'mkl', 'gpu': 'cu92mkl'] + +def wrapStep(nodeToRun, workspaceName, step) { + return { + node(nodeToRun) { + ws("workspace/${workspaceName}") { + timeout(time: max_time, unit: 'MINUTES') { + step() + } + } + } + } +} + +def toBuild = [:] +def labels = ['cpu', 'gpu'] +for (x in labels) { + def label = x // Required due to language + toBuild["Scala Build ${label}"] = wrapStep(nodeMap['cpu'], "build-scala-${label}") { + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) { + utils.init_git() + utils.docker_run("publish.ubuntu1404_cpu", 'publish_scala_build', false, '500m', 'MAVEN_PUBLISH_OS_TYPE mxnet_variant') + utils.pack_lib("scala_${label}", mx_scala_pub, false) + } + } +} + +def toTest = [:] +def systems = ['ubuntu1604', 'ubuntu1804', 'centos7'] +for (x in labels) { + def label = x // Required due to language + for (y in systems) { + def system = y // Required due to language + toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") { + withEnv(["mxnet_variant=${scalaVariantMap[label]}"]) { + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu', '500m', 'mxnet_variant') + } + } + } +} + +def toDeploy = [:] +for (x in labels) { + def label = x // Required due to language + toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap['cpu'], "deploy-scala-${label}") { + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) { + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("publish.ubuntu1604_${label}", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant') + } + } +} + +utils.main_wrapper( +core_logic: { + stage('Build Packages') { + parallel toBuild + } + stage('Test Packages') { + parallel toTest + } + stage('Deploy Packages') { + parallel toDeploy + } +} +, +failure_handler: { + if (currentBuild.result == "FAILURE") { + emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/publish/README.md b/ci/publish/README.md new file mode 100644 index 000000000000..2344cdff4cd2 --- /dev/null +++ b/ci/publish/README.md @@ -0,0 +1,33 @@ +# MXNet Publish Settings + +This folder contains the configuration of restricted node on Jenkins for the publish. It also contains a folder called `scala` that contains everything required for scala publish. In this `README`, we would bring a brief walkthrough of the Jenkins configuration as well as the usages of the scala deployment files. + +## Jenkins +Currently, Jenkins contains three build stages, namely `Build Packages`, `Test Packages` and `Deploy Packages`. During the `build package` stages, all dependencies will be built and a Scala package would be created. In the second stage, the package created from the previous stage would move to this stage to specifically run the tests. In the final stage, the packages passed the test would be deployed by the instances. + +The job is scheduled to be triggered every 24 hours on a [restricted instance](http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/restricted-publish-artifacts). + +Currently, we are supporting tests in the following systems: + +- Ubuntu 16.04 +- Ubuntu 18.04 +- Cent OS 7 + +All packages are currently built in `Ubuntu 14.04`. All Dockerfile used for publishing are available in `ci/docker/` with prefix `Dockerfile.publish`. + +Apart from that, the script used to create the environment and publish are available under `ci/docker/install`: + +- `ubuntu_publish.sh` install all required dependencies for Ubuntu 14.04 for publishing +- `ubuntu_base.sh` install minimum dependencies required to run the published packages + +## Scala publish +Currently Scala publish on Linux is fully supported on jenkins. The `scala/` folder contains all files needed to do the publish. Here is a breif instroduction of the files: + +- `build.sh` Main executable files to build the backend as well as scala package +- `buildkey.py` Main file used to extract password from the system and configure the maven +- `deploy.sh` Script to deploy the package +- `fullDeploy.sh` Used by CI to make full publish +- `test.sh` Make Scala test on CI + +## Python +We plans to support Python build on Jenkins soon \ No newline at end of file diff --git a/ci/publish/scala/build.sh b/ci/publish/scala/build.sh new file mode 100755 index 000000000000..11386ce2114f --- /dev/null +++ b/ci/publish/scala/build.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Setup Environment Variables +# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu +# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu + +source tools/staticbuild/build.sh $mxnet_variant maven + +set -ex + +# Compile tests for discovery later +cd scala-package +mvn -B deploy diff --git a/ci/publish/scala/buildkey.py b/ci/publish/scala/buildkey.py new file mode 100644 index 000000000000..8a1b7bf63286 --- /dev/null +++ b/ci/publish/scala/buildkey.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import json +import logging +import subprocess + +HOME = os.environ['HOME'] +KEY_PATH = os.path.join(HOME, ".m2") + + +''' +This file would do the following items: + Import keys from AWS Credential services + Create settings.xml in .m2 with pass phrase + Create security-settings.xml in .m2 with master password + Import keys.asc the encrypted keys in gpg +''' + + +def getCredentials(): + import boto3 + import botocore + endpoint_url = os.environ['MAVEN_PUBLISH_SECRET_ENDPOINT_URL'] + secret_creds_name = os.environ['MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS'] + secret_key_name = os.environ['MAVEN_PUBLISH_SECRET_NAME_GPG'] + region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION'] + + session = boto3.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name, + endpoint_url=endpoint_url + ) + try: + get_secret_value_response = client.get_secret_value( + SecretId=secret_creds_name + ) + get_secret_key_response = client.get_secret_value( + SecretId=secret_key_name + ) + except botocore.exceptions.ClientError as client_error: + if client_error.response['Error']['Code'] == 'ResourceNotFoundException': + name = (secret_key_name if get_secret_value_response + else secret_creds_name) + logging.exception("The requested secret %s was not found", name) + elif client_error.response['Error']['Code'] == 'InvalidRequestException': + logging.exception("The request was invalid due to:") + elif client_error.response['Error']['Code'] == 'InvalidParameterException': + logging.exception("The request had invalid params:") + raise + else: + secret = get_secret_value_response['SecretString'] + secret_dict = json.loads(secret) + secret_key = get_secret_key_response['SecretString'] + return secret_dict, secret_key + + +def importASC(key, gpgPassphrase): + filename = os.path.join(KEY_PATH, "key.asc") + with open(filename, 'w') as f: + f.write(key) + subprocess.check_output(['gpg2', '--batch', '--yes', + '--passphrase-fd', '0', + "--import", "{}".format(filename)], + input=str.encode(gpgPassphrase)) + + +def encryptMasterPSW(password): + filename = os.path.join(KEY_PATH, "encryptMasterPassword.exp") + with open(filename, 'w') as f: + f.write(''' + spawn mvn --encrypt-master-password + expect -exact "Master password: " + send -- "{}\r" + expect eof + '''.format(password)) + result = subprocess.check_output(['expect', filename]) + return str(result).split('\r\n')[-1][2:-3] + + +def encryptPSW(password): + filename = os.path.join(KEY_PATH, "encryptPassword.exp") + with open(filename, 'w') as f: + f.write(''' + spawn mvn --encrypt-password + expect -exact "Password: " + send -- "{}\r" + expect eof + '''.format(password)) + result = subprocess.check_output(['expect', filename]) + return str(result).split('\r\n')[-1][2:-3] + + +def masterPSW(password): + with open(os.path.join(KEY_PATH, "settings-security.xml"), "w") as f: + f.write("\n {}\n" + .format(password)) + + +def serverPSW(username, password, gpgPassphrase): + with open(os.path.join(KEY_PATH, "settings.xml"), "w") as f: + settingsString = ''' + + + + + + apache.snapshots.https + {} + {} + + + + apache.releases.https + {} + {} + + + + + + gpg + + gpg2 + {} + true + + + + + gpg + + '''.format(username, password, username, password, gpgPassphrase) + f.write(settingsString) + + +if __name__ == "__main__": + if not os.path.exists(KEY_PATH): + os.makedirs(KEY_PATH) + credentials, gpgKey = getCredentials() + masterPass = encryptMasterPSW(credentials['masterpass']) + masterPSW(masterPass) + passwordEncrypted = encryptPSW(credentials['password']) + serverPSW(credentials['user'], passwordEncrypted, + credentials['gpgPassphrase']) + importASC(gpgKey, credentials['gpgPassphrase']) diff --git a/ci/publish/scala/deploy.sh b/ci/publish/scala/deploy.sh new file mode 100755 index 000000000000..00c67214742b --- /dev/null +++ b/ci/publish/scala/deploy.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# On Jenkins, run python script to configure keys +if [[ $BUILD_ID ]]; then + python3 ci/publish/scala/buildkey.py +fi + +# Updating cache +mkdir -p ~/.gnupg +echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf +echo "max-cache-ttl 14400" >> ~/.gnupg/gpg-agent.conf +echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf +echo "pinentry-mode loopback" >> ~/.gnupg/gpg-agent.conf +export GPG_TTY=$(tty) + +cd scala-package + +mvn -B deploy -Pnightly + +# On Jenkins, clear all password .xml files, exp files, and gpg key files +if [[ $BUILD_ID ]]; then + rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp +fi diff --git a/ci/publish/scala/fullDeploy.sh b/ci/publish/scala/fullDeploy.sh new file mode 100755 index 000000000000..69d674a97497 --- /dev/null +++ b/ci/publish/scala/fullDeploy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +./ci/publish/scala/build.sh +./ci/publish/scala/test.sh +./ci/publish/scala/deploy.sh diff --git a/ci/publish/scala/test.sh b/ci/publish/scala/test.sh new file mode 100755 index 000000000000..32e36299a6fc --- /dev/null +++ b/ci/publish/scala/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +if [ -z "$JAVA_HOME" ]; then + source /etc/profile +fi + +# Test +cd scala-package/packageTest + +if [[ $mxnet_variant == cu* ]]; then + export SCALA_TEST_ON_GPU=1 + make testlocal USE_CUDA=1 CI=1 +else + make testlocal CI=1 +fi diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py index 358d54985aca..0a3bc4640c05 100644 --- a/ci/test_docker_cache.py +++ b/ci/test_docker_cache.py @@ -135,7 +135,7 @@ def test_full_cache(self): """ platform = 'test_full_cache' docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) - dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform) try: with open(dockerfile_path, 'w') as dockerfile_handle: dockerfile_handle.write(dockerfile_content) @@ -196,7 +196,7 @@ def test_partial_cache(self): """ platform = 'test_partial_cache' docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) - dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform) try: # Write initial Dockerfile with open(dockerfile_path, 'w') as dockerfile_handle: diff --git a/contrib/clojure-package/project.clj b/contrib/clojure-package/project.clj index 1216b239026f..d8bf152a1a7c 100644 --- a/contrib/clojure-package/project.clj +++ b/contrib/clojure-package/project.clj @@ -29,7 +29,7 @@ ;[org.apache.mxnet/mxnet-full_2.11-linux-x86_64-gpu "1.2.1"] ;;; CI - [org.apache.mxnet/mxnet-full_2.11-linux-x86_64-cpu "1.4.0-SNAPSHOT"] + [org.apache.mxnet/mxnet-full_2.11 "INTERNAL"] [org.clojure/tools.logging "0.4.0"] [org.apache.logging.log4j/log4j-core "2.8.1"] diff --git a/docs/mxdoc.py b/docs/mxdoc.py index 7be4b90e799b..35fd175c54aa 100644 --- a/docs/mxdoc.py +++ b/docs/mxdoc.py @@ -113,9 +113,9 @@ def build_scala_docs(app): scala_doc_sources = 'find . -type f -name "*.scala" | egrep \"\.\/core|\.\/infer\" | egrep -v \"\/javaapi\" | egrep -v \"Suite\"' scala_doc_classpath = ':'.join([ '`find native -name "*.jar" | grep "target/lib/" | tr "\\n" ":" `', - '`find macros -name "*-SNAPSHOT.jar" | tr "\\n" ":" `', - '`find core -name "*-SNAPSHOT.jar" | tr "\\n" ":" `', - '`find infer -name "*-SNAPSHOT.jar" | tr "\\n" ":" `' + '`find macros -name "*-INTERNAL.jar" | tr "\\n" ":" `', + '`find core -name "*-INTERNAL.jar" | tr "\\n" ":" `', + '`find infer -name "*-INTERNAL.jar" | tr "\\n" ":" `' ]) # There are unresolvable errors on mxnet 1.2.x. We are ignoring those errors while aborting the ci on newer versions scala_ignore_errors = '; exit 0' if '1.2.' in _BUILD_VER else '' @@ -135,9 +135,9 @@ def build_java_docs(app): java_doc_sources = 'find . -type f -name "*.scala" | egrep \"\.\/core|\.\/infer\" | egrep \"\/javaapi\" | egrep -v \"Suite\"' java_doc_classpath = ':'.join([ '`find native -name "*.jar" | grep "target/lib/" | tr "\\n" ":" `', - '`find macros -name "*-SNAPSHOT.jar" | tr "\\n" ":" `', - '`find core -name "*-SNAPSHOT.jar" | tr "\\n" ":" `', - '`find infer -name "*-SNAPSHOT.jar" | tr "\\n" ":" `' + '`find macros -name "*-INTERNAL.jar" | tr "\\n" ":" `', + '`find core -name "*-INTERNAL.jar" | tr "\\n" ":" `', + '`find infer -name "*-INTERNAL.jar" | tr "\\n" ":" `' ]) _run_cmd('cd {}; scaladoc `{}` -classpath {} -feature -deprecation' .format(java_path, java_doc_sources, java_doc_classpath)) diff --git a/make/config/libmxnet.sym b/make/config/libmxnet.sym new file mode 100644 index 000000000000..a472161baeb0 --- /dev/null +++ b/make/config/libmxnet.sym @@ -0,0 +1,12 @@ +MX* +NN* +_MX* +_NN* +mx* +nn* +_mx* +_nn* +Java_org_apache_mxnet* +*NDArray* +*Engine*Get* +*Storage*Get* diff --git a/make/config/libmxnet.ver b/make/config/libmxnet.ver new file mode 100644 index 000000000000..c83721307e37 --- /dev/null +++ b/make/config/libmxnet.ver @@ -0,0 +1,16 @@ +{ + global: + NN*; + MX*; + _NN*; + _MX*; + nn*; + mx*; + _nn*; + _mx*; + Java_org_apache_mxnet*; + *NDArray*; + *Engine*Get*; + *Storage*Get*; + local: *; +}; diff --git a/make/maven/maven_darwin_mkl.mk b/make/maven/maven_darwin_mkl.mk new file mode 100644 index 000000000000..f5b77ae784ad --- /dev/null +++ b/make/maven/maven_darwin_mkl.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making maven package +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=apple + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 0 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/maven/maven_linux_cu90mkl.mk b/make/maven/maven_linux_cu90mkl.mk new file mode 100644 index 000000000000..661f444ceac3 --- /dev/null +++ b/make/maven/maven_linux_cu90mkl.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making maven package +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/maven/maven_linux_cu92mkl.mk b/make/maven/maven_linux_cu92mkl.mk new file mode 100644 index 000000000000..ecd252769216 --- /dev/null +++ b/make/maven/maven_linux_cu92mkl.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making maven package +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/maven/maven_linux_mkl.mk b/make/maven/maven_linux_mkl.mk new file mode 100644 index 000000000000..6cb9f326bb7d --- /dev/null +++ b/make/maven/maven_linux_mkl.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making maven package +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_darwin_cpu.mk b/make/pip/pip_darwin_cpu.mk new file mode 100644 index 000000000000..bf240ad128cd --- /dev/null +++ b/make/pip/pip_darwin_cpu.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=apple + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 0 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# whether use MKL-DNN library +USE_MKLDNN = 0 + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_darwin_mkl.mk b/make/pip/pip_darwin_mkl.mk new file mode 100644 index 000000000000..30b328c6420e --- /dev/null +++ b/make/pip/pip_darwin_mkl.mk @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=apple + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 0 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cpu.mk b/make/pip/pip_linux_cpu.mk new file mode 100644 index 000000000000..22bb32b4d1b8 --- /dev/null +++ b/make/pip/pip_linux_cpu.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu100.mk b/make/pip/pip_linux_cu100.mk new file mode 100644 index 000000000000..514603b94ab6 --- /dev/null +++ b/make/pip/pip_linux_cu100.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-10.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu100mkl.mk b/make/pip/pip_linux_cu100mkl.mk new file mode 100644 index 000000000000..da4127f0dd07 --- /dev/null +++ b/make/pip/pip_linux_cu100mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-10.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu75.mk b/make/pip/pip_linux_cu75.mk new file mode 100644 index 000000000000..02468aa9613a --- /dev/null +++ b/make/pip/pip_linux_cu75.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-7.5 + +# whether use CuDNN R3 library +USE_CUDNN = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu75mkl.mk b/make/pip/pip_linux_cu75mkl.mk new file mode 100644 index 000000000000..6895383d4fec --- /dev/null +++ b/make/pip/pip_linux_cu75mkl.mk @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-7.5 + +# whether use CuDNN R3 library +USE_CUDNN = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu80.mk b/make/pip/pip_linux_cu80.mk new file mode 100644 index 000000000000..2a2b88de73a1 --- /dev/null +++ b/make/pip/pip_linux_cu80.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-8.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu80mkl.mk b/make/pip/pip_linux_cu80mkl.mk new file mode 100644 index 000000000000..40cef4ed2110 --- /dev/null +++ b/make/pip/pip_linux_cu80mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-8.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu90.mk b/make/pip/pip_linux_cu90.mk new file mode 100644 index 000000000000..88e55180b738 --- /dev/null +++ b/make/pip/pip_linux_cu90.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu90mkl.mk b/make/pip/pip_linux_cu90mkl.mk new file mode 100644 index 000000000000..9c4adf07bb6a --- /dev/null +++ b/make/pip/pip_linux_cu90mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu91.mk b/make/pip/pip_linux_cu91.mk new file mode 100644 index 000000000000..ea62510396ef --- /dev/null +++ b/make/pip/pip_linux_cu91.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.1 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu91mkl.mk b/make/pip/pip_linux_cu91mkl.mk new file mode 100644 index 000000000000..232ec47979cc --- /dev/null +++ b/make/pip/pip_linux_cu91mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.1 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu92.mk b/make/pip/pip_linux_cu92.mk new file mode 100644 index 000000000000..01d02cfefba9 --- /dev/null +++ b/make/pip/pip_linux_cu92.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu92mkl.mk b/make/pip/pip_linux_cu92mkl.mk new file mode 100644 index 000000000000..1abde3d71d7f --- /dev/null +++ b/make/pip/pip_linux_cu92mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_mkl.mk b/make/pip/pip_linux_mkl.mk new file mode 100644 index 000000000000..96b88850c21a --- /dev/null +++ b/make/pip/pip_linux_mkl.mk @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/scala-package/.gitignore b/scala-package/.gitignore index 6aa4da6b1cfc..22b12b31d050 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -1,8 +1,10 @@ .flattened-pom.xml core/src/main/scala/org/apache/mxnet/NDArrayAPIBase.scala core/src/main/scala/org/apache/mxnet/NDArrayBase.scala +core/src/main/scala/org/apache/mxnet/NDArrayRandomAPIBase.scala core/src/main/scala/org/apache/mxnet/javaapi/NDArrayBase.scala core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala core/src/main/scala/org/apache/mxnet/SymbolBase.scala +core/src/main/scala/org/apache/mxnet/SymbolRandomAPIBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ diff --git a/scala-package/README.md b/scala-package/README.md index 72b3d4d731a5..38b03b9d417b 100644 --- a/scala-package/README.md +++ b/scala-package/README.md @@ -27,6 +27,63 @@ It brings flexible and efficient GPU/CPU computing and state-of-art deep learnin in Scala, Java and other languages built on JVM. - It also enables you to construct and customize the state-of-art deep learning models in JVM languages, and apply them to tasks such as image classification and data science challenges. +- The Scala/Java Inferece APIs provides an easy out of the box solution for loading pre-trained MXNet models and running inference on them. + +Pre-Built Maven Packages +------------------------ + +### Stable ### + +The MXNet Scala/Java packages can be easily included in your Maven managed project. +The stable jar files for the packages are available on the [MXNet Maven Package Repository](https://search.maven.org/search?q=g:org.apache.mxnet) +Currently we provide packages for Linux (Ubuntu 16.04) (CPU and GPU) and macOS (CPU only). Stable packages for Windows and CentOS will come soon. For now, if you have a CentOS machine, follow the ```Build From Source``` section below. + +To add MXNet Scala/Java package to your project, add the dependency as shown below corresponding to your platform, under the ```dependencies``` tag in your project's ```pom.xml``` : + +**Linux GPU** + +maven badge + +```HTML + + org.apache.mxnet + mxnet-full_2.11-linux-x86_64-gpu + [1.3.1,) + +``` + +**Linux CPU** + +maven badge + +```HTML + + org.apache.mxnet + mxnet-full_2.11-linux-x86_64-cpu + [1.3.1,) + +``` + +**macOS CPU** + +maven badge + +```HTML + + org.apache.mxnet + mxnet-full_2.11-osx-x86_64-cpu + [1.3.1,) + +``` + +**Note:** ```[1.3.1,)<\version>``` indicates that we will fetch packages with version 1.3.1 or higher. This will always ensure that the pom.xml is able to fetch the latest and greatest jar files from Maven. + +### Nightly ### + +Apart from these, the nightly builds representing the bleeding edge development on Scala/Java packages are also available on the [MXNet Maven Nexus Package Repository](https://repository.apache.org/#nexus-search;gav~org.apache.mxnet~~~~). +Currently we provide nightly packages for Linux (CPU and GPU) and MacOS (CPU only). The Linux nightly jar files also work on CentOS. Nightly packages for Windows will come soon. + +Add the following ```repository``` to your project's ```pom.xml``` file : Install ------------ @@ -60,31 +117,52 @@ You can also use `mxnet-core_2.10-0.1.1.jar` and put the compiled native library If you have some native libraries conflict with the ones in the provided 'full' jar (e.g., you use openblas instead of atlas), this is a recommended way. Refer to the next section for how to build it from the very source. -Build ------------- +Build From Source +----------------- -Checkout the [Installation Guide](http://mxnet.incubator.apache.org/install/index.html) contains instructions to install mxnet. -Then you can compile the Scala Package by +Checkout the [Installation Guide](http://mxnet.incubator.apache.org/install/index.html) contains instructions to install mxnet package and build it from source. Scala maven build assume you already have a ``lib/libmxnet.so`` file. +If you have built MXNet from source and are looking to setup Scala from that point, you may simply run the following from the MXNet source root, Scala build will detect your platform (OSX/Linux) and libmxnet.so flavor (CPU/GPU): ```bash -make scalapkg +cd scala-package +mvn install ``` (Optional) run unit/integration tests by ```bash -make scalaunittest -make scalaintegrationtest +cd scala-package +mvn integration-test -DskipTests=false ``` Or run a subset of unit tests by, e.g., ```bash -make SCALA_TEST_ARGS=-Dsuites=org.apache.mxnet.NDArraySuite scalaunittest +cd scala-package +mvn -Dsuites=org.apache.mxnet.NDArraySuite integration-test ``` If everything goes well, you will find jars for `assembly`, `core` and `example` modules. -Also it produces the native library in `native/{your-architecture}/target`, which you can use to cooperate with the `core` module. +Also it produces the native library in `native/target`, which you can use to cooperate with the `core` module. + +Deploy to repository +-------------------- + +By default, `maven deploy` will deploy artifacts to local file system, you can file then in: ``scala-package/deploy/target/repo`` folder. + +For nightly build in CI, a snapshot build will be uploaded to apache repository with follow command: + +```bash +cd scala-package +mvn deploy -Pnightly +``` + +Use following command to deploy release build (push artifacts to apache staging repository): + +```bash +cd scala-package +mvn deploy -Pstaging +``` Once you've downloaded and unpacked MNIST dataset to `./data/`, run the training example by @@ -207,3 +285,7 @@ Release License ------- MXNet Scala Package is licensed under [Apache-2](https://github.com/apache/incubator-mxnet/blob/master/scala-package/LICENSE) license. + +MXNet uses some 3rd party softwares. Following 3rd party license files are bundled inside Scala jar file: +* cub/LICENSE.TXT +* mkldnn/external/mklml_mac_2019.0.1.20180928/license.txt diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml deleted file mode 100644 index 05db3a5f70b9..000000000000 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ /dev/null @@ -1,143 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-full-parent_2.11 - 1.4.0-SNAPSHOT - ../pom.xml - - - mxnet-full_2.11-linux-x86_64-cpu - MXNet Scala Package - Full Linux-x86_64 CPU-only - jar - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - - - org.apache.mxnet - libmxnet-scala-linux-x86_64-cpu - 1.4.0-SNAPSHOT - so - - - org.apache.mxnet - mxnet-infer_${scala.binary.version} - 1.4.0-SNAPSHOT - - - - - - - org.codehaus.mojo - flatten-maven-plugin - - - - remove - - - - - - flatten - process-resources - - flatten - - - - - flatten.clean - clean - - clean - - - - - - org.apache.maven.plugins - maven-deploy-plugin - false - - false - - - - org.apache.maven.plugins - maven-install-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - - - binary-jar - package - - single - - - false - - src/main/assembly/assembly.xml - - - - - sources-jar - package - - single - - - true - sources - - ${project.parent.basedir}/src/source.xml - - - - - javadoc-jar - package - - single - - - true - javadoc - - ${project.parent.basedir}/src/javadoc.xml - - - - - - - - diff --git a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml deleted file mode 100644 index 9f28706ac596..000000000000 --- a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - libmxnet-scala.so - false - false - false - - org.apache.mxnet:libmxnet-scala-linux-x86_64-cpu:so - - - - diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml deleted file mode 100644 index 708b1c48e98b..000000000000 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ /dev/null @@ -1,143 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-full-parent_2.11 - 1.4.0-SNAPSHOT - ../pom.xml - - - mxnet-full_2.11-linux-x86_64-gpu - MXNet Scala Package - Full Linux-x86_64 GPU - jar - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - - - org.apache.mxnet - libmxnet-scala-linux-x86_64-gpu - 1.4.0-SNAPSHOT - so - - - org.apache.mxnet - mxnet-infer_${scala.binary.version} - 1.4.0-SNAPSHOT - - - - - - - org.codehaus.mojo - flatten-maven-plugin - - - - remove - - - - - - flatten - process-resources - - flatten - - - - - flatten.clean - clean - - clean - - - - - - org.apache.maven.plugins - maven-deploy-plugin - false - - false - - - - org.apache.maven.plugins - maven-install-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - - - binary-jar - package - - single - - - false - - src/main/assembly/assembly.xml - - - - - sources-jar - package - - single - - - true - sources - - ${project.parent.basedir}/src/source.xml - - - - - javadoc-jar - package - - single - - - true - javadoc - - ${project.parent.basedir}/src/javadoc.xml - - - - - - - - diff --git a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml deleted file mode 100644 index 2b65a8c7ad66..000000000000 --- a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - libmxnet-scala.so - false - false - false - - org.apache.mxnet:libmxnet-scala-linux-x86_64-gpu:so - - - - diff --git a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml deleted file mode 100644 index d0550a3623b1..000000000000 --- a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - ${artifact.artifactId}${dashClassifier?}.${artifact.extension} - false - false - false - - *:*:dll:* - *:*:so:* - *:*:jnilib:* - - - - diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml deleted file mode 100644 index 2f80dd7fb707..000000000000 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ /dev/null @@ -1,139 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-full-parent_2.11 - 1.4.0-SNAPSHOT - ../pom.xml - - - mxnet-full_2.11-osx-x86_64-cpu - MXNet Scala Package - Full OSX-x86_64 CPU-only - jar - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - - - org.apache.mxnet - libmxnet-scala-osx-x86_64-cpu - 1.4.0-SNAPSHOT - jnilib - - - org.apache.mxnet - mxnet-infer_${scala.binary.version} - 1.4.0-SNAPSHOT - - - - - - - org.codehaus.mojo - flatten-maven-plugin - - - - remove - - - - - - flatten - process-resources - - flatten - - - - - flatten.clean - clean - - clean - - - - - - org.apache.maven.plugins - maven-deploy-plugin - false - - false - - - - org.apache.maven.plugins - maven-assembly-plugin - - - binary-jar - package - - single - - - false - - src/main/assembly/assembly.xml - - - - - sources-jar - package - - single - - - true - sources - - ${project.parent.basedir}/src/source.xml - - - - - javadoc-jar - package - - single - - - true - javadoc - - ${project.parent.basedir}/src/javadoc.xml - - - - - - - - diff --git a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml deleted file mode 100644 index 7a6c3ea9f525..000000000000 --- a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - libmxnet-scala.jnilib - false - false - false - - org.apache.mxnet:libmxnet-scala-osx-x86_64-cpu:jnilib - - - - diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index 2d7366fc83e2..b93c181bc1d4 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -21,106 +21,109 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-full-parent_2.11 - MXNet Scala Package - Full Parent + mxnet-full_2.11 + Assembly Scala Package pom + + ${project.parent.basedir}/.. + + + + + org.apache.mxnet + mxnet-core + INTERNAL + + + org.apache.mxnet + libmxnet-scala + INTERNAL + ${libtype} + + + org.apache.mxnet + mxnet-infer + INTERNAL + + + - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - release + staging - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-source-plugin - - - package - - jar-no-fork - - - true - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - package - - jar - - - true - - commons-codec:* - org.scala-lang:* - log4j:* - org.slf4j:* - - - - - org.apache.maven.plugins maven-gpg-plugin + 1.6 sign-artifacts - verify + deploy sign - - org.sonatype.plugins - nexus-staging-maven-plugin - true - - ossrh - https://oss.sonatype.org/ - true - - + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + binary-jar + package + + single + + + false + + src/main/assembly/assembly.xml + + + + + sources-jar + package + + single + + + true + + src/main/assembly/source.xml + + + + + javadoc-jar + package + + single + + + true + + src/main/assembly/javadoc.xml + + + + + + + diff --git a/scala-package/assembly/src/javadoc.xml b/scala-package/assembly/src/javadoc.xml deleted file mode 100644 index 8a91221372ac..000000000000 --- a/scala-package/assembly/src/javadoc.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - bundle - - jar - - false - - - - - - - org.apache.mxnet:mxnet-core_${scala.binary.version} - - true - - false - javadoc - / - true - - - - \ No newline at end of file diff --git a/scala-package/assembly/src/main/assembly/assembly.xml b/scala-package/assembly/src/main/assembly/assembly.xml new file mode 100644 index 000000000000..7525df883dab --- /dev/null +++ b/scala-package/assembly/src/main/assembly/assembly.xml @@ -0,0 +1,57 @@ + + full + + jar + + false + + + + *:*:jar + + + org.scala-lang:* + org.scala-lang.modules:* + + / + true + true + runtime + + + lib/native + libmxnet-scala.${libtype} + false + false + false + + org.apache.mxnet:libmxnet-scala:${libtype} + + + + + + ${MXNET_DIR}/lib + + libmxnet.so + libgfortran.so.3 + libquadmath.so.0 + libiomp5.so + libiomp5.dylib + libmklml_intel.so + libmklml.dylib + libmkldnn.so.0 + libmkldnn.0.dylib + + lib/native + + + ${MXNET_DIR}/3rdparty + + cub/LICENSE.TXT + mkldnn/external/mklml_mac_2019.0.1.20180928/license.txt + + / + + + diff --git a/scala-package/assembly/src/main/assembly/javadoc.xml b/scala-package/assembly/src/main/assembly/javadoc.xml new file mode 100644 index 000000000000..8f30a261811c --- /dev/null +++ b/scala-package/assembly/src/main/assembly/javadoc.xml @@ -0,0 +1,15 @@ + + bundle + + jar + + false + + + ${rootdir}/core/target/site/scaladocs + / + + + diff --git a/scala-package/assembly/src/main/assembly/source.xml b/scala-package/assembly/src/main/assembly/source.xml new file mode 100644 index 000000000000..87fcde360c48 --- /dev/null +++ b/scala-package/assembly/src/main/assembly/source.xml @@ -0,0 +1,19 @@ + + src + + + jar + + false + + + ${rootdir}/core/src/main/scala + + **\/*.scala + + / + + + diff --git a/scala-package/assembly/src/source.xml b/scala-package/assembly/src/source.xml deleted file mode 100644 index 003c0e0c87f5..000000000000 --- a/scala-package/assembly/src/source.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - src - - - jar - - false - - - true - - - org.apache.mxnet:mxnet-core_${scala.binary.version} - - - false - - - - src/main/scala - - **\/*.scala - - / - - - - - - \ No newline at end of file diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 1b1a570c3386..0bc672d41bd8 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -21,54 +21,70 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - - true - - - mxnet-core_2.11 + mxnet-core MXNet Scala Package - Core - - - unittest - - false - - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - + org.codehaus.mojo + native-maven-plugin + true + + + javah + verify + + default + ${project.build.directory}/custom-javah + ${basedir} + org_apache_mxnet_native_c_api.h + + org.apache.mxnet.LibInfo + + + + javah + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + verify-javah + verify + + exec + + + diff + ${project.build.directory}/custom-javah/org_apache_mxnet_native_c_api.h ${project.parent.basedir}/native/src/main/native/org_apache_mxnet_native_c_api.h + + + + apidoc-generation + generate-sources + + exec + + + java + ${project.parent.basedir} + -classpath %classpath:${rootdir}/init/target/classes:${rootdir}/macros/target/classes -Djava.library.path=${rootdir}/native/target org.apache.mxnet.APIDocGenerator ${rootdir}/core/src/main/scala/org/apache/mxnet/ + + + + + org.apache.maven.plugins maven-jar-plugin @@ -84,13 +100,33 @@ org.apache.maven.plugins maven-compiler-plugin + + org.apache.maven.plugins + maven-clean-plugin + 3.1.0 + + + + src/main/scala/org/apache/mxnet + + NDArrayAPIBase.scala + NDArrayBase.scala + NDArrayRandomAPIBase.scala + javaapi/NDArrayBase.scala + SymbolAPIBase.scala + SymbolRandomAPIBase.scala + + false + + + + org.scalatest scalatest-maven-plugin - ${skipTests} - -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ + -Djava.library.path=${project.parent.basedir}/native/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties @@ -101,9 +137,8 @@ 2.22.0 - -Djava.library.path=${project.parent.basedir}/native/${platform}/target + -Djava.library.path=${project.parent.basedir}/native/target - ${skipTests} @@ -115,14 +150,14 @@ org.apache.mxnet - mxnet-init_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-macros + INTERNAL provided org.apache.mxnet - mxnet-macros_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-scala-init + INTERNAL provided diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/Base.scala b/scala-package/core/src/main/scala/org/apache/mxnet/Base.scala index b2a53fd9f2dd..3515fcee8e70 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/Base.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/Base.scala @@ -67,6 +67,8 @@ private[mxnet] object Base { "Consider installing the library somewhere in the path " + "(for Windows: PATH, for Linux: LD_LIBRARY_PATH), " + "or specifying by Java cmd option -Djava.library.path=[lib path].") + logger.warn("LD_LIBRARY_PATH=" + System.getenv("LD_LIBRARY_PATH")) + logger.warn("java.library.path=" + System.getProperty("java.library.path")) NativeLibraryLoader.loadLibrary("mxnet-scala") } } catch { @@ -87,35 +89,8 @@ private[mxnet] object Base { @throws(classOf[UnsatisfiedLinkError]) private def tryLoadLibraryOS(libname: String): Unit = { - try { - logger.info(s"Try loading $libname from native path.") - System.loadLibrary(libname) - } catch { - case e: UnsatisfiedLinkError => - val os = System.getProperty("os.name") - // ref: http://lopica.sourceforge.net/os.html - if (os.startsWith("Linux")) { - tryLoadLibraryXPU(libname, "linux-x86_64") - } else if (os.startsWith("Mac")) { - tryLoadLibraryXPU(libname, "osx-x86_64") - } else { - // TODO(yizhi) support windows later - throw new UnsatisfiedLinkError() - } - } - } - - @throws(classOf[UnsatisfiedLinkError]) - private def tryLoadLibraryXPU(libname: String, arch: String): Unit = { - try { - // try gpu first - logger.info(s"Try loading $libname-$arch-gpu from native path.") - System.loadLibrary(s"$libname-$arch-gpu") - } catch { - case e: UnsatisfiedLinkError => - logger.info(s"Try loading $libname-$arch-cpu from native path.") - System.loadLibrary(s"$libname-$arch-cpu") - } + logger.info(s"Try loading $libname from native path.") + System.loadLibrary(libname) } // helper function definitions diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala b/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala index 198102d2377f..cbd5a26845de 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala @@ -209,6 +209,8 @@ class NDArray private[mxnet] (val nd: org.apache.mxnet.NDArray ) { this(NDArray.array(arr, shape, ctx)) } + override def toString: String = nd.toString + def serialize(): Array[Byte] = nd.serialize() /** diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala index e94d320391fa..1e6d9c4f9fce 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala @@ -85,12 +85,18 @@ private[mxnet] object NativeLibraryLoader { } logger.debug(s"Attempting to load $loadLibname") val libFileInJar = libPathInJar + loadLibname - val is: InputStream = getClass.getResourceAsStream(libFileInJar) - if (is == null) { - throw new UnsatisfiedLinkError(s"Couldn't find the resource $loadLibname") - } - logger.info(s"Loading $loadLibname from $libPathInJar copying to $libname") - loadLibraryFromStream(libname, is) + saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so", true) + saveLibraryToTemp("libgfortran.so.3", "/lib/native/libgfortran.so.3", false) + saveLibraryToTemp("libquadmath.so.0", "/lib/native/libquadmath.so.0", false) + saveLibraryToTemp("libiomp5.so", "/lib/native/libiomp5.so", false) + saveLibraryToTemp("libiomp5.dylib", "/lib/native/libiomp5.dylib", false) + saveLibraryToTemp("libmklml_intel.so", "/lib/native/libmklml_intel.so", false) + saveLibraryToTemp("libmklml.dylib", "/lib/native/libmklml.dylib", false) + saveLibraryToTemp("libmkldnn.so.0", "/lib/native/libmkldnn.so.0", false) + saveLibraryToTemp("libmkldnn.0.dylib", "/lib/native/libmkldnn.0.dylib", false) + val tempfile: File = saveLibraryToTemp(libname, libFileInJar, true) + + loadLibraryFromFile(libname, tempfile) } /** @@ -109,7 +115,7 @@ private[mxnet] object NativeLibraryLoader { @throws(classOf[IOException]) private def createTempFile(name: String): File = { - new File(_tempDir + File.separator + name) + new File(_tempDir, name) } /** @@ -117,34 +123,56 @@ private[mxnet] object NativeLibraryLoader { * and loads from there. * * @param libname name of the library (just used in constructing the library name) - * @param is InputStream pointing to the library + * @param tempfile File pointing to the library */ - private def loadLibraryFromStream(libname: String, is: InputStream) { + private def loadLibraryFromFile(libname: String, tempfile: File) { try { - val tempfile: File = createTempFile(libname) - val os: OutputStream = new FileOutputStream(tempfile) - logger.debug("tempfile.getPath() = {}", tempfile.getPath) - val savedTime: Long = System.currentTimeMillis - val buf: Array[Byte] = new Array[Byte](8192) - var len: Int = is.read(buf) - while (len > 0) { - os.write(buf, 0, len) - len = is.read(buf) - } - os.flush() - val lock: InputStream = new FileInputStream(tempfile) - os.close() - val seconds: Double = (System.currentTimeMillis - savedTime).toDouble / 1e3 - logger.debug(s"Copying took $seconds seconds.") logger.debug("Loading library from {}", tempfile.getPath) System.load(tempfile.getPath) - lock.close() } catch { - case io: IOException => - logger.error("Could not create the temp file: {}", io.toString) case ule: UnsatisfiedLinkError => logger.error("Couldn't load copied link file: {}", ule.toString) throw ule } } + + /** + * Load a system library from a stream. Copies the library to a temp file + * and loads from there. + * + * @param libname name of the library (just used in constructing the library name) + * @param resource String resource path in the jar file + * @param required true if library is required + */ + private def saveLibraryToTemp(libname: String, resource: String, required: Boolean): File = { + try { + val is: InputStream = getClass.getResourceAsStream(resource) + if (is == null) { + if (required) { + throw new UnsatisfiedLinkError(s"Couldn't find the resource $resource") + } else { + null + } + } else { + val tempfile: File = new File(_tempDir, libname) + val os: OutputStream = new FileOutputStream(tempfile) + logger.debug("tempfile.getPath() = {}", tempfile.getPath) + val savedTime: Long = System.currentTimeMillis + val buf: Array[Byte] = new Array[Byte](8192) + var len: Int = is.read(buf) + while (len > 0) { + os.write(buf, 0, len) + len = is.read(buf) + } + os.close() + is.close() + val seconds: Double = (System.currentTimeMillis - savedTime).toDouble / 1e3 + logger.debug(s"Copying $libname took $seconds seconds.") + tempfile + } + } catch { + case io: IOException => + throw new UnsatisfiedLinkError(s"Could not create temp file for $libname") + } + } } diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml new file mode 100644 index 000000000000..4290174e5b9b --- /dev/null +++ b/scala-package/deploy/pom.xml @@ -0,0 +1,144 @@ + + + 4.0.0 + + org.apache.mxnet + mxnet-parent + INTERNAL + ../pom.xml + + + mxnet-deployment + ${revision} + MXNet Scala Package - Full ${platform}-only + pom + + Scala Package for Apache MXNet (Incubating) - flexible and efficient library for deep learning. + + + + ${project.parent.basedir}/.. + mxnet-full_2.11-${platform}-${flavor} + 1.5.0-SNAPSHOT + apache.snapshots.https + + + + + + + + org.apache.mxnet + mxnet-full_2.11 + INTERNAL + + + + + + staging + + 1.5.0 + apache.releases.https + https://repository.apache.org/content/repositories/staging + jar.asc,asc,asc + ,sources.jar,javadoc.jar + + ../assembly/target/mxnet-full_2.11-INTERNAL.jar.asc,../assembly/target/mxnet-full_2.11-INTERNAL-src.jar.asc,../assembly/target/mxnet-full_2.11-INTERNAL-bundle.jar.asc + + + + + nightly + + apache.snapshots.https + https://repository.apache.org/content/repositories/snapshots + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + false + + + + + com.google.code.maven-replacer-plugin + replacer + 1.5.3 + + + deploy + + replace + + + + + ${basedir}/src/main/deploy/deploy.xml + ${project.build.directory}/deploy.xml + + + DESCRIPTION + ${project.description} + + + ARTIFACT_ID + ${ARTIFACT_ID} + + + PROJECT_VERSION + ${project.version} + + + SCALA_VERSION + ${scala.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + deploy-file + deploy + + deploy-file + + + ${project.description} + false + ${repositoryId} + ${repo_url} + ${project.groupId} + ${ARTIFACT_ID} + ${project.version} + jar + ${project.build.directory}/deploy.xml + ${rootdir}/assembly/target/mxnet-full_2.11-INTERNAL.jar + ${rootdir}/assembly/target/mxnet-full_2.11-INTERNAL-src.jar + ${rootdir}/assembly/target/mxnet-full_2.11-INTERNAL-bundle.jar + ${deploy_asc_types} + ${deploy_asc_classifers} + ${deploy_asc_files} + + + + + true + + + + + + diff --git a/scala-package/deploy/src/main/deploy/deploy.xml b/scala-package/deploy/src/main/deploy/deploy.xml new file mode 100644 index 000000000000..7bfd20737537 --- /dev/null +++ b/scala-package/deploy/src/main/deploy/deploy.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + org.apache.mxnet + ARTIFACT_ID + PROJECT_VERSION + DESCRIPTION + + + org.scala-lang + scala-library + SCALA_VERSION + + + org.scala-lang + scala-reflect + SCALA_VERSION + + + org.scala-lang.modules + scala-parser-combinators_2.11 + 1.0.4 + + + org.scala-lang + scala-compiler + SCALA_VERSION + + + diff --git a/scala-package/dev/compile-mxnet-backend.sh b/scala-package/dev/compile-mxnet-backend.sh index b065e01afc8e..114bf0766444 100755 --- a/scala-package/dev/compile-mxnet-backend.sh +++ b/scala-package/dev/compile-mxnet-backend.sh @@ -33,7 +33,7 @@ MXNETDIR=$2 # below routine shamelessly copied from # https://github.com/apache/incubator-mxnet/blob/master/setup-utils/install-mxnet-osx-python.sh -# This routine executes a command, +# This routine executes a command, # prints error message on the console on non-zero exit codes and # returns the exit code to the caller. chkret() { @@ -51,7 +51,10 @@ chkret() { UNAME=`uname -s` chkret pushd $MXNETDIR -chkret git submodule update --init --recursive + +set +e +git submodule update --init --recursive +set -e # don't want to overwrite an existing config file cp make/config.mk ./config.mk diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index e34f0984ea02..549622d15fb7 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -21,73 +21,18 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-examples_2.11 + mxnet-examples MXNet Scala Package - Examples true - - - integrationtest - - false - - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - release - - - - org.apache.maven.plugins - maven-source-plugin - - true - - - - org.apache.maven.plugins - maven-gpg-plugin - - true - - - - org.sonatype.plugins - nexus-staging-maven-plugin - - true - - - - - - - @@ -150,7 +95,7 @@ ${skipTests} - -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ + -Djava.library.path=${project.parent.basedir}/native/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties @@ -164,14 +109,14 @@ org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-core + INTERNAL provided org.apache.mxnet - mxnet-infer_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-infer + INTERNAL provided diff --git a/scala-package/examples/scripts/module/mnist_mlp.sh b/scala-package/examples/scripts/module/mnist_mlp.sh index 29306706dbbe..e16e16e246b2 100755 --- a/scala-package/examples/scripts/module/mnist_mlp.sh +++ b/scala-package/examples/scripts/module/mnist_mlp.sh @@ -18,7 +18,7 @@ # under the License. ROOT_DIR=$(cd `dirname $0`/../../..; pwd) -CLASSPATH=$ROOT_DIR/assembly/osx-x86_64-cpu/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* +CLASSPATH=$ROOT_DIR/assembly/osx-x86_64/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* mkdir -p model java -Xmx4G -cp $CLASSPATH \ diff --git a/scala-package/examples/scripts/module/run_sequential_module.sh b/scala-package/examples/scripts/module/run_sequential_module.sh index 9e1f30e3687b..1e40d2dd83f7 100644 --- a/scala-package/examples/scripts/module/run_sequential_module.sh +++ b/scala-package/examples/scripts/module/run_sequential_module.sh @@ -18,7 +18,7 @@ # under the License. ROOT_DIR=$(cd `dirname $0`/../../..; pwd) -CLASSPATH=$ROOT_DIR/assembly/linux-x86_64-cpu/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* +CLASSPATH=$ROOT_DIR/assembly/linux-x86_64/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* DATA_DIR=$ROOT_DIR/core/data diff --git a/scala-package/examples/scripts/run_visualization.sh b/scala-package/examples/scripts/run_visualization.sh index 3e8f5cd784f9..7f5b94fbe2b5 100644 --- a/scala-package/examples/scripts/run_visualization.sh +++ b/scala-package/examples/scripts/run_visualization.sh @@ -19,7 +19,7 @@ MXNET_ROOT=$(cd "$(dirname $0)/../../.."; pwd) -CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* +CLASS_PATH=$MXNET_ROOT/scala-package/assembly/assembly/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* # please install the Graphviz library # if you are using ubuntu, use the following command: diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala index 14fb7b85e9b3..2ccd38fc4f9c 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala @@ -49,14 +49,19 @@ class ExampleRNNSuite extends FunSuite with BeforeAndAfterAll { System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { ctx = Context.gpu() } - LstmBucketing.runTraining(tempDirPath + "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/RNN/sherlockholmes.train.txt", - tempDirPath + "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/RNN/sherlockholmes.valid.txt", Array(ctx), 1) + if (!System.getenv().containsKey("CI")) { + LstmBucketing.runTraining(tempDirPath + "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/RNN/sherlockholmes.train.txt", + tempDirPath + "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/RNN/sherlockholmes.valid.txt", Array(ctx), 1) + } else { + logger.info("Skipping test on CI...") + } } test("Example CI: Test TrainCharRNN") { val tempDirPath = System.getProperty("java.io.tmpdir") if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && - System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { + System.getenv("SCALA_TEST_ON_GPU").toInt == 1 && + !System.getenv().containsKey("CI")) { val ctx = Context.gpu() TrainCharRnn.runTrainCharRnn(tempDirPath + "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/RNN/obama.txt", tempDirPath, ctx, 1) diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index d8be5584677c..1daf79ee6cc0 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -18,102 +18,63 @@ - 4.0.0 - - mxnet-parent_2.11 - org.apache.mxnet - 1.4.0-SNAPSHOT - ../pom.xml - + 4.0.0 + + mxnet-parent + org.apache.mxnet + INTERNAL + ../pom.xml + - mxnet-infer_2.11 - MXNet Scala Package - Inference + mxnet-infer + MXNet Scala Package - Inference - - true - - - - - unittest - - false - - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-jar-plugin - - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.scalatest - scalatest-maven-plugin - - ${skipTests} - - -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ - -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties - - - - - org.scalastyle - scalastyle-maven-plugin - - - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - provided - - - - org.mockito - mockito-all - 1.10.19 - test - - + + + + org.apache.maven.plugins + maven-jar-plugin + + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.scalatest + scalatest-maven-plugin + + + -Djava.library.path=${project.parent.basedir}/native/target \ + -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + + + + org.scalastyle + scalastyle-maven-plugin + + + + + + org.apache.mxnet + mxnet-core + INTERNAL + provided + + + + org.mockito + mockito-all + 1.10.19 + test + + diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml deleted file mode 100644 index c1f61fca01aa..000000000000 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ /dev/null @@ -1,122 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-init-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-init-scala-linux-x86_64 - MXNet Scala Package - Initializer Native Linux-x86_64 - http://maven.apache.org - - so - - - - org.apache.mxnet - mxnet-init_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - linux - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_init_native_c_api.cc - - - - - -std=c++0x - - - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} - - - -shared - - - ${all_ldpaths} - -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive - - - ${ldflags} - -fopenmp - - - - - - javah - generate-sources - - linux - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_init_native_c_api.h - - org.apache.mxnet.init.LibInfo - - - - javah - - - - - - - diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml deleted file mode 100644 index 1c1342eea149..000000000000 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-init-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-init-scala-osx-x86_64 - MXNet Scala Package - Initializer Native OSX-x86_64 - http://maven.apache.org - - jnilib - - - - org.apache.mxnet - mxnet-init_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - darwin - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_init_native_c_api.cc - - - - - -std=c++0x - - - -I${project.basedir}/../../../include - ${cflags} - - - -shared - - - -framework JavaVM - -Wl,-exported_symbol,_Java_* - -Wl,-x - ${lddeps} - -force_load ${project.basedir}/../../../lib/libmxnet.a - - - ${ldflags} - - - - - - javah - generate-sources - - darwin - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_init_native_c_api.h - - org.apache.mxnet.init.LibInfo - - - - javah - - - - - - - diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index d70ec3132374..1254f615c64b 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -21,46 +21,170 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-scala-init-native-parent - MXNet Scala Package - Initializer Native Parent - pom + libmxnet-init-scala + MXNet Scala Package - Initializer Native + + + ${project.parent.basedir}/.. + + + ${libtype} - osx-x86_64-cpu - - osx-x86_64 - - - - linux-x86_64-cpu - - linux-x86_64 - + osx-x86_64 + + mac + + + + + org.codehaus.mojo + native-maven-plugin + true + + darwin + generic-classic + ${cxx} + ${cxx} + + + src/main/native + + org_apache_mxnet_init_native_c_api.cc + + + + + -std=c++0x + + + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs + + + -shared + + + -framework JavaVM + -Wl,-exported_symbol,_Java_* + -Wl,-x + + + -lmxnet -L${MXNET_DIR}/lib + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -add_rpath @loader_path ${project.build.directory}/${project.artifactId}.jnilib + + + + + + - linux-x86_64-gpu - - linux-x86_64 - + linux-x86_64 + + + unix + Linux + + + + + + org.codehaus.mojo + native-maven-plugin + true + + linux + generic-classic + ${cxx} + ${cxx} + + + src/main/native + + org_apache_mxnet_init_native_c_api.cc + + + + + -std=c++0x + + + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs + + + -shared + + + -Wl,--whole-archive + -Wl,--no-whole-archive -pthread -lm -fopenmp -lrt + + + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib + + + + + - org.apache.maven.plugins - maven-deploy-plugin - - true - + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + link-native-lib + generate-resources + + exec + + + bash + -c 'ln -sf ${MXNET_DIR}/lib/* ${project.build.directory}/' + + + - diff --git a/scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h b/scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h new file mode 100644 index 000000000000..6ff6ae6a107c --- /dev/null +++ b/scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_apache_mxnet_init_LibInfo */ + +#ifndef _Included_org_apache_mxnet_init_LibInfo +#define _Included_org_apache_mxnet_init_LibInfo +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_apache_mxnet_init_LibInfo + * Method: mxSymbolListAtomicSymbolCreators + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_init_LibInfo_mxSymbolListAtomicSymbolCreators + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_init_LibInfo + * Method: mxSymbolGetAtomicSymbolInfo + * Signature: (JLorg/apache/mxnet/init/Base/RefString;Lorg/apache/mxnet/init/Base/RefString;Lorg/apache/mxnet/init/Base/RefInt;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/init/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_init_LibInfo_mxSymbolGetAtomicSymbolInfo + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_init_LibInfo + * Method: mxListAllOpNames + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_init_LibInfo_mxListAllOpNames + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_init_LibInfo + * Method: nnGetOpHandle + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/init/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_init_LibInfo_nnGetOpHandle + (JNIEnv *, jobject, jstring, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index 3b46f83d0b92..fb9e3d3535fb 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -21,65 +21,69 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT - + mxnet-parent + INTERNAL + ../pom.xml - mxnet-init_2.11 + mxnet-scala-init MXNet Scala Package - Initializer - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - apache-release - - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - compile-mxnet-backend - compile - - exec - - - bash - ${project.parent.basedir}/dev/compile-mxnet-backend.sh ${build.platform} ${project.parent.basedir}/../ - - - - - - org.apache.maven.plugins - maven-deploy-plugin + + + + org.codehaus.mojo + native-maven-plugin + true + + + javah + verify - true + default + ${project.build.directory}/custom-javah + ${basedir} + org_apache_mxnet_init_native_c_api.h + + org.apache.mxnet.init.LibInfo + - - - - - + + javah + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + verify-javah + verify + + exec + + + diff + + ${project.build.directory}/custom-javah/org_apache_mxnet_init_native_c_api.h + ${project.parent.basedir}/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/scala-package/init/src/main/scala/org/apache/mxnet/init/Base.scala b/scala-package/init/src/main/scala/org/apache/mxnet/init/Base.scala index 7402dbd3bc1d..b5a6286af1b6 100644 --- a/scala-package/init/src/main/scala/org/apache/mxnet/init/Base.scala +++ b/scala-package/init/src/main/scala/org/apache/mxnet/init/Base.scala @@ -17,6 +17,8 @@ package org.apache.mxnet.init +import java.io.File + object Base { tryLoadInitLibrary() val _LIB = new LibInfo @@ -37,18 +39,22 @@ object Base { @throws(classOf[UnsatisfiedLinkError]) private def tryLoadInitLibrary(): Unit = { - var baseDir = System.getProperty("user.dir") + "/init-native" - // TODO(lanKing520) Update this to use relative path to the MXNet director. - // TODO(lanking520) baseDir = sys.env("MXNET_BASEDIR") + "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/scala-package/init-native" - if (System.getenv().containsKey("MXNET_BASEDIR")) { - baseDir = sys.env("MXNET_BASEDIR") + var userDir : File = new File(System.getProperty("user.dir")) + var nativeDir : File = new File(userDir, "init-native") + if (!nativeDir.exists()) { + nativeDir = new File(userDir.getParent, "init-native") + if (!nativeDir.exists()) { + throw new IllegalStateException("scala-init should be executed inside scala-package folder") + } } + val baseDir = nativeDir.getAbsolutePath + val os = System.getProperty("os.name") // ref: http://lopica.sourceforge.net/os.html if (os.startsWith("Linux")) { - System.load(s"$baseDir/linux-x86_64/target/libmxnet-init-scala-linux-x86_64.so") + System.load(s"$baseDir/target/libmxnet-init-scala.so") } else if (os.startsWith("Mac")) { - System.load(s"$baseDir/osx-x86_64/target/libmxnet-init-scala-osx-x86_64.jnilib") + System.load(s"$baseDir/target/libmxnet-init-scala.jnilib") } else { // TODO(yizhi) support windows later throw new UnsatisfiedLinkError() diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index c24403f5e630..d6dd0201897f 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -21,63 +21,20 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-macros_2.11 + mxnet-macros MXNet Scala Package - Macros - - - unittest - - false - - - - integrationtest - - true - - - - osx-x86_64-cpu - - osx-x86_64 - jnilib - - - - linux-x86_64-cpu - - linux-x86_64 - so - - - - linux-x86_64-gpu - - linux-x86_64 - so - - - - org.apache.mxnet - mxnet-init_${scala.binary.version} - 1.4.0-SNAPSHOT - provided - - - org.apache.mxnet - libmxnet-init-scala-${platform} - 1.4.0-SNAPSHOT + mxnet-scala-init + INTERNAL provided - ${libtype} commons-io @@ -86,16 +43,8 @@ - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - org.apache.maven.plugins maven-jar-plugin @@ -111,39 +60,15 @@ org.apache.maven.plugins maven-compiler-plugin - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - apidoc-generation - package - - java - - - - - - ${project.parent.basedir}/init/target/classes - - - ${project.parent.basedir}/core/src/main/scala/org/apache/mxnet/ - - org.apache.mxnet.APIDocGenerator - - org.scalatest scalatest-maven-plugin - ${skiptest} ${project.parent.basedir}/init-native - -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ + -Djava.library.path=${project.parent.basedir}/native/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties @@ -154,5 +79,4 @@ - diff --git a/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala b/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala index ce12dc7cd5a0..8f43421ca1b5 100644 --- a/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala +++ b/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala @@ -228,21 +228,21 @@ private[mxnet] object APIDocGenerator extends GeneratorBase { val finalStr = s"""/* - |* Licensed to the Apache Software Foundation (ASF) under one or more - |* contributor license agreements. See the NOTICE file distributed with - |* this work for additional information regarding copyright ownership. - |* The ASF licenses this file to You under the Apache License, Version 2.0 - |* (the "License"); you may not use this file except in compliance with - |* the License. You may obtain a copy of the License at - |* - |* http://www.apache.org/licenses/LICENSE-2.0 - |* - |* Unless required by applicable law or agreed to in writing, software - |* distributed under the License is distributed on an "AS IS" BASIS, - |* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - |* See the License for the specific language governing permissions and - |* limitations under the License. - |*/ + | * Licensed to the Apache Software Foundation (ASF) under one or more + | * contributor license agreements. See the NOTICE file distributed with + | * this work for additional information regarding copyright ownership. + | * The ASF licenses this file to You under the Apache License, Version 2.0 + | * (the "License"); you may not use this file except in compliance with + | * the License. You may obtain a copy of the License at + | * + | * http://www.apache.org/licenses/LICENSE-2.0 + | * + | * Unless required by applicable law or agreed to in writing, software + | * distributed under the License is distributed on an "AS IS" BASIS, + | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + | * See the License for the specific language governing permissions and + | * limitations under the License. + | */ | |$packageDef | diff --git a/scala-package/mxnet-demo/java-demo/Makefile b/scala-package/mxnet-demo/java-demo/Makefile index 6b76e272f9e0..4f2b5e938970 100644 --- a/scala-package/mxnet-demo/java-demo/Makefile +++ b/scala-package/mxnet-demo/java-demo/Makefile @@ -16,7 +16,7 @@ # under the License. SCALA_VERSION_PROFILE := 2.11 -MXNET_VERSION := 1.3.1-SNAPSHOT +MXNET_VERSION := [1.5.0-SNAPSHOT,\) ifeq ($(OS),Windows_NT) UNAME_S := Windows diff --git a/scala-package/mxnet-demo/java-demo/README.md b/scala-package/mxnet-demo/java-demo/README.md index 952775dd964e..5b39e15261f4 100644 --- a/scala-package/mxnet-demo/java-demo/README.md +++ b/scala-package/mxnet-demo/java-demo/README.md @@ -16,11 +16,17 @@ # MXNet Java Sample Project -This is an project created to use Maven-published Scala/Java package with two Java examples. +This is a project demonstrating how to use the Maven published Scala/Java MXNet package. +The examples provided include: +* NDArray creation +* NDArray operation +* Object Detection using the Inference API +* Image Classification using the Predictor API + ## Setup -You can use the `Makefile` to make the Java package. Simply do the following: -```Bash -make javademo +You are required to use Maven to build the package with the following commands under `java-demo`: +``` +mvn package ``` This will load the default parameter for all the environment variable. If you want to run with GPU on Linux, just simply add `USE_CUDA=1` when you run the make file @@ -29,7 +35,7 @@ You can use the following instruction as an alternative to achieve the same resu User are required to use `mvn package` to build the package, which are shown below: ```Bash -export SCALA_VERSION_PROFILE=2.11 MXNET_VERSION=1.3.1-SNAPSHOT +export SCALA_VERSION_PROFILE=2.11 MXNET_VERSION=1.5.0-SNAPSHOT export SCALA_PKG_PROFILE= mvn package -Dmxnet.profile=$SCALA_PKG_PROFILE \ -Dmxnet.scalaprofile=$SCALA_VERSION_PROFILE \ @@ -41,16 +47,16 @@ The `SCALA_PKG_PROFILE` should be chosen from `osx-x86_64-cpu`, `linux-x86_64-cp ## Run -### Hello World -The Scala file is being executed using Java. You can execute the helloWorld example as follows: +### NDArrayCreation +The Scala file is being executed using Java. You can execute the `NDArrayCreation` example as follows: ```Bash bash bin/java_sample.sh ``` You can also run the following command manually: ```Bash -java -cp $CLASSPATH sample.HelloWorld +java -cp $CLASSPATH sample.NDArrayCreation ``` -However, you have to define the Classpath before you run the demo code. More information can be found in the `java_sample.sh`. +However, you have to define the Classpath before you run the demo code. More information can be found in `bin/java_sample.sh`. The `CLASSPATH` should point to the jar file you have downloaded. It will load the library automatically and run the example @@ -97,5 +103,5 @@ sudo apt install libopencv-imgcodecs3.4 Is there any other version available? -You can find nightly release version from [here](https://repository.apache.org/#nexus-search;gav~org.apache.mxnet~~1.3.1-SNAPSHOT~~). +You can find nightly release version from [here](https://repository.apache.org/#nexus-search;gav~org.apache.mxnet~~1.5.0-SNAPSHOT~~). Please keep the same version in the Makefile or [above version](https://repository.apache.org/#nexus-search;gav~org.apache.mxnet~~~~) to run this demo. diff --git a/scala-package/mxnet-demo/java-demo/bin/java_sample.sh b/scala-package/mxnet-demo/java-demo/bin/java_sample.sh index 2ec9a78c3233..fb1795f20f9d 100644 --- a/scala-package/mxnet-demo/java-demo/bin/java_sample.sh +++ b/scala-package/mxnet-demo/java-demo/bin/java_sample.sh @@ -16,5 +16,5 @@ # under the License. #!/bin/bash CURR_DIR=$(cd $(dirname $0)/../; pwd) -CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/classes/lib/* -java -Xmx8G -cp $CLASSPATH mxnet.HelloWorld \ No newline at end of file +CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* +java -Xmx8G -cp $CLASSPATH mxnet.NDArrayCreation diff --git a/scala-package/mxnet-demo/java-demo/bin/run_od.sh b/scala-package/mxnet-demo/java-demo/bin/run_od.sh index e3c8fd545048..4370518dc8cd 100644 --- a/scala-package/mxnet-demo/java-demo/bin/run_od.sh +++ b/scala-package/mxnet-demo/java-demo/bin/run_od.sh @@ -16,5 +16,5 @@ # under the License. #!/bin/bash CURR_DIR=$(cd $(dirname $0)/../; pwd) -CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/classes/lib/* -java -Xmx8G -cp $CLASSPATH mxnet.ObjectDetection \ No newline at end of file +CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* +java -Xmx8G -cp $CLASSPATH mxnet.ObjectDetection diff --git a/scala-package/mxnet-demo/java-demo/pom.xml b/scala-package/mxnet-demo/java-demo/pom.xml index c12efa527c60..9dc7fcd431af 100644 --- a/scala-package/mxnet-demo/java-demo/pom.xml +++ b/scala-package/mxnet-demo/java-demo/pom.xml @@ -42,6 +42,12 @@ mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} ${mxnet.version} + + org.apache.mxnet + mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} + ${mxnet.version} + sources + commons-io commons-io diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java new file mode 100644 index 000000000000..8cb58da5c2e6 --- /dev/null +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mxnet; + +import org.apache.commons.io.FileUtils; +import org.apache.mxnet.infer.javaapi.Predictor; +import org.apache.mxnet.javaapi.*; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +public class ImageClassification { + private static String modelPath; + private static String imagePath; + + private static void downloadUrl(String url, String filePath) { + File tmpFile = new File(filePath); + if (!tmpFile.exists()) { + try { + FileUtils.copyURLToFile(new URL(url), tmpFile); + } catch (Exception exception) { + System.err.println(exception); + } + } + } + + public static void downloadModelImage() { + String tempDirPath = System.getProperty("java.io.tmpdir"); + String baseUrl = "https://s3.us-east-2.amazonaws.com/scala-infer-models"; + downloadUrl(baseUrl + "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/resnet-18/resnet-18-symbol.json", + tempDirPath + "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/resnet18/resnet-18-symbol.json"); + downloadUrl(baseUrl + "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/resnet-18/resnet-18-0000.params", + tempDirPath + "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/resnet18/resnet-18-0000.params"); + downloadUrl(baseUrl + "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/resnet-18/synset.txt", + tempDirPath + "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/resnet18/synset.txt"); + downloadUrl("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg", + tempDirPath + "/inputImages/resnet18/Pug-Cookie.jpg"); + modelPath = tempDirPath + File.separator + "resnet18/resnet-18"; + imagePath = tempDirPath + File.separator + + "inputImages/resnet18/Pug-Cookie.jpg"; + } + + /** + * Helper class to print the maximum prediction result + * @param probabilities The float array of probability + * @param modelPathPrefix model Path needs to load the synset.txt + */ + private static String printMaximumClass(float[] probabilities, + String modelPathPrefix) throws IOException { + String synsetFilePath = modelPathPrefix.substring(0, + 1 + modelPathPrefix.lastIndexOf(File.separator)) + "/synset.txt"; + BufferedReader reader = new BufferedReader(new FileReader(synsetFilePath)); + ArrayList list = new ArrayList<>(); + String line = reader.readLine(); + + while (line != null){ + list.add(line); + line = reader.readLine(); + } + reader.close(); + + int maxIdx = 0; + for (int i = 1;i probabilities[maxIdx]) { + maxIdx = i; + } + } + + return "Probability : " + probabilities[maxIdx] + " Class : " + list.get(maxIdx) ; + } + + public static void main(String[] args) { + // Download the model and Image + downloadModelImage(); + + // Prepare the model + List context = new ArrayList(); + context.add(Context.cpu()); + List inputDesc = new ArrayList<>(); + Shape inputShape = new Shape(new int[]{1, 3, 224, 224}); + inputDesc.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); + Predictor predictor = new Predictor(modelPath, inputDesc, context,0); + + // Prepare data + NDArray nd = Image.imRead(imagePath, 1, true); + nd = Image.imResize(nd, 224, 224, null); + nd = NDArray.transpose(nd, new Shape(new int[]{2, 0, 1}), null)[0]; // HWC to CHW + nd = NDArray.expand_dims(nd, 0, null)[0]; // Add N -> NCHW + nd = nd.asType(DType.Float32()); // Inference with Float32 + + // Predict directly + float[][] result = predictor.predict(new float[][]{nd.toArray()}); + try { + System.out.println("Predict with Float input"); + System.out.println(printMaximumClass(result[0], modelPath)); + } catch (IOException e) { + System.err.println(e); + } + + // predict with NDArray + List ndList = new ArrayList<>(); + ndList.add(nd); + List ndResult = predictor.predictWithNDArray(ndList); + try { + System.out.println("Predict with NDArray"); + System.out.println(printMaximumClass(ndResult.get(0).toArray(), modelPath)); + } catch (IOException e) { + System.err.println(e); + } + } +} diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java new file mode 100644 index 000000000000..32e2d84dcdbf --- /dev/null +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mxnet; + +import org.apache.mxnet.javaapi.*; + +public class NDArrayCreation { + static NDArray$ NDArray = NDArray$.MODULE$; + public static void main(String[] args) { + + // Create new NDArray + NDArray nd = new NDArray(new float[]{2.0f, 3.0f}, new Shape(new int[]{1, 2}), Context.cpu()); + System.out.println(nd); + + // create new Double NDArray + NDArray ndDouble = new NDArray(new double[]{2.0d, 3.0d}, new Shape(new int[]{2, 1}), Context.cpu()); + System.out.println(ndDouble); + + // create ones + NDArray ones = NDArray.ones(Context.cpu(), new int[] {1, 2, 3}); + System.out.println(ones); + + // random + NDArray random = NDArray.random_uniform( + NDArray.new random_uniformParam() + .setLow(0.0f) + .setHigh(2.0f) + .setShape(new Shape(new int[]{10, 10})) + )[0]; + System.out.println(random); + } +} diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/HelloWorld.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayOperation.java similarity index 63% rename from scala-package/mxnet-demo/java-demo/src/main/java/mxnet/HelloWorld.java rename to scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayOperation.java index 3f209a6c6c84..56a414307f46 100644 --- a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/HelloWorld.java +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayOperation.java @@ -14,15 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package mxnet; import org.apache.mxnet.javaapi.*; -import java.util.Arrays; -public class HelloWorld { +public class NDArrayOperation { + static NDArray$ NDArray = NDArray$.MODULE$; public static void main(String[] args) { - System.out.println("Hello World!"); NDArray nd = new NDArray(new float[]{2.0f, 3.0f}, new Shape(new int[]{1, 2}), Context.cpu()); - System.out.println(nd.shape()); + + // Transpose + NDArray ndT = nd.T(); + System.out.println(nd); + System.out.println(ndT); + + // change Data Type + NDArray ndInt = nd.asType(DType.Int32()); + System.out.println(ndInt); + + // element add + NDArray eleAdd = NDArray.elemwise_add(nd, nd, null)[0]; + System.out.println(eleAdd); + + // norm (L2 Norm) + NDArray normed = NDArray.norm(NDArray.new normParam(nd))[0]; + System.out.println(normed); } } diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java index cfe9b66c4b3f..65fe286aa2c7 100644 --- a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java @@ -68,20 +68,18 @@ public static void downloadModelImage() { public static void main(String[] args) { List context = new ArrayList(); - if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && - Integer.valueOf(System.getenv("SCALA_TEST_ON_GPU")) == 1) { - context.add(Context.gpu()); - } else { - context.add(Context.cpu()); - } + context.add(Context.cpu()); downloadModelImage(); + + List> output + = runObjectDetectionSingle(modelPath, imagePath, context); + Shape inputShape = new Shape(new int[] {1, 3, 512, 512}); Shape outputShape = new Shape(new int[] {1, 6132, 6}); int width = inputShape.get(2); int height = inputShape.get(3); - List> output - = runObjectDetectionSingle(modelPath, imagePath, context); String outputStr = "\n"; + for (List ele : output) { for (ObjectDetectorOutput i : ele) { outputStr += "Class: " + i.getClassName() + "\n"; @@ -98,4 +96,4 @@ public static void main(String[] args) { } System.out.println(outputStr); } -} \ No newline at end of file +} diff --git a/scala-package/mxnet-demo/scala-demo/pom.xml b/scala-package/mxnet-demo/scala-demo/pom.xml index 8d63d0f918ce..c2a14d878427 100644 --- a/scala-package/mxnet-demo/scala-demo/pom.xml +++ b/scala-package/mxnet-demo/scala-demo/pom.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 Demo - mxnet-scala-demo_2.11 + mxnet-scala-demo 1.0-SNAPSHOT MXNet Scala Demo pom diff --git a/scala-package/native/README.md b/scala-package/native/README.md new file mode 100644 index 000000000000..c87b064fff02 --- /dev/null +++ b/scala-package/native/README.md @@ -0,0 +1,67 @@ +# MXNet Scala JNI + +MXNet Scala JNI is a thin wrapper layer of underlying libmxnet.so. + +## javah +JNI native code requires a header file that matches the java/scala interface, +this file is usually generated with javah. + +In our case, org_apache_mxnet_native_c.h is generated and will be used to compile native code. + +To improve build performance, we check in generated org_apache_mxnet_native_c.h file. +And we added a check to detect mismatch with Scala code and generated header. The checker will +make sure we won't forget to update org_apache_mxnet_native_c.h file. + + +## Linker options + +Scala JNI (libmxnet-scala.so/libmxnet-scala.jnilib) is dynamically linked to libmxnet.so. +MXNet Scala will trying to load libmxnet.so from system LD_LIBRARY_PATH first. +If it failed, the try to resolve libmxnet.so in the same location as libmxnet-scala.so file. + +### Linux +``` +-Wl,-rpath=$ORIGIN -lmxnet +``` +Above option will tell system to looking for libmxnet.so from the same location. + + +### Mac OSX +On Mac, we have to execute install_name_tool command to change library loading path: +```bash +install_name_tool -change lib/libmxnet.so @loader_path/libmxnet.so libmxnet-scala.jnilib +``` + +Other linker options: +* -shared : link as shared library +* -Wl,-install_name,libmxnet-scala.jnilib : avoid use build machine's absolute path +* -framework JavaVM : Stand jni options for mac +* -Wl,-exported_symbol,_Java_* : Stand jni options for mac +* -Wl,-x : Do not put non-global symbols in the output file's symbol table. + + +## Compiler flags + +Scala JNI code technically doesn't need on any of MXNet make flags, +however c_api.h header links to many other dependencies header file, +which requires us to add DMSHADOW_USE_MKL and DMSHADOW_USE_CUDA to compile the JNI code. +These flags are not actually used by JNI and won't impact Scala's behavior. + + +### Linux + +``` +-DMSHADOW_USE_MKL=0 +-DMSHADOW_USE_CUDA=0 +-O3 -DNDEBUG=1 -fPIC -msse3 -mf16c +-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs +``` + +### Mac OSX + +``` +-DMSHADOW_USE_MKL=0 +-DMSHADOW_USE_CUDA=0 +-g -O0 -fPIC -msse3 -mf16c +-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs +``` diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml deleted file mode 100644 index 500e484cbb59..000000000000 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ /dev/null @@ -1,122 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-scala-linux-x86_64-cpu - MXNet Scala Package - Native Linux-x86_64 CPU-only - http://maven.apache.org - - so - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - linux - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_native_c_api.cc - - - - - -std=c++0x - - - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} - - - -shared - - - ${all_ldpaths} - -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive - - - ${ldflags} - -fopenmp - - - - - - javah - generate-sources - - linux - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_native_c_api.h - - org.apache.mxnet.LibInfo - - - - javah - - - - - - - diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml deleted file mode 100644 index da8b5f4a8183..000000000000 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ /dev/null @@ -1,122 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-scala-linux-x86_64-gpu - MXNet Scala Package - Native Linux-x86_64 GPU - http://maven.apache.org - - so - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - linux - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_native_c_api.cc - - - - - -std=c++0x - - - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} - - - -shared - - - ${all_ldpaths} - -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive - - - ${ldflags} - -fopenmp - - - - - - javah - generate-sources - - linux - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_native_c_api.h - - org.apache.mxnet.LibInfo - - - - javah - - - - - - - diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml deleted file mode 100644 index fc367ca9bfc5..000000000000 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ /dev/null @@ -1,122 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-scala-osx-x86_64-cpu - MXNet Scala Package - Native OSX-x86_64 CPU-only - http://maven.apache.org - - jnilib - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - darwin - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_native_c_api.cc - - - - - -std=c++0x - - - -I../../../include - ${cflags} - - - -shared - - - -framework JavaVM - -Wl,-exported_symbol,_Java_* - -Wl,-x - ${lddeps} - -force_load ${project.basedir}/../../../lib/libmxnet.a - -force_load ${project.basedir}/../../../3rdparty/tvm/nnvm/lib/libnnvm.a - - - ${ldflags} - - - - - - javah - generate-sources - - darwin - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_native_c_api.h - - org.apache.mxnet.LibInfo - - - - javah - - - - - - - diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index 2f1a211668a8..cc8c1cc86daf 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -21,46 +21,165 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-scala-native-parent - MXNet Scala Package - Native Parent - pom + libmxnet-scala + MXNet Scala Package - Native + ${libtype} + + + ${project.parent.basedir}/.. + - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - + osx-x86_64 + + mac + + + + + org.codehaus.mojo + native-maven-plugin + true + + darwin + generic-classic + ${cxx} + ${cxx} + + + src/main/native + + org_apache_mxnet_native_c_api.cc + + + + + -std=c++0x + + + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs + + + -shared + + + -framework JavaVM + -Wl,-exported_symbol,_Java_* + -Wl,-x + + + -Wl,-install_name,libmxnet-scala.jnilib -lmxnet -L${MXNET_DIR}/lib + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -add_rpath @loader_path ${project.build.directory}/${project.artifactId}.jnilib + + + + + + - linux-x86_64-gpu - - linux-x86_64-gpu - + linux-x86_64 + + + unix + Linux + + + + + + org.codehaus.mojo + native-maven-plugin + true + + linux + generic-classic + ${cxx} + ${cxx} + + + src/main/native + + org_apache_mxnet_native_c_api.cc + + + + + -std=c++0x + + + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs + + + -shared + + + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib + + + + + - org.apache.maven.plugins - maven-deploy-plugin - - true - + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + link-native-lib + generate-resources + + exec + + + bash + -c 'ln -sf ${MXNET_DIR}/lib/* ${project.build.directory}/' + + + - diff --git a/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.cc b/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.cc index 17d166eac345..d684c6d13564 100644 --- a/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include "jni_helper_func.h" JavaVM *_jvm; diff --git a/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h b/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h new file mode 100644 index 000000000000..40230ac6daae --- /dev/null +++ b/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h @@ -0,0 +1,853 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_apache_mxnet_LibInfo */ + +#ifndef _Included_org_apache_mxnet_LibInfo +#define _Included_org_apache_mxnet_LibInfo +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_apache_mxnet_LibInfo + * Method: nativeLibInit + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_nativeLibInit + (JNIEnv *, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxGetLastError + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_apache_mxnet_LibInfo_mxGetLastError + (JNIEnv *, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxListAllOpNames + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxListAllOpNames + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: nnGetOpHandle + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_nnGetOpHandle + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxImperativeInvoke + * Signature: (J[J[JLscala/collection/mutable/ArrayBuffer;I[Ljava/lang/String;[Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxImperativeInvoke + (JNIEnv *, jobject, jlong, jlongArray, jlongArray, jobject, jint, jobjectArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayCreateNone + * Signature: (Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayCreateNone + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayCreateEx + * Signature: ([IIIIIILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayCreateEx + (JNIEnv *, jobject, jintArray, jint, jint, jint, jint, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayWaitAll + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayWaitAll + (JNIEnv *, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayWaitToRead + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayWaitToRead + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxListFunctions + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxListFunctions + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxFuncDescribe + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lorg/apache/mxnet/Base/RefInt;Lorg/apache/mxnet/Base/RefInt;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxFuncDescribe + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxFuncGetInfo + * Signature: (JLorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxFuncGetInfo + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxFuncInvoke + * Signature: (J[J[F[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxFuncInvoke + (JNIEnv *, jobject, jlong, jlongArray, jfloatArray, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxFuncInvokeEx + * Signature: (J[J[F[JI[[B[[B)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxFuncInvokeEx + (JNIEnv *, jobject, jlong, jlongArray, jfloatArray, jlongArray, jint, jobjectArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayGetShape + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayGetShape + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySyncCopyToCPU + * Signature: (J[BI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySyncCopyToCPU + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySlice + * Signature: (JIILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySlice + (JNIEnv *, jobject, jlong, jint, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayAt + * Signature: (JILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayAt + (JNIEnv *, jobject, jlong, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayReshape + * Signature: (JI[ILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayReshape + (JNIEnv *, jobject, jlong, jint, jintArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySyncCopyFromCPU + * Signature: (J[FI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySyncCopyFromCPU + (JNIEnv *, jobject, jlong, jfloatArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayLoad + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;Lorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayLoad + (JNIEnv *, jobject, jstring, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySave + * Signature: (Ljava/lang/String;[J[Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySave + (JNIEnv *, jobject, jstring, jlongArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayGetContext + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayGetContext + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySaveRawBytes + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySaveRawBytes + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayLoadFromRawBytes + * Signature: ([BLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayLoadFromRawBytes + (JNIEnv *, jobject, jbyteArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayGetDType + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayGetDType + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxInitPSEnv + * Signature: ([Ljava/lang/String;[Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxInitPSEnv + (JNIEnv *, jobject, jobjectArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreRunServer + * Signature: (JLorg/apache/mxnet/KVServerControllerCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreRunServer + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreGetNumDeadNode + * Signature: (JILorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreGetNumDeadNode + (JNIEnv *, jobject, jlong, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreCreate + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreCreate + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreInit + * Signature: (JI[I[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreInit + (JNIEnv *, jobject, jlong, jint, jintArray, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreInitEx + * Signature: (JI[Ljava/lang/String;[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreInitEx + (JNIEnv *, jobject, jlong, jint, jobjectArray, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStorePush + * Signature: (JI[I[JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStorePush + (JNIEnv *, jobject, jlong, jint, jintArray, jlongArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStorePushEx + * Signature: (JI[Ljava/lang/String;[JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStorePushEx + (JNIEnv *, jobject, jlong, jint, jobjectArray, jlongArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStorePull + * Signature: (JI[I[JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStorePull + (JNIEnv *, jobject, jlong, jint, jintArray, jlongArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStorePullEx + * Signature: (JI[Ljava/lang/String;[JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStorePullEx + (JNIEnv *, jobject, jlong, jint, jobjectArray, jlongArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreSetUpdater + * Signature: (JLorg/apache/mxnet/MXKVStoreUpdater;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreSetUpdater + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreIsWorkerNode + * Signature: (Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreIsWorkerNode + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreGetType + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreGetType + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreSendCommmandToServers + * Signature: (JILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreSendCommmandToServers + (JNIEnv *, jobject, jlong, jint, jstring); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreBarrier + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreBarrier + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreGetGroupSize + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreGetGroupSize + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreGetRank + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreGetRank + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreSetBarrierBeforeExit + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreSetBarrierBeforeExit + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxListDataIters + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxListDataIters + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterCreateIter + * Signature: (J[Ljava/lang/String;[Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterCreateIter + (JNIEnv *, jobject, jlong, jobjectArray, jobjectArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetIterInfo + * Signature: (JLorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefString;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetIterInfo + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterBeforeFirst + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterBeforeFirst + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterNext + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterNext + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetLabel + * Signature: (JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetLabel + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetData + * Signature: (JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetData + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetIndex + * Signature: (JLscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetIndex + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetPadNum + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetPadNum + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorOutputs + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorOutputs + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorForward + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorForward + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorBackward + * Signature: (J[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorBackward + (JNIEnv *, jobject, jlong, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorPrint + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorPrint + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorSetMonitorCallback + * Signature: (JLorg/apache/mxnet/MXMonitorCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorSetMonitorCallback + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListAtomicSymbolCreators + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListAtomicSymbolCreators + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolGetAtomicSymbolInfo + * Signature: (JLorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolGetAtomicSymbolInfo + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateAtomicSymbol + * Signature: (J[Ljava/lang/String;[Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateAtomicSymbol + (JNIEnv *, jobject, jlong, jobjectArray, jobjectArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolSetAttr + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolSetAttr + (JNIEnv *, jobject, jlong, jstring, jstring); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListAttrShallow + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListAttrShallow + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListAttr + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListAttr + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCompose + * Signature: (JLjava/lang/String;[Ljava/lang/String;[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCompose + (JNIEnv *, jobject, jlong, jstring, jobjectArray, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateVariable + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateVariable + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolGetAttr + * Signature: (JLjava/lang/String;Lorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolGetAttr + (JNIEnv *, jobject, jlong, jstring, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListArguments + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListArguments + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCopy + * Signature: (JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCopy + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListAuxiliaryStates + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListAuxiliaryStates + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListOutputs + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListOutputs + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateGroup + * Signature: ([JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateGroup + (JNIEnv *, jobject, jlongArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolPrint + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolPrint + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolGetInternals + * Signature: (JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolGetInternals + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolInferType + * Signature: (J[Ljava/lang/String;[ILscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolInferType + (JNIEnv *, jobject, jlong, jobjectArray, jintArray, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolInferShape + * Signature: (JI[Ljava/lang/String;[I[ILscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolInferShape + (JNIEnv *, jobject, jlong, jint, jobjectArray, jintArray, jintArray, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolGetOutput + * Signature: (JILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolGetOutput + (JNIEnv *, jobject, jlong, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolSaveToJSON + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolSaveToJSON + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateFromJSON + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateFromJSON + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorBindX + * Signature: (JIII[Ljava/lang/String;[I[II[J[J[I[JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorBindX + (JNIEnv *, jobject, jlong, jint, jint, jint, jobjectArray, jintArray, jintArray, jint, jlongArray, jlongArray, jintArray, jlongArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorBindEX + * Signature: (JIII[Ljava/lang/String;[I[II[J[J[I[JJLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorBindEX + (JNIEnv *, jobject, jlong, jint, jint, jint, jobjectArray, jintArray, jintArray, jint, jlongArray, jlongArray, jintArray, jlongArray, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolSaveToFile + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolSaveToFile + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateFromFile + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateFromFile + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRandomSeed + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRandomSeed + (JNIEnv *, jobject, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNotifyShutdown + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNotifyShutdown + (JNIEnv *, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOWriterCreate + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOWriterCreate + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOReaderCreate + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOReaderCreate + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOWriterFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOWriterFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOReaderFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOReaderFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOWriterWriteRecord + * Signature: (JLjava/lang/String;I)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOWriterWriteRecord + (JNIEnv *, jobject, jlong, jstring, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOReaderReadRecord + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOReaderReadRecord + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOWriterTell + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOWriterTell + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOReaderSeek + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOReaderSeek + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRtcCreate + * Signature: (Ljava/lang/String;[Ljava/lang/String;[Ljava/lang/String;[J[JLjava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRtcCreate + (JNIEnv *, jobject, jstring, jobjectArray, jobjectArray, jlongArray, jlongArray, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRtcPush + * Signature: (J[J[JIIIIII)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRtcPush + (JNIEnv *, jobject, jlong, jlongArray, jlongArray, jint, jint, jint, jint, jint, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRtcFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRtcFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxCustomOpRegister + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/CustomOpProp;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxCustomOpRegister + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSetProfilerConfig + * Signature: ([Ljava/lang/String;[Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSetProfilerConfig + (JNIEnv *, jobject, jobjectArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSetProfilerState + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSetProfilerState + (JNIEnv *, jobject, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDumpProfile + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDumpProfile + (JNIEnv *, jobject, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/scala-package/packageTest/Makefile b/scala-package/packageTest/Makefile new file mode 100644 index 000000000000..8c12c1d04189 --- /dev/null +++ b/scala-package/packageTest/Makefile @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SCALA_VERSION_PROFILE := 2.11 +SCALA_VERSION := 2.11.8 +MXNET_VERSION := "[1.3.0-SNAPSHOT,)" + +MXNET_REPO = https://repository.apache.org/content/repositories/snapshots + +ifeq ($(OS),Windows_NT) + UNAME_S := Windows +else + UNAME_S := $(shell uname -s) +endif + +ifeq ($(UNAME_S), Windows) + # TODO: currently scala package does not support windows + SCALA_PKG_PROFILE := windows +else + ifeq ($(UNAME_S), Darwin) + SCALA_PKG_PROFILE := osx-x86_64-cpu + else + SCALA_PKG_PROFILE := linux-x86_64 + ifeq ($(USE_CUDA), 1) + SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu + else + SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu + endif + endif +endif + +ifeq ($(CI), 1) + MAVEN_ARGS := -B +endif + +PROFILES := -Ptest +ifeq ($(UNIT), 1) + PROFILES := "$(PROFILES),unittest" +endif +ifeq ($(INTEGRATION), 1) + PROFILES := "$(PROFILES),integrationtest" +endif + +ifneq ($(UNIT), 1) + ifneq ($(INTEGRATION), 1) + PROFILES := "$(PROFILES),unittest,integrationtest" + endif +endif + + +clean: + (mvn $(MAVEN_ARGS) clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +testinstall: + (mvn $(MAVEN_ARGS) integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + $(PROFILES) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +testlocal: + (mvn $(MAVEN_ARGS) integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + $(PROFILES),fromLocal \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +testsnapshot: + (mvn $(MAVEN_ARGS) integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + $(PROFILES),fromSnapshots \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.repo=$(MXNET_REPO) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md new file mode 100644 index 000000000000..e9980f353759 --- /dev/null +++ b/scala-package/packageTest/README.md @@ -0,0 +1,74 @@ +# MXNet Scala Package Test + +This is an project created to run the test suite on a fully packaged mxnet jar. The test suite is found locally but mxnet is from the target jarfile. + +## General Setup + +To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory: + +1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source) +2. Build the tests by running `mvn test-compile`. +3. Follow setup instructions below for your testing goal + +## Running + +There are three different modes of operation for testing based on the location of the jar and where it is coming from: + +### Test Installed Jars + +If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`). This might be useful if you acquire the .jar file from elsewhere. To install, it is easiest to use `mvn install:install-file -Dfile= -DpomFile=`. If the pom file is not available, you can also run `mvn install:install-file -Dfile= -DgroupId= -DartifactId= -Dversion= -Dpackaging=`. With the full mxnet jar, this might look like `mvn install:install-file -Dfile= -DgroupId=org.apache.mxnet -DartifactId=mxnet-full_2.11-linux-x86_64-cpu -Dversion=1.3.0 -Dpackaging=jar`. + +You can also run `mvn install` to install from a local build. + +After installing, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests. + +### Test Local Deployment + +To test the jars that would be produced by a deployment, you can run `mvn deploy` from the main mxnet directory. This produces a local snapshot located at `scala-package/deploy/target/repo`. To test this local snapshot, run `make testlocal`. It also installs the component packages needed for testing the examples in `scala-package/*/target/repo`. + +### Remote Repository Snapshot + +This mode is to test a jar located in a remote repository. The default repository is the apache snapshot repisotory located at `https://repository.apache.org/content/repositories/snapshots`. Note that the actual jar in a repisotory should be located at `$repoUrl/org/apache/mxnet/mxnet-full_$scalaVersion-$osMode/$version/*.jar`. + +Test the snapshot repo using `make testsnapshot` or a different repo using `make testsnapshot MXNET_REPO=$NEW_REPO_URL`. + +### Options + +You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above). + +For running on GPU, add the flag `USE_CUDA=1`. + +An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`. + +## Cleaning Up + +You can clean temporary files and target artifacts by running `make clean`. + +## Troubleshooting + +### Missing Examples + +If you fail with the following error +``` +[ERROR] Failed to execute goal org.scalatest:scalatest-maven-plugin:1.0:test (test) on project mxnet-scala-packagetest-examples_2.11: There are test failures -> [Help 1] +[ERROR] +[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch. +[ERROR] Re-run Maven using the -X switch to enable full debug logging. +[ERROR] +[ERROR] For more information about the errors and possible solutions, please read the following articles: +[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException +[ERROR] +[ERROR] After correcting the problems, you can resume the build with the command +[ERROR] mvn -rf :mxnet-scala-packagetest-examples_2.11 +Makefile:57: recipe for target 'scalaintegrationtest' failed +make: *** [scalaintegrationtest] Error 1 +``` + +and stacktrace begins with the following, + +``` +*** RUN ABORTED *** + java.lang.NoClassDefFoundError: org/apache/mxnetexamples/Util$ +``` + +you are missing the mxnetexamples package. See your test mode installation section for details. diff --git a/scala-package/packageTest/core/pom.xml b/scala-package/packageTest/core/pom.xml new file mode 100644 index 000000000000..bdcd7662f082 --- /dev/null +++ b/scala-package/packageTest/core/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-core_2.11 + MXNet Scala Package Test + pom + + + + unittest + + false + + + + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../core/target/test-classes + + + + + + diff --git a/scala-package/packageTest/core/scripts b/scala-package/packageTest/core/scripts new file mode 120000 index 000000000000..f806668aa847 --- /dev/null +++ b/scala-package/packageTest/core/scripts @@ -0,0 +1 @@ +../../core/scripts \ No newline at end of file diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml new file mode 100644 index 000000000000..070b78dda99b --- /dev/null +++ b/scala-package/packageTest/examples/pom.xml @@ -0,0 +1,95 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-examples_2.11 + MXNet Scala Package Test + pom + + + + integrationtest + + false + + + + fromLocal + + + parent + file://${basedir}/../../target/repo + + true + + + + init + file://${basedir}/../../init/target/repo + + true + + + + macros + file://${basedir}/../../macros/target/repo + + true + + + + core + file://${basedir}/../../core/target/repo + + true + + + + infer + file://${basedir}/../../infer/target/repo + + true + + + + examples + file://${basedir}/../../examples/target/repo + + true + + + + + + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../examples/target/test-classes + + + + + + + + org.apache.mxnet + mxnet-examples + INTERNAL + test + + + + diff --git a/scala-package/packageTest/examples/scripts b/scala-package/packageTest/examples/scripts new file mode 120000 index 000000000000..2bba4eeece74 --- /dev/null +++ b/scala-package/packageTest/examples/scripts @@ -0,0 +1 @@ +../../examples/scripts \ No newline at end of file diff --git a/scala-package/packageTest/infer/pom.xml b/scala-package/packageTest/infer/pom.xml new file mode 100644 index 000000000000..7c5a096d6e14 --- /dev/null +++ b/scala-package/packageTest/infer/pom.xml @@ -0,0 +1,38 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-infer_2.11 + MXNet Scala Package Test + pom + + + + unittest + + false + + + + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../infer/target/test-classes + + + + + diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml new file mode 100644 index 000000000000..f6a16dd77c9d --- /dev/null +++ b/scala-package/packageTest/pom.xml @@ -0,0 +1,196 @@ + + + 4.0.0 + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + MXNet Scala Package Test + pom + + + core + infer + + + + + test + + + integrationtest + + examples + + + + fromSnapshots + + + apache-snapshots + ${mxnet.repo} + default + + true + + + + + + fromLocal + + + full + file://${basedir}/../deploy/target/repo + + true + + + + + + + + true + + + + + org.apache.mxnet + mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} + ${mxnet.version} + + + org.scala-lang + scala-library + ${scala.version} + + + commons-io + commons-io + 2.4 + + + org.scalatest + scalatest_${mxnet.scalaprofile} + 3.0.4 + test + + + org.scalacheck + scalacheck_${mxnet.scalaprofile} + 1.13.5 + test + + + org.mockito + mockito-all + 1.10.19 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.6 + 1.6 + UTF-8 + + + + maven-resources-plugin + 2.7 + + + org.apache.maven.plugins + maven-dependency-plugin + 2.9 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.outputDirectory}/lib + runtime + test,provided + false + false + true + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.5 + + + package + + jar + + + + **/* + + + + + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + compile + + compile + + compile + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.19 + + true + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${skipTests} + ${project.build.directory}/surefire-reports + . + F + WDF TestSuite.txt + + + + test + integration-test + + test + + + + + + + + diff --git a/scala-package/pom.xml b/scala-package/pom.xml index fe061e22e484..9ead5805f1d4 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -22,11 +22,12 @@ org.apache apache - 19 + 19 + org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL MXNet Scala Package - Parent https://github.com/apache/incubator-mxnet/tree/master/scala-package @@ -53,8 +54,12 @@ 2.11.8 - 2.11 - + + g++ + $ + ${project.basedir}/.. + true + file://${project.build.directory}/repo pom @@ -62,46 +67,18 @@ init init-native macros - core native + core infer examples spark assembly + deploy - - release - - - - org.codehaus.mojo - build-helper-maven-plugin - - - generate-sources - - add-source - - - - ${project.build.directory}/genjavadoc - - - - - - - - - scala-2.11 - - 2.11.8 - 2.11 - @@ -131,31 +108,66 @@ - scala-2.12 + osx-x86_64 + + + mac + + - 2.12.4 - 2.12 + osx-x86_64 + jnilib + cpu + + + linux-x86_64 + + + unix + Linux + + + + linux-x86_64 + so + + - org.apache.maven.plugins - maven-enforcer-plugin + org.codehaus.mojo + exec-maven-plugin + 1.6.0 - enforce-versions + init-build-flavor + initialize - enforce + exec - - - - *:*_2.11 - *:*_2.10 - - - + bash + -c 'mkdir -p ${project.build.directory}; if [[ $(ldd ${MXNET_DIR}/lib/libmxnet.so | grep libcuda | wc -l) == "0" ]]; then echo flavor=cpu > ${project.build.directory}/flavor.properties; else echo flavor=gpu > ${project.build.directory}/flavor.properties; fi' + + + + + + org.codehaus.mojo + properties-maven-plugin + 1.0.0 + + + read-properties + initialize + + read-project-properties + + + + ${project.build.directory}/flavor.properties + @@ -163,24 +175,58 @@ + + staging + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + + + nightly + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + - org.apache.maven.plugins - maven-release-plugin - - true - false - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - + org.commonjava.maven.plugins + directory-maven-plugin + 0.1 + + + directories + + directory-of + + initialize + + rootdir + + org.apache.mxnet + mxnet-parent + + + + org.apache.maven.plugins @@ -223,14 +269,13 @@ org.apache.maven.plugins maven-assembly-plugin - 2.5.5 + 3.1.0 org.apache.maven.plugins maven-surefire-plugin 2.19 - true false @@ -245,7 +290,6 @@ scalatest-maven-plugin 1.0 - ${skipTests} ${project.build.directory}/surefire-reports . F @@ -270,7 +314,7 @@ org.scalastyle scalastyle-maven-plugin - 0.8.0 + 1.0.0 false true @@ -278,7 +322,7 @@ false ${basedir}/src/main/scala ${basedir}/src/test/scala - scalastyle-config.xml + ${rootdir}/scalastyle-config.xml ${basedir}/target/scalastyle-output.xml UTF-8 @@ -329,19 +373,18 @@ + + + org.apache.maven.plugins + maven-deploy-plugin + + false + deployrepo::default::${repo_url} + + - - org.scala-lang - scala-library - ${scala.version} - - - org.scala-lang - scala-reflect - ${scala.version} - commons-codec commons-codec @@ -366,7 +409,7 @@ org.scalatest - scalatest_${scala.binary.version} + scalatest_2.11 3.0.4 test @@ -377,13 +420,25 @@ org.scalacheck - scalacheck_${scala.binary.version} + scalacheck_2.11 1.13.5 test + + + + org.scala-lang + scala-library + ${scala.version} + + + org.scala-lang + scala-reflect + ${scala.version} + org.scala-lang.modules - scala-parser-combinators_${scala.binary.version} + scala-parser-combinators_2.11 1.0.4 diff --git a/scala-package/spark/bin/run-mnist-example.sh b/scala-package/spark/bin/run-mnist-example.sh index 4ebd6c61d56b..4f747f2c91a1 100755 --- a/scala-package/spark/bin/run-mnist-example.sh +++ b/scala-package/spark/bin/run-mnist-example.sh @@ -27,9 +27,9 @@ OS="" if [ "$(uname)" == "Darwin" ]; then # Do something under Mac OS X platform - OS='osx-x86_64-cpu' + OS='osx-x86_64' elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then - OS='linux-x86_64-cpu' + OS='linux-x86_64' fi LIB_DIR=${SPARK_MODULE_DIR}/target/classes/lib diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index 8c0ca087855c..2acb70b43303 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -21,47 +21,28 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-spark_2.11 + mxnet-spark MXNet Scala Package - Spark ML 1.6.3 - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - + org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-core + INTERNAL provided org.apache.spark - spark-mllib_${scala.binary.version} + spark-mllib_2.11 ${spark.version} diff --git a/src/initialize.cc b/src/initialize.cc index de7edd1b1455..8d0e3c304216 100644 --- a/src/initialize.cc +++ b/src/initialize.cc @@ -44,7 +44,11 @@ class LibraryInitializer { LibraryInitializer() { dmlc::InitLogging("mxnet"); #if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE - signal(SIGSEGV, SegfaultLogger); + struct sigaction sa; + sigaction(SIGSEGV, NULL, &sa); + if (sa.sa_handler == NULL) { + signal(SIGSEGV, SegfaultLogger); + } #endif // disable openmp for multithreaded workers diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md new file mode 100644 index 000000000000..c30e85d519c3 --- /dev/null +++ b/tools/dependencies/README.md @@ -0,0 +1,25 @@ +# Overview + +This folder contains scripts for building the dependencies from source. The static libraries from +the build artifacts can be used to create self-contained shared object for mxnet through static +linking. + +# Settings + +The scripts use the following environment variables for setting behavior: + +`DEPS_PATH`: the location in which the libraries are downloaded, built, and installed. +`PLATFORM`: name of the OS in lower case. Supported options are 'linux' and 'darwin'. + +It also expects the following build tools in path: make, cmake, tar, unzip, autoconf, nasm + +# FAQ + +## Build failure regarding to gcc, g++, gfortran +Currently, we only support gcc-4.8 build. It's your own choice to use a higher version of gcc. Please make sure your gcc, g++ and gfortran always have the same version in order to eliminate build failure. + +## idn2 not found +This issue appeared in the OSX build with XCode version 8.0 above (reproduced on 9.2). Please add the following build flag in `curl.sh` if your XCode version is more than 8.0: +``` +--without-libidn2 +``` \ No newline at end of file diff --git a/tools/dependencies/cityhash.sh b/tools/dependencies/cityhash.sh new file mode 100755 index 000000000000..6bc663e906fa --- /dev/null +++ b/tools/dependencies/cityhash.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of cityhash that can be used as dependency of mxnet. +set -ex +CITYHASH_VERSION=1.1.1 +if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then + # Download and build cityhash + >&2 echo "Building cityhash..." + git clone https://github.com/google/cityhash $DEPS_PATH/cityhash-$CITYHASH_VERSION + pushd . + cd $DEPS_PATH/cityhash-$CITYHASH_VERSION + git reset --hard 8af9b8c2b889d80c22d6bc26ba0df1afb79a30db + ./configure -prefix=$DEPS_PATH --enable-sse4.2 + $MAKE CXXFLAGS="-g -O3 -msse4.2" + $MAKE install + popd +fi diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh new file mode 100755 index 000000000000..bb947715f2ac --- /dev/null +++ b/tools/dependencies/curl.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libcurl that can be used as dependency of mxnet. +set -ex +LIBCURL_VERSION=7.61.0 +if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then + # download and build libcurl + >&2 echo "Building libcurl..." + curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o $DEPS_PATH/libcurl.zip + unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH + pushd . + cd $DEPS_PATH/curl-$LIBCURL_VERSION + if [[ $PLATFORM == 'linux' ]]; then + CONFIG_FLAG="" + elif [[ $PLATFORM == 'darwin' ]]; then + CONFIG_FLAG="--with-darwinssl" + fi + ./configure $CONFIG_FLAG \ + --with-zlib \ + --with-nghttps2 \ + --without-zsh-functions-dir \ + --without-librtmp \ + --without-libssh2 \ + --disable-debug \ + --disable-curldebug \ + --enable-symbol-hiding=yes \ + --enable-optimize=yes \ + --enable-shared=no \ + --enable-http=yes \ + --enable-ipv6=yes \ + --disable-ftp \ + --disable-ldap \ + --disable-ldaps \ + --disable-rtsp \ + --disable-proxy \ + --disable-dict \ + --disable-telnet \ + --disable-tftp \ + --disable-pop3 \ + --disable-imap \ + --disable-smb \ + --disable-smtp \ + --disable-gopher \ + --disable-manual \ + --prefix=$DEPS_PATH + $MAKE + $MAKE install + popd +fi diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh new file mode 100755 index 000000000000..a0cd8fcc95ef --- /dev/null +++ b/tools/dependencies/eigen.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script imports the headers from eigen3 that can be used to in opencv. +set -ex +EIGEN_VERSION=3.3.4 +if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then + # download eigen + >&2 echo "Loading eigen..." + curl -s -L https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o $DEPS_PATH/eigen.zip + unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build + pushd . + cd $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build + cmake \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. + $MAKE install + popd +fi diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh new file mode 100755 index 000000000000..3faa9f027dcb --- /dev/null +++ b/tools/dependencies/libpng.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libpng that can be used as dependency of mxnet/opencv. +set -ex +PNG_VERSION=1.6.34 +if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then + # download and build libpng + >&2 echo "Building libpng..." + curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip + unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build + pushd . + cd $DEPS_PATH/libpng-$PNG_VERSION/build + cmake \ + -D PNG_SHARED=OFF \ + -D PNG_STATIC=ON \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ + -D CMAKE_C_FLAGS=-fPIC .. + $MAKE + $MAKE install + mkdir -p $DEPS_PATH/include/libpng + ln -s $DEPS_PATH/include/png.h $DEPS_PATH/include/libpng/png.h + popd +fi diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh new file mode 100755 index 000000000000..2a402bbcac27 --- /dev/null +++ b/tools/dependencies/libtiff.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libtiff that can be used as dependency of mxnet/opencv. +set -ex +TIFF_VERSION="4-0-9" +if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then + # download and build libtiff + >&2 echo "Building libtiff..." + curl -s -L https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip + unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH + pushd . + cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION + ./configure --quiet --disable-shared --disable-jpeg --disable-zlib --disable-jbig --disable-lzma --prefix=$DEPS_PATH + $MAKE + $MAKE install + popd +fi diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh new file mode 100755 index 000000000000..ac813ebec1c7 --- /dev/null +++ b/tools/dependencies/libturbojpeg.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libturbojpeg that can be used as dependency of +# mxnet/opencv. +set -ex +TURBO_JPEG_VERSION=1.5.90 +if [[ $PLATFORM == 'darwin' ]]; then + JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm" +fi + +if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then + # download and build libjpeg + >&2 echo "Building libjpeg-turbo..." + curl -s -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip + unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build + pushd . + cd $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build + cmake \ + -G"Unix Makefiles" \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ + -D CMAKE_C_FLAGS=-fPIC \ + -D WITH_JAVA=FALSE \ + -D WITH_JPEG7=TRUE \ + -D WITH_JPEG8=TRUE \ + $JPEG_NASM_OPTION \ + -D ENABLE_SHARED=FALSE .. + $MAKE + $MAKE install + popd +fi diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh new file mode 100755 index 000000000000..c5f9953c8d1c --- /dev/null +++ b/tools/dependencies/libz.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libz that can be used as dependency of mxnet. +set -ex +ZLIB_VERSION=1.2.6 +if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then + # Download and build zlib + >&2 echo "Building zlib..." + curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip + unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build + pushd . + cd $DEPS_PATH/zlib-$ZLIB_VERSION/build + cmake \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ + -D BUILD_SHARED_LIBS=OFF .. + $MAKE + $MAKE install + popd +fi diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh new file mode 100755 index 000000000000..478c9925e967 --- /dev/null +++ b/tools/dependencies/lz4.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of lz4 that can be used as dependency of mxnet. +set -ex +LZ4_VERSION=r130 +if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then + # Download and build lz4 + >&2 echo "Building lz4..." + curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o $DEPS_PATH/lz4.zip + unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH + pushd . + cd $DEPS_PATH/lz4-$LZ4_VERSION + $MAKE + $MAKE PREFIX=$DEPS_PATH install + popd +fi diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh new file mode 100755 index 000000000000..16508586169a --- /dev/null +++ b/tools/dependencies/make_shared_dependencies.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This is a convenience script for calling the build scripts of all dependency libraries. +# Environment variables should be set beforehand. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + + +if [[ ! $PLATFORM == 'darwin' ]]; then + source $DIR/openblas.sh +fi +source $DIR/libz.sh +source $DIR/libturbojpeg.sh +source $DIR/libpng.sh +source $DIR/libtiff.sh +source $DIR/openssl.sh +source $DIR/curl.sh +source $DIR/eigen.sh +source $DIR/opencv.sh +source $DIR/protobuf.sh +source $DIR/cityhash.sh +source $DIR/zmq.sh +source $DIR/lz4.sh + +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh new file mode 100755 index 000000000000..27f473919a5b --- /dev/null +++ b/tools/dependencies/openblas.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of openblas that can be used as dependency of mxnet. +set +e # This script throws an error but otherwise works +set -x +OPENBLAS_VERSION=0.3.3 +if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then + # download and build openblas + >&2 echo "Building openblas..." + + curl -s -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip + unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH + pushd . + cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION + + $MAKE DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1 + $MAKE PREFIX=$DEPS_PATH install + popd + ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a + ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a +fi diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh new file mode 100755 index 000000000000..11c9c2155f91 --- /dev/null +++ b/tools/dependencies/opencv.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of opencv that can be used as dependency of mxnet. +# It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in $DEPS_PATH. +set -ex +OPENCV_VERSION=3.4.2 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +if [[ $PLATFORM == 'linux' ]]; then + OPENCV_LAPACK_OPTIONS=" \ + -D OpenBLAS_HOME=$DEPS_PATH \ + -D OpenBLAS_INCLUDE_DIR=$DEPS_PATH/include \ + -D OpenBLAS_LIB=$DEPS_PATH/lib/libopenblas.a \ + -D LAPACK_INCLUDE_DIR=$DEPS_PATH/include \ + -D LAPACK_LINK_LIBRARIES=$DEPS_PATH/lib/ \ + -D LAPACK_LIBRARIES=$DEPS_PATH/lib/libopenblas.a \ + -D LAPACK_CBLAS_H='cblas.h' \ + -D LAPACK_LAPACKE_H='lapacke.h' \ + -D LAPACK_IMPL='OpenBLAS' \ + -D HAVE_LAPACK=1" +fi + +if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgcodecs.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgproc.a ]]; then + # download and build opencv since we need the static library + >&2 echo "Building opencv..." + curl -s -L https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip + unzip -q $DEPS_PATH/opencv.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build + pushd . + cd $DEPS_PATH/opencv-$OPENCV_VERSION/build + cmake \ + -D OPENCV_ENABLE_NONFREE=OFF \ + -D WITH_1394=OFF \ + -D WITH_ARAVIS=OFF \ + -D WITH_AVFOUNDATION=OFF \ + -D WITH_CAROTENE=OFF \ + -D WITH_CLP=OFF \ + -D WITH_CSTRIPES=OFF \ + -D WITH_CPUFEATURES=OFF \ + -D WITH_CUBLAS=OFF \ + -D WITH_CUDA=OFF \ + -D WITH_CUFFT=OFF \ + -D WITH_DIRECTX=OFF \ + -D WITH_DSHOW=OFF \ + -D WITH_EIGEN=ON \ + -D WITH_FFMPEG=OFF \ + -D WITH_GDAL=OFF \ + -D WITH_GDCM=OFF \ + -D WITH_GIGEAPI=OFF \ + -D WITH_GPHOTO2=OFF \ + -D WITH_GSTREAMER=OFF \ + -D WITH_GSTREAMER_0_10=OFF \ + -D WITH_GTK=OFF \ + -D WITH_GTK_2_X=OFF \ + -D WITH_HALIDE=OFF \ + -D WITH_IMAGEIO=OFF \ + -D WITH_IMGCODEC_HDR=OFF \ + -D WITH_IMGCODEC_PXM=OFF \ + -D WITH_IMGCODEC_SUNRASTER=OFF \ + -D WITH_INF_ENGINE=OFF \ + -D WITH_INTELPERC=OFF \ + -D WITH_IPP=OFF \ + -D WITH_IPP_A=OFF \ + -D WITH_ITT=OFF \ + -D WITH_JASPER=OFF \ + -D WITH_JPEG=ON \ + -D WITH_LAPACK=ON \ + -D WITH_LIBREALSENSE=OFF \ + -D WITH_LIBV4L=OFF \ + -D WITH_MATLAB=OFF \ + -D WITH_MFX=OFF \ + -D WITH_MSMF=OFF \ + -D WITH_NVCUVID=OFF \ + -D WITH_OPENCL=OFF \ + -D WITH_OPENCLAMDBLAS=OFF \ + -D WITH_OPENCLAMDFFT=OFF \ + -D WITH_OPENCL_SVM=OFF \ + -D WITH_OPENEXR=OFF \ + -D WITH_OPENGL=OFF \ + -D WITH_OPENMP=OFF \ + -D WITH_OPENNI=OFF \ + -D WITH_OPENNI2=OFF \ + -D WITH_OPENVX=OFF \ + -D WITH_PNG=ON \ + -D WITH_PROTOBUF=OFF \ + -D WITH_PTHREADS_PF=ON \ + -D WITH_PVAPI=OFF \ + -D WITH_QT=OFF \ + -D WITH_QTKIT=OFF \ + -D WITH_QUICKTIME=OFF \ + -D WITH_TBB=OFF \ + -D WITH_TIFF=ON \ + -D WITH_UNICAP=OFF \ + -D WITH_V4L=OFF \ + -D WITH_VA=OFF \ + -D WITH_VA_INTEL=OFF \ + -D WITH_VFW=OFF \ + -D WITH_VTK=OFF \ + -D WITH_WEBP=OFF \ + -D WITH_WIN32UI=OFF \ + -D WITH_XIMEA=OFF \ + -D WITH_XINE=OFF \ + -D BUILD_ANDROID_EXAMPLES=OFF \ + -D BUILD_ANDROID_PROJECTS=OFF \ + -D BUILD_ANDROID_SERVICE=OFF \ + -D BUILD_CUDA_STUBS=OFF \ + -D BUILD_DOCS=OFF \ + -D BUILD_EXAMPLES=OFF \ + -D BUILD_FAT_JAVA_LIB=OFF \ + -D BUILD_IPP_IW=OFF \ + -D BUILD_ITT_IW=OFF \ + -D BUILD_JAVA=OFF \ + -D BUILD_JASPER=OFF \ + -D BUILD_JPEG=OFF \ + -D BUILD_OPENEXR=OFF \ + -D BUILD_PACKAGE=OFF \ + -D BUILD_PERF_TESTS=OFF \ + -D BUILD_PNG=OFF \ + -D BUILD_SHARED_LIBS=OFF \ + -D BUILD_TBB=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_TIFF=OFF \ + -D BUILD_WEBP=OFF \ + -D BUILD_WITH_DEBUG_INFO=OFF \ + -D BUILD_WITH_DYNAMIC_IPP=OFF \ + -D BUILD_WITH_STATIC_CRT=OFF \ + -D BUILD_ZLIB=OFF \ + -D BUILD_opencv_apps=OFF \ + -D BUILD_opencv_aruco=OFF \ + -D BUILD_opencv_calib3d=OFF \ + -D BUILD_opencv_contrib=OFF \ + -D BUILD_opencv_dnn=OFF \ + -D BUILD_opencv_features2d=OFF \ + -D BUILD_opencv_flann=OFF \ + -D BUILD_opencv_gpu=OFF \ + -D BUILD_opencv_gpuarithm=OFF \ + -D BUILD_opencv_gpubgsegm=OFF \ + -D BUILD_opencv_gpucodec=OFF \ + -D BUILD_opencv_gpufeatures2d=OFF \ + -D BUILD_opencv_gpufilters=OFF \ + -D BUILD_opencv_gpuimgproc=OFF \ + -D BUILD_opencv_gpulegacy=OFF \ + -D BUILD_opencv_gpuoptflow=OFF \ + -D BUILD_opencv_gpustereo=OFF \ + -D BUILD_opencv_gpuwarping=OFF \ + -D BUILD_opencv_highgui=OFF \ + -D BUILD_opencv_java=OFF \ + -D BUILD_opencv_js=OFF \ + -D BUILD_opencv_ml=OFF \ + -D BUILD_opencv_ml=OFF \ + -D BUILD_opencv_nonfree=OFF \ + -D BUILD_opencv_objdetect=OFF \ + -D BUILD_opencv_photo=OFF \ + -D BUILD_opencv_python=OFF \ + -D BUILD_opencv_python2=OFF \ + -D BUILD_opencv_python3=OFF \ + -D BUILD_opencv_superres=OFF \ + -D BUILD_opencv_video=OFF \ + -D BUILD_opencv_videoio=OFF \ + -D BUILD_opencv_videostab=OFF \ + -D BUILD_opencv_viz=OFF \ + -D BUILD_opencv_world=OFF \ + $OPENCV_LAPACK_OPTIONS \ + -D OPENCV_LIB_INSTALL_PATH=lib \ + -D OPENCV_INCLUDE_INSTALL_PATH=include \ + -D CMAKE_LIBRARY_PATH=$DEPS_PATH/lib \ + -D CMAKE_INCLUDE_PATH=$DEPS_PATH/include \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. + if [[ $PLATFORM == 'linux' ]]; then + cp $DIR/patch/opencv_lapack.h ./ + fi + $MAKE + $MAKE install + popd + # @szha: compatibility header + cat $DEPS_PATH/include/opencv2/imgcodecs/imgcodecs_c.h >> $DEPS_PATH/include/opencv2/imgcodecs.hpp +fi diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh new file mode 100755 index 000000000000..93284db3e39f --- /dev/null +++ b/tools/dependencies/openssl.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of openssl that can be used as dependency of mxnet. +set -ex +OPENSSL_VERSION=1.0.2l +if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; then + # download and build openssl + >&2 echo "Building openssl..." + OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g') + curl -s -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.zip -o $DEPS_PATH/openssl.zip + unzip -q $DEPS_PATH/openssl.zip -d $DEPS_PATH + pushd . + cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION + if [[ $PLATFORM == 'linux' ]]; then + TARGET=linux-x86_64 + elif [[ $PLATFORM == 'darwin' ]]; then + TARGET=darwin64-x86_64-cc + fi + ./Configure no-shared no-zlib --prefix=$DEPS_PATH --openssldir=$DEPS_PATH/ssl $TARGET + $MAKE + $MAKE install + popd +fi diff --git a/tools/dependencies/patch/opencv_lapack.h b/tools/dependencies/patch/opencv_lapack.h new file mode 100644 index 000000000000..97af9d67ea31 --- /dev/null +++ b/tools/dependencies/patch/opencv_lapack.h @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +extern "C" { +#include "cblas.h" +#include "lapacke.h" +} diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh new file mode 100755 index 000000000000..76ce1de8e822 --- /dev/null +++ b/tools/dependencies/protobuf.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of protobuf along with protoc, that can be used as dependency of mxnet. +set -ex +PROTOBUF_VERSION=3.5.1 +if [[ $PLATFORM == 'darwin' ]]; then + DY_EXT="dylib" +else + DY_EXT="so" +fi + +LIBPROTOBUF="$DEPS_PATH/lib/libprotobuf.$DY_EXT" +LIBPROTOC="$DEPS_PATH/lib/libprotoc.$DY_EXT" +if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then + # Download and build protobuf + >&2 echo "Building protobuf..." + curl -s -L https://github.com/google/protobuf/archive/v$PROTOBUF_VERSION.zip -o $DEPS_PATH/protobuf.zip + unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH + pushd . + cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION + ./autogen.sh + ./configure -prefix=$DEPS_PATH + $MAKE + $MAKE install + popd +fi + diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh new file mode 100755 index 000000000000..0042d6bcd073 --- /dev/null +++ b/tools/dependencies/zmq.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of zeroMQ that can be used as dependency of mxnet. +set -ex +ZEROMQ_VERSION=4.2.2 +if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then + # Download and build zmq + >&2 echo "Building zmq..." + curl -s -L https://github.com/zeromq/libzmq/archive/v$ZEROMQ_VERSION.zip -o $DEPS_PATH/zeromq.zip + unzip -q $DEPS_PATH/zeromq.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build + pushd . + cd $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build + cmake \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ + -D WITH_LIBSODIUM=OFF \ + -D BUILD_SHARED_LIBS=OFF .. + $MAKE + $MAKE install + popd +fi diff --git a/tools/license_header.py b/tools/license_header.py index 32a093e66a20..410d5c16cbf5 100755 --- a/tools/license_header.py +++ b/tools/license_header.py @@ -60,31 +60,26 @@ _LICENSE_PATTERNS = ['Licensed to the Apache Software Foundation'] # the folders or files that will be ignored -_WHITE_LIST = [ - # Licensed under docker/Dockerfiles/License.md +_WHITE_LIST = ['3rdparty', 'docker/Dockerfiles', - - # Git submodules under different licenses - '3rdparty', - - # Code shared with project by author - see file for details - 'src/operator/special_functions-inl.h', - - # Licensed under Caffe header - 'src/operator/nn/pool.h', - 'src/operator/contrib/psroi_pooling-inl.h', - 'src/operator/contrib/nn/deformable_im2col.h', + 'example/image-classification/predict-cpp/image-classification-predict.cc', + 'example/rcnn/rcnn/cython/nms_kernel.cu', + 'example/rcnn/rcnn/cython/setup.py', + 'example/ssd/dataset/pycocotools/coco.py', + 'julia/REQUIRE', + 'prepare_mkl.sh', + 'R-package/', + 'scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h', + 'scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h', + 'src/operator/contrib/ctc_include/', 'src/operator/contrib/nn/deformable_im2col.cuh', - 'src/operator/nn/im2col.h', + 'src/operator/contrib/nn/deformable_im2col.h', + 'src/operator/contrib/psroi_pooling-inl.h', + 'src/operator/mkl/', 'src/operator/nn/im2col.cuh', - - # Licenses in headers - 'docs/_static/searchtools_custom.js', - 'docs/_static/js/clipboard.js', - 'docs/_static/js/clipboard.min.js', - - # Licensed under 2-Clause BSD in header - 'example/ssd/dataset/pycocotools/coco.py', + 'src/operator/nn/im2col.h', + 'src/operator/nn/pool.h', + 'src/operator/special_functions-inl.h' ] # language extensions and the according commment mark diff --git a/tools/pip/MANIFEST.in b/tools/pip/MANIFEST.in new file mode 100644 index 000000000000..5e072064193c --- /dev/null +++ b/tools/pip/MANIFEST.in @@ -0,0 +1,11 @@ +include README +include mxnet/COMMIT_HASH +recursive-include mxnet/tools * +recursive-include mxnet *.py +recursive-include mxnet *.so +recursive-include mxnet *.so.* +recursive-include mxnet *.dylib +recursive-include mxnet *_LICENSE +recursive-include mxnet *.h +recursive-include mxnet *.cuh +recursive-include dmlc_tracker *.py diff --git a/tools/pip/sanity_test.py b/tools/pip/sanity_test.py new file mode 100644 index 000000000000..dc51e479906b --- /dev/null +++ b/tools/pip/sanity_test.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +"""Sanity test.""" +from __future__ import print_function +import sys +from base64 import b64decode + +try: + import mxnet as mx + mx.img.imdecode(b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==')).asnumpy() + print('Test succeeded') +except: + import traceback + print('Test failed') + traceback.print_exc() + sys.exit(1) diff --git a/tools/pip/setup.py b/tools/pip/setup.py new file mode 100644 index 000000000000..d5db6d87fc1d --- /dev/null +++ b/tools/pip/setup.py @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +# pylint: disable=invalid-name, exec-used +"""Setup mxnet package for pip.""" +from __future__ import absolute_import +from datetime import datetime +import os +import sys +import shutil +import platform + +if platform.system() == 'Linux': + sys.argv.append('--universal') + sys.argv.append('--plat-name=manylinux1_x86_64') + +from setuptools import setup, find_packages +from setuptools.dist import Distribution + +# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py +# Will be invoked which introduces dependences +CURRENT_DIR = os.path.dirname(__file__) +libinfo_py = os.path.join(CURRENT_DIR, 'mxnet-build/python/mxnet/libinfo.py') +libinfo = {'__file__': libinfo_py} +exec(compile(open(libinfo_py, "rb").read(), libinfo_py, 'exec'), libinfo, libinfo) + +LIB_PATH = libinfo['find_lib_path']() +__version__ = libinfo['__version__'] +if 'TRAVIS_TAG' not in os.environ or not os.environ['TRAVIS_TAG'].strip(): + __version__ += 'b{0}'.format(datetime.today().strftime('%Y%m%d')) +elif 'TRAVIS_TAG' in os.environ and os.environ['TRAVIS_TAG'].startswith('patch-'): + __version__ = os.environ['TRAVIS_TAG'].split('-')[1] + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return platform.system() == 'Darwin' + + +DEPENDENCIES = [ + 'numpy<1.15.0,>=1.8.2', + 'requests>=2.20.0', + 'graphviz<0.9.0,>=0.8.1' +] + +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True) +shutil.rmtree(os.path.join(CURRENT_DIR, 'dmlc_tracker'), ignore_errors=True) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/python/mxnet'), + os.path.join(CURRENT_DIR, 'mxnet')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dmlc-core/tracker/dmlc_tracker'), + os.path.join(CURRENT_DIR, 'dmlc_tracker')) +shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet')) + +# copy tools to mxnet package +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet/tools'), ignore_errors=True) +os.mkdir(os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/launch.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/im2rec.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/kill-mxnet.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/parse_log.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/diagnose.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/tools/caffe_converter'), os.path.join(CURRENT_DIR, 'mxnet/tools/caffe_converter')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/tools/bandwidth'), os.path.join(CURRENT_DIR, 'mxnet/tools/bandwidth')) + +# copy headers to mxnet package +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet/include'), ignore_errors=True) +os.mkdir(os.path.join(CURRENT_DIR, 'mxnet/include')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/include/mxnet'), + os.path.join(CURRENT_DIR, 'mxnet/include/mxnet')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dlpack/include/dlpack'), + os.path.join(CURRENT_DIR, 'mxnet/include/dlpack')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dmlc-core/include/dmlc'), + os.path.join(CURRENT_DIR, 'mxnet/include/dmlc')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/mshadow/mshadow'), + os.path.join(CURRENT_DIR, 'mxnet/include/mshadow')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/tvm/nnvm/include/nnvm'), + os.path.join(CURRENT_DIR, 'mxnet/include/nnvm')) + +package_name = 'mxnet' + +variant = os.environ['mxnet_variant'].upper() +if variant != 'CPU': + package_name = 'mxnet_{0}'.format(variant.lower()) + +with open('doc/PYPI_README.md') as readme_file: + long_description = readme_file.read() + +with open('doc/{0}_ADDITIONAL.md'.format(variant)) as variant_doc: + long_description = long_description + variant_doc.read() + +# pypi only supports rst, so use pandoc to convert +import pypandoc +if platform.system() == 'Darwin': + pypandoc.download_pandoc() +long_description = pypandoc.convert_text(long_description, 'rst', 'md') +short_description = 'MXNet is an ultra-scalable deep learning framework.' +libraries = [] +if variant == 'CPU': + libraries.append('openblas') +else: + if variant.startswith('CU92'): + libraries.append('CUDA-9.2') + elif variant.startswith('CU91'): + libraries.append('CUDA-9.1') + elif variant.startswith('CU90'): + libraries.append('CUDA-9.0') + elif variant.startswith('CU80'): + libraries.append('CUDA-8.0') + elif variant.startswith('CU75'): + libraries.append('CUDA-7.5') + if variant.endswith('MKL'): + libraries.append('MKLDNN') + +short_description += ' This version uses {0}.'.format(' and '.join(libraries)) + +package_data = {'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))], + 'dmlc_tracker': []} +if variant.endswith('MKL'): + if platform.system() == 'Darwin': + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmklml.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libiomp5.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmkldnn.0.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libmklml.dylib') + package_data['mxnet'].append('mxnet/libiomp5.dylib') + package_data['mxnet'].append('mxnet/libmkldnn.0.dylib') + else: + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmklml_intel.so'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libiomp5.so'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmkldnn.so.0'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libmklml_intel.so') + package_data['mxnet'].append('mxnet/libiomp5.so') + package_data['mxnet'].append('mxnet/libmkldnn.so.0') + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), '../MKLML_LICENSE'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/MKLML_LICENSE') +if platform.system() == 'Linux': + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libgfortran.so.3'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libgfortran.so.3') + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libquadmath.so.0'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libquadmath.so.0') + +from mxnet.base import _generate_op_module_signature +from mxnet.ndarray.register import _generate_ndarray_function_code +from mxnet.symbol.register import _generate_symbol_function_code +_generate_op_module_signature('mxnet', 'symbol', _generate_symbol_function_code) +_generate_op_module_signature('mxnet', 'ndarray', _generate_ndarray_function_code) + +setup(name=package_name, + version=__version__, + long_description=long_description, + description=short_description, + zip_safe=False, + packages=find_packages(), + package_data=package_data, + include_package_data=True, + install_requires=DEPENDENCIES, + distclass=BinaryDistribution, + license='Apache 2.0', + classifiers=[ # https://pypi.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: C++', + 'Programming Language :: Cython', + 'Programming Language :: Other', # R, Scala + 'Programming Language :: Perl', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + url='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/apache/incubator-mxnet') diff --git a/tools/pip_package/README.md b/tools/pip_package/README.md deleted file mode 100644 index 6d044167fcf8..000000000000 --- a/tools/pip_package/README.md +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - -MXNet Python Package -==================== -MXNet is a deep learning framework designed for both *efficiency* and *flexibility*. -It allows you to mix the flavours of deep learning programs together to maximize the efficiency and your productivity. - - -Installation ------------- -To install, check [Build Instruction](http://mxnet.io/get_started/setup.html) diff --git a/tools/pip_package/make_pip_package.sh b/tools/pip_package/make_pip_package.sh deleted file mode 100755 index 46b4938b0785..000000000000 --- a/tools/pip_package/make_pip_package.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# Assuming the script is run at mxnet/tools/pip_package -# This script builds from scratch the dependencies of mxnet into static -# librareis and statically links them to produce a (mostly) standalone -# libmxnet.so, then packages it into the python wheel. -# It assumes the build environment to be a sandbox that doesn't have the .so -# objects for the dependencies, i.e. zlib, openblas, libjpeg, libpng, libtiff -# and opencv. - -# Install necessary build tools -if [ -n "$(command -v apt-get)" ]; then - sudo apt-get update; - sudo apt-get install -y build-essential git python-pip zip pkg-config cmake -elif [ -n "$(command -v yum)" ]; then - sudo yum install -y cmake - sudo yum groupinstall -y "Development Tools" - sudo yum install -y python27 python27-setuptools python27-tools python-pip -else - echo "Need a package manager to install build tools, e.g. apt/yum" - exit 1 -fi -sudo pip install -U pip setuptools wheel - -# Set up path as temporary working directory -DEPS_PATH=$PWD/../../deps -mkdir $DEPS_PATH - -# Dependencies can be updated here. Be sure to verify the download link before -# changing. The dependencies are: -ZLIB_VERSION=1.2.6 -OPENBLAS_VERSION=0.2.19 -JPEG_VERSION=8.4.0 -PNG_VERSION=1.5.10 -TIFF_VERSION=3.8.2 -OPENCV_VERSION=2.4.13 - -# Setup path to dependencies -export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$PKG_CONFIG_PATH -export CPATH=$DEPS_PATH/include:$CPATH - -# Position Independent code must be turned on for statically linking .a -export CC="gcc -fPIC" -export CXX="g++ -fPIC" - -# Download and build zlib -curl -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip -unzip $DEPS_PATH/zlib.zip -d $DEPS_PATH -mkdir $DEPS_PATH/zlib-$ZLIB_VERSION/build -cd $DEPS_PATH/zlib-$ZLIB_VERSION/build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ - -D BUILD_SHARED_LIBS=OFF .. -make -j$(nproc) -make install -cd - - -# download and build openblas -curl -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip -unzip $DEPS_PATH/openblas.zip -d $DEPS_PATH -cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION -make FC=gfortran -j $(($(nproc) + 1)) -make PREFIX=$DEPS_PATH install -cd - -ln -s $DEPS_PATH/lib/libopenblas_haswellp-r0.2.19.a $DEPS_PATH/lib/libcblas.a - -# download and build libjpeg -curl -L https://github.com/LuaDist/libjpeg/archive/$JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip -unzip $DEPS_PATH/libjpeg.zip -d $DEPS_PATH -cd $DEPS_PATH/libjpeg-$JPEG_VERSION -./configure --disable-shared --prefix=$DEPS_PATH -make -j$(nproc) -make test -make install -cd - - -# download and build libpng -curl -L https://github.com/LuaDist/libpng/archive/$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip -unzip $DEPS_PATH/libpng.zip -d $DEPS_PATH -mkdir $DEPS_PATH/libpng-$PNG_VERSION/build -cd $DEPS_PATH/libpng-$PNG_VERSION/build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ - -D PNG_CONFIGURE_LIBPNG=-fPIC \ - -D BUILD_SHARED_LIBS=OFF .. -make -j$(nproc) -make install -cd - - -# download and build libtiff -curl -L https://github.com/LuaDist/libtiff/archive/$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip -unzip $DEPS_PATH/libtiff.zip -d $DEPS_PATH -cd $DEPS_PATH/libtiff-$TIFF_VERSION -./configure --disable-shared --prefix=$DEPS_PATH -make -j$(nproc) -make install -cd - - -# download and build opencv since we need the static library -curl -L https://github.com/Itseez/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip -unzip $DEPS_PATH/opencv.zip -d $DEPS_PATH -mkdir $DEPS_PATH/opencv-$OPENCV_VERSION/build -cd $DEPS_PATH/opencv-$OPENCV_VERSION/build -cmake -D WITH_1394=OFF \ - -D WITH_AVFOUNDATION=OFF \ - -D WITH_CUDA=OFF \ - -D WITH_VTK=OFF \ - -D WITH_CUFFT=OFF \ - -D WITH_CUBLAS=OFF \ - -D WITH_NVCUVID=OFF \ - -D WITH_EIGEN=ON \ - -D WITH_VFW=OFF \ - -D WITH_FFMPEG=OFF \ - -D WITH_GSTREAMER=OFF \ - -D WITH_GTK=OFF \ - -D WITH_JASPER=OFF \ - -D WITH_JPEG=ON \ - -D WITH_PNG=ON \ - -D WITH_QUICKTIME=OFF \ - -D WITH_TBB=ON \ - -D WITH_TIFF=OFF \ - -D WITH_V4L=OFF \ - -D WITH_LIBV4L=OFF \ - -D WITH_DSHOW=OFF \ - -D WITH_MSMF=OFF \ - -D WITH_OPENCL=OFF \ - -D WITH_OPENCLAMDFFT=OFF \ - -D WITH_OPENCLAMDBLAS=OFF \ - -D BUILD_SHARED_LIBS=OFF \ - -D BUILD_opencv_apps=OFF \ - -D BUILD_opencv_gpu=OFF \ - -D BUILD_opencv_video=OFF \ - -D BUILD_opencv_contrib=OFF \ - -D BUILD_opencv_nonfree=OFF \ - -D BUILD_opencv_flann=OFF \ - -D BUILD_opencv_features2d=OFF \ - -D BUILD_opencv_calib3d=OFF \ - -D BUILD_opencv_objdetect=OFF \ - -D BUILD_opencv_ml=OFF \ - -D BUILD_opencv_photo=OFF \ - -D BUILD_DOCS=OFF \ - -D BUILD_PACKAGE=OFF \ - -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. -make -j $(nproc) -make install # user will always have access to home, so no sudo needed -cd - - -# Although .so building is explicitly turned off for most libraries, sometimes -# they still get created. So, remove them just to make sure they don't -# interfere, or otherwise we might get libmxnet.so that is not self-contained. -rm $DEPS_PATH/{lib,lib64}/*.{so,so.0} - -# Go to the parent path and build mxnet -cd ../../ -cp make/pip_$(uname | tr '[:upper:]' '[:lower:]')_cpu.mk config.mk -make -j $(nproc) - -# Generate wheel. The output is in the mxnet/tools/pip_package/dist path. -cd tools/pip_package -python setup.py bdist_wheel diff --git a/tools/pip_package/setup.py b/tools/pip_package/setup.py deleted file mode 100644 index e4bf48236bde..000000000000 --- a/tools/pip_package/setup.py +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# pylint: disable=invalid-name, exec-used -"""Setup mxnet package.""" -from __future__ import absolute_import -import os -import shutil - -from setuptools import setup, find_packages -from setuptools.dist import Distribution - -# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py -# Will be invoked which introduces dependences -CURRENT_DIR = os.path.dirname(__file__) -libinfo_py = os.path.join(CURRENT_DIR, '../../python/mxnet/libinfo.py') -libinfo = {'__file__': libinfo_py} -exec(compile(open(libinfo_py, "rb").read(), libinfo_py, 'exec'), libinfo, libinfo) - -LIB_PATH = libinfo['find_lib_path']() -__version__ = libinfo['__version__'] - -class BinaryDistribution(Distribution): - def has_ext_modules(self): - return True - - -DEPENDENCIES = [ - 'numpy', -] - -shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True) -shutil.copytree(os.path.join(CURRENT_DIR, '../../python/mxnet'), - os.path.join(CURRENT_DIR, 'mxnet')) -shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet')) - -setup(name='mxnet', - version=__version__, - description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), - zip_safe=False, - packages=find_packages(), - package_data={'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))]}, - include_package_data=True, - install_requires=DEPENDENCIES, - distclass=BinaryDistribution, - url='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/dmlc/mxnet') diff --git a/tools/setup_gpu_build_tools.sh b/tools/setup_gpu_build_tools.sh new file mode 100755 index 000000000000..167d4c6a6e13 --- /dev/null +++ b/tools/setup_gpu_build_tools.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script installs the tools and libraries for CUDA GPU on Ubuntu. +# Usage: VARIANT=cu92mkl; DEPS_PATH=$HOME; setup_gpu_build_tools.sh $VARIANT $DEPS_PATH; +# It installs the tools into DEPS_PATH as specified by the second argument, and will set +# the following environment variables: +# PATH, CPLUS_INCLUDE_PATH, C_INCLUDE_PATH, LIBRARY_PATH, LD_LIBRARY_PATH, NVCC + +VARIANT=$1 +DEPS_PATH=$2 + +>&2 echo "Setting CUDA versions for $VARIANT" +if [[ $VARIANT == cu100* ]]; then + CUDA_VERSION='10.0.130-1' + CUDA_PATCH_VERSION='10.0.130-1' + LIBCUDA_VERSION='410.48-0ubuntu1' + LIBCUDNN_VERSION='7.3.1.20-1+cuda10.0' + LIBNCCL_VERSION='2.3.4-1+cuda9.2' +elif [[ $VARIANT == cu92* ]]; then + CUDA_VERSION='9.2.148-1' + CUDA_PATCH_VERSION='9.2.148.1-1' + LIBCUDA_VERSION='396.44-0ubuntu1' + LIBCUDNN_VERSION='7.3.1.20-1+cuda9.2' + LIBNCCL_VERSION='2.3.4-1+cuda9.2' +elif [[ $VARIANT == cu91* ]]; then + CUDA_VERSION='9.1.85-1' + CUDA_PATCH_VERSION='9.1.85.3-1' + LIBCUDA_VERSION='396.44-0ubuntu1' + LIBCUDNN_VERSION='7.1.3.16-1+cuda9.1' + LIBNCCL_VERSION='2.2.12-1+cuda9.1' +elif [[ $VARIANT == cu90* ]]; then + CUDA_VERSION='9.0.176-1' + CUDA_PATCH_VERSION='9.0.176.3-1' + LIBCUDA_VERSION='384.145-0ubuntu1' + LIBCUDNN_VERSION='7.3.1.20-1+cuda9.0' + LIBNCCL_VERSION='2.3.4-1+cuda9.0' +elif [[ $VARIANT == cu80* ]]; then + CUDA_VERSION='8.0.61-1' + CUDA_PATCH_VERSION='8.0.61.2-1' + LIBCUDA_VERSION='375.88-0ubuntu1' + LIBCUDNN_VERSION='7.2.1.38-1+cuda8.0' + LIBNCCL_VERSION='2.3.4-1+cuda8.0' +elif [[ $VARIANT == cu75* ]]; then + CUDA_VERSION='7.5-18' + CUDA_PATCH_VERSION='7.5-18' + LIBCUDA_VERSION='375.88-0ubuntu1' + LIBCUDNN_VERSION='6.0.21-1+cuda7.5' + LIBNCCL_VERSION='' +fi +if [[ $VARIANT == cu* ]]; then + CUDA_MAJOR_VERSION=$(echo $CUDA_VERSION | tr '-' '.' | cut -d. -f1,2) + CUDA_MAJOR_DASH=$(echo $CUDA_VERSION | tr '-' '.' | cut -d. -f1,2 | tr '.' '-') + NVIDIA_MAJOR_VERSION=$(echo $LIBCUDA_VERSION | cut -d. -f1) + LIBCUDA_MAJOR=$(echo $LIBCUDA_VERSION | cut -d. -f1) + LIBCUDNN_MAJOR=$(echo $LIBCUDNN_VERSION | cut -d. -f1) + os_name=$(cat /etc/*release | grep '^ID=' | sed 's/^.*=//g') + os_version=$(cat /etc/*release | grep VERSION_ID | sed 's/^.*"\([0-9]*\)\.\([0-9]*\)"/\1\2/g') + os_id="${os_name}${os_version}" + if [[ $CUDA_MAJOR_DASH == 9-* ]] || [[ $CUDA_MAJOR_DASH == 10-* ]]; then + os_id="ubuntu1604" + fi + export PATH=/usr/lib/binutils-2.26/bin/:${PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/bin + export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/include + export C_INCLUDE_PATH=${C_INCLUDE_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/include + export LIBRARY_PATH=${LIBRARY_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/lib64:$DEPS_PATH/usr/lib/x86_64-linux-gnu:$DEPS_PATH/usr/lib/nvidia-$NVIDIA_MAJOR_VERSION + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/lib64:$DEPS_PATH/usr/lib/x86_64-linux-gnu:$DEPS_PATH/usr/lib/nvidia-$NVIDIA_MAJOR_VERSION + export NVCC=$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/bin/nvcc +fi + +# list of debs to download from nvidia +if [[ $VARIANT == cu100* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu92* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu91* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvcc-${CUDA_MAJOR_DASH}_9.1.85.2-1_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu90* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_9.0.176.4-1_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_9.0.176.4-1_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu80* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu75* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + ) +fi + + +if [[ ! -d $DEPS_PATH/usr/local/cuda-${CUDA_MAJOR_VERSION} ]]; then + prefix=$DEPS_PATH + + for item in ${cuda_files[*]} + do + echo "Installing $item" + curl -sL "http://developer.download.nvidia.com/compute/cuda/repos/${os_id}/x86_64/${item}" -o package.deb + dpkg -X package.deb ${prefix} + rm package.deb + done + for item in ${ml_files[*]} + do + echo "Installing $item" + curl -sL "http://developer.download.nvidia.com/compute/machine-learning/repos/${os_id}/x86_64/${item}" -o package.deb + dpkg -X package.deb ${prefix} + rm package.deb + done + + mkdir -p ${prefix}/include + mkdir -p ${prefix}/lib + cp ${prefix}/usr/include/x86_64-linux-gnu/cudnn_v${LIBCUDNN_MAJOR}.h ${prefix}/include/cudnn.h + ln -s libcudnn_static_v${LIBCUDNN_MAJOR}.a ${prefix}/usr/lib/x86_64-linux-gnu/libcudnn.a + cp ${prefix}/usr/local/cuda-${CUDA_MAJOR_VERSION}/lib64/*.a ${prefix}/lib/ + cp ${prefix}/usr/include/nccl.h ${prefix}/include/nccl.h + ln -s libnccl_static.a ${prefix}/usr/lib/x86_64-linux-gnu/libnccl.a +fi diff --git a/tools/staticbuild/README.md b/tools/staticbuild/README.md new file mode 100644 index 000000000000..7cfeb7855a55 --- /dev/null +++ b/tools/staticbuild/README.md @@ -0,0 +1,32 @@ +# MXNet Static build + +This folder contains the core script used to build the static library. This README would bring you the information and usages of the script in here. Please be aware, all of the scripts are designed to be run under the root folder. + +## `build.sh` +This script is a wrapper around `build_lib.sh` aimed to simplify the usage of it. It would automatically identify the system version, number of cores and all environment variable settings. Here are the examples you can run this script: + +``` +tools/staticbuild/build.sh cu92 maven +``` +This would build the mxnet package based on CUDA9.2 and Maven (Scala) build setttings. +``` +tools/staticbuild/build.sh mkl pip +``` +This would build the mxnet package based on MKLDNN and and pypi configuration settings. + +As the result, users would have a complete static dependencies in `/staticdeps` in the root folder as well as a static-linked `libmxnet.so` file lives in `lib`. You can build your language binding by using the `libmxnet.so`. + +## `build_lib.sh` +This script would clone the most up-to-date master and build the MXNet backend with static library. In order to run that, you should have prepare the the following environment variable: + +- `DEPS_PATH` Path to your static dependencies +- `STATIC_BUILD_TARGET` Either `pip` or `maven` as your publish platform +- `PLATFORM` linux, darwin +- `VARIANT` cpu, cu*, cu*mkl, mkl + +It is not recommended to run this file alone since there are a bunch of variables need to be set. + +After running this script, you would have everything you need ready in the `/lib` folder. + +## `build_wheel.sh` +This script is used to build the python package as well as running a sanity test \ No newline at end of file diff --git a/tools/staticbuild/build.sh b/tools/staticbuild/build.sh new file mode 100755 index 000000000000..4f8a78d70490 --- /dev/null +++ b/tools/staticbuild/build.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ $# -lt 1 ]; then + >&2 echo "Usage: build.sh " +fi + +export CURDIR=$PWD +export DEPS_PATH=$PWD/staticdeps +export VARIANT=$(echo $1 | tr '[:upper:]' '[:lower:]') +export STATIC_BUILD_TARGET=$(echo $2 | tr '[:upper:]' '[:lower:]') +export PLATFORM=$(uname | tr '[:upper:]' '[:lower:]') + +if [[ $VARIANT == darwin* ]]; then + export VARIANT="darwin" +fi + +NUM_PROC=1 +if [[ ! -z $(command -v nproc) ]]; then + NUM_PROC=$(nproc) +elif [[ ! -z $(command -v sysctl) ]]; then + NUM_PROC=$(sysctl -n hw.ncpu) +else + >&2 echo "Can't discover number of cores." +fi +export NUM_PROC +>&2 echo "Using $NUM_PROC parallel jobs in building." + +if [[ $DEBUG -eq 1 ]]; then + export ADD_MAKE_FLAG="-j $NUM_PROC" +else + export ADD_MAKE_FLAG="--quiet -j $NUM_PROC" +fi +export MAKE="make $ADD_MAKE_FLAG" + +export CC="gcc -fPIC" +export CXX="g++ -fPIC" +export FC="gfortran" +export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$DEPS_PATH/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH +export CPATH=$DEPS_PATH/include:$CPATH + +if [[ $PLATFORM == 'linux' && $VARIANT == cu* ]]; then + source tools/setup_gpu_build_tools.sh $VARIANT $DEPS_PATH +fi + +mkdir -p $DEPS_PATH + +# Build Dependencies +source tools/dependencies/make_shared_dependencies.sh + +# Build mxnet +source tools/staticbuild/build_lib.sh diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh new file mode 100755 index 000000000000..b08057321bf7 --- /dev/null +++ b/tools/staticbuild/build_lib.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the libraries of mxnet. +make_config=make/${STATIC_BUILD_TARGET}/${STATIC_BUILD_TARGET}_${PLATFORM}_${VARIANT}.mk +if [[ ! -f $make_config ]]; then + >&2 echo "Couldn't find make config $make_config for the current settings." + exit 1 +fi + +>&2 echo "Now building mxnet modules..." +cp $make_config config.mk + +git submodule update --init --recursive || true + +$MAKE DEPS_PATH=$DEPS_PATH DMLCCORE +$MAKE DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a +$MAKE DEPS_PATH=$DEPS_PATH PSLITE + +if [[ $VARIANT == *mkl ]]; then + MKLDNN_LICENSE='license.txt' + if [[ $PLATFORM == 'linux' ]]; then + IOMP_LIBFILE='libiomp5.so' + MKLML_LIBFILE='libmklml_intel.so' + MKLDNN_LIBFILE='libmkldnn.so.0' + else + IOMP_LIBFILE='libiomp5.dylib' + MKLML_LIBFILE='libmklml.dylib' + MKLDNN_LIBFILE='libmkldnn.0.dylib' + fi + $MAKE DEPS_PATH=$DEPS_PATH mkldnn + cp 3rdparty/mkldnn/build/install/lib/$IOMP_LIBFILE lib + cp 3rdparty/mkldnn/build/install/lib/$MKLML_LIBFILE lib + cp 3rdparty/mkldnn/build/install/lib/$MKLDNN_LIBFILE lib + cp 3rdparty/mkldnn/build/install/$MKLDNN_LICENSE lib + cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE +fi + +>&2 echo "Now building mxnet..." +$MAKE DEPS_PATH=$DEPS_PATH + +if [[ $PLATFORM == 'linux' ]]; then + cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3 + cp -L /usr/lib/x86_64-linux-gnu/libquadmath.so.0 lib/libquadmath.so.0 +fi + +# Print the linked objects on libmxnet.so +>&2 echo "Checking linked objects on libmxnet.so..." +if [[ ! -z $(command -v readelf) ]]; then + readelf -d lib/libmxnet.so + strip --strip-unneeded lib/libmxnet.so +elif [[ ! -z $(command -v otool) ]]; then + otool -L lib/libmxnet.so + strip -u -r -x lib/libmxnet.so +else + >&2 echo "Not available" +fi + +ln -s staticdeps/ deps diff --git a/tools/pip_package/MANIFEST.in b/tools/staticbuild/build_wheel.sh old mode 100644 new mode 100755 similarity index 69% rename from tools/pip_package/MANIFEST.in rename to tools/staticbuild/build_wheel.sh index 5c6a72377e9f..a79634117c21 --- a/tools/pip_package/MANIFEST.in +++ b/tools/staticbuild/build_wheel.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,6 +17,15 @@ # specific language governing permissions and limitations # under the License. -include README -recursive-include * *.py -recursive-include * *.so +# This script builds the wheel for binary distribution and performs sanity check. + +cd mxnet-build +echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH +cd - + +# Make wheel for testing +python setup.py bdist_wheel + +wheel_name=$(ls -t dist | head -n 1) +pip install -U --user --force-reinstall dist/$wheel_name +python sanity_test.py