From b2b49a563c31e46329f857e4033c4d6e07363719 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Tue, 11 Dec 2018 15:19:06 -0800 Subject: [PATCH 01/16] [MXNET-1155] Add scala packageTest utility (#13046) * [MXNET-1155] Add scala packageTest utility * Clean up utility * Safe change directory in Makefile for scala * mvn install file instructions with details --- Makefile | 35 +++- scala-package/.gitignore | 1 + scala-package/examples/pom.xml | 14 ++ scala-package/packageTest/Makefile | 87 +++++++++ scala-package/packageTest/README.md | 72 ++++++++ scala-package/packageTest/core/pom.xml | 39 ++++ scala-package/packageTest/core/scripts | 1 + scala-package/packageTest/examples/pom.xml | 48 +++++ scala-package/packageTest/examples/scripts | 1 + scala-package/packageTest/infer/pom.xml | 38 ++++ scala-package/packageTest/pom.xml | 196 +++++++++++++++++++++ 11 files changed, 523 insertions(+), 9 deletions(-) create mode 100644 scala-package/packageTest/Makefile create mode 100644 scala-package/packageTest/README.md create mode 100644 scala-package/packageTest/core/pom.xml create mode 120000 scala-package/packageTest/core/scripts create mode 100644 scala-package/packageTest/examples/pom.xml create mode 120000 scala-package/packageTest/examples/scripts create mode 100644 scala-package/packageTest/infer/pom.xml create mode 100644 scala-package/packageTest/pom.xml diff --git a/Makefile b/Makefile index 16ea59f3d585..822704e26752 100644 --- a/Makefile +++ b/Makefile @@ -600,11 +600,19 @@ rpkgtest: Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)' scalaclean: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE)) +scalatestcompile: + (cd $(ROOTDIR)/scala-package && \ + mvn test-compile -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ + -Dcurrent_libdir="$(ROOTDIR)/lib" \ + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + scalapkg: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ @@ -612,49 +620,58 @@ scalapkg: -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scalaunittest: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalaintegrationtest: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalainstall: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scalarelease-dryrun: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scalarelease-prepare: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn release:clean release:prepare -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scalarelease-perform: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn release:perform -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scaladeploy: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") +scaladeploylocal: + (cd $(ROOTDIR)/scala-package && \ + mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ + -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \ + -Dgpg.skip \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + jnilint: 3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src diff --git a/scala-package/.gitignore b/scala-package/.gitignore index 6aa4da6b1cfc..8bc87f53e802 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -6,3 +6,4 @@ core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala core/src/main/scala/org/apache/mxnet/SymbolBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ +local-snapshot \ No newline at end of file diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index e34f0984ea02..666ea1c9fb5c 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -58,6 +58,20 @@ linux-x86_64-gpu + + deployLocal + + + + org.apache.maven.plugins + maven-deploy-plugin + + false + + + + + release diff --git a/scala-package/packageTest/Makefile b/scala-package/packageTest/Makefile new file mode 100644 index 000000000000..6073ff8a722f --- /dev/null +++ b/scala-package/packageTest/Makefile @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SCALA_VERSION_PROFILE := 2.11 +SCALA_VERSION := 2.11.8 +MXNET_VERSION := "[1.3.0-SNAPSHOT,)" + +MXNET_REPO = https://repository.apache.org/content/repositories/snapshots + +ifeq ($(OS),Windows_NT) + UNAME_S := Windows +else + UNAME_S := $(shell uname -s) +endif + +ifeq ($(UNAME_S), Windows) + # TODO: currently scala package does not support windows + SCALA_PKG_PROFILE := windows +else + ifeq ($(UNAME_S), Darwin) + SCALA_PKG_PROFILE := osx-x86_64-cpu + else + SCALA_PKG_PROFILE := linux-x86_64 + ifeq ($(USE_CUDA), 1) + SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu + else + SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu + endif + endif +endif + +PROFILES := -Ptest +ifeq ($(UNIT), 1) + PROFILES := "$(PROFILES),unittest" +endif +ifeq ($(INTEGRATION), 1) + PROFILES := "$(PROFILES),integrationtest" +endif + +ifneq ($(UNIT), 1) + ifneq ($(INTEGRATION), 1) + PROFILES := "$(PROFILES),unittest,integrationtest" + endif +endif + + +clean: + (mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +testinstall: + (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + $(PROFILES) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +testlocal: + (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + $(PROFILES),fromLocal \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +testsnapshot: + (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + $(PROFILES),fromSnapshots \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.repo=$(MXNET_REPO) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md new file mode 100644 index 000000000000..3f1eeb842f07 --- /dev/null +++ b/scala-package/packageTest/README.md @@ -0,0 +1,72 @@ +# MXNet Scala Package Test + +This is an project created to run the test suite on a fully packaged mxnet jar. The test suite is found locally but mxnet is from the target jarfile. + +## General Setup + +To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory: + +1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source) +2. Build the tests by running `make scalatestcompile`. +3. Follow setup instructions below for your testing goal + +## Running + +There are three different modes of operation for testing based on the location of the jar and where it is coming from: + +### Test Installed Jars + +If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`). This might be useful if you acquire the .jar file from elsewhere. To install, it is easiest to use `mvn install:install-file -Dfile= -DpomFile=`. If the pom file is not available, you can also run `mvn install:install-file -Dfile= -DgroupId= -DartifactId= -Dversion= -Dpackaging=`. With the full mxnet jar, this might look like `mvn install:install-file -Dfile= -DgroupId=org.apache.mxnet -DartifactId=mxnet-full_2.11-linux-x86_64-cpu -Dversion=1.3.0 -Dpackaging=jar`. + +You can also run `make scalainstall` to install from a local build. + +After installing, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests. + +### Test Local Deployment + +To test the jars that would be produced by a deployment, you can run `make scaladeploylocal` from the main mxnet directory. This produces a local snapshot located at `scala-package/local-snapshot`. To test this local snapshot, run `make testlocal`. + +### Remote Repository Snapshot + +This mode is to test a jar located in a remote repository. The default repository is the apache snapshot repisotory located at `https://repository.apache.org/content/repositories/snapshots`. Note that the actual jar in a repisotory should be located at `$repoUrl/org/apache/mxnet/mxnet-full_$scalaVersion-$osMode/$version/*.jar`. + +Test the snapshot repo using `make testsnapshot` or a different repo using `make testsnapshot MXNET_REPO=$NEW_REPO_URL`. + +### Options + +You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above). + +An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`. + +## Cleaning Up + +You can clean temporary files and target artifacts by running `make scalaclean`. + +## Troubleshooting + +### Missing Examples + +If you fail with the following error +``` +[ERROR] Failed to execute goal org.scalatest:scalatest-maven-plugin:1.0:test (test) on project mxnet-scala-packagetest-examples_2.11: There are test failures -> [Help 1] +[ERROR] +[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch. +[ERROR] Re-run Maven using the -X switch to enable full debug logging. +[ERROR] +[ERROR] For more information about the errors and possible solutions, please read the following articles: +[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException +[ERROR] +[ERROR] After correcting the problems, you can resume the build with the command +[ERROR] mvn -rf :mxnet-scala-packagetest-examples_2.11 +Makefile:57: recipe for target 'scalaintegrationtest' failed +make: *** [scalaintegrationtest] Error 1 +``` + +and stacktrace begins with the following, + +``` +*** RUN ABORTED *** + java.lang.NoClassDefFoundError: org/apache/mxnetexamples/Util$ +``` + +you are missing the mxnetexamples package. See your test mode installation section for details. diff --git a/scala-package/packageTest/core/pom.xml b/scala-package/packageTest/core/pom.xml new file mode 100644 index 000000000000..bdcd7662f082 --- /dev/null +++ b/scala-package/packageTest/core/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-core_2.11 + MXNet Scala Package Test + pom + + + + unittest + + false + + + + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../core/target/test-classes + + + + + + diff --git a/scala-package/packageTest/core/scripts b/scala-package/packageTest/core/scripts new file mode 120000 index 000000000000..f806668aa847 --- /dev/null +++ b/scala-package/packageTest/core/scripts @@ -0,0 +1 @@ +../../core/scripts \ No newline at end of file diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml new file mode 100644 index 000000000000..e11be657e225 --- /dev/null +++ b/scala-package/packageTest/examples/pom.xml @@ -0,0 +1,48 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-examples_2.11 + MXNet Scala Package Test + pom + + + + integrationtest + + false + + + + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../examples/target/test-classes + + + + + + + + org.apache.mxnet + mxnet-examples_${mxnet.scalaprofile} + ${mxnet.version} + test + + + + diff --git a/scala-package/packageTest/examples/scripts b/scala-package/packageTest/examples/scripts new file mode 120000 index 000000000000..2bba4eeece74 --- /dev/null +++ b/scala-package/packageTest/examples/scripts @@ -0,0 +1 @@ +../../examples/scripts \ No newline at end of file diff --git a/scala-package/packageTest/infer/pom.xml b/scala-package/packageTest/infer/pom.xml new file mode 100644 index 000000000000..7c5a096d6e14 --- /dev/null +++ b/scala-package/packageTest/infer/pom.xml @@ -0,0 +1,38 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-infer_2.11 + MXNet Scala Package Test + pom + + + + unittest + + false + + + + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../infer/target/test-classes + + + + + diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml new file mode 100644 index 000000000000..9c5c11cf2779 --- /dev/null +++ b/scala-package/packageTest/pom.xml @@ -0,0 +1,196 @@ + + + 4.0.0 + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + MXNet Scala Package Test + pom + + + core + infer + + + + + test + + + integrationtest + + examples + + + + fromSnapshots + + + apache-snapshots + ${mxnet.repo} + default + + true + + + + + + fromLocal + + + local-snapshot + file://${basedir}/../local-snapshot + + true + + + + + + + + true + + + + + org.apache.mxnet + mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} + ${mxnet.version} + + + org.scala-lang + scala-library + ${scala.version} + + + commons-io + commons-io + 2.4 + + + org.scalatest + scalatest_${mxnet.scalaprofile} + 3.0.4 + test + + + org.scalacheck + scalacheck_${mxnet.scalaprofile} + 1.13.5 + test + + + org.mockito + mockito-all + 1.10.19 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.6 + 1.6 + UTF-8 + + + + maven-resources-plugin + 2.7 + + + org.apache.maven.plugins + maven-dependency-plugin + 2.9 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.outputDirectory}/lib + runtime + test,provided + false + false + true + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.5 + + + package + + jar + + + + **/* + + + + + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + compile + + compile + + compile + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.19 + + true + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${skipTests} + ${project.build.directory}/surefire-reports + . + F + WDF TestSuite.txt + + + + test + integration-test + + test + + + + + + + + From 2e226ac05814033e9810fbda28358578ce6a9beb Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Tue, 11 Dec 2018 15:21:05 -0800 Subject: [PATCH 02/16] [MXNET-1224]: improve scala maven jni build and packing. (#13493) Major JNI feature changes. Please find more info here: https://cwiki.apache.org/confluence/display/MXNET/Scala+maven+build+improvement --- .../assembly/linux-x86_64-cpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 ++ .../assembly/linux-x86_64-gpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 ++ .../osx-x86_64-cpu/main/assembly/assembly.xml | 46 -------------- scala-package/assembly/osx-x86_64-cpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 ++ scala-package/core/pom.xml | 8 +++ .../mxnet/util/NativeLibraryLoader.scala | 55 ++++++++++------ scala-package/examples/pom.xml | 4 ++ scala-package/infer/pom.xml | 4 ++ .../init-native/linux-x86_64/pom.xml | 42 ++++++++++--- scala-package/init-native/osx-x86_64/pom.xml | 49 +++++++++++++-- scala-package/native/README.md | 63 +++++++++++++++++++ scala-package/native/linux-x86_64-cpu/pom.xml | 25 ++++---- scala-package/native/linux-x86_64-gpu/pom.xml | 25 ++++---- scala-package/native/osx-x86_64-cpu/pom.xml | 50 ++++++++++++--- scala-package/pom.xml | 2 + 18 files changed, 291 insertions(+), 112 deletions(-) delete mode 100644 scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml create mode 100644 scala-package/native/README.md diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index 05db3a5f70b9..d3c95a60639a 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -30,6 +30,10 @@ MXNet Scala Package - Full Linux-x86_64 CPU-only jar + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet diff --git a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml index 9f28706ac596..6b5a5b238216 100644 --- a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml @@ -41,4 +41,10 @@ + + + ${MXNET_DIR}/lib/libmxnet.so + lib/native + + diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index 708b1c48e98b..122beb0977ba 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -30,6 +30,10 @@ MXNet Scala Package - Full Linux-x86_64 GPU jar + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet diff --git a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml index 2b65a8c7ad66..471f07cf4bfc 100644 --- a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml @@ -41,4 +41,10 @@ + + + ${MXNET_DIR}/lib/libmxnet.so + lib/native + + diff --git a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml deleted file mode 100644 index d0550a3623b1..000000000000 --- a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - ${artifact.artifactId}${dashClassifier?}.${artifact.extension} - false - false - false - - *:*:dll:* - *:*:so:* - *:*:jnilib:* - - - - diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index 2f80dd7fb707..68fb09df68e3 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -30,6 +30,10 @@ MXNet Scala Package - Full OSX-x86_64 CPU-only jar + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet diff --git a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml index 7a6c3ea9f525..3f632a76daf6 100644 --- a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml @@ -41,4 +41,10 @@ + + + ${MXNET_DIR}/lib/libmxnet.so + lib/native + + diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 1b1a570c3386..7b3cffc96f6b 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -28,6 +28,7 @@ true + ${project.parent.basedir}/.. mxnet-core_2.11 @@ -93,6 +94,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + ${MXNET_DIR}/lib + @@ -104,6 +108,10 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target ${skipTests} + always + + ${MXNET_DIR}/lib + diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala index e94d320391fa..2ce893b478ed 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala @@ -85,12 +85,10 @@ private[mxnet] object NativeLibraryLoader { } logger.debug(s"Attempting to load $loadLibname") val libFileInJar = libPathInJar + loadLibname - val is: InputStream = getClass.getResourceAsStream(libFileInJar) - if (is == null) { - throw new UnsatisfiedLinkError(s"Couldn't find the resource $loadLibname") - } - logger.info(s"Loading $loadLibname from $libPathInJar copying to $libname") - loadLibraryFromStream(libname, is) + saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so") + val tempfile: File = saveLibraryToTemp(libname, libFileInJar) + + loadLibraryFromFile(libname, tempfile) } /** @@ -109,7 +107,7 @@ private[mxnet] object NativeLibraryLoader { @throws(classOf[IOException]) private def createTempFile(name: String): File = { - new File(_tempDir + File.separator + name) + new File(_tempDir, name) } /** @@ -117,11 +115,34 @@ private[mxnet] object NativeLibraryLoader { * and loads from there. * * @param libname name of the library (just used in constructing the library name) - * @param is InputStream pointing to the library + * @param tempfile File pointing to the library */ - private def loadLibraryFromStream(libname: String, is: InputStream) { + private def loadLibraryFromFile(libname: String, tempfile: File) { + try { + logger.debug("Loading library from {}", tempfile.getPath) + System.load(tempfile.getPath) + } catch { + case ule: UnsatisfiedLinkError => + logger.error("Couldn't load copied link file: {}", ule.toString) + throw ule + } + } + + /** + * Load a system library from a stream. Copies the library to a temp file + * and loads from there. + * + * @param libname name of the library (just used in constructing the library name) + * @param resource String resource path in the jar file + */ + private def saveLibraryToTemp(libname: String, resource: String): File = { try { - val tempfile: File = createTempFile(libname) + val is: InputStream = getClass.getResourceAsStream(resource) + if (is == null) { + throw new UnsatisfiedLinkError(s"Couldn't find the resource $resource") + } + + val tempfile: File = new File(_tempDir, libname) val os: OutputStream = new FileOutputStream(tempfile) logger.debug("tempfile.getPath() = {}", tempfile.getPath) val savedTime: Long = System.currentTimeMillis @@ -131,20 +152,14 @@ private[mxnet] object NativeLibraryLoader { os.write(buf, 0, len) len = is.read(buf) } - os.flush() - val lock: InputStream = new FileInputStream(tempfile) os.close() + is.close() val seconds: Double = (System.currentTimeMillis - savedTime).toDouble / 1e3 - logger.debug(s"Copying took $seconds seconds.") - logger.debug("Loading library from {}", tempfile.getPath) - System.load(tempfile.getPath) - lock.close() + logger.debug(s"Copying $libname took $seconds seconds.") + tempfile } catch { case io: IOException => - logger.error("Could not create the temp file: {}", io.toString) - case ule: UnsatisfiedLinkError => - logger.error("Couldn't load copied link file: {}", ule.toString) - throw ule + throw new UnsatisfiedLinkError(s"Could not create temp file for $libname") } } } diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 666ea1c9fb5c..1ebc37a0d456 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -31,6 +31,7 @@ true + ${project.parent.basedir}/.. @@ -167,6 +168,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + ${MXNET_DIR}/lib + diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index d8be5584677c..6a8e77d63e11 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -31,6 +31,7 @@ true + ${project.parent.basedir}/.. @@ -93,6 +94,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + ${MXNET_DIR}/lib + diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml index c1f61fca01aa..2f0f586ee2e0 100644 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ b/scala-package/init-native/linux-x86_64/pom.xml @@ -32,6 +32,10 @@ so + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -78,22 +82,24 @@ -std=c++0x - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared - ${all_ldpaths} -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive + -Wl,--no-whole-archive -pthread -lm -fopenmp -lrt - ${ldflags} - -fopenmp + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib @@ -102,7 +108,6 @@ javah generate-sources - linux default ${project.build.directory}/custom-javah ${basedir} @@ -117,6 +122,25 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + link-native-lib + generate-resources + + exec + + + ln + -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so + + + + diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index 1c1342eea149..59a04192ad5c 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -32,6 +32,10 @@ jnilib + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -78,8 +82,14 @@ -std=c++0x - -I${project.basedir}/../../../include - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared @@ -88,11 +98,9 @@ -framework JavaVM -Wl,-exported_symbol,_Java_* -Wl,-x - ${lddeps} - -force_load ${project.basedir}/../../../lib/libmxnet.a - ${ldflags} + -lmxnet -L${MXNET_DIR}/lib @@ -101,7 +109,6 @@ javah generate-sources - darwin default ${project.build.directory}/custom-javah ${basedir} @@ -116,6 +123,36 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib + + + + link-native-lib + generate-resources + + exec + + + ln + -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so + + + + diff --git a/scala-package/native/README.md b/scala-package/native/README.md new file mode 100644 index 000000000000..cb6dd3890dd2 --- /dev/null +++ b/scala-package/native/README.md @@ -0,0 +1,63 @@ +# MXNet Scala JNI + +MXNet Scala JNI is a thin wrapper layer of underlying libmxnet.so. + +## javah +JNI native code requires a header file that matches the java/scala interface, +this file is usually generated with javah. + +In our case, jni_helper_func.h is generated and will be used to compile native code. + + +## Linker options + +Scala JNI (libmxnet-scala.so/libmxnet-scala.jnilib) is dynamically linked to libmxnet.so. +MXNet Scala will trying to load libmxnet.so from system LD_LIBRARY_PATH first. +If it failed, the try to resolve libmxnet.so in the same location as libmxnet-scala.so file. + +### Linux +``` +-Wl,-rpath=$ORIGIN -lmxnet +``` +Above option will tell system to looking for libmxnet.so from the same location. + + +### Mac OSX +On Mac, we have to execute install_name_tool command to change library loading path: +```bash +install_name_tool -change lib/libmxnet.so @loader_path/libmxnet.so libmxnet-scala.jnilib +``` + +Other linker options: +* -shared : link as shared library +* -Wl,-install_name,libmxnet-scala.jnilib : avoid use build machine's absolute path +* -framework JavaVM : Stand jni options for mac +* -Wl,-exported_symbol,_Java_* : Stand jni options for mac +* -Wl,-x : Do not put non-global symbols in the output file's symbol table. + + +## Compiler flags + +Scala JNI code technically doesn't need on any of MXNet make flags, +however c_api.h header links to many other dependencies header file, +which requires us to add DMSHADOW_USE_MKL and DMSHADOW_USE_CUDA to compile the JNI code. +These flags are not actually used by JNI and won't impact Scala's behavior. + + +### Linux + +``` +-DMSHADOW_USE_MKL=0 +-DMSHADOW_USE_CUDA=0 +-O3 -DNDEBUG=1 -fPIC -msse3 -mf16c +-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs +``` + +### Mac OSX + +``` +-DMSHADOW_USE_MKL=0 +-DMSHADOW_USE_CUDA=0 +-g -O0 -fPIC -msse3 -mf16c +-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs +``` diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml index 500e484cbb59..ce1e04e918a4 100644 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ b/scala-package/native/linux-x86_64-cpu/pom.xml @@ -32,6 +32,10 @@ so + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -78,22 +82,20 @@ -std=c++0x - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared - - ${all_ldpaths} - -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive - - ${ldflags} - -fopenmp + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib @@ -102,7 +104,6 @@ javah generate-sources - linux default ${project.build.directory}/custom-javah ${basedir} diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml index da8b5f4a8183..052140d5e07d 100644 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ b/scala-package/native/linux-x86_64-gpu/pom.xml @@ -32,6 +32,10 @@ so + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -78,22 +82,20 @@ -std=c++0x - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared - - ${all_ldpaths} - -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive - - ${ldflags} - -fopenmp + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib @@ -102,7 +104,6 @@ javah generate-sources - linux default ${project.build.directory}/custom-javah ${basedir} diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index fc367ca9bfc5..ddc358425f76 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -32,6 +32,10 @@ jnilib + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -78,8 +82,14 @@ -std=c++0x - -I../../../include - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared @@ -88,12 +98,9 @@ -framework JavaVM -Wl,-exported_symbol,_Java_* -Wl,-x - ${lddeps} - -force_load ${project.basedir}/../../../lib/libmxnet.a - -force_load ${project.basedir}/../../../3rdparty/tvm/nnvm/lib/libnnvm.a - ${ldflags} + -Wl,-install_name,libmxnet-scala.jnilib -lmxnet -L${MXNET_DIR}/lib @@ -102,7 +109,6 @@ javah generate-sources - darwin default ${project.build.directory}/custom-javah ${basedir} @@ -117,6 +123,36 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib + + + + link-native-lib + generate-resources + + exec + + + ln + -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so + + + + diff --git a/scala-package/pom.xml b/scala-package/pom.xml index fe061e22e484..f0e86ec3d630 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -55,6 +55,8 @@ 2.11.8 2.11 + g++ + $ pom From bdb23d432c9462078ba46a25236403a030f60f43 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Wed, 12 Dec 2018 17:14:13 -0800 Subject: [PATCH 03/16] Scripts for building dependency libraries of MXNet (#13282) * openblas script * ps-lite dependencies * USE_S3 dependencies * image libraries * license --- tools/dependencies/README.md | 14 +++ tools/dependencies/cityhash.sh | 32 +++++ tools/dependencies/curl.sh | 64 ++++++++++ tools/dependencies/eigen.sh | 34 +++++ tools/dependencies/libpng.sh | 40 ++++++ tools/dependencies/libtiff.sh | 32 +++++ tools/dependencies/libturbojpeg.sh | 47 +++++++ tools/dependencies/libz.sh | 36 ++++++ tools/dependencies/lz4.sh | 31 +++++ tools/dependencies/openblas.sh | 35 ++++++ tools/dependencies/opencv.sh | 191 +++++++++++++++++++++++++++++ tools/dependencies/openssl.sh | 38 ++++++ tools/dependencies/protobuf.sh | 41 +++++++ tools/dependencies/zmq.sh | 38 ++++++ 14 files changed, 673 insertions(+) create mode 100644 tools/dependencies/README.md create mode 100755 tools/dependencies/cityhash.sh create mode 100755 tools/dependencies/curl.sh create mode 100755 tools/dependencies/eigen.sh create mode 100755 tools/dependencies/libpng.sh create mode 100755 tools/dependencies/libtiff.sh create mode 100755 tools/dependencies/libturbojpeg.sh create mode 100755 tools/dependencies/libz.sh create mode 100755 tools/dependencies/lz4.sh create mode 100755 tools/dependencies/openblas.sh create mode 100755 tools/dependencies/opencv.sh create mode 100755 tools/dependencies/openssl.sh create mode 100755 tools/dependencies/protobuf.sh create mode 100755 tools/dependencies/zmq.sh diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md new file mode 100644 index 000000000000..cfe3d6c75dc9 --- /dev/null +++ b/tools/dependencies/README.md @@ -0,0 +1,14 @@ +# Overview + +This folder contains scripts for building the dependencies from source. The static libraries from +the build artifacts can be used to create self-contained shared object for mxnet through static +linking. + +# Settings + +The scripts use the following environment variables for setting behavior: + +`DEPS_PATH`: the location in which the libraries are downloaded, built, and installed. +`PLATFORM`: name of the OS in lower case. Supported options are 'linux' and 'darwin'. + +It also expects the following build tools in path: make, cmake, tar, unzip, autoconf, nasm diff --git a/tools/dependencies/cityhash.sh b/tools/dependencies/cityhash.sh new file mode 100755 index 000000000000..81cc9cbaad3a --- /dev/null +++ b/tools/dependencies/cityhash.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of cityhash that can be used as dependency of mxnet. +CITYHASH_VERSION=1.1.1 +if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then + # Download and build cityhash + >&2 echo "Building cityhash..." + git clone https://github.com/google/cityhash $DEPS_PATH/cityhash-$CITYHASH_VERSION + cd $DEPS_PATH/cityhash-$CITYHASH_VERSION + git reset --hard 8af9b8c2b889d80c22d6bc26ba0df1afb79a30db + ./configure -prefix=$DEPS_PATH --enable-sse4.2 + make CXXFLAGS="-g -O3 -msse4.2" + make install + cd - +fi diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh new file mode 100755 index 000000000000..9633edb78538 --- /dev/null +++ b/tools/dependencies/curl.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libcurl that can be used as dependency of mxnet. +LIBCURL_VERSION=7.61.0 +if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then + # download and build libcurl + >&2 echo "Building libcurl..." + curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o $DEPS_PATH/libcurl.zip + unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH + cd $DEPS_PATH/curl-$LIBCURL_VERSION + if [[ $PLATFORM == 'linux' ]]; then + CONFIG_FLAG="" + elif [[ $PLATFORM == 'darwin' ]]; then + CONFIG_FLAG="--with-darwinssl" + fi + ./configure $CONFIG_FLAG \ + --with-zlib \ + --with-nghttps2 \ + --without-zsh-functions-dir \ + --without-librtmp \ + --without-libssh2 \ + --disable-debug \ + --disable-curldebug \ + --enable-symbol-hiding=yes \ + --enable-optimize=yes \ + --enable-shared=no \ + --enable-http=yes \ + --enable-ipv6=yes \ + --disable-ftp \ + --disable-ldap \ + --disable-ldaps \ + --disable-rtsp \ + --disable-proxy \ + --disable-dict \ + --disable-telnet \ + --disable-tftp \ + --disable-pop3 \ + --disable-imap \ + --disable-smb \ + --disable-smtp \ + --disable-gopher \ + --disable-manual \ + --prefix=$DEPS_PATH + make + make install + cd - +fi diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh new file mode 100755 index 000000000000..ac2f75a03a52 --- /dev/null +++ b/tools/dependencies/eigen.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script imports the headers from eigen3 that can be used to in opencv. +EIGEN_VERSION=3.3.4 +if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then + # download eigen + >&2 echo "Loading eigen..." + curl -s -L https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o $DEPS_PATH/eigen.zip + unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build + cd $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build + cmake \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. + make install + cd - +fi diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh new file mode 100755 index 000000000000..d1523c654478 --- /dev/null +++ b/tools/dependencies/libpng.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libpng that can be used as dependency of mxnet/opencv. +PNG_VERSION=1.6.34 +if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then + # download and build libpng + >&2 echo "Building libpng..." + curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip + unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build + cd $DEPS_PATH/libpng-$PNG_VERSION/build + cmake \ + -D PNG_SHARED=OFF \ + -D PNG_STATIC=ON \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ + -D CMAKE_C_FLAGS=-fPIC .. + make + make install + mkdir -p $DEPS_PATH/include/libpng + ln -s $DEPS_PATH/include/png.h $DEPS_PATH/include/libpng/png.h + cd - +fi diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh new file mode 100755 index 000000000000..14dcb2d7bde0 --- /dev/null +++ b/tools/dependencies/libtiff.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libtiff that can be used as dependency of mxnet/opencv. +TIFF_VERSION="4-0-9" +if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then + # download and build libtiff + >&2 echo "Building libtiff..." + curl -s -L https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip + unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH + cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION + ./configure --quiet --disable-shared --disable-jpeg --disable-zlib --disable-jbig --disable-lzma --prefix=$DEPS_PATH + make + make install + cd - +fi diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh new file mode 100755 index 000000000000..4991906f8878 --- /dev/null +++ b/tools/dependencies/libturbojpeg.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libturbojpeg that can be used as dependency of +# mxnet/opencv. +TURBO_JPEG_VERSION=1.5.90 +if [[ $PLATFORM == 'darwin' ]]; then + JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm" +fi + +if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then + # download and build libjpeg + >&2 echo "Building libjpeg-turbo..." + curl -s -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip + unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build + cd $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build + cmake \ + -G"Unix Makefiles" \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ + -D CMAKE_C_FLAGS=-fPIC \ + -D WITH_JAVA=FALSE \ + -D WITH_JPEG7=TRUE \ + -D WITH_JPEG8=TRUE \ + $JPEG_NASM_OPTION \ + -D ENABLE_SHARED=FALSE .. + make + make install + cd - +fi diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh new file mode 100755 index 000000000000..927f1de82e72 --- /dev/null +++ b/tools/dependencies/libz.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of libz that can be used as dependency of mxnet. +ZLIB_VERSION=1.2.6 +if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then + # Download and build zlib + >&2 echo "Building zlib..." + curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip + unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build + cd $DEPS_PATH/zlib-$ZLIB_VERSION/build + cmake \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ + -D BUILD_SHARED_LIBS=OFF .. + make + make install + cd - +fi diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh new file mode 100755 index 000000000000..a4269bf29bb9 --- /dev/null +++ b/tools/dependencies/lz4.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of lz4 that can be used as dependency of mxnet. +LZ4_VERSION=r130 +if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then + # Download and build lz4 + >&2 echo "Building lz4..." + curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o $DEPS_PATH/lz4.zip + unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH + cd $DEPS_PATH/lz4-$LZ4_VERSION + make + make PREFIX=$DEPS_PATH install + cd - +fi diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh new file mode 100755 index 000000000000..9463e3325e0d --- /dev/null +++ b/tools/dependencies/openblas.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of openblas that can be used as dependency of mxnet. +OPENBLAS_VERSION=0.3.3 +if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then + # download and build openblas + >&2 echo "Building openblas..." + + curl -s -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip + unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH + cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION + + make DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1 + make PREFIX=$DEPS_PATH install + cd - + ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a + ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a +fi diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh new file mode 100755 index 000000000000..98ff115f1765 --- /dev/null +++ b/tools/dependencies/opencv.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of opencv that can be used as dependency of mxnet. +# It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in $DEPS_PATH. +OPENCV_VERSION=3.4.2 +if [[ $PLATFORM == 'linux' ]]; then + OPENCV_LAPACK_OPTIONS=" \ + -D OpenBLAS_HOME=$DEPS_PATH \ + -D OpenBLAS_INCLUDE_DIR=$DEPS_PATH/include \ + -D OpenBLAS_LIB=$DEPS_PATH/lib/libopenblas.a \ + -D LAPACK_INCLUDE_DIR=$DEPS_PATH/include \ + -D LAPACK_LINK_LIBRARIES=$DEPS_PATH/lib/ \ + -D LAPACK_LIBRARIES=$DEPS_PATH/lib/libopenblas.a \ + -D LAPACK_CBLAS_H='cblas.h' \ + -D LAPACK_LAPACKE_H='lapacke.h' \ + -D LAPACK_IMPL='OpenBLAS' \ + -D HAVE_LAPACK=1" +fi + +if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgcodecs.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgproc.a ]]; then + # download and build opencv since we need the static library + >&2 echo "Building opencv..." + curl -s -L https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip + unzip -q $DEPS_PATH/opencv.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build + cd $DEPS_PATH/opencv-$OPENCV_VERSION/build + cmake \ + -D OPENCV_ENABLE_NONFREE=OFF \ + -D WITH_1394=OFF \ + -D WITH_ARAVIS=OFF \ + -D WITH_AVFOUNDATION=OFF \ + -D WITH_CAROTENE=OFF \ + -D WITH_CLP=OFF \ + -D WITH_CSTRIPES=OFF \ + -D WITH_CPUFEATURES=OFF \ + -D WITH_CUBLAS=OFF \ + -D WITH_CUDA=OFF \ + -D WITH_CUFFT=OFF \ + -D WITH_DIRECTX=OFF \ + -D WITH_DSHOW=OFF \ + -D WITH_EIGEN=ON \ + -D WITH_FFMPEG=OFF \ + -D WITH_GDAL=OFF \ + -D WITH_GDCM=OFF \ + -D WITH_GIGEAPI=OFF \ + -D WITH_GPHOTO2=OFF \ + -D WITH_GSTREAMER=OFF \ + -D WITH_GSTREAMER_0_10=OFF \ + -D WITH_GTK=OFF \ + -D WITH_GTK_2_X=OFF \ + -D WITH_HALIDE=OFF \ + -D WITH_IMAGEIO=OFF \ + -D WITH_IMGCODEC_HDR=OFF \ + -D WITH_IMGCODEC_PXM=OFF \ + -D WITH_IMGCODEC_SUNRASTER=OFF \ + -D WITH_INF_ENGINE=OFF \ + -D WITH_INTELPERC=OFF \ + -D WITH_IPP=OFF \ + -D WITH_IPP_A=OFF \ + -D WITH_ITT=OFF \ + -D WITH_JASPER=OFF \ + -D WITH_JPEG=ON \ + -D WITH_LAPACK=ON \ + -D WITH_LIBREALSENSE=OFF \ + -D WITH_LIBV4L=OFF \ + -D WITH_MATLAB=OFF \ + -D WITH_MFX=OFF \ + -D WITH_MSMF=OFF \ + -D WITH_NVCUVID=OFF \ + -D WITH_OPENCL=OFF \ + -D WITH_OPENCLAMDBLAS=OFF \ + -D WITH_OPENCLAMDFFT=OFF \ + -D WITH_OPENCL_SVM=OFF \ + -D WITH_OPENEXR=OFF \ + -D WITH_OPENGL=OFF \ + -D WITH_OPENMP=OFF \ + -D WITH_OPENNI=OFF \ + -D WITH_OPENNI2=OFF \ + -D WITH_OPENVX=OFF \ + -D WITH_PNG=ON \ + -D WITH_PROTOBUF=OFF \ + -D WITH_PTHREADS_PF=ON \ + -D WITH_PVAPI=OFF \ + -D WITH_QT=OFF \ + -D WITH_QTKIT=OFF \ + -D WITH_QUICKTIME=OFF \ + -D WITH_TBB=OFF \ + -D WITH_TIFF=ON \ + -D WITH_UNICAP=OFF \ + -D WITH_V4L=OFF \ + -D WITH_VA=OFF \ + -D WITH_VA_INTEL=OFF \ + -D WITH_VFW=OFF \ + -D WITH_VTK=OFF \ + -D WITH_WEBP=OFF \ + -D WITH_WIN32UI=OFF \ + -D WITH_XIMEA=OFF \ + -D WITH_XINE=OFF \ + -D BUILD_ANDROID_EXAMPLES=OFF \ + -D BUILD_ANDROID_PROJECTS=OFF \ + -D BUILD_ANDROID_SERVICE=OFF \ + -D BUILD_CUDA_STUBS=OFF \ + -D BUILD_DOCS=OFF \ + -D BUILD_EXAMPLES=OFF \ + -D BUILD_FAT_JAVA_LIB=OFF \ + -D BUILD_IPP_IW=OFF \ + -D BUILD_ITT_IW=OFF \ + -D BUILD_JAVA=OFF \ + -D BUILD_JASPER=OFF \ + -D BUILD_JPEG=OFF \ + -D BUILD_OPENEXR=OFF \ + -D BUILD_PACKAGE=OFF \ + -D BUILD_PERF_TESTS=OFF \ + -D BUILD_PNG=OFF \ + -D BUILD_SHARED_LIBS=OFF \ + -D BUILD_TBB=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_TIFF=OFF \ + -D BUILD_WEBP=OFF \ + -D BUILD_WITH_DEBUG_INFO=OFF \ + -D BUILD_WITH_DYNAMIC_IPP=OFF \ + -D BUILD_WITH_STATIC_CRT=OFF \ + -D BUILD_ZLIB=OFF \ + -D BUILD_opencv_apps=OFF \ + -D BUILD_opencv_aruco=OFF \ + -D BUILD_opencv_calib3d=OFF \ + -D BUILD_opencv_contrib=OFF \ + -D BUILD_opencv_dnn=OFF \ + -D BUILD_opencv_features2d=OFF \ + -D BUILD_opencv_flann=OFF \ + -D BUILD_opencv_gpu=OFF \ + -D BUILD_opencv_gpuarithm=OFF \ + -D BUILD_opencv_gpubgsegm=OFF \ + -D BUILD_opencv_gpucodec=OFF \ + -D BUILD_opencv_gpufeatures2d=OFF \ + -D BUILD_opencv_gpufilters=OFF \ + -D BUILD_opencv_gpuimgproc=OFF \ + -D BUILD_opencv_gpulegacy=OFF \ + -D BUILD_opencv_gpuoptflow=OFF \ + -D BUILD_opencv_gpustereo=OFF \ + -D BUILD_opencv_gpuwarping=OFF \ + -D BUILD_opencv_highgui=OFF \ + -D BUILD_opencv_java=OFF \ + -D BUILD_opencv_js=OFF \ + -D BUILD_opencv_ml=OFF \ + -D BUILD_opencv_ml=OFF \ + -D BUILD_opencv_nonfree=OFF \ + -D BUILD_opencv_objdetect=OFF \ + -D BUILD_opencv_photo=OFF \ + -D BUILD_opencv_python=OFF \ + -D BUILD_opencv_python2=OFF \ + -D BUILD_opencv_python3=OFF \ + -D BUILD_opencv_superres=OFF \ + -D BUILD_opencv_video=OFF \ + -D BUILD_opencv_videoio=OFF \ + -D BUILD_opencv_videostab=OFF \ + -D BUILD_opencv_viz=OFF \ + -D BUILD_opencv_world=OFF \ + $OPENCV_LAPACK_OPTIONS \ + -D OPENCV_LIB_INSTALL_PATH=lib \ + -D OPENCV_INCLUDE_INSTALL_PATH=include \ + -D CMAKE_LIBRARY_PATH=$DEPS_PATH/lib \ + -D CMAKE_INCLUDE_PATH=$DEPS_PATH/include \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. + if [[ $PLATFORM == 'linux' ]]; then + cp $DEPS_PATH/../patch/opencv_lapack.h ./ + fi + make + make install + cd - + # @szha: compatibility header + cat $DEPS_PATH/include/opencv2/imgcodecs/imgcodecs_c.h >> $DEPS_PATH/include/opencv2/imgcodecs.hpp +fi diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh new file mode 100755 index 000000000000..b7e4317d4a89 --- /dev/null +++ b/tools/dependencies/openssl.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of openssl that can be used as dependency of mxnet. +OPENSSL_VERSION=1.0.2l +if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; then + # download and build openssl + >&2 echo "Building openssl..." + OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g') + curl -s -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.zip -o $DEPS_PATH/openssl.zip + unzip -q $DEPS_PATH/openssl.zip -d $DEPS_PATH + cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION + if [[ $PLATFORM == 'linux' ]]; then + TARGET=linux-x86_64 + elif [[ $PLATFORM == 'darwin' ]]; then + TARGET=darwin64-x86_64-cc + fi + ./Configure no-shared no-zlib --prefix=$DEPS_PATH --openssldir=$DEPS_PATH/ssl $TARGET + make + make install + cd - +fi diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh new file mode 100755 index 000000000000..dfa3d71f3750 --- /dev/null +++ b/tools/dependencies/protobuf.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of protobuf along with protoc, that can be used as dependency of mxnet. +PROTOBUF_VERSION=3.5.1 +if [[ $PLATFORM == 'darwin' ]]; then + DY_EXT="dylib" +else + DY_EXT="so" +fi + +LIBPROTOBUF="$DEPS_PATH/lib/libprotobuf.$DY_EXT" +LIBPROTOC="$DEPS_PATH/lib/libprotoc.$DY_EXT" +if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then + # Download and build protobuf + >&2 echo "Building protobuf..." + curl -s -L https://github.com/google/protobuf/archive/v$PROTOBUF_VERSION.zip -o $DEPS_PATH/protobuf.zip + unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH + cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION + ./autogen.sh + ./configure -prefix=$DEPS_PATH + make + make install + cd - +fi diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh new file mode 100755 index 000000000000..55e17798c2d3 --- /dev/null +++ b/tools/dependencies/zmq.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the static library of zeroMQ that can be used as dependency of mxnet. +ZEROMQ_VERSION=4.2.2 +if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then + # Download and build zmq + >&2 echo "Building zmq..." + curl -s -L https://github.com/zeromq/libzmq/archive/v$ZEROMQ_VERSION.zip -o $DEPS_PATH/zeromq.zip + unzip -q $DEPS_PATH/zeromq.zip -d $DEPS_PATH + mkdir -p $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build + cd $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build + cmake \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ + -D WITH_LIBSODIUM=OFF \ + -D BUILD_SHARED_LIBS=OFF .. + make + make install + cp $DEPS_PATH/lib/x86_64-linux-gnu/libzmq.a $DEPS_PATH/lib/libzmq.a + cd - +fi From 7010ad100d66030e01bb9c90bf2d14831c9dd92e Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 13 Dec 2018 10:04:50 -0800 Subject: [PATCH 04/16] build config for maven and pip (#13556) * config for pip * symbol whitelist * maven build config --- make/config/libmxnet.sym | 12 +++ make/config/libmxnet.ver | 16 +++ make/maven/maven_darwin_cpu.mk | 182 ++++++++++++++++++++++++++++++++ make/maven/maven_linux_cpu.mk | 182 ++++++++++++++++++++++++++++++++ make/maven/maven_linux_cu90.mk | 185 +++++++++++++++++++++++++++++++++ make/pip/pip_darwin_cpu.mk | 182 ++++++++++++++++++++++++++++++++ make/pip/pip_darwin_mkl.mk | 166 +++++++++++++++++++++++++++++ make/pip/pip_linux_cpu.mk | 182 ++++++++++++++++++++++++++++++++ make/pip/pip_linux_cu100.mk | 185 +++++++++++++++++++++++++++++++++ make/pip/pip_linux_cu100mkl.mk | 169 ++++++++++++++++++++++++++++++ make/pip/pip_linux_cu75.mk | 182 ++++++++++++++++++++++++++++++++ make/pip/pip_linux_cu75mkl.mk | 166 +++++++++++++++++++++++++++++ make/pip/pip_linux_cu80.mk | 185 +++++++++++++++++++++++++++++++++ make/pip/pip_linux_cu80mkl.mk | 169 ++++++++++++++++++++++++++++++ make/pip/pip_linux_cu90.mk | 185 +++++++++++++++++++++++++++++++++ make/pip/pip_linux_cu90mkl.mk | 169 ++++++++++++++++++++++++++++++ make/pip/pip_linux_cu91.mk | 185 +++++++++++++++++++++++++++++++++ make/pip/pip_linux_cu91mkl.mk | 169 ++++++++++++++++++++++++++++++ make/pip/pip_linux_cu92.mk | 185 +++++++++++++++++++++++++++++++++ make/pip/pip_linux_cu92mkl.mk | 169 ++++++++++++++++++++++++++++++ make/pip/pip_linux_mkl.mk | 166 +++++++++++++++++++++++++++++ 21 files changed, 3391 insertions(+) create mode 100644 make/config/libmxnet.sym create mode 100644 make/config/libmxnet.ver create mode 100644 make/maven/maven_darwin_cpu.mk create mode 100644 make/maven/maven_linux_cpu.mk create mode 100644 make/maven/maven_linux_cu90.mk create mode 100644 make/pip/pip_darwin_cpu.mk create mode 100644 make/pip/pip_darwin_mkl.mk create mode 100644 make/pip/pip_linux_cpu.mk create mode 100644 make/pip/pip_linux_cu100.mk create mode 100644 make/pip/pip_linux_cu100mkl.mk create mode 100644 make/pip/pip_linux_cu75.mk create mode 100644 make/pip/pip_linux_cu75mkl.mk create mode 100644 make/pip/pip_linux_cu80.mk create mode 100644 make/pip/pip_linux_cu80mkl.mk create mode 100644 make/pip/pip_linux_cu90.mk create mode 100644 make/pip/pip_linux_cu90mkl.mk create mode 100644 make/pip/pip_linux_cu91.mk create mode 100644 make/pip/pip_linux_cu91mkl.mk create mode 100644 make/pip/pip_linux_cu92.mk create mode 100644 make/pip/pip_linux_cu92mkl.mk create mode 100644 make/pip/pip_linux_mkl.mk diff --git a/make/config/libmxnet.sym b/make/config/libmxnet.sym new file mode 100644 index 000000000000..a472161baeb0 --- /dev/null +++ b/make/config/libmxnet.sym @@ -0,0 +1,12 @@ +MX* +NN* +_MX* +_NN* +mx* +nn* +_mx* +_nn* +Java_org_apache_mxnet* +*NDArray* +*Engine*Get* +*Storage*Get* diff --git a/make/config/libmxnet.ver b/make/config/libmxnet.ver new file mode 100644 index 000000000000..c83721307e37 --- /dev/null +++ b/make/config/libmxnet.ver @@ -0,0 +1,16 @@ +{ + global: + NN*; + MX*; + _NN*; + _MX*; + nn*; + mx*; + _nn*; + _mx*; + Java_org_apache_mxnet*; + *NDArray*; + *Engine*Get*; + *Storage*Get*; + local: *; +}; diff --git a/make/maven/maven_darwin_cpu.mk b/make/maven/maven_darwin_cpu.mk new file mode 100644 index 000000000000..b8bd97fbc8ed --- /dev/null +++ b/make/maven/maven_darwin_cpu.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making maven package +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 0 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=apple + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 0 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# whether use MKL-DNN library +USE_MKLDNN = 0 + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/maven/maven_linux_cpu.mk b/make/maven/maven_linux_cpu.mk new file mode 100644 index 000000000000..07177fbe3064 --- /dev/null +++ b/make/maven/maven_linux_cpu.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making maven package +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 0 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/maven/maven_linux_cu90.mk b/make/maven/maven_linux_cu90.mk new file mode 100644 index 000000000000..097d252f0e4e --- /dev/null +++ b/make/maven/maven_linux_cu90.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making maven package +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 0 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_darwin_cpu.mk b/make/pip/pip_darwin_cpu.mk new file mode 100644 index 000000000000..bf240ad128cd --- /dev/null +++ b/make/pip/pip_darwin_cpu.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=apple + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 0 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# whether use MKL-DNN library +USE_MKLDNN = 0 + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_darwin_mkl.mk b/make/pip/pip_darwin_mkl.mk new file mode 100644 index 000000000000..30b328c6420e --- /dev/null +++ b/make/pip/pip_darwin_mkl.mk @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=apple + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 0 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cpu.mk b/make/pip/pip_linux_cpu.mk new file mode 100644 index 000000000000..22bb32b4d1b8 --- /dev/null +++ b/make/pip/pip_linux_cpu.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu100.mk b/make/pip/pip_linux_cu100.mk new file mode 100644 index 000000000000..514603b94ab6 --- /dev/null +++ b/make/pip/pip_linux_cu100.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-10.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu100mkl.mk b/make/pip/pip_linux_cu100mkl.mk new file mode 100644 index 000000000000..da4127f0dd07 --- /dev/null +++ b/make/pip/pip_linux_cu100mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-10.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu75.mk b/make/pip/pip_linux_cu75.mk new file mode 100644 index 000000000000..02468aa9613a --- /dev/null +++ b/make/pip/pip_linux_cu75.mk @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-7.5 + +# whether use CuDNN R3 library +USE_CUDNN = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu75mkl.mk b/make/pip/pip_linux_cu75mkl.mk new file mode 100644 index 000000000000..6895383d4fec --- /dev/null +++ b/make/pip/pip_linux_cu75mkl.mk @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-7.5 + +# whether use CuDNN R3 library +USE_CUDNN = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu80.mk b/make/pip/pip_linux_cu80.mk new file mode 100644 index 000000000000..2a2b88de73a1 --- /dev/null +++ b/make/pip/pip_linux_cu80.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-8.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu80mkl.mk b/make/pip/pip_linux_cu80mkl.mk new file mode 100644 index 000000000000..40cef4ed2110 --- /dev/null +++ b/make/pip/pip_linux_cu80mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-8.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu90.mk b/make/pip/pip_linux_cu90.mk new file mode 100644 index 000000000000..88e55180b738 --- /dev/null +++ b/make/pip/pip_linux_cu90.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu90mkl.mk b/make/pip/pip_linux_cu90mkl.mk new file mode 100644 index 000000000000..9c4adf07bb6a --- /dev/null +++ b/make/pip/pip_linux_cu90mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.0 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu91.mk b/make/pip/pip_linux_cu91.mk new file mode 100644 index 000000000000..ea62510396ef --- /dev/null +++ b/make/pip/pip_linux_cu91.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.1 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu91mkl.mk b/make/pip/pip_linux_cu91mkl.mk new file mode 100644 index 000000000000..232ec47979cc --- /dev/null +++ b/make/pip/pip_linux_cu91mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.1 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu92.mk b/make/pip/pip_linux_cu92.mk new file mode 100644 index 000000000000..01d02cfefba9 --- /dev/null +++ b/make/pip/pip_linux_cu92.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_cu92mkl.mk b/make/pip/pip_linux_cu92mkl.mk new file mode 100644 index 000000000000..1abde3d71d7f --- /dev/null +++ b/make/pip/pip_linux_cu92mkl.mk @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/pip/pip_linux_mkl.mk b/make/pip/pip_linux_mkl.mk new file mode 100644 index 000000000000..96b88850c21a --- /dev/null +++ b/make/pip/pip_linux_mkl.mk @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making python wheel +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 1 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CuDNN R3 library +USE_CUDNN = 0 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 0 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 1 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk From 0e67958513b68df527ccd9f7a547f38983b91770 Mon Sep 17 00:00:00 2001 From: Lanking Date: Fri, 14 Dec 2018 16:24:48 -0800 Subject: [PATCH 05/16] [MXNET-1251] Basic configuration to do static-linking (#13621) * Basic configuration to do static-linking * update build script and place it in the install part * clean up the code further * revert maven into build-from-source * add curl to deps --- ci/docker/install/ubuntu_publish.sh | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 ci/docker/install/ubuntu_publish.sh diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh new file mode 100644 index 000000000000..bc3513dd13e5 --- /dev/null +++ b/ci/docker/install/ubuntu_publish.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Build on Ubuntu 14.04 LTS for LINUX CPU/GPU +apt-get update +apt-get install -y software-properties-common +add-apt-repository ppa:ubuntu-toolchain-r/test -y +add-apt-repository ppa:openjdk-r/ppa -y # Java lib +apt-get update +apt-get install -y git \ + cmake3 \ + libcurl4-openssl-dev \ + unzip \ + gcc-4.8 \ + g++-4.8 \ + gfortran \ + gfortran-4.8 \ + binutils \ + nasm \ + libtool \ + curl \ + pandoc \ + python3-pip \ + automake \ + pkg-config \ + openjdk-8-jdk +curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +tar xzf apache-maven-3.3.9-bin.tar.gz +mkdir /usr/local/maven +mv apache-maven-3.3.9/ /usr/local/maven/ +update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +update-ca-certificates -f From b78d97ded18d4e1bdf9c818c9baf696a62f3693c Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Fri, 14 Dec 2018 18:17:41 -0800 Subject: [PATCH 06/16] scripts for building libmxnet binary and wheel (#13648) * add script for making all dependencies * tools for building pip package * build scripts for lib and wheel --- tools/build/build_lib.sh | 80 +++++++ .../MANIFEST.in => build/build_wheel.sh} | 17 +- .../dependencies/make_shared_dependencies.sh | 40 ++++ tools/dependencies/opencv.sh | 3 +- tools/dependencies/patch/opencv_lapack.h | 23 +++ tools/dependencies/protobuf.sh | 2 + tools/pip/MANIFEST.in | 11 + tools/pip/sanity_test.py | 32 +++ tools/pip/setup.py | 195 ++++++++++++++++++ tools/pip_package/README.md | 26 --- tools/pip_package/make_pip_package.sh | 179 ---------------- tools/pip_package/setup.py | 60 ------ 12 files changed, 399 insertions(+), 269 deletions(-) create mode 100755 tools/build/build_lib.sh rename tools/{pip_package/MANIFEST.in => build/build_wheel.sh} (69%) mode change 100644 => 100755 create mode 100755 tools/dependencies/make_shared_dependencies.sh create mode 100644 tools/dependencies/patch/opencv_lapack.h create mode 100644 tools/pip/MANIFEST.in create mode 100644 tools/pip/sanity_test.py create mode 100644 tools/pip/setup.py delete mode 100644 tools/pip_package/README.md delete mode 100755 tools/pip_package/make_pip_package.sh delete mode 100644 tools/pip_package/setup.py diff --git a/tools/build/build_lib.sh b/tools/build/build_lib.sh new file mode 100755 index 000000000000..032fcb92045f --- /dev/null +++ b/tools/build/build_lib.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the libraries of mxnet. +make_config=config/pip_${PLATFORM}_${VARIANT}.mk +if [[ ! -f $make_config ]]; then + >&2 echo "Couldn't find make config $make_config for the current settings." + exit 1 +fi + +git clone --recursive https://github.com/apache/incubator-mxnet mxnet-build + +>&2 echo "Now building mxnet modules..." +cp $make_config mxnet-build/config.mk + +cd mxnet-build + +make DEPS_PATH=$DEPS_PATH DMLCCORE +make DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a +make DEPS_PATH=$DEPS_PATH PSLITE + +if [[ $VARIANT == *mkl ]]; then + MKLDNN_LICENSE='license.txt' + if [[ $PLATFORM == 'linux' ]]; then + IOMP_LIBFILE='libiomp5.so' + MKLML_LIBFILE='libmklml_intel.so' + MKLDNN_LIBFILE='libmkldnn.so.0' + else + IOMP_LIBFILE='libiomp5.dylib' + MKLML_LIBFILE='libmklml.dylib' + MKLDNN_LIBFILE='libmkldnn.0.dylib' + fi + make DEPS_PATH=$DEPS_PATH mkldnn + cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE +fi + +if [[ $VARIANT == *mkl ]]; then + >&2 echo "Copying MKL license." + rm lib/libmkldnn.{so,dylib} + rm lib/libmkldnn.0.*.dylib + rm lib/libmkldnn.so.0.* +fi + +>&2 echo "Now building mxnet..." +make DEPS_PATH=$DEPS_PATH || exit 1; + +if [[ $PLATFORM == 'linux' ]]; then + cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3 + cp -L /usr/lib/x86_64-linux-gnu/libquadmath.so.0 lib/libquadmath.so.0 +fi + +# Print the linked objects on libmxnet.so +>&2 echo "Checking linked objects on libmxnet.so..." +if [[ ! -z $(command -v readelf) ]]; then + readelf -d lib/libmxnet.so + strip --strip-unneeded lib/libmxnet.so +elif [[ ! -z $(command -v otool) ]]; then + otool -L lib/libmxnet.so + strip -u -r -x lib/libmxnet.so +else + >&2 echo "Not available" +fi + +cd ../ diff --git a/tools/pip_package/MANIFEST.in b/tools/build/build_wheel.sh old mode 100644 new mode 100755 similarity index 69% rename from tools/pip_package/MANIFEST.in rename to tools/build/build_wheel.sh index 5c6a72377e9f..a79634117c21 --- a/tools/pip_package/MANIFEST.in +++ b/tools/build/build_wheel.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,6 +17,15 @@ # specific language governing permissions and limitations # under the License. -include README -recursive-include * *.py -recursive-include * *.so +# This script builds the wheel for binary distribution and performs sanity check. + +cd mxnet-build +echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH +cd - + +# Make wheel for testing +python setup.py bdist_wheel + +wheel_name=$(ls -t dist | head -n 1) +pip install -U --user --force-reinstall dist/$wheel_name +python sanity_test.py diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh new file mode 100755 index 000000000000..d678fddcc02d --- /dev/null +++ b/tools/dependencies/make_shared_dependencies.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This is a convenience script for calling the build scripts of all dependency libraries. +# Environment variables should be set beforehand. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + + +if [[ ! $PLATFORM == 'darwin' ]]; then + source $DIR/openblas.sh +fi +source $DIR/libz.sh +source $DIR/libturbojpeg.sh +source $DIR/libpng.sh +source $DIR/libtiff.sh +source $DIR/openssl.sh +source $DIR/curl.sh +source $DIR/eigen.sh +source $DIR/opencv.sh +source $DIR/protobuf.sh +source $DIR/cityhash.sh +source $DIR/zmq.sh +source $DIR/lz4.sh diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh index 98ff115f1765..99d0ecb71c36 100755 --- a/tools/dependencies/opencv.sh +++ b/tools/dependencies/opencv.sh @@ -20,6 +20,7 @@ # This script builds the static library of opencv that can be used as dependency of mxnet. # It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in $DEPS_PATH. OPENCV_VERSION=3.4.2 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" if [[ $PLATFORM == 'linux' ]]; then OPENCV_LAPACK_OPTIONS=" \ -D OpenBLAS_HOME=$DEPS_PATH \ @@ -181,7 +182,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. if [[ $PLATFORM == 'linux' ]]; then - cp $DEPS_PATH/../patch/opencv_lapack.h ./ + cp $DIR/patch/opencv_lapack.h ./ fi make make install diff --git a/tools/dependencies/patch/opencv_lapack.h b/tools/dependencies/patch/opencv_lapack.h new file mode 100644 index 000000000000..97af9d67ea31 --- /dev/null +++ b/tools/dependencies/patch/opencv_lapack.h @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +extern "C" { +#include "cblas.h" +#include "lapacke.h" +} diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh index dfa3d71f3750..1564701042af 100755 --- a/tools/dependencies/protobuf.sh +++ b/tools/dependencies/protobuf.sh @@ -39,3 +39,5 @@ if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then make install cd - fi + +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib diff --git a/tools/pip/MANIFEST.in b/tools/pip/MANIFEST.in new file mode 100644 index 000000000000..5e072064193c --- /dev/null +++ b/tools/pip/MANIFEST.in @@ -0,0 +1,11 @@ +include README +include mxnet/COMMIT_HASH +recursive-include mxnet/tools * +recursive-include mxnet *.py +recursive-include mxnet *.so +recursive-include mxnet *.so.* +recursive-include mxnet *.dylib +recursive-include mxnet *_LICENSE +recursive-include mxnet *.h +recursive-include mxnet *.cuh +recursive-include dmlc_tracker *.py diff --git a/tools/pip/sanity_test.py b/tools/pip/sanity_test.py new file mode 100644 index 000000000000..dc51e479906b --- /dev/null +++ b/tools/pip/sanity_test.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +"""Sanity test.""" +from __future__ import print_function +import sys +from base64 import b64decode + +try: + import mxnet as mx + mx.img.imdecode(b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==')).asnumpy() + print('Test succeeded') +except: + import traceback + print('Test failed') + traceback.print_exc() + sys.exit(1) diff --git a/tools/pip/setup.py b/tools/pip/setup.py new file mode 100644 index 000000000000..d5db6d87fc1d --- /dev/null +++ b/tools/pip/setup.py @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +# pylint: disable=invalid-name, exec-used +"""Setup mxnet package for pip.""" +from __future__ import absolute_import +from datetime import datetime +import os +import sys +import shutil +import platform + +if platform.system() == 'Linux': + sys.argv.append('--universal') + sys.argv.append('--plat-name=manylinux1_x86_64') + +from setuptools import setup, find_packages +from setuptools.dist import Distribution + +# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py +# Will be invoked which introduces dependences +CURRENT_DIR = os.path.dirname(__file__) +libinfo_py = os.path.join(CURRENT_DIR, 'mxnet-build/python/mxnet/libinfo.py') +libinfo = {'__file__': libinfo_py} +exec(compile(open(libinfo_py, "rb").read(), libinfo_py, 'exec'), libinfo, libinfo) + +LIB_PATH = libinfo['find_lib_path']() +__version__ = libinfo['__version__'] +if 'TRAVIS_TAG' not in os.environ or not os.environ['TRAVIS_TAG'].strip(): + __version__ += 'b{0}'.format(datetime.today().strftime('%Y%m%d')) +elif 'TRAVIS_TAG' in os.environ and os.environ['TRAVIS_TAG'].startswith('patch-'): + __version__ = os.environ['TRAVIS_TAG'].split('-')[1] + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return platform.system() == 'Darwin' + + +DEPENDENCIES = [ + 'numpy<1.15.0,>=1.8.2', + 'requests>=2.20.0', + 'graphviz<0.9.0,>=0.8.1' +] + +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True) +shutil.rmtree(os.path.join(CURRENT_DIR, 'dmlc_tracker'), ignore_errors=True) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/python/mxnet'), + os.path.join(CURRENT_DIR, 'mxnet')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dmlc-core/tracker/dmlc_tracker'), + os.path.join(CURRENT_DIR, 'dmlc_tracker')) +shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet')) + +# copy tools to mxnet package +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet/tools'), ignore_errors=True) +os.mkdir(os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/launch.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/im2rec.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/kill-mxnet.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/parse_log.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/diagnose.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/tools/caffe_converter'), os.path.join(CURRENT_DIR, 'mxnet/tools/caffe_converter')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/tools/bandwidth'), os.path.join(CURRENT_DIR, 'mxnet/tools/bandwidth')) + +# copy headers to mxnet package +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet/include'), ignore_errors=True) +os.mkdir(os.path.join(CURRENT_DIR, 'mxnet/include')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/include/mxnet'), + os.path.join(CURRENT_DIR, 'mxnet/include/mxnet')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dlpack/include/dlpack'), + os.path.join(CURRENT_DIR, 'mxnet/include/dlpack')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dmlc-core/include/dmlc'), + os.path.join(CURRENT_DIR, 'mxnet/include/dmlc')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/mshadow/mshadow'), + os.path.join(CURRENT_DIR, 'mxnet/include/mshadow')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/tvm/nnvm/include/nnvm'), + os.path.join(CURRENT_DIR, 'mxnet/include/nnvm')) + +package_name = 'mxnet' + +variant = os.environ['mxnet_variant'].upper() +if variant != 'CPU': + package_name = 'mxnet_{0}'.format(variant.lower()) + +with open('doc/PYPI_README.md') as readme_file: + long_description = readme_file.read() + +with open('doc/{0}_ADDITIONAL.md'.format(variant)) as variant_doc: + long_description = long_description + variant_doc.read() + +# pypi only supports rst, so use pandoc to convert +import pypandoc +if platform.system() == 'Darwin': + pypandoc.download_pandoc() +long_description = pypandoc.convert_text(long_description, 'rst', 'md') +short_description = 'MXNet is an ultra-scalable deep learning framework.' +libraries = [] +if variant == 'CPU': + libraries.append('openblas') +else: + if variant.startswith('CU92'): + libraries.append('CUDA-9.2') + elif variant.startswith('CU91'): + libraries.append('CUDA-9.1') + elif variant.startswith('CU90'): + libraries.append('CUDA-9.0') + elif variant.startswith('CU80'): + libraries.append('CUDA-8.0') + elif variant.startswith('CU75'): + libraries.append('CUDA-7.5') + if variant.endswith('MKL'): + libraries.append('MKLDNN') + +short_description += ' This version uses {0}.'.format(' and '.join(libraries)) + +package_data = {'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))], + 'dmlc_tracker': []} +if variant.endswith('MKL'): + if platform.system() == 'Darwin': + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmklml.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libiomp5.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmkldnn.0.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libmklml.dylib') + package_data['mxnet'].append('mxnet/libiomp5.dylib') + package_data['mxnet'].append('mxnet/libmkldnn.0.dylib') + else: + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmklml_intel.so'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libiomp5.so'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmkldnn.so.0'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libmklml_intel.so') + package_data['mxnet'].append('mxnet/libiomp5.so') + package_data['mxnet'].append('mxnet/libmkldnn.so.0') + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), '../MKLML_LICENSE'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/MKLML_LICENSE') +if platform.system() == 'Linux': + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libgfortran.so.3'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libgfortran.so.3') + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libquadmath.so.0'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libquadmath.so.0') + +from mxnet.base import _generate_op_module_signature +from mxnet.ndarray.register import _generate_ndarray_function_code +from mxnet.symbol.register import _generate_symbol_function_code +_generate_op_module_signature('mxnet', 'symbol', _generate_symbol_function_code) +_generate_op_module_signature('mxnet', 'ndarray', _generate_ndarray_function_code) + +setup(name=package_name, + version=__version__, + long_description=long_description, + description=short_description, + zip_safe=False, + packages=find_packages(), + package_data=package_data, + include_package_data=True, + install_requires=DEPENDENCIES, + distclass=BinaryDistribution, + license='Apache 2.0', + classifiers=[ # https://pypi.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: C++', + 'Programming Language :: Cython', + 'Programming Language :: Other', # R, Scala + 'Programming Language :: Perl', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + url='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/apache/incubator-mxnet') diff --git a/tools/pip_package/README.md b/tools/pip_package/README.md deleted file mode 100644 index 6d044167fcf8..000000000000 --- a/tools/pip_package/README.md +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - -MXNet Python Package -==================== -MXNet is a deep learning framework designed for both *efficiency* and *flexibility*. -It allows you to mix the flavours of deep learning programs together to maximize the efficiency and your productivity. - - -Installation ------------- -To install, check [Build Instruction](http://mxnet.io/get_started/setup.html) diff --git a/tools/pip_package/make_pip_package.sh b/tools/pip_package/make_pip_package.sh deleted file mode 100755 index 46b4938b0785..000000000000 --- a/tools/pip_package/make_pip_package.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# Assuming the script is run at mxnet/tools/pip_package -# This script builds from scratch the dependencies of mxnet into static -# librareis and statically links them to produce a (mostly) standalone -# libmxnet.so, then packages it into the python wheel. -# It assumes the build environment to be a sandbox that doesn't have the .so -# objects for the dependencies, i.e. zlib, openblas, libjpeg, libpng, libtiff -# and opencv. - -# Install necessary build tools -if [ -n "$(command -v apt-get)" ]; then - sudo apt-get update; - sudo apt-get install -y build-essential git python-pip zip pkg-config cmake -elif [ -n "$(command -v yum)" ]; then - sudo yum install -y cmake - sudo yum groupinstall -y "Development Tools" - sudo yum install -y python27 python27-setuptools python27-tools python-pip -else - echo "Need a package manager to install build tools, e.g. apt/yum" - exit 1 -fi -sudo pip install -U pip setuptools wheel - -# Set up path as temporary working directory -DEPS_PATH=$PWD/../../deps -mkdir $DEPS_PATH - -# Dependencies can be updated here. Be sure to verify the download link before -# changing. The dependencies are: -ZLIB_VERSION=1.2.6 -OPENBLAS_VERSION=0.2.19 -JPEG_VERSION=8.4.0 -PNG_VERSION=1.5.10 -TIFF_VERSION=3.8.2 -OPENCV_VERSION=2.4.13 - -# Setup path to dependencies -export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$PKG_CONFIG_PATH -export CPATH=$DEPS_PATH/include:$CPATH - -# Position Independent code must be turned on for statically linking .a -export CC="gcc -fPIC" -export CXX="g++ -fPIC" - -# Download and build zlib -curl -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip -unzip $DEPS_PATH/zlib.zip -d $DEPS_PATH -mkdir $DEPS_PATH/zlib-$ZLIB_VERSION/build -cd $DEPS_PATH/zlib-$ZLIB_VERSION/build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ - -D BUILD_SHARED_LIBS=OFF .. -make -j$(nproc) -make install -cd - - -# download and build openblas -curl -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip -unzip $DEPS_PATH/openblas.zip -d $DEPS_PATH -cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION -make FC=gfortran -j $(($(nproc) + 1)) -make PREFIX=$DEPS_PATH install -cd - -ln -s $DEPS_PATH/lib/libopenblas_haswellp-r0.2.19.a $DEPS_PATH/lib/libcblas.a - -# download and build libjpeg -curl -L https://github.com/LuaDist/libjpeg/archive/$JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip -unzip $DEPS_PATH/libjpeg.zip -d $DEPS_PATH -cd $DEPS_PATH/libjpeg-$JPEG_VERSION -./configure --disable-shared --prefix=$DEPS_PATH -make -j$(nproc) -make test -make install -cd - - -# download and build libpng -curl -L https://github.com/LuaDist/libpng/archive/$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip -unzip $DEPS_PATH/libpng.zip -d $DEPS_PATH -mkdir $DEPS_PATH/libpng-$PNG_VERSION/build -cd $DEPS_PATH/libpng-$PNG_VERSION/build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ - -D PNG_CONFIGURE_LIBPNG=-fPIC \ - -D BUILD_SHARED_LIBS=OFF .. -make -j$(nproc) -make install -cd - - -# download and build libtiff -curl -L https://github.com/LuaDist/libtiff/archive/$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip -unzip $DEPS_PATH/libtiff.zip -d $DEPS_PATH -cd $DEPS_PATH/libtiff-$TIFF_VERSION -./configure --disable-shared --prefix=$DEPS_PATH -make -j$(nproc) -make install -cd - - -# download and build opencv since we need the static library -curl -L https://github.com/Itseez/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip -unzip $DEPS_PATH/opencv.zip -d $DEPS_PATH -mkdir $DEPS_PATH/opencv-$OPENCV_VERSION/build -cd $DEPS_PATH/opencv-$OPENCV_VERSION/build -cmake -D WITH_1394=OFF \ - -D WITH_AVFOUNDATION=OFF \ - -D WITH_CUDA=OFF \ - -D WITH_VTK=OFF \ - -D WITH_CUFFT=OFF \ - -D WITH_CUBLAS=OFF \ - -D WITH_NVCUVID=OFF \ - -D WITH_EIGEN=ON \ - -D WITH_VFW=OFF \ - -D WITH_FFMPEG=OFF \ - -D WITH_GSTREAMER=OFF \ - -D WITH_GTK=OFF \ - -D WITH_JASPER=OFF \ - -D WITH_JPEG=ON \ - -D WITH_PNG=ON \ - -D WITH_QUICKTIME=OFF \ - -D WITH_TBB=ON \ - -D WITH_TIFF=OFF \ - -D WITH_V4L=OFF \ - -D WITH_LIBV4L=OFF \ - -D WITH_DSHOW=OFF \ - -D WITH_MSMF=OFF \ - -D WITH_OPENCL=OFF \ - -D WITH_OPENCLAMDFFT=OFF \ - -D WITH_OPENCLAMDBLAS=OFF \ - -D BUILD_SHARED_LIBS=OFF \ - -D BUILD_opencv_apps=OFF \ - -D BUILD_opencv_gpu=OFF \ - -D BUILD_opencv_video=OFF \ - -D BUILD_opencv_contrib=OFF \ - -D BUILD_opencv_nonfree=OFF \ - -D BUILD_opencv_flann=OFF \ - -D BUILD_opencv_features2d=OFF \ - -D BUILD_opencv_calib3d=OFF \ - -D BUILD_opencv_objdetect=OFF \ - -D BUILD_opencv_ml=OFF \ - -D BUILD_opencv_photo=OFF \ - -D BUILD_DOCS=OFF \ - -D BUILD_PACKAGE=OFF \ - -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. -make -j $(nproc) -make install # user will always have access to home, so no sudo needed -cd - - -# Although .so building is explicitly turned off for most libraries, sometimes -# they still get created. So, remove them just to make sure they don't -# interfere, or otherwise we might get libmxnet.so that is not self-contained. -rm $DEPS_PATH/{lib,lib64}/*.{so,so.0} - -# Go to the parent path and build mxnet -cd ../../ -cp make/pip_$(uname | tr '[:upper:]' '[:lower:]')_cpu.mk config.mk -make -j $(nproc) - -# Generate wheel. The output is in the mxnet/tools/pip_package/dist path. -cd tools/pip_package -python setup.py bdist_wheel diff --git a/tools/pip_package/setup.py b/tools/pip_package/setup.py deleted file mode 100644 index e4bf48236bde..000000000000 --- a/tools/pip_package/setup.py +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# pylint: disable=invalid-name, exec-used -"""Setup mxnet package.""" -from __future__ import absolute_import -import os -import shutil - -from setuptools import setup, find_packages -from setuptools.dist import Distribution - -# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py -# Will be invoked which introduces dependences -CURRENT_DIR = os.path.dirname(__file__) -libinfo_py = os.path.join(CURRENT_DIR, '../../python/mxnet/libinfo.py') -libinfo = {'__file__': libinfo_py} -exec(compile(open(libinfo_py, "rb").read(), libinfo_py, 'exec'), libinfo, libinfo) - -LIB_PATH = libinfo['find_lib_path']() -__version__ = libinfo['__version__'] - -class BinaryDistribution(Distribution): - def has_ext_modules(self): - return True - - -DEPENDENCIES = [ - 'numpy', -] - -shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True) -shutil.copytree(os.path.join(CURRENT_DIR, '../../python/mxnet'), - os.path.join(CURRENT_DIR, 'mxnet')) -shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet')) - -setup(name='mxnet', - version=__version__, - description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), - zip_safe=False, - packages=find_packages(), - package_data={'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))]}, - include_package_data=True, - install_requires=DEPENDENCIES, - distclass=BinaryDistribution, - url='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/dmlc/mxnet') From 15cb2721f6252893ecc401378c028c37a6dfbb33 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Tue, 8 Jan 2019 11:31:35 -0800 Subject: [PATCH 07/16] [MXNET-862] Basic maven jenkins pipeline (#13450) * Jenkins Publish Nightly Maven Progress * Seperate Build, Test, and Deploy Stages with parallel --- Makefile | 16 +- ci/Jenkinsfile_utils.groovy | 9 +- ci/build.py | 20 ++- ci/docker/install/ubuntu_scala.sh | 3 + ci/docker/runtime_functions.sh | 47 +++++- ci/docker_cache.py | 3 +- ci/publish/Jenkinsfile | 105 +++++++++++++ scala-package/.gitignore | 2 +- scala-package/dev/build.sh | 34 +++++ scala-package/dev/buildkey.py | 165 +++++++++++++++++++++ scala-package/dev/compile-mxnet-backend.sh | 7 +- scala-package/dev/deploy.sh | 44 ++++++ scala-package/dev/test.sh | 23 +++ scala-package/packageTest/Makefile | 12 +- 14 files changed, 457 insertions(+), 33 deletions(-) create mode 100644 ci/publish/Jenkinsfile create mode 100755 scala-package/dev/build.sh create mode 100644 scala-package/dev/buildkey.py create mode 100755 scala-package/dev/deploy.sh create mode 100755 scala-package/dev/test.sh diff --git a/Makefile b/Makefile index 822704e26752..3ae3a33e3b42 100644 --- a/Makefile +++ b/Makefile @@ -454,6 +454,10 @@ else CFLAGS += -DMXNET_USE_LIBJPEG_TURBO=0 endif +ifeq ($(CI), 1) + MAVEN_ARGS := -B +endif + # For quick compile test, used smaller subset ALLX_DEP= $(ALL_DEP) @@ -613,7 +617,7 @@ scalatestcompile: scalapkg: (cd $(ROOTDIR)/scala-package && \ - mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ + mvn package $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dcurrent_libdir="$(ROOTDIR)/lib" \ @@ -621,19 +625,19 @@ scalapkg: scalaunittest: (cd $(ROOTDIR)/scala-package && \ - mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \ + mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalaintegrationtest: (cd $(ROOTDIR)/scala-package && \ - mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \ + mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalainstall: (cd $(ROOTDIR)/scala-package && \ - mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \ + mvn install $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") @@ -658,14 +662,14 @@ scalarelease-perform: scaladeploy: (cd $(ROOTDIR)/scala-package && \ - mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ + mvn deploy $(MAVEN_ARGS) -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scaladeploylocal: (cd $(ROOTDIR)/scala-package && \ - mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ + mvn deploy $(MAVEN_ARGS) -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \ -Dgpg.skip \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index fb84e0045d53..e36d353d873c 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -64,7 +64,7 @@ def init_git_win() { // pack libraries for later use def pack_lib(name, libs, include_gcov_data = false) { - sh """ + sh returnStatus: true, script: """ set +e echo "Packing ${libs} into ${name}" echo ${libs} | sed -e 's/,/ /g' | xargs md5sum @@ -83,7 +83,7 @@ return 0 def unpack_and_init(name, libs, include_gcov_data = false) { init_git() unstash name - sh """ + sh returnStatus: true, script: """ set +e echo "Unpacked ${libs} from ${name}" echo ${libs} | sed -e 's/,/ /g' | xargs md5sum @@ -147,8 +147,9 @@ def collect_test_results_windows(original_file_name, new_file_name) { } -def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" +def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "") { + def command = "ci/build.py %ENV_VARS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" + command = command.replaceAll('%ENV_VARS%', env_vars.length() > 0 ? "-e ${env_vars}" : '') command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') command = command.replaceAll('%PLATFORM%', platform) command = command.replaceAll('%FUNCTION_NAME%', function_name) diff --git a/ci/build.py b/ci/build.py index acf277dd2c32..7f1df6cc41c3 100755 --- a/ci/build.py +++ b/ci/build.py @@ -215,20 +215,21 @@ def container_run(platform: str, local_ccache_dir: str, command: List[str], cleanup: Cleanup, + environment: Dict[str, str], dry_run: bool = False) -> int: """Run command in a container""" container_wait_s = 600 # # Environment setup # - environment = { + environment.update({ 'CCACHE_MAXSIZE': '500G', 'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared 'CCACHE_DIR': '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/work/ccache', # this path is inside the container as /work/ccache is # mounted 'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache # verification. - } + }) # These variables are passed to the container to the process tree killer can find runaway # process inside the container # https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller @@ -446,6 +447,10 @@ def main() -> int: parser.add_argument("--no-cache", action="store_true", help="passes --no-cache to docker build") + parser.add_argument("-e", "--environment", nargs="*", default=[], + help="Environment variables for the docker container. " + "Specify with a list containing either names or name=value") + parser.add_argument("command", help="command to run in the container", nargs='*', action='append', type=str) @@ -474,6 +479,9 @@ def signal_handler(signum, _): signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) + environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e])) + for e in args.environment]) + if args.list: print(list_platforms()) elif args.platform: @@ -493,13 +501,13 @@ def signal_handler(signum, _): ret = container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment) elif args.print_docker_run: command = [] ret = container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup, environment=environment) else: # With no commands, execute a build function for the target platform command = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)] @@ -507,7 +515,7 @@ def signal_handler(signum, _): ret = container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment) if ret != 0: logging.critical("Execution of %s failed with status: %d", command, ret) @@ -535,7 +543,7 @@ def signal_handler(signum, _): container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment) shutil.move(buildir(), plat_buildir) logging.info("Built files left in: %s", plat_buildir) diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 6ecb8d801186..22be230efdd3 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -33,4 +33,7 @@ apt-get install -y openjdk-8-jre apt-get update || true apt-get install -y \ maven \ + gnupg \ + gnupg2 \ + gnupg-agent \ scala diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index b4564e0b41ef..715b76a0eb5c 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -36,6 +36,11 @@ clean_repo() { git submodule update --init --recursive } +scala_prepare() { + # Clean up maven logs + export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" +} + build_ccache_wrappers() { set -ex @@ -273,7 +278,6 @@ build_amzn_linux_cpu() { ninja -v } - build_centos7_mkldnn() { set -ex cd /work/mxnet @@ -759,21 +763,24 @@ unittest_ubuntu_python3_quantization_gpu() { unittest_ubuntu_cpu_scala() { set -ex - make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 - make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + scala_prepare + make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 + make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 } unittest_centos7_cpu_scala() { set -ex cd /work/mxnet + scala_prepare make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 } unittest_ubuntu_cpu_clojure() { set -ex - make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 - make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + scala_prepare + make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 + make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 ./contrib/clojure-package/ci-test.sh } @@ -917,8 +924,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() { integrationtest_ubuntu_gpu_scala() { set -ex - make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 - make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + scala_prepare + make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1 + make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 } integrationtest_ubuntu_gpu_dist_kvstore() { @@ -982,7 +990,6 @@ build_docs() { popd } - # Functions that run the nightly Tests: #Runs Apache RAT Check on MXNet Source for License Headers @@ -1171,6 +1178,30 @@ deploy_jl_docs() { # ... } +publish_scala_build() { + set -ex + pushd . + scala_prepare + ./scala-package/dev/build.sh + popd +} + +publish_scala_test() { + set -ex + pushd . + scala_prepare + ./scala-package/dev/test.sh + popd +} + +publish_scala_deploy() { + set -ex + pushd . + scala_prepare + ./scala-package/dev/deploy.sh + popd +} + # broken_link_checker broken_link_checker() { diff --git a/ci/docker_cache.py b/ci/docker_cache.py index fe1882a567aa..24a806c7dc97 100755 --- a/ci/docker_cache.py +++ b/ci/docker_cache.py @@ -211,8 +211,7 @@ def _get_dockerhub_credentials(): # pragma: no cover logging.exception("The request was invalid due to:") elif client_error.response['Error']['Code'] == 'InvalidParameterException': logging.exception("The request had invalid params:") - else: - raise + raise else: secret = get_secret_value_response['SecretString'] secret_dict = json.loads(secret) diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile new file mode 100644 index 000000000000..49e7d7a88d40 --- /dev/null +++ b/ci/publish/Jenkinsfile @@ -0,0 +1,105 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +//mxnet libraries +mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/test-classes/**, scala-package/local-snapshot/**' + +// timeout in minutes +max_time = 120 + +node('restricted-utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') +} +utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') + +// CPU and GPU. OSX nodes are not currently supported by Jenkins +def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU] +def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu'] + +def wrapStep(nodeToRun, workspaceName, step) { + return { + node(nodeToRun) { + ws("workspace/${workspaceName}") { + timeout(time: max_time, unit: 'MINUTES') { + step() + } + } + } + } +} + +def toBuild = [:] +def labels = ['cpu'] // , 'gpu'] +for (x in labels) { + def label = x // Required due to language + toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") { + env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] + utils.init_git() + utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') + utils.pack_lib("scala_${label}", mx_scala_pub, false) + } +} + +def toTest = [:] +def systems = ['ubuntu'] // , 'centos7'] +for (x in labels) { + def label = x // Required due to language + for (y in systems) { + def system = y // Required due to language + toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") { + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("${system}_${label}", 'publish_scala_test', label == 'gpu') + } + } +} + +def toDeploy = [:] +for (x in labels) { + def label = x // Required due to language + toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") { + env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + } +} + +utils.main_wrapper( +core_logic: { + stage('Build Packages') { + parallel toBuild + } + stage('Test Packages') { + parallel toTest + } + stage('Deploy Packages') { + parallel toDeploy + } +} +, +failure_handler: { + if (currentBuild.result == "FAILURE") { + // emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/scala-package/.gitignore b/scala-package/.gitignore index 8bc87f53e802..f69ea0bbe9a4 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -6,4 +6,4 @@ core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala core/src/main/scala/org/apache/mxnet/SymbolBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ -local-snapshot \ No newline at end of file +local-snapshot diff --git a/scala-package/dev/build.sh b/scala-package/dev/build.sh new file mode 100755 index 000000000000..c336fd84e5e2 --- /dev/null +++ b/scala-package/dev/build.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Setup Environment Variables +# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu +# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu + +bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./ + +# Scala steps to deploy +make scalapkg CI=1 + +# Compile tests for discovery later +export GPG_TTY=$(tty) +make scalatestcompile CI=1 +# make scalainstall CI=1 +make scaladeploylocal CI=1 diff --git a/scala-package/dev/buildkey.py b/scala-package/dev/buildkey.py new file mode 100644 index 000000000000..8a1b7bf63286 --- /dev/null +++ b/scala-package/dev/buildkey.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import json +import logging +import subprocess + +HOME = os.environ['HOME'] +KEY_PATH = os.path.join(HOME, ".m2") + + +''' +This file would do the following items: + Import keys from AWS Credential services + Create settings.xml in .m2 with pass phrase + Create security-settings.xml in .m2 with master password + Import keys.asc the encrypted keys in gpg +''' + + +def getCredentials(): + import boto3 + import botocore + endpoint_url = os.environ['MAVEN_PUBLISH_SECRET_ENDPOINT_URL'] + secret_creds_name = os.environ['MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS'] + secret_key_name = os.environ['MAVEN_PUBLISH_SECRET_NAME_GPG'] + region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION'] + + session = boto3.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name, + endpoint_url=endpoint_url + ) + try: + get_secret_value_response = client.get_secret_value( + SecretId=secret_creds_name + ) + get_secret_key_response = client.get_secret_value( + SecretId=secret_key_name + ) + except botocore.exceptions.ClientError as client_error: + if client_error.response['Error']['Code'] == 'ResourceNotFoundException': + name = (secret_key_name if get_secret_value_response + else secret_creds_name) + logging.exception("The requested secret %s was not found", name) + elif client_error.response['Error']['Code'] == 'InvalidRequestException': + logging.exception("The request was invalid due to:") + elif client_error.response['Error']['Code'] == 'InvalidParameterException': + logging.exception("The request had invalid params:") + raise + else: + secret = get_secret_value_response['SecretString'] + secret_dict = json.loads(secret) + secret_key = get_secret_key_response['SecretString'] + return secret_dict, secret_key + + +def importASC(key, gpgPassphrase): + filename = os.path.join(KEY_PATH, "key.asc") + with open(filename, 'w') as f: + f.write(key) + subprocess.check_output(['gpg2', '--batch', '--yes', + '--passphrase-fd', '0', + "--import", "{}".format(filename)], + input=str.encode(gpgPassphrase)) + + +def encryptMasterPSW(password): + filename = os.path.join(KEY_PATH, "encryptMasterPassword.exp") + with open(filename, 'w') as f: + f.write(''' + spawn mvn --encrypt-master-password + expect -exact "Master password: " + send -- "{}\r" + expect eof + '''.format(password)) + result = subprocess.check_output(['expect', filename]) + return str(result).split('\r\n')[-1][2:-3] + + +def encryptPSW(password): + filename = os.path.join(KEY_PATH, "encryptPassword.exp") + with open(filename, 'w') as f: + f.write(''' + spawn mvn --encrypt-password + expect -exact "Password: " + send -- "{}\r" + expect eof + '''.format(password)) + result = subprocess.check_output(['expect', filename]) + return str(result).split('\r\n')[-1][2:-3] + + +def masterPSW(password): + with open(os.path.join(KEY_PATH, "settings-security.xml"), "w") as f: + f.write("\n {}\n" + .format(password)) + + +def serverPSW(username, password, gpgPassphrase): + with open(os.path.join(KEY_PATH, "settings.xml"), "w") as f: + settingsString = ''' + + + + + + apache.snapshots.https + {} + {} + + + + apache.releases.https + {} + {} + + + + + + gpg + + gpg2 + {} + true + + + + + gpg + + '''.format(username, password, username, password, gpgPassphrase) + f.write(settingsString) + + +if __name__ == "__main__": + if not os.path.exists(KEY_PATH): + os.makedirs(KEY_PATH) + credentials, gpgKey = getCredentials() + masterPass = encryptMasterPSW(credentials['masterpass']) + masterPSW(masterPass) + passwordEncrypted = encryptPSW(credentials['password']) + serverPSW(credentials['user'], passwordEncrypted, + credentials['gpgPassphrase']) + importASC(gpgKey, credentials['gpgPassphrase']) diff --git a/scala-package/dev/compile-mxnet-backend.sh b/scala-package/dev/compile-mxnet-backend.sh index b065e01afc8e..114bf0766444 100755 --- a/scala-package/dev/compile-mxnet-backend.sh +++ b/scala-package/dev/compile-mxnet-backend.sh @@ -33,7 +33,7 @@ MXNETDIR=$2 # below routine shamelessly copied from # https://github.com/apache/incubator-mxnet/blob/master/setup-utils/install-mxnet-osx-python.sh -# This routine executes a command, +# This routine executes a command, # prints error message on the console on non-zero exit codes and # returns the exit code to the caller. chkret() { @@ -51,7 +51,10 @@ chkret() { UNAME=`uname -s` chkret pushd $MXNETDIR -chkret git submodule update --init --recursive + +set +e +git submodule update --init --recursive +set -e # don't want to overwrite an existing config file cp make/config.mk ./config.mk diff --git a/scala-package/dev/deploy.sh b/scala-package/dev/deploy.sh new file mode 100755 index 000000000000..6f845ba5d78f --- /dev/null +++ b/scala-package/dev/deploy.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Setup Environment Variables +# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu +# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu + +# Run python to configure keys +python3 $PWD/scala-package/dev/buildkey.py + +# Updating cache +mkdir -p ~/.gnupg +echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf +echo "max-cache-ttl 14400" >> ~/.gnupg/gpg-agent.conf +echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf +echo "pinentry-mode loopback" >> ~/.gnupg/gpg-agent.conf +export GPG_TTY=$(tty) + +cd scala-package +VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec) +cd .. + +# echo "\n\n$VERSION\n" | make scalarelease-dryrun +make scaladeploy CI=1 + +# Clear all password .xml files, exp files, and gpg key files +rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp diff --git a/scala-package/dev/test.sh b/scala-package/dev/test.sh new file mode 100755 index 000000000000..03810fbf8d55 --- /dev/null +++ b/scala-package/dev/test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Test +cd scala-package/packageTest +make scalaintegrationtestlocal CI=1 diff --git a/scala-package/packageTest/Makefile b/scala-package/packageTest/Makefile index 6073ff8a722f..8c12c1d04189 100644 --- a/scala-package/packageTest/Makefile +++ b/scala-package/packageTest/Makefile @@ -43,6 +43,10 @@ else endif endif +ifeq ($(CI), 1) + MAVEN_ARGS := -B +endif + PROFILES := -Ptest ifeq ($(UNIT), 1) PROFILES := "$(PROFILES),unittest" @@ -59,27 +63,27 @@ endif clean: - (mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + (mvn $(MAVEN_ARGS) clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) testinstall: - (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + (mvn $(MAVEN_ARGS) integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ $(PROFILES) \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) testlocal: - (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + (mvn $(MAVEN_ARGS) integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ $(PROFILES),fromLocal \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) testsnapshot: - (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + (mvn $(MAVEN_ARGS) integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ $(PROFILES),fromSnapshots \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.repo=$(MXNET_REPO) \ From 63755febfa12eff3cecd923ab5b23ec9ef3d43c6 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Tue, 8 Jan 2019 17:25:53 -0800 Subject: [PATCH 08/16] Re-organize Scala maven build (#13626) * Re-organize scala maven build 1. Automatically detect which platform to build for scala. 2. Remove platform dependend submodules 3. Fix cyclic module dependencies 4. Fix scalatype style check 5. Now mvn can be executed in submodule 6. Maven build can be executed from any directory not only in root project 7. Checkin javah header file, and use verify task to detect native API changes 8. Improve incremental build performance 9. Remove unittest and integration-test profile, use proper task instead 10. Delete generated scala file during maven clean. * Redo maven deploy related tasks. 1. Removed maven release plugin. 2. Make maven build friendly to CI, allow cli override version. 3. Moved gpg signing to deploy stage. 4. Created a separeated deploy module. 5. Updated Makefile to new maven build change. 6. Remove unused nexus-staging-plugin 7. Added nightly and staging profile for CI. * Support mkldnn for Scala. --- Makefile | 85 +- contrib/clojure-package/project.clj | 2 +- docs/mxdoc.py | 12 +- scala-package/.gitignore | 2 + scala-package/README.md | 100 +- .../assembly/linux-x86_64-cpu/pom.xml | 147 --- .../src/main/assembly/assembly.xml | 50 - .../assembly/linux-x86_64-gpu/pom.xml | 147 --- .../src/main/assembly/assembly.xml | 50 - scala-package/assembly/osx-x86_64-cpu/pom.xml | 143 --- .../src/main/assembly/assembly.xml | 50 - scala-package/assembly/pom.xml | 157 ++-- scala-package/assembly/src/javadoc.xml | 40 - .../assembly/src/main/assembly/assembly.xml | 57 ++ .../assembly/src/main/assembly/javadoc.xml | 15 + .../assembly/src/main/assembly/source.xml | 19 + scala-package/assembly/src/source.xml | 48 - scala-package/core/pom.xml | 137 +-- .../main/scala/org/apache/mxnet/Base.scala | 33 +- .../mxnet/util/NativeLibraryLoader.scala | 53 +- scala-package/deploy/pom.xml | 140 +++ .../deploy/src/main/deploy/deploy.xml | 37 + scala-package/examples/pom.xml | 89 +- .../examples/scripts/module/mnist_mlp.sh | 2 +- .../scripts/module/run_sequential_module.sh | 2 +- .../examples/scripts/run_visualization.sh | 2 +- scala-package/infer/pom.xml | 157 ++-- .../init-native/linux-x86_64/pom.xml | 146 --- scala-package/init-native/osx-x86_64/pom.xml | 158 ---- scala-package/init-native/pom.xml | 174 +++- .../org_apache_mxnet_init_native_c_api.h | 45 + scala-package/init/pom.xml | 107 ++- .../scala/org/apache/mxnet/init/Base.scala | 20 +- scala-package/macros/pom.xml | 88 +- .../org/apache/mxnet/APIDocGenerator.scala | 30 +- scala-package/mxnet-demo/java-demo/Makefile | 2 +- scala-package/mxnet-demo/java-demo/README.md | 4 +- scala-package/mxnet-demo/scala-demo/pom.xml | 2 +- scala-package/native/README.md | 6 +- scala-package/native/linux-x86_64-cpu/pom.xml | 123 --- scala-package/native/linux-x86_64-gpu/pom.xml | 123 --- scala-package/native/osx-x86_64-cpu/pom.xml | 158 ---- scala-package/native/pom.xml | 169 +++- .../native/org_apache_mxnet_native_c_api.cc | 1 + .../native/org_apache_mxnet_native_c_api.h | 853 ++++++++++++++++++ scala-package/pom.xml | 184 ++-- scala-package/spark/bin/run-mnist-example.sh | 4 +- scala-package/spark/pom.xml | 33 +- tools/license_header.py | 39 +- 49 files changed, 2060 insertions(+), 2185 deletions(-) delete mode 100644 scala-package/assembly/linux-x86_64-cpu/pom.xml delete mode 100644 scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml delete mode 100644 scala-package/assembly/linux-x86_64-gpu/pom.xml delete mode 100644 scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml delete mode 100644 scala-package/assembly/osx-x86_64-cpu/pom.xml delete mode 100644 scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml delete mode 100644 scala-package/assembly/src/javadoc.xml create mode 100644 scala-package/assembly/src/main/assembly/assembly.xml create mode 100644 scala-package/assembly/src/main/assembly/javadoc.xml create mode 100644 scala-package/assembly/src/main/assembly/source.xml delete mode 100644 scala-package/assembly/src/source.xml create mode 100644 scala-package/deploy/pom.xml create mode 100644 scala-package/deploy/src/main/deploy/deploy.xml delete mode 100644 scala-package/init-native/linux-x86_64/pom.xml delete mode 100644 scala-package/init-native/osx-x86_64/pom.xml create mode 100644 scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h delete mode 100644 scala-package/native/linux-x86_64-cpu/pom.xml delete mode 100644 scala-package/native/linux-x86_64-gpu/pom.xml delete mode 100644 scala-package/native/osx-x86_64-cpu/pom.xml create mode 100644 scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h diff --git a/Makefile b/Makefile index 3ae3a33e3b42..4f8bedb7e44b 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,6 @@ ROOTDIR = $(CURDIR) TPARTYDIR = $(ROOTDIR)/3rdparty -SCALA_VERSION_PROFILE := scala-2.11 - ifeq ($(OS),Windows_NT) UNAME_S := Windows else @@ -397,18 +395,13 @@ PLUGIN_OBJ = PLUGIN_CUOBJ = include $(MXNET_PLUGINS) -ifeq ($(UNAME_S), Windows) - # TODO(yizhi) currently scala package does not support windows - SCALA_PKG_PROFILE := windows -else +ifneq ($(UNAME_S), Windows) ifeq ($(UNAME_S), Darwin) WHOLE_ARCH= -all_load NO_WHOLE_ARCH= -noall_load - SCALA_PKG_PROFILE := osx-x86_64 else WHOLE_ARCH= --whole-archive NO_WHOLE_ARCH= --no-whole-archive - SCALA_PKG_PROFILE := linux-x86_64 endif endif @@ -427,7 +420,6 @@ ifeq ($(USE_CUDA), 1) # Make sure to add stubs as fallback in order to be able to build # without full CUDA install (especially if run without nvidia-docker) LDFLAGS += -L/usr/local/cuda/lib64/stubs - SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu ifeq ($(USE_NCCL), 1) ifneq ($(USE_NCCL_PATH), NONE) CFLAGS += -I$(USE_NCCL_PATH)/include @@ -439,7 +431,6 @@ ifeq ($(USE_CUDA), 1) CFLAGS += -DMXNET_USE_NCCL=0 endif else - SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu CFLAGS += -DMXNET_USE_NCCL=0 endif @@ -604,80 +595,22 @@ rpkgtest: Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)' scalaclean: - (cd $(ROOTDIR)/scala-package && \ - mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE)) - -scalatestcompile: - (cd $(ROOTDIR)/scala-package && \ - mvn test-compile -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ - -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dcurrent_libdir="$(ROOTDIR)/lib" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + (cd $(ROOTDIR)/scala-package && mvn clean) scalapkg: - (cd $(ROOTDIR)/scala-package && \ - mvn package $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ - -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dcurrent_libdir="$(ROOTDIR)/lib" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + (cd $(ROOTDIR)/scala-package && mvn install -DskipTests) + +scalainstall: + (cd $(ROOTDIR)/scala-package && mvn install) scalaunittest: - (cd $(ROOTDIR)/scala-package && \ - mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) + (cd $(ROOTDIR)/scala-package && mvn install) scalaintegrationtest: - (cd $(ROOTDIR)/scala-package && \ - mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) - -scalainstall: - (cd $(ROOTDIR)/scala-package && \ - mvn install $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \ - -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") - -scalarelease-dryrun: - (cd $(ROOTDIR)/scala-package && \ - mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \ - -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") - -scalarelease-prepare: - (cd $(ROOTDIR)/scala-package && \ - mvn release:clean release:prepare -DautoVersionSubmodules=true \ - -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") - -scalarelease-perform: - (cd $(ROOTDIR)/scala-package && \ - mvn release:perform -DautoVersionSubmodules=true \ - -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") - -scaladeploy: - (cd $(ROOTDIR)/scala-package && \ - mvn deploy $(MAVEN_ARGS) -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ - -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") - -scaladeploylocal: - (cd $(ROOTDIR)/scala-package && \ - mvn deploy $(MAVEN_ARGS) -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ - -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \ - -Dgpg.skip \ - -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + (cd $(ROOTDIR)/scala-package && mvn integration-test -DskipTests=false) jnilint: - 3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src + 3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src --exclude_path scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h rclean: $(RM) -r R-package/src/image_recordio.h R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \ diff --git a/contrib/clojure-package/project.clj b/contrib/clojure-package/project.clj index 1216b239026f..d8bf152a1a7c 100644 --- a/contrib/clojure-package/project.clj +++ b/contrib/clojure-package/project.clj @@ -29,7 +29,7 @@ ;[org.apache.mxnet/mxnet-full_2.11-linux-x86_64-gpu "1.2.1"] ;;; CI - [org.apache.mxnet/mxnet-full_2.11-linux-x86_64-cpu "1.4.0-SNAPSHOT"] + [org.apache.mxnet/mxnet-full_2.11 "INTERNAL"] [org.clojure/tools.logging "0.4.0"] [org.apache.logging.log4j/log4j-core "2.8.1"] diff --git a/docs/mxdoc.py b/docs/mxdoc.py index 7be4b90e799b..35fd175c54aa 100644 --- a/docs/mxdoc.py +++ b/docs/mxdoc.py @@ -113,9 +113,9 @@ def build_scala_docs(app): scala_doc_sources = 'find . -type f -name "*.scala" | egrep \"\.\/core|\.\/infer\" | egrep -v \"\/javaapi\" | egrep -v \"Suite\"' scala_doc_classpath = ':'.join([ '`find native -name "*.jar" | grep "target/lib/" | tr "\\n" ":" `', - '`find macros -name "*-SNAPSHOT.jar" | tr "\\n" ":" `', - '`find core -name "*-SNAPSHOT.jar" | tr "\\n" ":" `', - '`find infer -name "*-SNAPSHOT.jar" | tr "\\n" ":" `' + '`find macros -name "*-INTERNAL.jar" | tr "\\n" ":" `', + '`find core -name "*-INTERNAL.jar" | tr "\\n" ":" `', + '`find infer -name "*-INTERNAL.jar" | tr "\\n" ":" `' ]) # There are unresolvable errors on mxnet 1.2.x. We are ignoring those errors while aborting the ci on newer versions scala_ignore_errors = '; exit 0' if '1.2.' in _BUILD_VER else '' @@ -135,9 +135,9 @@ def build_java_docs(app): java_doc_sources = 'find . -type f -name "*.scala" | egrep \"\.\/core|\.\/infer\" | egrep \"\/javaapi\" | egrep -v \"Suite\"' java_doc_classpath = ':'.join([ '`find native -name "*.jar" | grep "target/lib/" | tr "\\n" ":" `', - '`find macros -name "*-SNAPSHOT.jar" | tr "\\n" ":" `', - '`find core -name "*-SNAPSHOT.jar" | tr "\\n" ":" `', - '`find infer -name "*-SNAPSHOT.jar" | tr "\\n" ":" `' + '`find macros -name "*-INTERNAL.jar" | tr "\\n" ":" `', + '`find core -name "*-INTERNAL.jar" | tr "\\n" ":" `', + '`find infer -name "*-INTERNAL.jar" | tr "\\n" ":" `' ]) _run_cmd('cd {}; scaladoc `{}` -classpath {} -feature -deprecation' .format(java_path, java_doc_sources, java_doc_classpath)) diff --git a/scala-package/.gitignore b/scala-package/.gitignore index f69ea0bbe9a4..bf60e7bef0c5 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -1,9 +1,11 @@ .flattened-pom.xml core/src/main/scala/org/apache/mxnet/NDArrayAPIBase.scala core/src/main/scala/org/apache/mxnet/NDArrayBase.scala +core/src/main/scala/org/apache/mxnet/NDArrayRandomAPIBase.scala core/src/main/scala/org/apache/mxnet/javaapi/NDArrayBase.scala core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala core/src/main/scala/org/apache/mxnet/SymbolBase.scala +core/src/main/scala/org/apache/mxnet/SymbolRandomAPIBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ local-snapshot diff --git a/scala-package/README.md b/scala-package/README.md index 72b3d4d731a5..38b03b9d417b 100644 --- a/scala-package/README.md +++ b/scala-package/README.md @@ -27,6 +27,63 @@ It brings flexible and efficient GPU/CPU computing and state-of-art deep learnin in Scala, Java and other languages built on JVM. - It also enables you to construct and customize the state-of-art deep learning models in JVM languages, and apply them to tasks such as image classification and data science challenges. +- The Scala/Java Inferece APIs provides an easy out of the box solution for loading pre-trained MXNet models and running inference on them. + +Pre-Built Maven Packages +------------------------ + +### Stable ### + +The MXNet Scala/Java packages can be easily included in your Maven managed project. +The stable jar files for the packages are available on the [MXNet Maven Package Repository](https://search.maven.org/search?q=g:org.apache.mxnet) +Currently we provide packages for Linux (Ubuntu 16.04) (CPU and GPU) and macOS (CPU only). Stable packages for Windows and CentOS will come soon. For now, if you have a CentOS machine, follow the ```Build From Source``` section below. + +To add MXNet Scala/Java package to your project, add the dependency as shown below corresponding to your platform, under the ```dependencies``` tag in your project's ```pom.xml``` : + +**Linux GPU** + +maven badge + +```HTML + + org.apache.mxnet + mxnet-full_2.11-linux-x86_64-gpu + [1.3.1,) + +``` + +**Linux CPU** + +maven badge + +```HTML + + org.apache.mxnet + mxnet-full_2.11-linux-x86_64-cpu + [1.3.1,) + +``` + +**macOS CPU** + +maven badge + +```HTML + + org.apache.mxnet + mxnet-full_2.11-osx-x86_64-cpu + [1.3.1,) + +``` + +**Note:** ```[1.3.1,)<\version>``` indicates that we will fetch packages with version 1.3.1 or higher. This will always ensure that the pom.xml is able to fetch the latest and greatest jar files from Maven. + +### Nightly ### + +Apart from these, the nightly builds representing the bleeding edge development on Scala/Java packages are also available on the [MXNet Maven Nexus Package Repository](https://repository.apache.org/#nexus-search;gav~org.apache.mxnet~~~~). +Currently we provide nightly packages for Linux (CPU and GPU) and MacOS (CPU only). The Linux nightly jar files also work on CentOS. Nightly packages for Windows will come soon. + +Add the following ```repository``` to your project's ```pom.xml``` file : Install ------------ @@ -60,31 +117,52 @@ You can also use `mxnet-core_2.10-0.1.1.jar` and put the compiled native library If you have some native libraries conflict with the ones in the provided 'full' jar (e.g., you use openblas instead of atlas), this is a recommended way. Refer to the next section for how to build it from the very source. -Build ------------- +Build From Source +----------------- -Checkout the [Installation Guide](http://mxnet.incubator.apache.org/install/index.html) contains instructions to install mxnet. -Then you can compile the Scala Package by +Checkout the [Installation Guide](http://mxnet.incubator.apache.org/install/index.html) contains instructions to install mxnet package and build it from source. Scala maven build assume you already have a ``lib/libmxnet.so`` file. +If you have built MXNet from source and are looking to setup Scala from that point, you may simply run the following from the MXNet source root, Scala build will detect your platform (OSX/Linux) and libmxnet.so flavor (CPU/GPU): ```bash -make scalapkg +cd scala-package +mvn install ``` (Optional) run unit/integration tests by ```bash -make scalaunittest -make scalaintegrationtest +cd scala-package +mvn integration-test -DskipTests=false ``` Or run a subset of unit tests by, e.g., ```bash -make SCALA_TEST_ARGS=-Dsuites=org.apache.mxnet.NDArraySuite scalaunittest +cd scala-package +mvn -Dsuites=org.apache.mxnet.NDArraySuite integration-test ``` If everything goes well, you will find jars for `assembly`, `core` and `example` modules. -Also it produces the native library in `native/{your-architecture}/target`, which you can use to cooperate with the `core` module. +Also it produces the native library in `native/target`, which you can use to cooperate with the `core` module. + +Deploy to repository +-------------------- + +By default, `maven deploy` will deploy artifacts to local file system, you can file then in: ``scala-package/deploy/target/repo`` folder. + +For nightly build in CI, a snapshot build will be uploaded to apache repository with follow command: + +```bash +cd scala-package +mvn deploy -Pnightly +``` + +Use following command to deploy release build (push artifacts to apache staging repository): + +```bash +cd scala-package +mvn deploy -Pstaging +``` Once you've downloaded and unpacked MNIST dataset to `./data/`, run the training example by @@ -207,3 +285,7 @@ Release License ------- MXNet Scala Package is licensed under [Apache-2](https://github.com/apache/incubator-mxnet/blob/master/scala-package/LICENSE) license. + +MXNet uses some 3rd party softwares. Following 3rd party license files are bundled inside Scala jar file: +* cub/LICENSE.TXT +* mkldnn/external/mklml_mac_2019.0.1.20180928/license.txt diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml deleted file mode 100644 index d3c95a60639a..000000000000 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-full-parent_2.11 - 1.4.0-SNAPSHOT - ../pom.xml - - - mxnet-full_2.11-linux-x86_64-cpu - MXNet Scala Package - Full Linux-x86_64 CPU-only - jar - - - ${project.parent.parent.basedir}/.. - - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - - - org.apache.mxnet - libmxnet-scala-linux-x86_64-cpu - 1.4.0-SNAPSHOT - so - - - org.apache.mxnet - mxnet-infer_${scala.binary.version} - 1.4.0-SNAPSHOT - - - - - - - org.codehaus.mojo - flatten-maven-plugin - - - - remove - - - - - - flatten - process-resources - - flatten - - - - - flatten.clean - clean - - clean - - - - - - org.apache.maven.plugins - maven-deploy-plugin - false - - false - - - - org.apache.maven.plugins - maven-install-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - - - binary-jar - package - - single - - - false - - src/main/assembly/assembly.xml - - - - - sources-jar - package - - single - - - true - sources - - ${project.parent.basedir}/src/source.xml - - - - - javadoc-jar - package - - single - - - true - javadoc - - ${project.parent.basedir}/src/javadoc.xml - - - - - - - - diff --git a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml deleted file mode 100644 index 6b5a5b238216..000000000000 --- a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - libmxnet-scala.so - false - false - false - - org.apache.mxnet:libmxnet-scala-linux-x86_64-cpu:so - - - - - - ${MXNET_DIR}/lib/libmxnet.so - lib/native - - - diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml deleted file mode 100644 index 122beb0977ba..000000000000 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-full-parent_2.11 - 1.4.0-SNAPSHOT - ../pom.xml - - - mxnet-full_2.11-linux-x86_64-gpu - MXNet Scala Package - Full Linux-x86_64 GPU - jar - - - ${project.parent.parent.basedir}/.. - - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - - - org.apache.mxnet - libmxnet-scala-linux-x86_64-gpu - 1.4.0-SNAPSHOT - so - - - org.apache.mxnet - mxnet-infer_${scala.binary.version} - 1.4.0-SNAPSHOT - - - - - - - org.codehaus.mojo - flatten-maven-plugin - - - - remove - - - - - - flatten - process-resources - - flatten - - - - - flatten.clean - clean - - clean - - - - - - org.apache.maven.plugins - maven-deploy-plugin - false - - false - - - - org.apache.maven.plugins - maven-install-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - - - binary-jar - package - - single - - - false - - src/main/assembly/assembly.xml - - - - - sources-jar - package - - single - - - true - sources - - ${project.parent.basedir}/src/source.xml - - - - - javadoc-jar - package - - single - - - true - javadoc - - ${project.parent.basedir}/src/javadoc.xml - - - - - - - - diff --git a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml deleted file mode 100644 index 471f07cf4bfc..000000000000 --- a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - libmxnet-scala.so - false - false - false - - org.apache.mxnet:libmxnet-scala-linux-x86_64-gpu:so - - - - - - ${MXNET_DIR}/lib/libmxnet.so - lib/native - - - diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml deleted file mode 100644 index 68fb09df68e3..000000000000 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ /dev/null @@ -1,143 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-full-parent_2.11 - 1.4.0-SNAPSHOT - ../pom.xml - - - mxnet-full_2.11-osx-x86_64-cpu - MXNet Scala Package - Full OSX-x86_64 CPU-only - jar - - - ${project.parent.parent.basedir}/.. - - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - - - org.apache.mxnet - libmxnet-scala-osx-x86_64-cpu - 1.4.0-SNAPSHOT - jnilib - - - org.apache.mxnet - mxnet-infer_${scala.binary.version} - 1.4.0-SNAPSHOT - - - - - - - org.codehaus.mojo - flatten-maven-plugin - - - - remove - - - - - - flatten - process-resources - - flatten - - - - - flatten.clean - clean - - clean - - - - - - org.apache.maven.plugins - maven-deploy-plugin - false - - false - - - - org.apache.maven.plugins - maven-assembly-plugin - - - binary-jar - package - - single - - - false - - src/main/assembly/assembly.xml - - - - - sources-jar - package - - single - - - true - sources - - ${project.parent.basedir}/src/source.xml - - - - - javadoc-jar - package - - single - - - true - javadoc - - ${project.parent.basedir}/src/javadoc.xml - - - - - - - - diff --git a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml deleted file mode 100644 index 3f632a76daf6..000000000000 --- a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - libmxnet-scala.jnilib - false - false - false - - org.apache.mxnet:libmxnet-scala-osx-x86_64-cpu:jnilib - - - - - - ${MXNET_DIR}/lib/libmxnet.so - lib/native - - - diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index 2d7366fc83e2..b93c181bc1d4 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -21,106 +21,109 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-full-parent_2.11 - MXNet Scala Package - Full Parent + mxnet-full_2.11 + Assembly Scala Package pom + + ${project.parent.basedir}/.. + + + + + org.apache.mxnet + mxnet-core + INTERNAL + + + org.apache.mxnet + libmxnet-scala + INTERNAL + ${libtype} + + + org.apache.mxnet + mxnet-infer + INTERNAL + + + - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - release + staging - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-source-plugin - - - package - - jar-no-fork - - - true - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - package - - jar - - - true - - commons-codec:* - org.scala-lang:* - log4j:* - org.slf4j:* - - - - - org.apache.maven.plugins maven-gpg-plugin + 1.6 sign-artifacts - verify + deploy sign - - org.sonatype.plugins - nexus-staging-maven-plugin - true - - ossrh - https://oss.sonatype.org/ - true - - + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + binary-jar + package + + single + + + false + + src/main/assembly/assembly.xml + + + + + sources-jar + package + + single + + + true + + src/main/assembly/source.xml + + + + + javadoc-jar + package + + single + + + true + + src/main/assembly/javadoc.xml + + + + + + + diff --git a/scala-package/assembly/src/javadoc.xml b/scala-package/assembly/src/javadoc.xml deleted file mode 100644 index 8a91221372ac..000000000000 --- a/scala-package/assembly/src/javadoc.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - bundle - - jar - - false - - - - - - - org.apache.mxnet:mxnet-core_${scala.binary.version} - - true - - false - javadoc - / - true - - - - \ No newline at end of file diff --git a/scala-package/assembly/src/main/assembly/assembly.xml b/scala-package/assembly/src/main/assembly/assembly.xml new file mode 100644 index 000000000000..7525df883dab --- /dev/null +++ b/scala-package/assembly/src/main/assembly/assembly.xml @@ -0,0 +1,57 @@ + + full + + jar + + false + + + + *:*:jar + + + org.scala-lang:* + org.scala-lang.modules:* + + / + true + true + runtime + + + lib/native + libmxnet-scala.${libtype} + false + false + false + + org.apache.mxnet:libmxnet-scala:${libtype} + + + + + + ${MXNET_DIR}/lib + + libmxnet.so + libgfortran.so.3 + libquadmath.so.0 + libiomp5.so + libiomp5.dylib + libmklml_intel.so + libmklml.dylib + libmkldnn.so.0 + libmkldnn.0.dylib + + lib/native + + + ${MXNET_DIR}/3rdparty + + cub/LICENSE.TXT + mkldnn/external/mklml_mac_2019.0.1.20180928/license.txt + + / + + + diff --git a/scala-package/assembly/src/main/assembly/javadoc.xml b/scala-package/assembly/src/main/assembly/javadoc.xml new file mode 100644 index 000000000000..8f30a261811c --- /dev/null +++ b/scala-package/assembly/src/main/assembly/javadoc.xml @@ -0,0 +1,15 @@ + + bundle + + jar + + false + + + ${rootdir}/core/target/site/scaladocs + / + + + diff --git a/scala-package/assembly/src/main/assembly/source.xml b/scala-package/assembly/src/main/assembly/source.xml new file mode 100644 index 000000000000..87fcde360c48 --- /dev/null +++ b/scala-package/assembly/src/main/assembly/source.xml @@ -0,0 +1,19 @@ + + src + + + jar + + false + + + ${rootdir}/core/src/main/scala + + **\/*.scala + + / + + + diff --git a/scala-package/assembly/src/source.xml b/scala-package/assembly/src/source.xml deleted file mode 100644 index 003c0e0c87f5..000000000000 --- a/scala-package/assembly/src/source.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - src - - - jar - - false - - - true - - - org.apache.mxnet:mxnet-core_${scala.binary.version} - - - false - - - - src/main/scala - - **\/*.scala - - / - - - - - - \ No newline at end of file diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 7b3cffc96f6b..0bc672d41bd8 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -21,55 +21,70 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - - true - ${project.parent.basedir}/.. - - - mxnet-core_2.11 + mxnet-core MXNet Scala Package - Core - - - unittest - - false - - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - + org.codehaus.mojo + native-maven-plugin + true + + + javah + verify + + default + ${project.build.directory}/custom-javah + ${basedir} + org_apache_mxnet_native_c_api.h + + org.apache.mxnet.LibInfo + + + + javah + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + verify-javah + verify + + exec + + + diff + ${project.build.directory}/custom-javah/org_apache_mxnet_native_c_api.h ${project.parent.basedir}/native/src/main/native/org_apache_mxnet_native_c_api.h + + + + apidoc-generation + generate-sources + + exec + + + java + ${project.parent.basedir} + -classpath %classpath:${rootdir}/init/target/classes:${rootdir}/macros/target/classes -Djava.library.path=${rootdir}/native/target org.apache.mxnet.APIDocGenerator ${rootdir}/core/src/main/scala/org/apache/mxnet/ + + + + + org.apache.maven.plugins maven-jar-plugin @@ -85,18 +100,35 @@ org.apache.maven.plugins maven-compiler-plugin + + org.apache.maven.plugins + maven-clean-plugin + 3.1.0 + + + + src/main/scala/org/apache/mxnet + + NDArrayAPIBase.scala + NDArrayBase.scala + NDArrayRandomAPIBase.scala + javaapi/NDArrayBase.scala + SymbolAPIBase.scala + SymbolRandomAPIBase.scala + + false + + + + org.scalatest scalatest-maven-plugin - ${skipTests} - -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ + -Djava.library.path=${project.parent.basedir}/native/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties - - ${MXNET_DIR}/lib - @@ -105,13 +137,8 @@ 2.22.0 - -Djava.library.path=${project.parent.basedir}/native/${platform}/target + -Djava.library.path=${project.parent.basedir}/native/target - ${skipTests} - always - - ${MXNET_DIR}/lib - @@ -123,14 +150,14 @@ org.apache.mxnet - mxnet-init_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-macros + INTERNAL provided org.apache.mxnet - mxnet-macros_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-scala-init + INTERNAL provided diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/Base.scala b/scala-package/core/src/main/scala/org/apache/mxnet/Base.scala index b2a53fd9f2dd..3515fcee8e70 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/Base.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/Base.scala @@ -67,6 +67,8 @@ private[mxnet] object Base { "Consider installing the library somewhere in the path " + "(for Windows: PATH, for Linux: LD_LIBRARY_PATH), " + "or specifying by Java cmd option -Djava.library.path=[lib path].") + logger.warn("LD_LIBRARY_PATH=" + System.getenv("LD_LIBRARY_PATH")) + logger.warn("java.library.path=" + System.getProperty("java.library.path")) NativeLibraryLoader.loadLibrary("mxnet-scala") } } catch { @@ -87,35 +89,8 @@ private[mxnet] object Base { @throws(classOf[UnsatisfiedLinkError]) private def tryLoadLibraryOS(libname: String): Unit = { - try { - logger.info(s"Try loading $libname from native path.") - System.loadLibrary(libname) - } catch { - case e: UnsatisfiedLinkError => - val os = System.getProperty("os.name") - // ref: http://lopica.sourceforge.net/os.html - if (os.startsWith("Linux")) { - tryLoadLibraryXPU(libname, "linux-x86_64") - } else if (os.startsWith("Mac")) { - tryLoadLibraryXPU(libname, "osx-x86_64") - } else { - // TODO(yizhi) support windows later - throw new UnsatisfiedLinkError() - } - } - } - - @throws(classOf[UnsatisfiedLinkError]) - private def tryLoadLibraryXPU(libname: String, arch: String): Unit = { - try { - // try gpu first - logger.info(s"Try loading $libname-$arch-gpu from native path.") - System.loadLibrary(s"$libname-$arch-gpu") - } catch { - case e: UnsatisfiedLinkError => - logger.info(s"Try loading $libname-$arch-cpu from native path.") - System.loadLibrary(s"$libname-$arch-cpu") - } + logger.info(s"Try loading $libname from native path.") + System.loadLibrary(libname) } // helper function definitions diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala index 2ce893b478ed..1e6d9c4f9fce 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala @@ -85,8 +85,16 @@ private[mxnet] object NativeLibraryLoader { } logger.debug(s"Attempting to load $loadLibname") val libFileInJar = libPathInJar + loadLibname - saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so") - val tempfile: File = saveLibraryToTemp(libname, libFileInJar) + saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so", true) + saveLibraryToTemp("libgfortran.so.3", "/lib/native/libgfortran.so.3", false) + saveLibraryToTemp("libquadmath.so.0", "/lib/native/libquadmath.so.0", false) + saveLibraryToTemp("libiomp5.so", "/lib/native/libiomp5.so", false) + saveLibraryToTemp("libiomp5.dylib", "/lib/native/libiomp5.dylib", false) + saveLibraryToTemp("libmklml_intel.so", "/lib/native/libmklml_intel.so", false) + saveLibraryToTemp("libmklml.dylib", "/lib/native/libmklml.dylib", false) + saveLibraryToTemp("libmkldnn.so.0", "/lib/native/libmkldnn.so.0", false) + saveLibraryToTemp("libmkldnn.0.dylib", "/lib/native/libmkldnn.0.dylib", false) + val tempfile: File = saveLibraryToTemp(libname, libFileInJar, true) loadLibraryFromFile(libname, tempfile) } @@ -134,29 +142,34 @@ private[mxnet] object NativeLibraryLoader { * * @param libname name of the library (just used in constructing the library name) * @param resource String resource path in the jar file + * @param required true if library is required */ - private def saveLibraryToTemp(libname: String, resource: String): File = { + private def saveLibraryToTemp(libname: String, resource: String, required: Boolean): File = { try { val is: InputStream = getClass.getResourceAsStream(resource) if (is == null) { - throw new UnsatisfiedLinkError(s"Couldn't find the resource $resource") - } - - val tempfile: File = new File(_tempDir, libname) - val os: OutputStream = new FileOutputStream(tempfile) - logger.debug("tempfile.getPath() = {}", tempfile.getPath) - val savedTime: Long = System.currentTimeMillis - val buf: Array[Byte] = new Array[Byte](8192) - var len: Int = is.read(buf) - while (len > 0) { - os.write(buf, 0, len) - len = is.read(buf) + if (required) { + throw new UnsatisfiedLinkError(s"Couldn't find the resource $resource") + } else { + null + } + } else { + val tempfile: File = new File(_tempDir, libname) + val os: OutputStream = new FileOutputStream(tempfile) + logger.debug("tempfile.getPath() = {}", tempfile.getPath) + val savedTime: Long = System.currentTimeMillis + val buf: Array[Byte] = new Array[Byte](8192) + var len: Int = is.read(buf) + while (len > 0) { + os.write(buf, 0, len) + len = is.read(buf) + } + os.close() + is.close() + val seconds: Double = (System.currentTimeMillis - savedTime).toDouble / 1e3 + logger.debug(s"Copying $libname took $seconds seconds.") + tempfile } - os.close() - is.close() - val seconds: Double = (System.currentTimeMillis - savedTime).toDouble / 1e3 - logger.debug(s"Copying $libname took $seconds seconds.") - tempfile } catch { case io: IOException => throw new UnsatisfiedLinkError(s"Could not create temp file for $libname") diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml new file mode 100644 index 000000000000..c51aa9a92090 --- /dev/null +++ b/scala-package/deploy/pom.xml @@ -0,0 +1,140 @@ + + + 4.0.0 + + org.apache.mxnet + mxnet-parent + INTERNAL + ../pom.xml + + + mxnet-deployment + ${revision} + MXNet Scala Package - Full ${platform}-only + pom + + Scala Package for Apache MXNet (Incubating) - flexible and efficient library for deep learning. + + + + mxnet-full_2.11-${platform}-${flavor} + 1.5.0-SNAPSHOT + apache.snapshots.https + file://${project.build.directory}/repo + + + + + + + + org.apache.mxnet + mxnet-full_2.11 + INTERNAL + + + + + + staging + + 1.5.0 + apache.releases.https + https://repository.apache.org/content/repositories/staging + jar.asc,asc,asc + ,sources.jar,javadoc.jar + + ../assembly/target/mxnet-full_2.11-INTERNAL.jar.asc,../assembly/target/mxnet-full_2.11-INTERNAL-src.jar.asc,../assembly/target/mxnet-full_2.11-INTERNAL-bundle.jar.asc + + + + + nightly + + apache.snapshots.https + https://repository.apache.org/content/repositories/snapshots + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + false + + + + + com.google.code.maven-replacer-plugin + replacer + 1.5.3 + + + deploy + + replace + + + + + ${basedir}/src/main/deploy/deploy.xml + ${project.build.directory}/deploy.xml + + + DESCRIPTION + ${project.description} + + + ARTIFACT_ID + ${ARTIFACT_ID} + + + PROJECT_VERSION + ${project.version} + + + SCALA_VERSION + ${scala.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + deploy-file + deploy + + deploy-file + + + ${project.description} + ${repositoryId} + ${repo_url} + ${project.groupId} + ${ARTIFACT_ID} + ${project.version} + jar + ${project.build.directory}/deploy.xml + ${rootdir}/assembly/target/mxnet-full_2.11-INTERNAL.jar + ${rootdir}/assembly/target/mxnet-full_2.11-INTERNAL-src.jar + ${rootdir}/assembly/target/mxnet-full_2.11-INTERNAL-bundle.jar + ${deploy_asc_types} + ${deploy_asc_classifers} + ${deploy_asc_files} + + + + + + + + diff --git a/scala-package/deploy/src/main/deploy/deploy.xml b/scala-package/deploy/src/main/deploy/deploy.xml new file mode 100644 index 000000000000..97ba5f0ceb49 --- /dev/null +++ b/scala-package/deploy/src/main/deploy/deploy.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + org.apache.mxnet + ARTIFACT_ID + PROJECT_VERSION + DESCRIPTION + + + org.scala-lang + scala-library + SCALA_VERSION + runtime + + + org.scala-lang + scala-reflect + SCALA_VERSION + runtime + + + org.scala-lang.modules + scala-parser-combinators_2.11 + 1.0.4 + runtime + + + org.scala-lang + scala-compiler + SCALA_VERSION + runtime + + + diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 1ebc37a0d456..549622d15fb7 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -21,88 +21,18 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-examples_2.11 + mxnet-examples MXNet Scala Package - Examples true - ${project.parent.basedir}/.. - - - integrationtest - - false - - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - deployLocal - - - - org.apache.maven.plugins - maven-deploy-plugin - - false - - - - - - - release - - - - org.apache.maven.plugins - maven-source-plugin - - true - - - - org.apache.maven.plugins - maven-gpg-plugin - - true - - - - org.sonatype.plugins - nexus-staging-maven-plugin - - true - - - - - - - @@ -165,12 +95,9 @@ ${skipTests} - -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ + -Djava.library.path=${project.parent.basedir}/native/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties - - ${MXNET_DIR}/lib - @@ -182,14 +109,14 @@ org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-core + INTERNAL provided org.apache.mxnet - mxnet-infer_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-infer + INTERNAL provided diff --git a/scala-package/examples/scripts/module/mnist_mlp.sh b/scala-package/examples/scripts/module/mnist_mlp.sh index 29306706dbbe..e16e16e246b2 100755 --- a/scala-package/examples/scripts/module/mnist_mlp.sh +++ b/scala-package/examples/scripts/module/mnist_mlp.sh @@ -18,7 +18,7 @@ # under the License. ROOT_DIR=$(cd `dirname $0`/../../..; pwd) -CLASSPATH=$ROOT_DIR/assembly/osx-x86_64-cpu/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* +CLASSPATH=$ROOT_DIR/assembly/osx-x86_64/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* mkdir -p model java -Xmx4G -cp $CLASSPATH \ diff --git a/scala-package/examples/scripts/module/run_sequential_module.sh b/scala-package/examples/scripts/module/run_sequential_module.sh index 9e1f30e3687b..1e40d2dd83f7 100644 --- a/scala-package/examples/scripts/module/run_sequential_module.sh +++ b/scala-package/examples/scripts/module/run_sequential_module.sh @@ -18,7 +18,7 @@ # under the License. ROOT_DIR=$(cd `dirname $0`/../../..; pwd) -CLASSPATH=$ROOT_DIR/assembly/linux-x86_64-cpu/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* +CLASSPATH=$ROOT_DIR/assembly/linux-x86_64/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* DATA_DIR=$ROOT_DIR/core/data diff --git a/scala-package/examples/scripts/run_visualization.sh b/scala-package/examples/scripts/run_visualization.sh index 3e8f5cd784f9..7f5b94fbe2b5 100644 --- a/scala-package/examples/scripts/run_visualization.sh +++ b/scala-package/examples/scripts/run_visualization.sh @@ -19,7 +19,7 @@ MXNET_ROOT=$(cd "$(dirname $0)/../../.."; pwd) -CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* +CLASS_PATH=$MXNET_ROOT/scala-package/assembly/assembly/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* # please install the Graphviz library # if you are using ubuntu, use the following command: diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 6a8e77d63e11..1daf79ee6cc0 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -18,106 +18,63 @@ - 4.0.0 - - mxnet-parent_2.11 - org.apache.mxnet - 1.4.0-SNAPSHOT - ../pom.xml - + 4.0.0 + + mxnet-parent + org.apache.mxnet + INTERNAL + ../pom.xml + - mxnet-infer_2.11 - MXNet Scala Package - Inference + mxnet-infer + MXNet Scala Package - Inference - - true - ${project.parent.basedir}/.. - - - - - unittest - - false - - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-jar-plugin - - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.scalatest - scalatest-maven-plugin - - ${skipTests} - - -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ - -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties - - - ${MXNET_DIR}/lib - - - - - org.scalastyle - scalastyle-maven-plugin - - - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - provided - - - - org.mockito - mockito-all - 1.10.19 - test - - + + + + org.apache.maven.plugins + maven-jar-plugin + + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.scalatest + scalatest-maven-plugin + + + -Djava.library.path=${project.parent.basedir}/native/target \ + -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + + + + org.scalastyle + scalastyle-maven-plugin + + + + + + org.apache.mxnet + mxnet-core + INTERNAL + provided + + + + org.mockito + mockito-all + 1.10.19 + test + + diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml deleted file mode 100644 index 2f0f586ee2e0..000000000000 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ /dev/null @@ -1,146 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-init-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-init-scala-linux-x86_64 - MXNet Scala Package - Initializer Native Linux-x86_64 - http://maven.apache.org - - so - - - ${project.parent.parent.basedir}/.. - - - - - org.apache.mxnet - mxnet-init_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - linux - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_init_native_c_api.cc - - - - - -std=c++0x - - - -I${MXNET_DIR}/include - -I${MXNET_DIR}/3rdparty/dmlc-core/include - -I${MXNET_DIR}/3rdparty/mshadow - -I${MXNET_DIR}/3rdparty/dlpack/include - -I${MXNET_DIR}/3rdparty/tvm/nnvm/include - -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 - -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c - -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs - - - -shared - - - -Wl,--whole-archive - -Wl,--no-whole-archive -pthread -lm -fopenmp -lrt - - - -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib - - - - - - javah - generate-sources - - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_init_native_c_api.h - - org.apache.mxnet.init.LibInfo - - - - javah - - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - link-native-lib - generate-resources - - exec - - - ln - -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so - - - - - - - diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml deleted file mode 100644 index 59a04192ad5c..000000000000 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ /dev/null @@ -1,158 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-init-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-init-scala-osx-x86_64 - MXNet Scala Package - Initializer Native OSX-x86_64 - http://maven.apache.org - - jnilib - - - ${project.parent.parent.basedir}/.. - - - - - org.apache.mxnet - mxnet-init_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - darwin - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_init_native_c_api.cc - - - - - -std=c++0x - - - -I${MXNET_DIR}/include - -I${MXNET_DIR}/3rdparty/dmlc-core/include - -I${MXNET_DIR}/3rdparty/mshadow - -I${MXNET_DIR}/3rdparty/dlpack/include - -I${MXNET_DIR}/3rdparty/tvm/nnvm/include - -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 - -g -O0 -fPIC -msse3 -mf16c - -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs - - - -shared - - - -framework JavaVM - -Wl,-exported_symbol,_Java_* - -Wl,-x - - - -lmxnet -L${MXNET_DIR}/lib - - - - - - javah - generate-sources - - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_init_native_c_api.h - - org.apache.mxnet.init.LibInfo - - - - javah - - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - post-native-build - package - - exec - - - install_name_tool - -change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib - - - - link-native-lib - generate-resources - - exec - - - ln - -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so - - - - - - - diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index d70ec3132374..1254f615c64b 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -21,46 +21,170 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-scala-init-native-parent - MXNet Scala Package - Initializer Native Parent - pom + libmxnet-init-scala + MXNet Scala Package - Initializer Native + + + ${project.parent.basedir}/.. + + + ${libtype} - osx-x86_64-cpu - - osx-x86_64 - - - - linux-x86_64-cpu - - linux-x86_64 - + osx-x86_64 + + mac + + + + + org.codehaus.mojo + native-maven-plugin + true + + darwin + generic-classic + ${cxx} + ${cxx} + + + src/main/native + + org_apache_mxnet_init_native_c_api.cc + + + + + -std=c++0x + + + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs + + + -shared + + + -framework JavaVM + -Wl,-exported_symbol,_Java_* + -Wl,-x + + + -lmxnet -L${MXNET_DIR}/lib + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -add_rpath @loader_path ${project.build.directory}/${project.artifactId}.jnilib + + + + + + - linux-x86_64-gpu - - linux-x86_64 - + linux-x86_64 + + + unix + Linux + + + + + + org.codehaus.mojo + native-maven-plugin + true + + linux + generic-classic + ${cxx} + ${cxx} + + + src/main/native + + org_apache_mxnet_init_native_c_api.cc + + + + + -std=c++0x + + + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs + + + -shared + + + -Wl,--whole-archive + -Wl,--no-whole-archive -pthread -lm -fopenmp -lrt + + + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib + + + + + - org.apache.maven.plugins - maven-deploy-plugin - - true - + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + link-native-lib + generate-resources + + exec + + + bash + -c 'ln -sf ${MXNET_DIR}/lib/* ${project.build.directory}/' + + + - diff --git a/scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h b/scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h new file mode 100644 index 000000000000..6ff6ae6a107c --- /dev/null +++ b/scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_apache_mxnet_init_LibInfo */ + +#ifndef _Included_org_apache_mxnet_init_LibInfo +#define _Included_org_apache_mxnet_init_LibInfo +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_apache_mxnet_init_LibInfo + * Method: mxSymbolListAtomicSymbolCreators + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_init_LibInfo_mxSymbolListAtomicSymbolCreators + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_init_LibInfo + * Method: mxSymbolGetAtomicSymbolInfo + * Signature: (JLorg/apache/mxnet/init/Base/RefString;Lorg/apache/mxnet/init/Base/RefString;Lorg/apache/mxnet/init/Base/RefInt;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/init/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_init_LibInfo_mxSymbolGetAtomicSymbolInfo + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_init_LibInfo + * Method: mxListAllOpNames + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_init_LibInfo_mxListAllOpNames + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_init_LibInfo + * Method: nnGetOpHandle + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/init/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_init_LibInfo_nnGetOpHandle + (JNIEnv *, jobject, jstring, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index 3b46f83d0b92..30e24975a841 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -21,65 +21,62 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT - + mxnet-parent + INTERNAL + ../pom.xml - mxnet-init_2.11 + mxnet-scala-init MXNet Scala Package - Initializer - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - - apache-release - - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - compile-mxnet-backend - compile - - exec - - - bash - ${project.parent.basedir}/dev/compile-mxnet-backend.sh ${build.platform} ${project.parent.basedir}/../ - - - - - - org.apache.maven.plugins - maven-deploy-plugin + + + + org.codehaus.mojo + native-maven-plugin + true + + + javah + verify - true + default + ${project.build.directory}/custom-javah + ${basedir} + org_apache_mxnet_init_native_c_api.h + + org.apache.mxnet.init.LibInfo + - - - - - + + javah + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + verify-javah + verify + + exec + + + diff + + ${project.build.directory}/custom-javah/org_apache_mxnet_init_native_c_api.h + ${project.parent.basedir}/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h + + + + + + + + diff --git a/scala-package/init/src/main/scala/org/apache/mxnet/init/Base.scala b/scala-package/init/src/main/scala/org/apache/mxnet/init/Base.scala index 7402dbd3bc1d..b5a6286af1b6 100644 --- a/scala-package/init/src/main/scala/org/apache/mxnet/init/Base.scala +++ b/scala-package/init/src/main/scala/org/apache/mxnet/init/Base.scala @@ -17,6 +17,8 @@ package org.apache.mxnet.init +import java.io.File + object Base { tryLoadInitLibrary() val _LIB = new LibInfo @@ -37,18 +39,22 @@ object Base { @throws(classOf[UnsatisfiedLinkError]) private def tryLoadInitLibrary(): Unit = { - var baseDir = System.getProperty("user.dir") + "/init-native" - // TODO(lanKing520) Update this to use relative path to the MXNet director. - // TODO(lanking520) baseDir = sys.env("MXNET_BASEDIR") + "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/scala-package/init-native" - if (System.getenv().containsKey("MXNET_BASEDIR")) { - baseDir = sys.env("MXNET_BASEDIR") + var userDir : File = new File(System.getProperty("user.dir")) + var nativeDir : File = new File(userDir, "init-native") + if (!nativeDir.exists()) { + nativeDir = new File(userDir.getParent, "init-native") + if (!nativeDir.exists()) { + throw new IllegalStateException("scala-init should be executed inside scala-package folder") + } } + val baseDir = nativeDir.getAbsolutePath + val os = System.getProperty("os.name") // ref: http://lopica.sourceforge.net/os.html if (os.startsWith("Linux")) { - System.load(s"$baseDir/linux-x86_64/target/libmxnet-init-scala-linux-x86_64.so") + System.load(s"$baseDir/target/libmxnet-init-scala.so") } else if (os.startsWith("Mac")) { - System.load(s"$baseDir/osx-x86_64/target/libmxnet-init-scala-osx-x86_64.jnilib") + System.load(s"$baseDir/target/libmxnet-init-scala.jnilib") } else { // TODO(yizhi) support windows later throw new UnsatisfiedLinkError() diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index c24403f5e630..d6dd0201897f 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -21,63 +21,20 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-macros_2.11 + mxnet-macros MXNet Scala Package - Macros - - - unittest - - false - - - - integrationtest - - true - - - - osx-x86_64-cpu - - osx-x86_64 - jnilib - - - - linux-x86_64-cpu - - linux-x86_64 - so - - - - linux-x86_64-gpu - - linux-x86_64 - so - - - - org.apache.mxnet - mxnet-init_${scala.binary.version} - 1.4.0-SNAPSHOT - provided - - - org.apache.mxnet - libmxnet-init-scala-${platform} - 1.4.0-SNAPSHOT + mxnet-scala-init + INTERNAL provided - ${libtype} commons-io @@ -86,16 +43,8 @@ - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - org.apache.maven.plugins maven-jar-plugin @@ -111,39 +60,15 @@ org.apache.maven.plugins maven-compiler-plugin - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - apidoc-generation - package - - java - - - - - - ${project.parent.basedir}/init/target/classes - - - ${project.parent.basedir}/core/src/main/scala/org/apache/mxnet/ - - org.apache.mxnet.APIDocGenerator - - org.scalatest scalatest-maven-plugin - ${skiptest} ${project.parent.basedir}/init-native - -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ + -Djava.library.path=${project.parent.basedir}/native/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties @@ -154,5 +79,4 @@ - diff --git a/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala b/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala index ce12dc7cd5a0..8f43421ca1b5 100644 --- a/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala +++ b/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala @@ -228,21 +228,21 @@ private[mxnet] object APIDocGenerator extends GeneratorBase { val finalStr = s"""/* - |* Licensed to the Apache Software Foundation (ASF) under one or more - |* contributor license agreements. See the NOTICE file distributed with - |* this work for additional information regarding copyright ownership. - |* The ASF licenses this file to You under the Apache License, Version 2.0 - |* (the "License"); you may not use this file except in compliance with - |* the License. You may obtain a copy of the License at - |* - |* http://www.apache.org/licenses/LICENSE-2.0 - |* - |* Unless required by applicable law or agreed to in writing, software - |* distributed under the License is distributed on an "AS IS" BASIS, - |* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - |* See the License for the specific language governing permissions and - |* limitations under the License. - |*/ + | * Licensed to the Apache Software Foundation (ASF) under one or more + | * contributor license agreements. See the NOTICE file distributed with + | * this work for additional information regarding copyright ownership. + | * The ASF licenses this file to You under the Apache License, Version 2.0 + | * (the "License"); you may not use this file except in compliance with + | * the License. You may obtain a copy of the License at + | * + | * http://www.apache.org/licenses/LICENSE-2.0 + | * + | * Unless required by applicable law or agreed to in writing, software + | * distributed under the License is distributed on an "AS IS" BASIS, + | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + | * See the License for the specific language governing permissions and + | * limitations under the License. + | */ | |$packageDef | diff --git a/scala-package/mxnet-demo/java-demo/Makefile b/scala-package/mxnet-demo/java-demo/Makefile index 6b76e272f9e0..4f2b5e938970 100644 --- a/scala-package/mxnet-demo/java-demo/Makefile +++ b/scala-package/mxnet-demo/java-demo/Makefile @@ -16,7 +16,7 @@ # under the License. SCALA_VERSION_PROFILE := 2.11 -MXNET_VERSION := 1.3.1-SNAPSHOT +MXNET_VERSION := [1.5.0-SNAPSHOT,\) ifeq ($(OS),Windows_NT) UNAME_S := Windows diff --git a/scala-package/mxnet-demo/java-demo/README.md b/scala-package/mxnet-demo/java-demo/README.md index 952775dd964e..0f9a13b76447 100644 --- a/scala-package/mxnet-demo/java-demo/README.md +++ b/scala-package/mxnet-demo/java-demo/README.md @@ -29,7 +29,7 @@ You can use the following instruction as an alternative to achieve the same resu User are required to use `mvn package` to build the package, which are shown below: ```Bash -export SCALA_VERSION_PROFILE=2.11 MXNET_VERSION=1.3.1-SNAPSHOT +export SCALA_VERSION_PROFILE=2.11 MXNET_VERSION=1.5.0-SNAPSHOT export SCALA_PKG_PROFILE= mvn package -Dmxnet.profile=$SCALA_PKG_PROFILE \ -Dmxnet.scalaprofile=$SCALA_VERSION_PROFILE \ @@ -97,5 +97,5 @@ sudo apt install libopencv-imgcodecs3.4 Is there any other version available? -You can find nightly release version from [here](https://repository.apache.org/#nexus-search;gav~org.apache.mxnet~~1.3.1-SNAPSHOT~~). +You can find nightly release version from [here](https://repository.apache.org/#nexus-search;gav~org.apache.mxnet~~1.5.0-SNAPSHOT~~). Please keep the same version in the Makefile or [above version](https://repository.apache.org/#nexus-search;gav~org.apache.mxnet~~~~) to run this demo. diff --git a/scala-package/mxnet-demo/scala-demo/pom.xml b/scala-package/mxnet-demo/scala-demo/pom.xml index 8d63d0f918ce..c2a14d878427 100644 --- a/scala-package/mxnet-demo/scala-demo/pom.xml +++ b/scala-package/mxnet-demo/scala-demo/pom.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 Demo - mxnet-scala-demo_2.11 + mxnet-scala-demo 1.0-SNAPSHOT MXNet Scala Demo pom diff --git a/scala-package/native/README.md b/scala-package/native/README.md index cb6dd3890dd2..c87b064fff02 100644 --- a/scala-package/native/README.md +++ b/scala-package/native/README.md @@ -6,7 +6,11 @@ MXNet Scala JNI is a thin wrapper layer of underlying libmxnet.so. JNI native code requires a header file that matches the java/scala interface, this file is usually generated with javah. -In our case, jni_helper_func.h is generated and will be used to compile native code. +In our case, org_apache_mxnet_native_c.h is generated and will be used to compile native code. + +To improve build performance, we check in generated org_apache_mxnet_native_c.h file. +And we added a check to detect mismatch with Scala code and generated header. The checker will +make sure we won't forget to update org_apache_mxnet_native_c.h file. ## Linker options diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml deleted file mode 100644 index ce1e04e918a4..000000000000 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ /dev/null @@ -1,123 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-scala-linux-x86_64-cpu - MXNet Scala Package - Native Linux-x86_64 CPU-only - http://maven.apache.org - - so - - - ${project.parent.parent.basedir}/.. - - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - linux - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_native_c_api.cc - - - - - -std=c++0x - - - -I${MXNET_DIR}/include - -I${MXNET_DIR}/3rdparty/dmlc-core/include - -I${MXNET_DIR}/3rdparty/mshadow - -I${MXNET_DIR}/3rdparty/dlpack/include - -I${MXNET_DIR}/3rdparty/tvm/nnvm/include - -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 - -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c - -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs - - - -shared - - - -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib - - - - - - javah - generate-sources - - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_native_c_api.h - - org.apache.mxnet.LibInfo - - - - javah - - - - - - - diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml deleted file mode 100644 index 052140d5e07d..000000000000 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ /dev/null @@ -1,123 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-scala-linux-x86_64-gpu - MXNet Scala Package - Native Linux-x86_64 GPU - http://maven.apache.org - - so - - - ${project.parent.parent.basedir}/.. - - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - linux - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_native_c_api.cc - - - - - -std=c++0x - - - -I${MXNET_DIR}/include - -I${MXNET_DIR}/3rdparty/dmlc-core/include - -I${MXNET_DIR}/3rdparty/mshadow - -I${MXNET_DIR}/3rdparty/dlpack/include - -I${MXNET_DIR}/3rdparty/tvm/nnvm/include - -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 - -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c - -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs - - - -shared - - - -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib - - - - - - javah - generate-sources - - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_native_c_api.h - - org.apache.mxnet.LibInfo - - - - javah - - - - - - - diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml deleted file mode 100644 index ddc358425f76..000000000000 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ /dev/null @@ -1,158 +0,0 @@ - - - - 4.0.0 - - org.apache.mxnet - mxnet-scala-native-parent - 1.4.0-SNAPSHOT - ../pom.xml - - - libmxnet-scala-osx-x86_64-cpu - MXNet Scala Package - Native OSX-x86_64 CPU-only - http://maven.apache.org - - jnilib - - - ${project.parent.parent.basedir}/.. - - - - - org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT - jar - compile - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.codehaus.mojo - native-maven-plugin - true - - - darwin - generic-classic - ${cxx} - ${cxx} - - - ../src/main/native - - org_apache_mxnet_native_c_api.cc - - - - - -std=c++0x - - - -I${MXNET_DIR}/include - -I${MXNET_DIR}/3rdparty/dmlc-core/include - -I${MXNET_DIR}/3rdparty/mshadow - -I${MXNET_DIR}/3rdparty/dlpack/include - -I${MXNET_DIR}/3rdparty/tvm/nnvm/include - -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 - -g -O0 -fPIC -msse3 -mf16c - -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs - - - -shared - - - -framework JavaVM - -Wl,-exported_symbol,_Java_* - -Wl,-x - - - -Wl,-install_name,libmxnet-scala.jnilib -lmxnet -L${MXNET_DIR}/lib - - - - - - javah - generate-sources - - default - ${project.build.directory}/custom-javah - ${basedir} - org_apache_mxnet_native_c_api.h - - org.apache.mxnet.LibInfo - - - - javah - - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - post-native-build - package - - exec - - - install_name_tool - -change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib - - - - link-native-lib - generate-resources - - exec - - - ln - -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so - - - - - - - diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index 2f1a211668a8..cc8c1cc86daf 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -21,46 +21,165 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-scala-native-parent - MXNet Scala Package - Native Parent - pom + libmxnet-scala + MXNet Scala Package - Native + ${libtype} + + + ${project.parent.basedir}/.. + - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - + osx-x86_64 + + mac + + + + + org.codehaus.mojo + native-maven-plugin + true + + darwin + generic-classic + ${cxx} + ${cxx} + + + src/main/native + + org_apache_mxnet_native_c_api.cc + + + + + -std=c++0x + + + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs + + + -shared + + + -framework JavaVM + -Wl,-exported_symbol,_Java_* + -Wl,-x + + + -Wl,-install_name,libmxnet-scala.jnilib -lmxnet -L${MXNET_DIR}/lib + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -add_rpath @loader_path ${project.build.directory}/${project.artifactId}.jnilib + + + + + + - linux-x86_64-gpu - - linux-x86_64-gpu - + linux-x86_64 + + + unix + Linux + + + + + + org.codehaus.mojo + native-maven-plugin + true + + linux + generic-classic + ${cxx} + ${cxx} + + + src/main/native + + org_apache_mxnet_native_c_api.cc + + + + + -std=c++0x + + + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs + + + -shared + + + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib + + + + + - org.apache.maven.plugins - maven-deploy-plugin - - true - + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + link-native-lib + generate-resources + + exec + + + bash + -c 'ln -sf ${MXNET_DIR}/lib/* ${project.build.directory}/' + + + - diff --git a/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.cc b/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.cc index 17d166eac345..d684c6d13564 100644 --- a/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include "jni_helper_func.h" JavaVM *_jvm; diff --git a/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h b/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h new file mode 100644 index 000000000000..40230ac6daae --- /dev/null +++ b/scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h @@ -0,0 +1,853 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_apache_mxnet_LibInfo */ + +#ifndef _Included_org_apache_mxnet_LibInfo +#define _Included_org_apache_mxnet_LibInfo +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_apache_mxnet_LibInfo + * Method: nativeLibInit + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_nativeLibInit + (JNIEnv *, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxGetLastError + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_apache_mxnet_LibInfo_mxGetLastError + (JNIEnv *, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxListAllOpNames + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxListAllOpNames + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: nnGetOpHandle + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_nnGetOpHandle + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxImperativeInvoke + * Signature: (J[J[JLscala/collection/mutable/ArrayBuffer;I[Ljava/lang/String;[Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxImperativeInvoke + (JNIEnv *, jobject, jlong, jlongArray, jlongArray, jobject, jint, jobjectArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayCreateNone + * Signature: (Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayCreateNone + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayCreateEx + * Signature: ([IIIIIILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayCreateEx + (JNIEnv *, jobject, jintArray, jint, jint, jint, jint, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayWaitAll + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayWaitAll + (JNIEnv *, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayWaitToRead + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayWaitToRead + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxListFunctions + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxListFunctions + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxFuncDescribe + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lorg/apache/mxnet/Base/RefInt;Lorg/apache/mxnet/Base/RefInt;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxFuncDescribe + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxFuncGetInfo + * Signature: (JLorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxFuncGetInfo + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxFuncInvoke + * Signature: (J[J[F[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxFuncInvoke + (JNIEnv *, jobject, jlong, jlongArray, jfloatArray, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxFuncInvokeEx + * Signature: (J[J[F[JI[[B[[B)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxFuncInvokeEx + (JNIEnv *, jobject, jlong, jlongArray, jfloatArray, jlongArray, jint, jobjectArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayGetShape + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayGetShape + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySyncCopyToCPU + * Signature: (J[BI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySyncCopyToCPU + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySlice + * Signature: (JIILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySlice + (JNIEnv *, jobject, jlong, jint, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayAt + * Signature: (JILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayAt + (JNIEnv *, jobject, jlong, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayReshape + * Signature: (JI[ILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayReshape + (JNIEnv *, jobject, jlong, jint, jintArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySyncCopyFromCPU + * Signature: (J[FI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySyncCopyFromCPU + (JNIEnv *, jobject, jlong, jfloatArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayLoad + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;Lorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayLoad + (JNIEnv *, jobject, jstring, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySave + * Signature: (Ljava/lang/String;[J[Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySave + (JNIEnv *, jobject, jstring, jlongArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayGetContext + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayGetContext + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArraySaveRawBytes + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArraySaveRawBytes + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayLoadFromRawBytes + * Signature: ([BLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayLoadFromRawBytes + (JNIEnv *, jobject, jbyteArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNDArrayGetDType + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNDArrayGetDType + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxInitPSEnv + * Signature: ([Ljava/lang/String;[Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxInitPSEnv + (JNIEnv *, jobject, jobjectArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreRunServer + * Signature: (JLorg/apache/mxnet/KVServerControllerCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreRunServer + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreGetNumDeadNode + * Signature: (JILorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreGetNumDeadNode + (JNIEnv *, jobject, jlong, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreCreate + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreCreate + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreInit + * Signature: (JI[I[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreInit + (JNIEnv *, jobject, jlong, jint, jintArray, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreInitEx + * Signature: (JI[Ljava/lang/String;[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreInitEx + (JNIEnv *, jobject, jlong, jint, jobjectArray, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStorePush + * Signature: (JI[I[JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStorePush + (JNIEnv *, jobject, jlong, jint, jintArray, jlongArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStorePushEx + * Signature: (JI[Ljava/lang/String;[JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStorePushEx + (JNIEnv *, jobject, jlong, jint, jobjectArray, jlongArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStorePull + * Signature: (JI[I[JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStorePull + (JNIEnv *, jobject, jlong, jint, jintArray, jlongArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStorePullEx + * Signature: (JI[Ljava/lang/String;[JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStorePullEx + (JNIEnv *, jobject, jlong, jint, jobjectArray, jlongArray, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreSetUpdater + * Signature: (JLorg/apache/mxnet/MXKVStoreUpdater;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreSetUpdater + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreIsWorkerNode + * Signature: (Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreIsWorkerNode + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreGetType + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreGetType + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreSendCommmandToServers + * Signature: (JILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreSendCommmandToServers + (JNIEnv *, jobject, jlong, jint, jstring); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreBarrier + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreBarrier + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreGetGroupSize + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreGetGroupSize + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreGetRank + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreGetRank + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreSetBarrierBeforeExit + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreSetBarrierBeforeExit + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxKVStoreFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxKVStoreFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxListDataIters + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxListDataIters + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterCreateIter + * Signature: (J[Ljava/lang/String;[Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterCreateIter + (JNIEnv *, jobject, jlong, jobjectArray, jobjectArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetIterInfo + * Signature: (JLorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefString;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetIterInfo + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterBeforeFirst + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterBeforeFirst + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterNext + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterNext + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetLabel + * Signature: (JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetLabel + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetData + * Signature: (JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetData + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetIndex + * Signature: (JLscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetIndex + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDataIterGetPadNum + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDataIterGetPadNum + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorOutputs + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorOutputs + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorForward + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorForward + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorBackward + * Signature: (J[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorBackward + (JNIEnv *, jobject, jlong, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorPrint + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorPrint + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorSetMonitorCallback + * Signature: (JLorg/apache/mxnet/MXMonitorCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorSetMonitorCallback + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListAtomicSymbolCreators + * Signature: (Lscala/collection/mutable/ListBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListAtomicSymbolCreators + (JNIEnv *, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolGetAtomicSymbolInfo + * Signature: (JLorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolGetAtomicSymbolInfo + (JNIEnv *, jobject, jlong, jobject, jobject, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateAtomicSymbol + * Signature: (J[Ljava/lang/String;[Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateAtomicSymbol + (JNIEnv *, jobject, jlong, jobjectArray, jobjectArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolSetAttr + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolSetAttr + (JNIEnv *, jobject, jlong, jstring, jstring); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListAttrShallow + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListAttrShallow + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListAttr + * Signature: (JLorg/apache/mxnet/Base/RefInt;Lscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListAttr + (JNIEnv *, jobject, jlong, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCompose + * Signature: (JLjava/lang/String;[Ljava/lang/String;[J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCompose + (JNIEnv *, jobject, jlong, jstring, jobjectArray, jlongArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateVariable + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateVariable + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolGetAttr + * Signature: (JLjava/lang/String;Lorg/apache/mxnet/Base/RefString;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolGetAttr + (JNIEnv *, jobject, jlong, jstring, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListArguments + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListArguments + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCopy + * Signature: (JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCopy + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListAuxiliaryStates + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListAuxiliaryStates + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolListOutputs + * Signature: (JLscala/collection/mutable/ArrayBuffer;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolListOutputs + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateGroup + * Signature: ([JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateGroup + (JNIEnv *, jobject, jlongArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolPrint + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolPrint + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolGetInternals + * Signature: (JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolGetInternals + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolInferType + * Signature: (J[Ljava/lang/String;[ILscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolInferType + (JNIEnv *, jobject, jlong, jobjectArray, jintArray, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolInferShape + * Signature: (JI[Ljava/lang/String;[I[ILscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lscala/collection/mutable/ListBuffer;Lorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolInferShape + (JNIEnv *, jobject, jlong, jint, jobjectArray, jintArray, jintArray, jobject, jobject, jobject, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolGetOutput + * Signature: (JILorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolGetOutput + (JNIEnv *, jobject, jlong, jint, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolSaveToJSON + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolSaveToJSON + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateFromJSON + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateFromJSON + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorBindX + * Signature: (JIII[Ljava/lang/String;[I[II[J[J[I[JLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorBindX + (JNIEnv *, jobject, jlong, jint, jint, jint, jobjectArray, jintArray, jintArray, jint, jlongArray, jlongArray, jintArray, jlongArray, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxExecutorBindEX + * Signature: (JIII[Ljava/lang/String;[I[II[J[J[I[JJLorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxExecutorBindEX + (JNIEnv *, jobject, jlong, jint, jint, jint, jobjectArray, jintArray, jintArray, jint, jlongArray, jlongArray, jintArray, jlongArray, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolSaveToFile + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolSaveToFile + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolCreateFromFile + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolCreateFromFile + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSymbolFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSymbolFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRandomSeed + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRandomSeed + (JNIEnv *, jobject, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxNotifyShutdown + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxNotifyShutdown + (JNIEnv *, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOWriterCreate + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOWriterCreate + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOReaderCreate + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOReaderCreate + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOWriterFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOWriterFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOReaderFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOReaderFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOWriterWriteRecord + * Signature: (JLjava/lang/String;I)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOWriterWriteRecord + (JNIEnv *, jobject, jlong, jstring, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOReaderReadRecord + * Signature: (JLorg/apache/mxnet/Base/RefString;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOReaderReadRecord + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOWriterTell + * Signature: (JLorg/apache/mxnet/Base/RefInt;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOWriterTell + (JNIEnv *, jobject, jlong, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRecordIOReaderSeek + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRecordIOReaderSeek + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRtcCreate + * Signature: (Ljava/lang/String;[Ljava/lang/String;[Ljava/lang/String;[J[JLjava/lang/String;Lorg/apache/mxnet/Base/RefLong;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRtcCreate + (JNIEnv *, jobject, jstring, jobjectArray, jobjectArray, jlongArray, jlongArray, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRtcPush + * Signature: (J[J[JIIIIII)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRtcPush + (JNIEnv *, jobject, jlong, jlongArray, jlongArray, jint, jint, jint, jint, jint, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxRtcFree + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxRtcFree + (JNIEnv *, jobject, jlong); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxCustomOpRegister + * Signature: (Ljava/lang/String;Lorg/apache/mxnet/CustomOpProp;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxCustomOpRegister + (JNIEnv *, jobject, jstring, jobject); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSetProfilerConfig + * Signature: ([Ljava/lang/String;[Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSetProfilerConfig + (JNIEnv *, jobject, jobjectArray, jobjectArray); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxSetProfilerState + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxSetProfilerState + (JNIEnv *, jobject, jint); + +/* + * Class: org_apache_mxnet_LibInfo + * Method: mxDumpProfile + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_apache_mxnet_LibInfo_mxDumpProfile + (JNIEnv *, jobject, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/scala-package/pom.xml b/scala-package/pom.xml index f0e86ec3d630..c985ce9c453b 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -22,11 +22,12 @@ org.apache apache - 19 + 19 + org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL MXNet Scala Package - Parent https://github.com/apache/incubator-mxnet/tree/master/scala-package @@ -53,10 +54,10 @@ 2.11.8 - 2.11 - + g++ $ + ${project.basedir}/.. pom @@ -64,46 +65,18 @@ init init-native macros - core native + core infer examples spark assembly + deploy - - release - - - - org.codehaus.mojo - build-helper-maven-plugin - - - generate-sources - - add-source - - - - ${project.build.directory}/genjavadoc - - - - - - - - - scala-2.11 - - 2.11.8 - 2.11 - @@ -133,31 +106,66 @@ - scala-2.12 + osx-x86_64 + + + mac + + + + osx-x86_64 + jnilib + cpu + + + + linux-x86_64 + + + unix + Linux + + - 2.12.4 - 2.12 + linux-x86_64 + so + - org.apache.maven.plugins - maven-enforcer-plugin + org.codehaus.mojo + exec-maven-plugin + 1.6.0 - enforce-versions + init-build-flavor + initialize - enforce + exec - - - - *:*_2.11 - *:*_2.10 - - - + bash + -c 'mkdir -p ${project.build.directory}; if [[ $(ldd ${MXNET_DIR}/lib/libmxnet.so | grep libcuda.so | wc -l) == "0" ]]; then echo flavor=cpu > ${project.build.directory}/flavor.properties; else echo flavor=gpu > ${project.build.directory}/flavor.properties; fi' + + + + + + org.codehaus.mojo + properties-maven-plugin + 1.0.0 + + + read-properties + initialize + + read-project-properties + + + + ${project.build.directory}/flavor.properties + @@ -170,19 +178,25 @@ - org.apache.maven.plugins - maven-release-plugin - - true - false - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - + org.commonjava.maven.plugins + directory-maven-plugin + 0.1 + + + directories + + directory-of + + initialize + + rootdir + + org.apache.mxnet + mxnet-parent + + + + org.apache.maven.plugins @@ -225,14 +239,13 @@ org.apache.maven.plugins maven-assembly-plugin - 2.5.5 + 3.1.0 org.apache.maven.plugins maven-surefire-plugin 2.19 - true false @@ -247,7 +260,6 @@ scalatest-maven-plugin 1.0 - ${skipTests} ${project.build.directory}/surefire-reports . F @@ -272,7 +284,7 @@ org.scalastyle scalastyle-maven-plugin - 0.8.0 + 1.0.0 false true @@ -280,7 +292,7 @@ false ${basedir}/src/main/scala ${basedir}/src/test/scala - scalastyle-config.xml + ${rootdir}/scalastyle-config.xml ${basedir}/target/scalastyle-output.xml UTF-8 @@ -331,19 +343,17 @@ + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + - - org.scala-lang - scala-library - ${scala.version} - - - org.scala-lang - scala-reflect - ${scala.version} - commons-codec commons-codec @@ -368,7 +378,7 @@ org.scalatest - scalatest_${scala.binary.version} + scalatest_2.11 3.0.4 test @@ -379,13 +389,25 @@ org.scalacheck - scalacheck_${scala.binary.version} + scalacheck_2.11 1.13.5 test + + + + org.scala-lang + scala-library + ${scala.version} + + + org.scala-lang + scala-reflect + ${scala.version} + org.scala-lang.modules - scala-parser-combinators_${scala.binary.version} + scala-parser-combinators_2.11 1.0.4 diff --git a/scala-package/spark/bin/run-mnist-example.sh b/scala-package/spark/bin/run-mnist-example.sh index 4ebd6c61d56b..4f747f2c91a1 100755 --- a/scala-package/spark/bin/run-mnist-example.sh +++ b/scala-package/spark/bin/run-mnist-example.sh @@ -27,9 +27,9 @@ OS="" if [ "$(uname)" == "Darwin" ]; then # Do something under Mac OS X platform - OS='osx-x86_64-cpu' + OS='osx-x86_64' elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then - OS='linux-x86_64-cpu' + OS='linux-x86_64' fi LIB_DIR=${SPARK_MODULE_DIR}/target/classes/lib diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index 8c0ca087855c..2acb70b43303 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -21,47 +21,28 @@ 4.0.0 org.apache.mxnet - mxnet-parent_2.11 - 1.4.0-SNAPSHOT + mxnet-parent + INTERNAL ../pom.xml - mxnet-spark_2.11 + mxnet-spark MXNet Scala Package - Spark ML 1.6.3 - - - osx-x86_64-cpu - - osx-x86_64-cpu - - - - linux-x86_64-cpu - - linux-x86_64-cpu - - - - linux-x86_64-gpu - - linux-x86_64-gpu - - - + org.apache.mxnet - mxnet-core_${scala.binary.version} - 1.4.0-SNAPSHOT + mxnet-core + INTERNAL provided org.apache.spark - spark-mllib_${scala.binary.version} + spark-mllib_2.11 ${spark.version} diff --git a/tools/license_header.py b/tools/license_header.py index 32a093e66a20..410d5c16cbf5 100755 --- a/tools/license_header.py +++ b/tools/license_header.py @@ -60,31 +60,26 @@ _LICENSE_PATTERNS = ['Licensed to the Apache Software Foundation'] # the folders or files that will be ignored -_WHITE_LIST = [ - # Licensed under docker/Dockerfiles/License.md +_WHITE_LIST = ['3rdparty', 'docker/Dockerfiles', - - # Git submodules under different licenses - '3rdparty', - - # Code shared with project by author - see file for details - 'src/operator/special_functions-inl.h', - - # Licensed under Caffe header - 'src/operator/nn/pool.h', - 'src/operator/contrib/psroi_pooling-inl.h', - 'src/operator/contrib/nn/deformable_im2col.h', + 'example/image-classification/predict-cpp/image-classification-predict.cc', + 'example/rcnn/rcnn/cython/nms_kernel.cu', + 'example/rcnn/rcnn/cython/setup.py', + 'example/ssd/dataset/pycocotools/coco.py', + 'julia/REQUIRE', + 'prepare_mkl.sh', + 'R-package/', + 'scala-package/init-native/src/main/native/org_apache_mxnet_init_native_c_api.h', + 'scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h', + 'src/operator/contrib/ctc_include/', 'src/operator/contrib/nn/deformable_im2col.cuh', - 'src/operator/nn/im2col.h', + 'src/operator/contrib/nn/deformable_im2col.h', + 'src/operator/contrib/psroi_pooling-inl.h', + 'src/operator/mkl/', 'src/operator/nn/im2col.cuh', - - # Licenses in headers - 'docs/_static/searchtools_custom.js', - 'docs/_static/js/clipboard.js', - 'docs/_static/js/clipboard.min.js', - - # Licensed under 2-Clause BSD in header - 'example/ssd/dataset/pycocotools/coco.py', + 'src/operator/nn/im2col.h', + 'src/operator/nn/pool.h', + 'src/operator/special_functions-inl.h' ] # language extensions and the according commment mark From e2a6c17ece4dca984a73dade7d2b8a8664140d7f Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Wed, 9 Jan 2019 17:49:57 -0800 Subject: [PATCH 09/16] Dockerfiles for Publish Testing (#13707) --- ci/build.py | 11 +++-- ci/docker/Dockerfile.publish.test.centos7_cpu | 38 +++++++++++++++++ ci/docker/Dockerfile.publish.test.centos7_gpu | 38 +++++++++++++++++ .../Dockerfile.publish.test.ubuntu1404_cpu | 39 ++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1404_gpu | 40 ++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1604_cpu | 39 ++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1604_gpu | 39 ++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1804_cpu | 41 +++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1804_gpu | 41 +++++++++++++++++++ ci/docker/Dockerfile.publish.ubuntu1404_cpu | 36 ++++++++++++++++ ci/docker/Dockerfile.publish.ubuntu1404_gpu | 36 ++++++++++++++++ ci/docker/install/centos7_base.sh | 33 +++++++++++++++ ci/docker/install/centos7_scala.sh | 10 ++++- ci/docker/install/ubuntu_base.sh | 36 ++++++++++++++++ ci/docker/install/ubuntu_core.sh | 15 +++---- ci/docker/install/ubuntu_publish.sh | 25 +++++++++-- ci/docker/install/ubuntu_scala.sh | 27 +++++++++--- ci/docker/runtime_functions.sh | 6 +-- ci/publish/Jenkinsfile | 22 +++++----- .../dev => ci/publish/scala}/build.sh | 9 +--- .../dev => ci/publish/scala}/buildkey.py | 0 .../dev => ci/publish/scala}/deploy.sh | 7 +--- ci/publish/scala/fullDeploy.sh | 23 +++++++++++ .../dev => ci/publish/scala}/test.sh | 7 +++- ci/test_docker_cache.py | 4 +- scala-package/.gitignore | 1 - scala-package/packageTest/pom.xml | 2 +- 27 files changed, 571 insertions(+), 54 deletions(-) create mode 100644 ci/docker/Dockerfile.publish.test.centos7_cpu create mode 100644 ci/docker/Dockerfile.publish.test.centos7_gpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1404_cpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1404_gpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1604_cpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1604_gpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1804_cpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1804_gpu create mode 100644 ci/docker/Dockerfile.publish.ubuntu1404_cpu create mode 100644 ci/docker/Dockerfile.publish.ubuntu1404_gpu create mode 100755 ci/docker/install/centos7_base.sh create mode 100755 ci/docker/install/ubuntu_base.sh rename {scala-package/dev => ci/publish/scala}/build.sh (88%) rename {scala-package/dev => ci/publish/scala}/buildkey.py (100%) rename {scala-package/dev => ci/publish/scala}/deploy.sh (86%) create mode 100644 ci/publish/scala/fullDeploy.sh rename {scala-package/dev => ci/publish/scala}/test.sh (88%) diff --git a/ci/build.py b/ci/build.py index 7f1df6cc41c3..52d4623e682a 100755 --- a/ci/build.py +++ b/ci/build.py @@ -92,22 +92,24 @@ def get_dockerfiles_path(): def get_platforms(path: str = get_dockerfiles_path()) -> List[str]: """Get a list of architectures given our dockerfiles""" - dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*")) + dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*")) dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles)) - files = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles)) + files = list(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles)) platforms = list(map(lambda x: os.path.split(x)[1], sorted(files))) return platforms def get_docker_tag(platform: str, registry: str) -> str: """:return: docker tag to be used for the container""" + platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform) if not registry: registry = "mxnet_local" - return "{0}/build.{1}".format(registry, platform) + return "{0}/{1}".format(registry, platform) def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str: - return os.path.join(path, "Dockerfile.build.{0}".format(platform)) + platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform) + return os.path.join(path, "Dockerfile.{0}".format(platform)) def get_docker_binary(use_nvidia_docker: bool) -> str: @@ -523,6 +525,7 @@ def signal_handler(signum, _): elif args.all: platforms = get_platforms() + platforms = [platform for platform in platforms if 'build.' in platform] logging.info("Building for all architectures: %s", platforms) logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: diff --git a/ci/docker/Dockerfile.publish.test.centos7_cpu b/ci/docker/Dockerfile.publish.test.centos7_cpu new file mode 100644 index 000000000000..7d284452971b --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.centos7_cpu @@ -0,0 +1,38 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on CentOS 7 for CPU + +FROM centos:7 + +WORKDIR /work/deps + +COPY install/centos7_base.sh /work/ +RUN /work/centos7_base.sh + +COPY install/centos7_scala.sh /work/ +RUN /work/centos7_scala.sh + +ARG USER_ID=0 +COPY install/centos7_adduser.sh /work/ +RUN /work/centos7_adduser.sh + +ENV PYTHONPATH=./python/ +WORKDIR /work/mxnet + +COPY runtime_functions.sh /work/ diff --git a/ci/docker/Dockerfile.publish.test.centos7_gpu b/ci/docker/Dockerfile.publish.test.centos7_gpu new file mode 100644 index 000000000000..e7f584683109 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.centos7_gpu @@ -0,0 +1,38 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on CentOS 7 for CPU + +FROM nvidia/cuda:9.2-cudnn7-devel-centos7 + +WORKDIR /work/deps + +COPY install/centos7_base.sh /work/ +RUN /work/centos7_base.sh + +COPY install/centos7_scala.sh /work/ +RUN /work/centos7_scala.sh + +ARG USER_ID=0 +COPY install/centos7_adduser.sh /work/ +RUN /work/centos7_adduser.sh + +ENV PYTHONPATH=./python/ +WORKDIR /work/mxnet + +COPY runtime_functions.sh /work/ diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu new file mode 100644 index 000000000000..035837686554 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu new file mode 100644 index 000000000000..854dd68a63c1 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu @@ -0,0 +1,40 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 14.04 for GPU + +# Use CPU with setup_gpu script +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu new file mode 100644 index 000000000000..bbb7b6a0d7bd --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu new file mode 100644 index 000000000000..660461dc0cfa --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu new file mode 100644 index 000000000000..e3a8c193f234 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu @@ -0,0 +1,41 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU + +FROM ubuntu:18.04 + +WORKDIR /work/deps + +ENV DEBIAN_FRONTEND noninteractive + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu new file mode 100644 index 000000000000..99f7e0d3eff9 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu @@ -0,0 +1,41 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 18.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu18.04 + +WORKDIR /work/deps + +ENV DEBIAN_FRONTEND noninteractive + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1404_cpu b/ci/docker/Dockerfile.publish.ubuntu1404_cpu new file mode 100644 index 000000000000..04ce94f95eae --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1404_cpu @@ -0,0 +1,36 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_publish.sh /work/ +RUN /work/ubuntu_publish.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1404_gpu b/ci/docker/Dockerfile.publish.ubuntu1404_gpu new file mode 100644 index 000000000000..9855986a2891 --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1404_gpu @@ -0,0 +1,36 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 14.04 for GPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_publish.sh /work/ +RUN /work/ubuntu_publish.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_base.sh b/ci/docker/install/centos7_base.sh new file mode 100755 index 000000000000..3b84aeb57b06 --- /dev/null +++ b/ci/docker/install/centos7_base.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex + +# Multipackage installation does not fail in yum +yum -y install epel-release +yum -y install git +yum -y install wget +yum -y install make +yum -y install cmake +yum -y install unzip +yum -y install ninja-build +yum -y install gcc-gfortran diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_scala.sh index ea46de9b9311..5c43f011cbf1 100755 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/install/centos7_scala.sh @@ -23,9 +23,17 @@ set -ex yum install -y java-1.8.0-openjdk-devel +export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk +export PATH=$JAVA_HOME/bin:$PATH # Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +wget -q http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz tar xzf apache-maven-3.3.9-bin.tar.gz mkdir /usr/local/maven mv apache-maven-3.3.9/ /usr/local/maven/ alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 + +echo "export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk" >> /etc/profile.d/maven.sh +echo "export M3_HOME=/usr/local/src/apache-maven" >> /etc/profile.d/maven.sh +echo "export PATH=$M3_HOME/bin:$JAVA_HOME/bin:$PATH" >> /etc/profile.d/maven.sh +chmod +x /etc/profile.d/maven.sh +source /etc/profile.d/maven.sh diff --git a/ci/docker/install/ubuntu_base.sh b/ci/docker/install/ubuntu_base.sh new file mode 100755 index 000000000000..b34c0b3e18f1 --- /dev/null +++ b/ci/docker/install/ubuntu_base.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex +apt-get update || true +apt-get install -y \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + git \ + ninja-build \ + libgfortran3 \ + software-properties-common \ + sudo \ + unzip \ + wget diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 942b686c1602..c0b5e6a039bf 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -30,7 +30,6 @@ apt-get install -y \ apt-transport-https \ build-essential \ ca-certificates \ - cmake \ curl \ git \ libatlas-base-dev \ @@ -46,11 +45,9 @@ apt-get install -y \ unzip \ wget -# Note: we specify an exact cmake version to work around a cmake 3.10 CUDA 10 issue. -# Reference: https://github.com/clab/dynet/issues/1457 -mkdir /opt/cmake && cd /opt/cmake -wget -nv https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh -sh cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license -ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -rm cmake-3.12.4-Linux-x86_64.sh -cmake --version +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + apt-get install -y cmake3 +else + apt-get install -y cmake +fi diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh index bc3513dd13e5..1ad6ab947842 100644 --- a/ci/docker/install/ubuntu_publish.sh +++ b/ci/docker/install/ubuntu_publish.sh @@ -35,14 +35,31 @@ apt-get install -y git \ nasm \ libtool \ curl \ + wget \ + sudo \ + gnupg \ + gnupg2 \ + gnupg-agent \ pandoc \ python3-pip \ automake \ pkg-config \ openjdk-8-jdk -curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ +curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +tar xzf apache-maven-3.3.9-bin.tar.gz +mkdir /usr/local/maven +mv apache-maven-3.3.9/ /usr/local/maven/ update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 update-ca-certificates -f + +apt-get install -y python python3 + +# the version of the pip shipped with ubuntu may be too lower, install a recent version here +wget -nv https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py +python2 get-pip.py + +apt-get remove -y python3-urllib3 + +pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 22be230efdd3..5bade47463e2 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -24,16 +24,31 @@ set -ex cd "$(dirname "$0")" # install libraries for mxnet's scala package on ubuntu echo 'Installing Scala...' -apt-get update || true -apt-get install -y software-properties-common -apt-get update || true -apt-get install -y openjdk-8-jdk -apt-get install -y openjdk-8-jre +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + add-apt-repository -y ppa:openjdk-r/ppa +fi + +# All Ubuntu apt-get update || true apt-get install -y \ - maven \ + openjdk-8-jdk \ + openjdk-8-jre \ + software-properties-common \ gnupg \ gnupg2 \ gnupg-agent \ scala + +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz + tar xzf apache-maven-3.3.9-bin.tar.gz + mkdir /usr/local/maven + mv apache-maven-3.3.9/ /usr/local/maven/ + update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 + update-ca-certificates -f +else + apt-get install -y maven +fi diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 715b76a0eb5c..fc299187467c 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1182,7 +1182,7 @@ publish_scala_build() { set -ex pushd . scala_prepare - ./scala-package/dev/build.sh + ./ci/publish/scala/build.sh popd } @@ -1190,7 +1190,7 @@ publish_scala_test() { set -ex pushd . scala_prepare - ./scala-package/dev/test.sh + ./ci/publish/scala/test.sh popd } @@ -1198,7 +1198,7 @@ publish_scala_deploy() { set -ex pushd . scala_prepare - ./scala-package/dev/deploy.sh + ./ci/publish/scala/deploy.sh popd } diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile index 49e7d7a88d40..9a360c6b5bed 100644 --- a/ci/publish/Jenkinsfile +++ b/ci/publish/Jenkinsfile @@ -21,7 +21,7 @@ // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ //mxnet libraries -mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/test-classes/**, scala-package/local-snapshot/**' +mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**' // timeout in minutes max_time = 120 @@ -54,22 +54,23 @@ def labels = ['cpu'] // , 'gpu'] for (x in labels) { def label = x // Required due to language toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") { - env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] - utils.init_git() - utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') - utils.pack_lib("scala_${label}", mx_scala_pub, false) + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + utils.init_git() + utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') + utils.pack_lib("scala_${label}", mx_scala_pub, false) + } } } def toTest = [:] -def systems = ['ubuntu'] // , 'centos7'] +def systems = ['ubuntu1604', 'ubuntu1804', 'centos7'] for (x in labels) { def label = x // Required due to language for (y in systems) { def system = y // Required due to language toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") { utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("${system}_${label}", 'publish_scala_test', label == 'gpu') + utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu') } } } @@ -78,9 +79,10 @@ def toDeploy = [:] for (x in labels) { def label = x // Required due to language toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") { - env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] - utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + } } } diff --git a/scala-package/dev/build.sh b/ci/publish/scala/build.sh similarity index 88% rename from scala-package/dev/build.sh rename to ci/publish/scala/build.sh index c336fd84e5e2..17f969afe142 100755 --- a/scala-package/dev/build.sh +++ b/ci/publish/scala/build.sh @@ -24,11 +24,6 @@ set -ex bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./ -# Scala steps to deploy -make scalapkg CI=1 - # Compile tests for discovery later -export GPG_TTY=$(tty) -make scalatestcompile CI=1 -# make scalainstall CI=1 -make scaladeploylocal CI=1 +cd scala-package +mvn -B deploy diff --git a/scala-package/dev/buildkey.py b/ci/publish/scala/buildkey.py similarity index 100% rename from scala-package/dev/buildkey.py rename to ci/publish/scala/buildkey.py diff --git a/scala-package/dev/deploy.sh b/ci/publish/scala/deploy.sh similarity index 86% rename from scala-package/dev/deploy.sh rename to ci/publish/scala/deploy.sh index 6f845ba5d78f..4eb33907eeb5 100755 --- a/scala-package/dev/deploy.sh +++ b/ci/publish/scala/deploy.sh @@ -23,7 +23,7 @@ set -ex # export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu # Run python to configure keys -python3 $PWD/scala-package/dev/buildkey.py +python3 ci/publish/scala/buildkey.py # Updating cache mkdir -p ~/.gnupg @@ -34,11 +34,8 @@ echo "pinentry-mode loopback" >> ~/.gnupg/gpg-agent.conf export GPG_TTY=$(tty) cd scala-package -VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec) -cd .. -# echo "\n\n$VERSION\n" | make scalarelease-dryrun -make scaladeploy CI=1 +mvn -B deploy -Pnightly # Clear all password .xml files, exp files, and gpg key files rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp diff --git a/ci/publish/scala/fullDeploy.sh b/ci/publish/scala/fullDeploy.sh new file mode 100644 index 000000000000..69d674a97497 --- /dev/null +++ b/ci/publish/scala/fullDeploy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +./ci/publish/scala/build.sh +./ci/publish/scala/test.sh +./ci/publish/scala/deploy.sh diff --git a/scala-package/dev/test.sh b/ci/publish/scala/test.sh similarity index 88% rename from scala-package/dev/test.sh rename to ci/publish/scala/test.sh index 03810fbf8d55..5cef35ca3c2b 100755 --- a/scala-package/dev/test.sh +++ b/ci/publish/scala/test.sh @@ -18,6 +18,11 @@ set -ex +if [ -z "$JAVA_HOME" ]; then + source /etc/profile +fi + # Test cd scala-package/packageTest -make scalaintegrationtestlocal CI=1 +# make testlocal CI=1 +make testsnapshot UNIT=1 CI=1 diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py index 358d54985aca..0a3bc4640c05 100644 --- a/ci/test_docker_cache.py +++ b/ci/test_docker_cache.py @@ -135,7 +135,7 @@ def test_full_cache(self): """ platform = 'test_full_cache' docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) - dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform) try: with open(dockerfile_path, 'w') as dockerfile_handle: dockerfile_handle.write(dockerfile_content) @@ -196,7 +196,7 @@ def test_partial_cache(self): """ platform = 'test_partial_cache' docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) - dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform) try: # Write initial Dockerfile with open(dockerfile_path, 'w') as dockerfile_handle: diff --git a/scala-package/.gitignore b/scala-package/.gitignore index bf60e7bef0c5..22b12b31d050 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -8,4 +8,3 @@ core/src/main/scala/org/apache/mxnet/SymbolBase.scala core/src/main/scala/org/apache/mxnet/SymbolRandomAPIBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ -local-snapshot diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index 9c5c11cf2779..f7d9e3b180bc 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -42,7 +42,7 @@ local-snapshot - file://${basedir}/../local-snapshot + file://${basedir}/../deploy/target/repo true From b45c895f3dd50ce572b5a69db7e1c7595cd8b7b6 Mon Sep 17 00:00:00 2001 From: Lanking Date: Thu, 10 Jan 2019 19:56:19 -0500 Subject: [PATCH 10/16] change to compile time (#13835) --- scala-package/deploy/src/main/deploy/deploy.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scala-package/deploy/src/main/deploy/deploy.xml b/scala-package/deploy/src/main/deploy/deploy.xml index 97ba5f0ceb49..7bfd20737537 100644 --- a/scala-package/deploy/src/main/deploy/deploy.xml +++ b/scala-package/deploy/src/main/deploy/deploy.xml @@ -13,25 +13,21 @@ org.scala-lang scala-library SCALA_VERSION - runtime org.scala-lang scala-reflect SCALA_VERSION - runtime org.scala-lang.modules scala-parser-combinators_2.11 1.0.4 - runtime org.scala-lang scala-compiler SCALA_VERSION - runtime From 32979a9029d9e5f881648d44ebe20cf32c9c1385 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Tue, 15 Jan 2019 11:04:42 -0800 Subject: [PATCH 11/16] Avoid adding SegfaultLogger if process already has sig handler. (#13842) In current implemenation, we override signal handler regardless if MXNET_USE_SIGNAL_HANDLER=1. This breaks caller process behavior and cause process exit unexpectedly. The example use case is libmxnet.so is loadded into java process via JNI or JNA. JVM will crash due to SegfaultLogger. In this PR, we will not register SegfaultLogger if there is a signal handler registered. --- src/initialize.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/initialize.cc b/src/initialize.cc index de7edd1b1455..8d0e3c304216 100644 --- a/src/initialize.cc +++ b/src/initialize.cc @@ -44,7 +44,11 @@ class LibraryInitializer { LibraryInitializer() { dmlc::InitLogging("mxnet"); #if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE - signal(SIGSEGV, SegfaultLogger); + struct sigaction sa; + sigaction(SIGSEGV, NULL, &sa); + if (sa.sa_handler == NULL) { + signal(SIGSEGV, SegfaultLogger); + } #endif // disable openmp for multithreaded workers From 91280bb79856e5366125d501b0372b91434c11bd Mon Sep 17 00:00:00 2001 From: Lanking Date: Tue, 15 Jan 2019 16:02:31 -0500 Subject: [PATCH 12/16] fix the fetching GPU problem (#13889) --- scala-package/deploy/pom.xml | 1 + scala-package/pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml index c51aa9a92090..542bf6c16880 100644 --- a/scala-package/deploy/pom.xml +++ b/scala-package/deploy/pom.xml @@ -19,6 +19,7 @@ + ${project.parent.basedir}/.. mxnet-full_2.11-${platform}-${flavor} 1.5.0-SNAPSHOT apache.snapshots.https diff --git a/scala-package/pom.xml b/scala-package/pom.xml index c985ce9c453b..f4d5004021fc 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -146,7 +146,7 @@ bash - -c 'mkdir -p ${project.build.directory}; if [[ $(ldd ${MXNET_DIR}/lib/libmxnet.so | grep libcuda.so | wc -l) == "0" ]]; then echo flavor=cpu > ${project.build.directory}/flavor.properties; else echo flavor=gpu > ${project.build.directory}/flavor.properties; fi' + -c 'mkdir -p ${project.build.directory}; if [[ $(ldd ${MXNET_DIR}/lib/libmxnet.so | grep libcuda | wc -l) == "0" ]]; then echo flavor=cpu > ${project.build.directory}/flavor.properties; else echo flavor=gpu > ${project.build.directory}/flavor.properties; fi' From e62d84a4196ea031d0a45025e6c21b5b824d093e Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Tue, 15 Jan 2019 18:12:23 -0800 Subject: [PATCH 13/16] Jenkins nightly maven with static build script and gpu (#13767) --- .gitignore | 1 + ci/docker/Dockerfile.publish.ubuntu1604_cpu | 42 +++ ci/docker/Dockerfile.publish.ubuntu1604_gpu | 42 +++ ci/docker/install/ubuntu_base.sh | 4 + ci/docker/install/ubuntu_scala.sh | 3 - ci/publish/Jenkinsfile | 27 +- ci/publish/scala/build.sh | 4 +- ci/publish/scala/deploy.sh | 16 +- ci/publish/scala/fullDeploy.sh | 0 ci/publish/scala/test.sh | 9 +- make/maven/maven_linux_cu92.mk | 185 +++++++++++++ scala-package/deploy/pom.xml | 2 +- .../mxnetexamples/rnn/ExampleRNNSuite.scala | 11 +- scala-package/init/pom.xml | 7 + scala-package/packageTest/README.md | 10 +- scala-package/packageTest/examples/pom.xml | 51 +++- scala-package/packageTest/pom.xml | 2 +- scala-package/pom.xml | 33 ++- tools/dependencies/cityhash.sh | 8 +- tools/dependencies/curl.sh | 8 +- tools/dependencies/eigen.sh | 6 +- tools/dependencies/libpng.sh | 8 +- tools/dependencies/libtiff.sh | 8 +- tools/dependencies/libturbojpeg.sh | 8 +- tools/dependencies/libz.sh | 8 +- tools/dependencies/lz4.sh | 8 +- .../dependencies/make_shared_dependencies.sh | 2 + tools/dependencies/openblas.sh | 9 +- tools/dependencies/opencv.sh | 8 +- tools/dependencies/openssl.sh | 8 +- tools/dependencies/protobuf.sh | 9 +- tools/dependencies/zmq.sh | 9 +- tools/setup_gpu_build_tools.sh | 256 ++++++++++++++++++ tools/staticbuild/build.sh | 68 +++++ tools/{build => staticbuild}/build_lib.sh | 24 +- tools/{build => staticbuild}/build_wheel.sh | 0 36 files changed, 815 insertions(+), 89 deletions(-) create mode 100644 ci/docker/Dockerfile.publish.ubuntu1604_cpu create mode 100644 ci/docker/Dockerfile.publish.ubuntu1604_gpu mode change 100644 => 100755 ci/publish/scala/fullDeploy.sh create mode 100644 make/maven/maven_linux_cu92.mk create mode 100755 tools/setup_gpu_build_tools.sh create mode 100755 tools/staticbuild/build.sh rename tools/{build => staticbuild}/build_lib.sh (77%) rename tools/{build => staticbuild}/build_wheel.sh (100%) diff --git a/.gitignore b/.gitignore index c8a813649bb8..7cdbc260e6b3 100644 --- a/.gitignore +++ b/.gitignore @@ -146,6 +146,7 @@ tools/pip_package/mxnet # temporary path for building dependencies when building wheel ./deps/ +./staticdeps/ bld ./tmp/* *.jar diff --git a/ci/docker/Dockerfile.publish.ubuntu1604_cpu b/ci/docker/Dockerfile.publish.ubuntu1604_cpu new file mode 100644 index 000000000000..df284be2b37e --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1604_cpu @@ -0,0 +1,42 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1604_gpu b/ci/docker/Dockerfile.publish.ubuntu1604_gpu new file mode 100644 index 000000000000..2a1f8594e752 --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1604_gpu @@ -0,0 +1,42 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/ubuntu_base.sh b/ci/docker/install/ubuntu_base.sh index b34c0b3e18f1..f36e53279f57 100755 --- a/ci/docker/install/ubuntu_base.sh +++ b/ci/docker/install/ubuntu_base.sh @@ -33,4 +33,8 @@ apt-get install -y \ software-properties-common \ sudo \ unzip \ + expect \ + gnupg \ + gnupg2 \ + gnupg-agent \ wget diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 5bade47463e2..8b23d614b490 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -36,9 +36,6 @@ apt-get install -y \ openjdk-8-jdk \ openjdk-8-jre \ software-properties-common \ - gnupg \ - gnupg2 \ - gnupg-agent \ scala # Ubuntu 14.04 diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile index 9a360c6b5bed..c14ab2502cda 100644 --- a/ci/publish/Jenkinsfile +++ b/ci/publish/Jenkinsfile @@ -21,7 +21,7 @@ // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ //mxnet libraries -mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**' +mx_scala_pub = 'lib/**, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/**/target/**, scala-package/*/target/repo/**' // timeout in minutes max_time = 120 @@ -34,8 +34,9 @@ node('restricted-utility') { utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') // CPU and GPU. OSX nodes are not currently supported by Jenkins -def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU] +def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3] def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu'] +def scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92'] def wrapStep(nodeToRun, workspaceName, step) { return { @@ -50,13 +51,13 @@ def wrapStep(nodeToRun, workspaceName, step) { } def toBuild = [:] -def labels = ['cpu'] // , 'gpu'] +def labels = ['cpu', 'gpu'] for (x in labels) { def label = x // Required due to language - toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") { - withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + toBuild["Scala Build ${label}"] = wrapStep(nodeMap['cpu'], "build-scala-${label}") { + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) { utils.init_git() - utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') + utils.docker_run("publish.ubuntu1404_cpu", 'publish_scala_build', false, '500m', 'MAVEN_PUBLISH_OS_TYPE mxnet_variant') utils.pack_lib("scala_${label}", mx_scala_pub, false) } } @@ -69,8 +70,10 @@ for (x in labels) { for (y in systems) { def system = y // Required due to language toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") { - utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu') + withEnv(["mxnet_variant=${scalaVariantMap[label]}"]) { + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu', '500m', 'mxnet_variant') + } } } } @@ -78,10 +81,10 @@ for (x in labels) { def toDeploy = [:] for (x in labels) { def label = x // Required due to language - toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") { - withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap['cpu'], "deploy-scala-${label}") { + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) { utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + utils.docker_run("publish.ubuntu1604_cpu", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant') } } } @@ -101,7 +104,7 @@ core_logic: { , failure_handler: { if (currentBuild.result == "FAILURE") { - // emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' + emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' } } ) diff --git a/ci/publish/scala/build.sh b/ci/publish/scala/build.sh index 17f969afe142..11386ce2114f 100755 --- a/ci/publish/scala/build.sh +++ b/ci/publish/scala/build.sh @@ -22,7 +22,9 @@ set -ex # MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu # export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu -bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./ +source tools/staticbuild/build.sh $mxnet_variant maven + +set -ex # Compile tests for discovery later cd scala-package diff --git a/ci/publish/scala/deploy.sh b/ci/publish/scala/deploy.sh index 4eb33907eeb5..00c67214742b 100755 --- a/ci/publish/scala/deploy.sh +++ b/ci/publish/scala/deploy.sh @@ -18,12 +18,10 @@ set -ex -# Setup Environment Variables -# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu -# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu - -# Run python to configure keys -python3 ci/publish/scala/buildkey.py +# On Jenkins, run python script to configure keys +if [[ $BUILD_ID ]]; then + python3 ci/publish/scala/buildkey.py +fi # Updating cache mkdir -p ~/.gnupg @@ -37,5 +35,7 @@ cd scala-package mvn -B deploy -Pnightly -# Clear all password .xml files, exp files, and gpg key files -rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp +# On Jenkins, clear all password .xml files, exp files, and gpg key files +if [[ $BUILD_ID ]]; then + rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp +fi diff --git a/ci/publish/scala/fullDeploy.sh b/ci/publish/scala/fullDeploy.sh old mode 100644 new mode 100755 diff --git a/ci/publish/scala/test.sh b/ci/publish/scala/test.sh index 5cef35ca3c2b..32e36299a6fc 100755 --- a/ci/publish/scala/test.sh +++ b/ci/publish/scala/test.sh @@ -24,5 +24,10 @@ fi # Test cd scala-package/packageTest -# make testlocal CI=1 -make testsnapshot UNIT=1 CI=1 + +if [[ $mxnet_variant == cu* ]]; then + export SCALA_TEST_ON_GPU=1 + make testlocal USE_CUDA=1 CI=1 +else + make testlocal CI=1 +fi diff --git a/make/maven/maven_linux_cu92.mk b/make/maven/maven_linux_cu92.mk new file mode 100644 index 000000000000..0ed18a71b976 --- /dev/null +++ b/make/maven/maven_linux_cu92.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making maven package +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 0 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml index 542bf6c16880..4e9da8908be1 100644 --- a/scala-package/deploy/pom.xml +++ b/scala-package/deploy/pom.xml @@ -23,7 +23,6 @@ mxnet-full_2.11-${platform}-${flavor} 1.5.0-SNAPSHOT apache.snapshots.https - file://${project.build.directory}/repo @@ -118,6 +117,7 @@ ${project.description} + false ${repositoryId} ${repo_url} ${project.groupId} diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala index 14fb7b85e9b3..2ccd38fc4f9c 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala @@ -49,14 +49,19 @@ class ExampleRNNSuite extends FunSuite with BeforeAndAfterAll { System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { ctx = Context.gpu() } - LstmBucketing.runTraining(tempDirPath + "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/RNN/sherlockholmes.train.txt", - tempDirPath + "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/RNN/sherlockholmes.valid.txt", Array(ctx), 1) + if (!System.getenv().containsKey("CI")) { + LstmBucketing.runTraining(tempDirPath + "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/RNN/sherlockholmes.train.txt", + tempDirPath + "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/RNN/sherlockholmes.valid.txt", Array(ctx), 1) + } else { + logger.info("Skipping test on CI...") + } } test("Example CI: Test TrainCharRNN") { val tempDirPath = System.getProperty("java.io.tmpdir") if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && - System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { + System.getenv("SCALA_TEST_ON_GPU").toInt == 1 && + !System.getenv().containsKey("CI")) { val ctx = Context.gpu() TrainCharRnn.runTrainCharRnn(tempDirPath + "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/RNN/obama.txt", tempDirPath, ctx, 1) diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index 30e24975a841..fb9e3d3535fb 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -76,6 +76,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md index 3f1eeb842f07..e9980f353759 100644 --- a/scala-package/packageTest/README.md +++ b/scala-package/packageTest/README.md @@ -7,7 +7,7 @@ This is an project created to run the test suite on a fully packaged mxnet jar. To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory: 1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source) -2. Build the tests by running `make scalatestcompile`. +2. Build the tests by running `mvn test-compile`. 3. Follow setup instructions below for your testing goal ## Running @@ -18,13 +18,13 @@ There are three different modes of operation for testing based on the location o If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`). This might be useful if you acquire the .jar file from elsewhere. To install, it is easiest to use `mvn install:install-file -Dfile= -DpomFile=`. If the pom file is not available, you can also run `mvn install:install-file -Dfile= -DgroupId= -DartifactId= -Dversion= -Dpackaging=`. With the full mxnet jar, this might look like `mvn install:install-file -Dfile= -DgroupId=org.apache.mxnet -DartifactId=mxnet-full_2.11-linux-x86_64-cpu -Dversion=1.3.0 -Dpackaging=jar`. -You can also run `make scalainstall` to install from a local build. +You can also run `mvn install` to install from a local build. After installing, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests. ### Test Local Deployment -To test the jars that would be produced by a deployment, you can run `make scaladeploylocal` from the main mxnet directory. This produces a local snapshot located at `scala-package/local-snapshot`. To test this local snapshot, run `make testlocal`. +To test the jars that would be produced by a deployment, you can run `mvn deploy` from the main mxnet directory. This produces a local snapshot located at `scala-package/deploy/target/repo`. To test this local snapshot, run `make testlocal`. It also installs the component packages needed for testing the examples in `scala-package/*/target/repo`. ### Remote Repository Snapshot @@ -36,11 +36,13 @@ Test the snapshot repo using `make testsnapshot` or a different repo using `make You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above). +For running on GPU, add the flag `USE_CUDA=1`. + An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`. ## Cleaning Up -You can clean temporary files and target artifacts by running `make scalaclean`. +You can clean temporary files and target artifacts by running `make clean`. ## Troubleshooting diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml index e11be657e225..070b78dda99b 100644 --- a/scala-package/packageTest/examples/pom.xml +++ b/scala-package/packageTest/examples/pom.xml @@ -21,6 +21,53 @@ false + + fromLocal + + + parent + file://${basedir}/../../target/repo + + true + + + + init + file://${basedir}/../../init/target/repo + + true + + + + macros + file://${basedir}/../../macros/target/repo + + true + + + + core + file://${basedir}/../../core/target/repo + + true + + + + infer + file://${basedir}/../../infer/target/repo + + true + + + + examples + file://${basedir}/../../examples/target/repo + + true + + + + @@ -39,8 +86,8 @@ org.apache.mxnet - mxnet-examples_${mxnet.scalaprofile} - ${mxnet.version} + mxnet-examples + INTERNAL test diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index f7d9e3b180bc..f6a16dd77c9d 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -41,7 +41,7 @@ fromLocal - local-snapshot + full file://${basedir}/../deploy/target/repo true diff --git a/scala-package/pom.xml b/scala-package/pom.xml index f4d5004021fc..9ead5805f1d4 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -58,6 +58,8 @@ g++ $ ${project.basedir}/.. + true + file://${project.build.directory}/repo pom @@ -173,6 +175,34 @@ + + staging + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + + + nightly + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + @@ -348,7 +378,8 @@ org.apache.maven.plugins maven-deploy-plugin - true + false + deployrepo::default::${repo_url} diff --git a/tools/dependencies/cityhash.sh b/tools/dependencies/cityhash.sh index 81cc9cbaad3a..6bc663e906fa 100755 --- a/tools/dependencies/cityhash.sh +++ b/tools/dependencies/cityhash.sh @@ -18,15 +18,17 @@ # under the License. # This script builds the static library of cityhash that can be used as dependency of mxnet. +set -ex CITYHASH_VERSION=1.1.1 if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then # Download and build cityhash >&2 echo "Building cityhash..." git clone https://github.com/google/cityhash $DEPS_PATH/cityhash-$CITYHASH_VERSION + pushd . cd $DEPS_PATH/cityhash-$CITYHASH_VERSION git reset --hard 8af9b8c2b889d80c22d6bc26ba0df1afb79a30db ./configure -prefix=$DEPS_PATH --enable-sse4.2 - make CXXFLAGS="-g -O3 -msse4.2" - make install - cd - + $MAKE CXXFLAGS="-g -O3 -msse4.2" + $MAKE install + popd fi diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh index 9633edb78538..bb947715f2ac 100755 --- a/tools/dependencies/curl.sh +++ b/tools/dependencies/curl.sh @@ -18,12 +18,14 @@ # under the License. # This script builds the static library of libcurl that can be used as dependency of mxnet. +set -ex LIBCURL_VERSION=7.61.0 if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then # download and build libcurl >&2 echo "Building libcurl..." curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o $DEPS_PATH/libcurl.zip unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/curl-$LIBCURL_VERSION if [[ $PLATFORM == 'linux' ]]; then CONFIG_FLAG="" @@ -58,7 +60,7 @@ if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then --disable-gopher \ --disable-manual \ --prefix=$DEPS_PATH - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh index ac2f75a03a52..a0cd8fcc95ef 100755 --- a/tools/dependencies/eigen.sh +++ b/tools/dependencies/eigen.sh @@ -18,6 +18,7 @@ # under the License. # This script imports the headers from eigen3 that can be used to in opencv. +set -ex EIGEN_VERSION=3.3.4 if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then # download eigen @@ -25,10 +26,11 @@ if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then curl -s -L https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o $DEPS_PATH/eigen.zip unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build + pushd . cd $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build cmake \ -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. - make install - cd - + $MAKE install + popd fi diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh index d1523c654478..3faa9f027dcb 100755 --- a/tools/dependencies/libpng.sh +++ b/tools/dependencies/libpng.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of libpng that can be used as dependency of mxnet/opencv. +set -ex PNG_VERSION=1.6.34 if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then # download and build libpng @@ -25,6 +26,7 @@ if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build + pushd . cd $DEPS_PATH/libpng-$PNG_VERSION/build cmake \ -D PNG_SHARED=OFF \ @@ -32,9 +34,9 @@ if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ -D CMAKE_C_FLAGS=-fPIC .. - make - make install + $MAKE + $MAKE install mkdir -p $DEPS_PATH/include/libpng ln -s $DEPS_PATH/include/png.h $DEPS_PATH/include/libpng/png.h - cd - + popd fi diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh index 14dcb2d7bde0..2a402bbcac27 100755 --- a/tools/dependencies/libtiff.sh +++ b/tools/dependencies/libtiff.sh @@ -18,15 +18,17 @@ # under the License. # This script builds the static library of libtiff that can be used as dependency of mxnet/opencv. +set -ex TIFF_VERSION="4-0-9" if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then # download and build libtiff >&2 echo "Building libtiff..." curl -s -L https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION ./configure --quiet --disable-shared --disable-jpeg --disable-zlib --disable-jbig --disable-lzma --prefix=$DEPS_PATH - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh index 4991906f8878..ac813ebec1c7 100755 --- a/tools/dependencies/libturbojpeg.sh +++ b/tools/dependencies/libturbojpeg.sh @@ -19,6 +19,7 @@ # This script builds the static library of libturbojpeg that can be used as dependency of # mxnet/opencv. +set -ex TURBO_JPEG_VERSION=1.5.90 if [[ $PLATFORM == 'darwin' ]]; then JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm" @@ -30,6 +31,7 @@ if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a curl -s -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build + pushd . cd $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build cmake \ -G"Unix Makefiles" \ @@ -41,7 +43,7 @@ if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a -D WITH_JPEG8=TRUE \ $JPEG_NASM_OPTION \ -D ENABLE_SHARED=FALSE .. - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh index 927f1de82e72..c5f9953c8d1c 100755 --- a/tools/dependencies/libz.sh +++ b/tools/dependencies/libz.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of libz that can be used as dependency of mxnet. +set -ex ZLIB_VERSION=1.2.6 if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then # Download and build zlib @@ -25,12 +26,13 @@ if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build + pushd . cd $DEPS_PATH/zlib-$ZLIB_VERSION/build cmake \ -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ -D BUILD_SHARED_LIBS=OFF .. - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh index a4269bf29bb9..478c9925e967 100755 --- a/tools/dependencies/lz4.sh +++ b/tools/dependencies/lz4.sh @@ -18,14 +18,16 @@ # under the License. # This script builds the static library of lz4 that can be used as dependency of mxnet. +set -ex LZ4_VERSION=r130 if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then # Download and build lz4 >&2 echo "Building lz4..." curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o $DEPS_PATH/lz4.zip unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/lz4-$LZ4_VERSION - make - make PREFIX=$DEPS_PATH install - cd - + $MAKE + $MAKE PREFIX=$DEPS_PATH install + popd fi diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh index d678fddcc02d..16508586169a 100755 --- a/tools/dependencies/make_shared_dependencies.sh +++ b/tools/dependencies/make_shared_dependencies.sh @@ -38,3 +38,5 @@ source $DIR/protobuf.sh source $DIR/cityhash.sh source $DIR/zmq.sh source $DIR/lz4.sh + +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh index 9463e3325e0d..27f473919a5b 100755 --- a/tools/dependencies/openblas.sh +++ b/tools/dependencies/openblas.sh @@ -18,6 +18,8 @@ # under the License. # This script builds the static library of openblas that can be used as dependency of mxnet. +set +e # This script throws an error but otherwise works +set -x OPENBLAS_VERSION=0.3.3 if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then # download and build openblas @@ -25,11 +27,12 @@ if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then curl -s -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION - make DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1 - make PREFIX=$DEPS_PATH install - cd - + $MAKE DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1 + $MAKE PREFIX=$DEPS_PATH install + popd ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a fi diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh index 99d0ecb71c36..11c9c2155f91 100755 --- a/tools/dependencies/opencv.sh +++ b/tools/dependencies/opencv.sh @@ -19,6 +19,7 @@ # This script builds the static library of opencv that can be used as dependency of mxnet. # It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in $DEPS_PATH. +set -ex OPENCV_VERSION=3.4.2 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" if [[ $PLATFORM == 'linux' ]]; then @@ -41,6 +42,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc curl -s -L https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip unzip -q $DEPS_PATH/opencv.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build + pushd . cd $DEPS_PATH/opencv-$OPENCV_VERSION/build cmake \ -D OPENCV_ENABLE_NONFREE=OFF \ @@ -184,9 +186,9 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc if [[ $PLATFORM == 'linux' ]]; then cp $DIR/patch/opencv_lapack.h ./ fi - make - make install - cd - + $MAKE + $MAKE install + popd # @szha: compatibility header cat $DEPS_PATH/include/opencv2/imgcodecs/imgcodecs_c.h >> $DEPS_PATH/include/opencv2/imgcodecs.hpp fi diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh index b7e4317d4a89..93284db3e39f 100755 --- a/tools/dependencies/openssl.sh +++ b/tools/dependencies/openssl.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of openssl that can be used as dependency of mxnet. +set -ex OPENSSL_VERSION=1.0.2l if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; then # download and build openssl @@ -25,6 +26,7 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g') curl -s -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.zip -o $DEPS_PATH/openssl.zip unzip -q $DEPS_PATH/openssl.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION if [[ $PLATFORM == 'linux' ]]; then TARGET=linux-x86_64 @@ -32,7 +34,7 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; TARGET=darwin64-x86_64-cc fi ./Configure no-shared no-zlib --prefix=$DEPS_PATH --openssldir=$DEPS_PATH/ssl $TARGET - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh index 1564701042af..76ce1de8e822 100755 --- a/tools/dependencies/protobuf.sh +++ b/tools/dependencies/protobuf.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of protobuf along with protoc, that can be used as dependency of mxnet. +set -ex PROTOBUF_VERSION=3.5.1 if [[ $PLATFORM == 'darwin' ]]; then DY_EXT="dylib" @@ -32,12 +33,12 @@ if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then >&2 echo "Building protobuf..." curl -s -L https://github.com/google/protobuf/archive/v$PROTOBUF_VERSION.zip -o $DEPS_PATH/protobuf.zip unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION ./autogen.sh ./configure -prefix=$DEPS_PATH - make - make install - cd - + $MAKE + $MAKE install + popd fi -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh index 55e17798c2d3..0042d6bcd073 100755 --- a/tools/dependencies/zmq.sh +++ b/tools/dependencies/zmq.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of zeroMQ that can be used as dependency of mxnet. +set -ex ZEROMQ_VERSION=4.2.2 if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then # Download and build zmq @@ -25,14 +26,14 @@ if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then curl -s -L https://github.com/zeromq/libzmq/archive/v$ZEROMQ_VERSION.zip -o $DEPS_PATH/zeromq.zip unzip -q $DEPS_PATH/zeromq.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build + pushd . cd $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build cmake \ -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ -D WITH_LIBSODIUM=OFF \ -D BUILD_SHARED_LIBS=OFF .. - make - make install - cp $DEPS_PATH/lib/x86_64-linux-gnu/libzmq.a $DEPS_PATH/lib/libzmq.a - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/setup_gpu_build_tools.sh b/tools/setup_gpu_build_tools.sh new file mode 100755 index 000000000000..167d4c6a6e13 --- /dev/null +++ b/tools/setup_gpu_build_tools.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script installs the tools and libraries for CUDA GPU on Ubuntu. +# Usage: VARIANT=cu92mkl; DEPS_PATH=$HOME; setup_gpu_build_tools.sh $VARIANT $DEPS_PATH; +# It installs the tools into DEPS_PATH as specified by the second argument, and will set +# the following environment variables: +# PATH, CPLUS_INCLUDE_PATH, C_INCLUDE_PATH, LIBRARY_PATH, LD_LIBRARY_PATH, NVCC + +VARIANT=$1 +DEPS_PATH=$2 + +>&2 echo "Setting CUDA versions for $VARIANT" +if [[ $VARIANT == cu100* ]]; then + CUDA_VERSION='10.0.130-1' + CUDA_PATCH_VERSION='10.0.130-1' + LIBCUDA_VERSION='410.48-0ubuntu1' + LIBCUDNN_VERSION='7.3.1.20-1+cuda10.0' + LIBNCCL_VERSION='2.3.4-1+cuda9.2' +elif [[ $VARIANT == cu92* ]]; then + CUDA_VERSION='9.2.148-1' + CUDA_PATCH_VERSION='9.2.148.1-1' + LIBCUDA_VERSION='396.44-0ubuntu1' + LIBCUDNN_VERSION='7.3.1.20-1+cuda9.2' + LIBNCCL_VERSION='2.3.4-1+cuda9.2' +elif [[ $VARIANT == cu91* ]]; then + CUDA_VERSION='9.1.85-1' + CUDA_PATCH_VERSION='9.1.85.3-1' + LIBCUDA_VERSION='396.44-0ubuntu1' + LIBCUDNN_VERSION='7.1.3.16-1+cuda9.1' + LIBNCCL_VERSION='2.2.12-1+cuda9.1' +elif [[ $VARIANT == cu90* ]]; then + CUDA_VERSION='9.0.176-1' + CUDA_PATCH_VERSION='9.0.176.3-1' + LIBCUDA_VERSION='384.145-0ubuntu1' + LIBCUDNN_VERSION='7.3.1.20-1+cuda9.0' + LIBNCCL_VERSION='2.3.4-1+cuda9.0' +elif [[ $VARIANT == cu80* ]]; then + CUDA_VERSION='8.0.61-1' + CUDA_PATCH_VERSION='8.0.61.2-1' + LIBCUDA_VERSION='375.88-0ubuntu1' + LIBCUDNN_VERSION='7.2.1.38-1+cuda8.0' + LIBNCCL_VERSION='2.3.4-1+cuda8.0' +elif [[ $VARIANT == cu75* ]]; then + CUDA_VERSION='7.5-18' + CUDA_PATCH_VERSION='7.5-18' + LIBCUDA_VERSION='375.88-0ubuntu1' + LIBCUDNN_VERSION='6.0.21-1+cuda7.5' + LIBNCCL_VERSION='' +fi +if [[ $VARIANT == cu* ]]; then + CUDA_MAJOR_VERSION=$(echo $CUDA_VERSION | tr '-' '.' | cut -d. -f1,2) + CUDA_MAJOR_DASH=$(echo $CUDA_VERSION | tr '-' '.' | cut -d. -f1,2 | tr '.' '-') + NVIDIA_MAJOR_VERSION=$(echo $LIBCUDA_VERSION | cut -d. -f1) + LIBCUDA_MAJOR=$(echo $LIBCUDA_VERSION | cut -d. -f1) + LIBCUDNN_MAJOR=$(echo $LIBCUDNN_VERSION | cut -d. -f1) + os_name=$(cat /etc/*release | grep '^ID=' | sed 's/^.*=//g') + os_version=$(cat /etc/*release | grep VERSION_ID | sed 's/^.*"\([0-9]*\)\.\([0-9]*\)"/\1\2/g') + os_id="${os_name}${os_version}" + if [[ $CUDA_MAJOR_DASH == 9-* ]] || [[ $CUDA_MAJOR_DASH == 10-* ]]; then + os_id="ubuntu1604" + fi + export PATH=/usr/lib/binutils-2.26/bin/:${PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/bin + export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/include + export C_INCLUDE_PATH=${C_INCLUDE_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/include + export LIBRARY_PATH=${LIBRARY_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/lib64:$DEPS_PATH/usr/lib/x86_64-linux-gnu:$DEPS_PATH/usr/lib/nvidia-$NVIDIA_MAJOR_VERSION + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/lib64:$DEPS_PATH/usr/lib/x86_64-linux-gnu:$DEPS_PATH/usr/lib/nvidia-$NVIDIA_MAJOR_VERSION + export NVCC=$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/bin/nvcc +fi + +# list of debs to download from nvidia +if [[ $VARIANT == cu100* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu92* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu91* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvcc-${CUDA_MAJOR_DASH}_9.1.85.2-1_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu90* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_9.0.176.4-1_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_9.0.176.4-1_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu80* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ + ) +elif [[ $VARIANT == cu75* ]]; then + cuda_files=( \ + "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ + "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ + "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ + ) + ml_files=( \ + "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ + ) +fi + + +if [[ ! -d $DEPS_PATH/usr/local/cuda-${CUDA_MAJOR_VERSION} ]]; then + prefix=$DEPS_PATH + + for item in ${cuda_files[*]} + do + echo "Installing $item" + curl -sL "http://developer.download.nvidia.com/compute/cuda/repos/${os_id}/x86_64/${item}" -o package.deb + dpkg -X package.deb ${prefix} + rm package.deb + done + for item in ${ml_files[*]} + do + echo "Installing $item" + curl -sL "http://developer.download.nvidia.com/compute/machine-learning/repos/${os_id}/x86_64/${item}" -o package.deb + dpkg -X package.deb ${prefix} + rm package.deb + done + + mkdir -p ${prefix}/include + mkdir -p ${prefix}/lib + cp ${prefix}/usr/include/x86_64-linux-gnu/cudnn_v${LIBCUDNN_MAJOR}.h ${prefix}/include/cudnn.h + ln -s libcudnn_static_v${LIBCUDNN_MAJOR}.a ${prefix}/usr/lib/x86_64-linux-gnu/libcudnn.a + cp ${prefix}/usr/local/cuda-${CUDA_MAJOR_VERSION}/lib64/*.a ${prefix}/lib/ + cp ${prefix}/usr/include/nccl.h ${prefix}/include/nccl.h + ln -s libnccl_static.a ${prefix}/usr/lib/x86_64-linux-gnu/libnccl.a +fi diff --git a/tools/staticbuild/build.sh b/tools/staticbuild/build.sh new file mode 100755 index 000000000000..4f8a78d70490 --- /dev/null +++ b/tools/staticbuild/build.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ $# -lt 1 ]; then + >&2 echo "Usage: build.sh " +fi + +export CURDIR=$PWD +export DEPS_PATH=$PWD/staticdeps +export VARIANT=$(echo $1 | tr '[:upper:]' '[:lower:]') +export STATIC_BUILD_TARGET=$(echo $2 | tr '[:upper:]' '[:lower:]') +export PLATFORM=$(uname | tr '[:upper:]' '[:lower:]') + +if [[ $VARIANT == darwin* ]]; then + export VARIANT="darwin" +fi + +NUM_PROC=1 +if [[ ! -z $(command -v nproc) ]]; then + NUM_PROC=$(nproc) +elif [[ ! -z $(command -v sysctl) ]]; then + NUM_PROC=$(sysctl -n hw.ncpu) +else + >&2 echo "Can't discover number of cores." +fi +export NUM_PROC +>&2 echo "Using $NUM_PROC parallel jobs in building." + +if [[ $DEBUG -eq 1 ]]; then + export ADD_MAKE_FLAG="-j $NUM_PROC" +else + export ADD_MAKE_FLAG="--quiet -j $NUM_PROC" +fi +export MAKE="make $ADD_MAKE_FLAG" + +export CC="gcc -fPIC" +export CXX="g++ -fPIC" +export FC="gfortran" +export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$DEPS_PATH/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH +export CPATH=$DEPS_PATH/include:$CPATH + +if [[ $PLATFORM == 'linux' && $VARIANT == cu* ]]; then + source tools/setup_gpu_build_tools.sh $VARIANT $DEPS_PATH +fi + +mkdir -p $DEPS_PATH + +# Build Dependencies +source tools/dependencies/make_shared_dependencies.sh + +# Build mxnet +source tools/staticbuild/build_lib.sh diff --git a/tools/build/build_lib.sh b/tools/staticbuild/build_lib.sh similarity index 77% rename from tools/build/build_lib.sh rename to tools/staticbuild/build_lib.sh index 032fcb92045f..dc6bd4e8b766 100755 --- a/tools/build/build_lib.sh +++ b/tools/staticbuild/build_lib.sh @@ -18,22 +18,20 @@ # under the License. # This script builds the libraries of mxnet. -make_config=config/pip_${PLATFORM}_${VARIANT}.mk +make_config=make/${STATIC_BUILD_TARGET}/${STATIC_BUILD_TARGET}_${PLATFORM}_${VARIANT}.mk if [[ ! -f $make_config ]]; then >&2 echo "Couldn't find make config $make_config for the current settings." exit 1 fi -git clone --recursive https://github.com/apache/incubator-mxnet mxnet-build - >&2 echo "Now building mxnet modules..." -cp $make_config mxnet-build/config.mk +cp $make_config config.mk -cd mxnet-build +git submodule update --init --recursive || true -make DEPS_PATH=$DEPS_PATH DMLCCORE -make DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a -make DEPS_PATH=$DEPS_PATH PSLITE +$MAKE DEPS_PATH=$DEPS_PATH DMLCCORE +$MAKE DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a +$MAKE DEPS_PATH=$DEPS_PATH PSLITE if [[ $VARIANT == *mkl ]]; then MKLDNN_LICENSE='license.txt' @@ -46,7 +44,11 @@ if [[ $VARIANT == *mkl ]]; then MKLML_LIBFILE='libmklml.dylib' MKLDNN_LIBFILE='libmkldnn.0.dylib' fi - make DEPS_PATH=$DEPS_PATH mkldnn + $MAKE DEPS_PATH=$DEPS_PATH mkldnn + cp 3rdparty/mkldnn/build/install/lib/$IOMP_LIBFILE lib + cp 3rdparty/mkldnn/build/install/lib/$MKLML_LIBFILE lib + cp 3rdparty/mkldnn/build/install/lib/$MKLDNN_LIBFILE lib + cp 3rdparty/mkldnn/build/install/$MKLDNN_LICENSE lib cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE fi @@ -58,7 +60,7 @@ if [[ $VARIANT == *mkl ]]; then fi >&2 echo "Now building mxnet..." -make DEPS_PATH=$DEPS_PATH || exit 1; +$MAKE DEPS_PATH=$DEPS_PATH if [[ $PLATFORM == 'linux' ]]; then cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3 @@ -77,4 +79,4 @@ else >&2 echo "Not available" fi -cd ../ +ln -s staticdeps/ deps diff --git a/tools/build/build_wheel.sh b/tools/staticbuild/build_wheel.sh similarity index 100% rename from tools/build/build_wheel.sh rename to tools/staticbuild/build_wheel.sh From 85fa8c915b11e0789970d53a30c702a35831b420 Mon Sep 17 00:00:00 2001 From: Lanking Date: Thu, 17 Jan 2019 19:25:06 -0500 Subject: [PATCH 14/16] Static build instruction for MXNet in general (#13914) * update scripts and tutorial * add the static test for scala package * kill publish test * fix build issue * address comments --- ci/docker/runtime_functions.sh | 10 ++++++ ci/jenkins/Jenkins_steps.groovy | 13 ++++++++ ci/jenkins/Jenkinsfile_unix_cpu | 3 +- ci/publish/Jenkinsfile | 4 +-- ci/publish/README.md | 33 +++++++++++++++++++ ...aven_darwin_cpu.mk => maven_darwin_mkl.mk} | 4 +-- ...n_linux_cu90.mk => maven_linux_cu90mkl.mk} | 4 +-- ...n_linux_cu92.mk => maven_linux_cu92mkl.mk} | 4 +-- ...{maven_linux_cpu.mk => maven_linux_mkl.mk} | 4 +-- tools/dependencies/README.md | 11 +++++++ tools/staticbuild/README.md | 32 ++++++++++++++++++ tools/staticbuild/build_lib.sh | 7 ---- 12 files changed, 111 insertions(+), 18 deletions(-) create mode 100644 ci/publish/README.md rename make/maven/{maven_darwin_cpu.mk => maven_darwin_mkl.mk} (99%) rename make/maven/{maven_linux_cu90.mk => maven_linux_cu90mkl.mk} (99%) rename make/maven/{maven_linux_cu92.mk => maven_linux_cu92mkl.mk} (99%) rename make/maven/{maven_linux_cpu.mk => maven_linux_mkl.mk} (99%) create mode 100644 tools/staticbuild/README.md diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index fc299187467c..c35cf8c22d89 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1178,6 +1178,16 @@ deploy_jl_docs() { # ... } +build_scala_static_mkl() { + set -ex + pushd . + scala_prepare + export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu + export mxnet_variant=mkl + ./ci/publish/scala/build.sh + popd +} + publish_scala_build() { set -ex pushd . diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 73c60290d32b..c0b038244636 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -489,6 +489,19 @@ def compile_windows_gpu_mkldnn() { }] } +def test_static_scala_cpu() { + return ['Static build CPU 14.04 Scala' : { + node(NODE_LINUX_CPU) { + ws('workspace/ut-publish-scala-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run("publish.ubuntu1404_cpu", 'build_scala_static_mkl', false) + } + } + } + }] +} + def test_unix_python2_cpu() { return ['Python2: CPU': { node(NODE_LINUX_CPU) { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 9c9a41503772..e93dcf576bcd 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -50,7 +50,8 @@ core_logic: { custom_steps.test_unix_r_cpu(), custom_steps.test_unix_julia_cpu(), custom_steps.test_unix_onnx_cpu(), - + custom_steps.test_unix_cpp_cpu(), + custom_steps.test_static_scala_cpu(), /* Disabled due to master build failure: * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/ * https://github.com/apache/incubator-mxnet/issues/11801 diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile index c14ab2502cda..2b91f4f749d4 100644 --- a/ci/publish/Jenkinsfile +++ b/ci/publish/Jenkinsfile @@ -36,7 +36,7 @@ utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-m // CPU and GPU. OSX nodes are not currently supported by Jenkins def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3] def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu'] -def scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92'] +def scalaVariantMap = ['cpu': 'mkl', 'gpu': 'cu92mkl'] def wrapStep(nodeToRun, workspaceName, step) { return { @@ -84,7 +84,7 @@ for (x in labels) { toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap['cpu'], "deploy-scala-${label}") { withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) { utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("publish.ubuntu1604_cpu", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant') + utils.docker_run("publish.ubuntu1604_${label}", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant') } } } diff --git a/ci/publish/README.md b/ci/publish/README.md new file mode 100644 index 000000000000..2344cdff4cd2 --- /dev/null +++ b/ci/publish/README.md @@ -0,0 +1,33 @@ +# MXNet Publish Settings + +This folder contains the configuration of restricted node on Jenkins for the publish. It also contains a folder called `scala` that contains everything required for scala publish. In this `README`, we would bring a brief walkthrough of the Jenkins configuration as well as the usages of the scala deployment files. + +## Jenkins +Currently, Jenkins contains three build stages, namely `Build Packages`, `Test Packages` and `Deploy Packages`. During the `build package` stages, all dependencies will be built and a Scala package would be created. In the second stage, the package created from the previous stage would move to this stage to specifically run the tests. In the final stage, the packages passed the test would be deployed by the instances. + +The job is scheduled to be triggered every 24 hours on a [restricted instance](http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/restricted-publish-artifacts). + +Currently, we are supporting tests in the following systems: + +- Ubuntu 16.04 +- Ubuntu 18.04 +- Cent OS 7 + +All packages are currently built in `Ubuntu 14.04`. All Dockerfile used for publishing are available in `ci/docker/` with prefix `Dockerfile.publish`. + +Apart from that, the script used to create the environment and publish are available under `ci/docker/install`: + +- `ubuntu_publish.sh` install all required dependencies for Ubuntu 14.04 for publishing +- `ubuntu_base.sh` install minimum dependencies required to run the published packages + +## Scala publish +Currently Scala publish on Linux is fully supported on jenkins. The `scala/` folder contains all files needed to do the publish. Here is a breif instroduction of the files: + +- `build.sh` Main executable files to build the backend as well as scala package +- `buildkey.py` Main file used to extract password from the system and configure the maven +- `deploy.sh` Script to deploy the package +- `fullDeploy.sh` Used by CI to make full publish +- `test.sh` Make Scala test on CI + +## Python +We plans to support Python build on Jenkins soon \ No newline at end of file diff --git a/make/maven/maven_darwin_cpu.mk b/make/maven/maven_darwin_mkl.mk similarity index 99% rename from make/maven/maven_darwin_cpu.mk rename to make/maven/maven_darwin_mkl.mk index b8bd97fbc8ed..f5b77ae784ad 100644 --- a/make/maven/maven_darwin_cpu.mk +++ b/make/maven/maven_darwin_mkl.mk @@ -37,7 +37,7 @@ DEBUG = 0 USE_PROFILER = 1 # whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 0 +USE_SIGNAL_HANDLER = 1 # the additional link flags you want to add ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip @@ -87,7 +87,7 @@ USE_LIBJPEG_TURBO = 1 # Please refer to MKL_README.md for details # whether use MKL-DNN library -USE_MKLDNN = 0 +USE_MKLDNN = 1 # MKL ML Library folder, need to be root for /usr/local # Change to User Home directory for standard user diff --git a/make/maven/maven_linux_cu90.mk b/make/maven/maven_linux_cu90mkl.mk similarity index 99% rename from make/maven/maven_linux_cu90.mk rename to make/maven/maven_linux_cu90mkl.mk index 097d252f0e4e..661f444ceac3 100644 --- a/make/maven/maven_linux_cu90.mk +++ b/make/maven/maven_linux_cu90mkl.mk @@ -37,7 +37,7 @@ DEBUG = 0 USE_PROFILER = 1 # whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 0 +USE_SIGNAL_HANDLER = 1 # the additional link flags you want to add ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections @@ -86,7 +86,7 @@ USE_OPERATOR_TUNING = 1 USE_LIBJPEG_TURBO = 1 # whether use MKL-DNN library -USE_MKLDNN = 0 +USE_MKLDNN = 1 # MKL ML Library for Intel CPU/Xeon Phi diff --git a/make/maven/maven_linux_cu92.mk b/make/maven/maven_linux_cu92mkl.mk similarity index 99% rename from make/maven/maven_linux_cu92.mk rename to make/maven/maven_linux_cu92mkl.mk index 0ed18a71b976..ecd252769216 100644 --- a/make/maven/maven_linux_cu92.mk +++ b/make/maven/maven_linux_cu92mkl.mk @@ -37,7 +37,7 @@ DEBUG = 0 USE_PROFILER = 1 # whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 0 +USE_SIGNAL_HANDLER = 1 # the additional link flags you want to add ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections @@ -86,7 +86,7 @@ USE_OPERATOR_TUNING = 1 USE_LIBJPEG_TURBO = 1 # whether use MKL-DNN library -USE_MKLDNN = 0 +USE_MKLDNN = 1 # MKL ML Library for Intel CPU/Xeon Phi diff --git a/make/maven/maven_linux_cpu.mk b/make/maven/maven_linux_mkl.mk similarity index 99% rename from make/maven/maven_linux_cpu.mk rename to make/maven/maven_linux_mkl.mk index 07177fbe3064..6cb9f326bb7d 100644 --- a/make/maven/maven_linux_cpu.mk +++ b/make/maven/maven_linux_mkl.mk @@ -37,7 +37,7 @@ DEBUG = 0 USE_PROFILER = 1 # whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 0 +USE_SIGNAL_HANDLER = 1 # the additional link flags you want to add ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections @@ -83,7 +83,7 @@ USE_OPERATOR_TUNING = 1 USE_LIBJPEG_TURBO = 1 # whether use MKL-DNN library -USE_MKLDNN = 0 +USE_MKLDNN = 1 # MKL ML Library for Intel CPU/Xeon Phi diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md index cfe3d6c75dc9..c30e85d519c3 100644 --- a/tools/dependencies/README.md +++ b/tools/dependencies/README.md @@ -12,3 +12,14 @@ The scripts use the following environment variables for setting behavior: `PLATFORM`: name of the OS in lower case. Supported options are 'linux' and 'darwin'. It also expects the following build tools in path: make, cmake, tar, unzip, autoconf, nasm + +# FAQ + +## Build failure regarding to gcc, g++, gfortran +Currently, we only support gcc-4.8 build. It's your own choice to use a higher version of gcc. Please make sure your gcc, g++ and gfortran always have the same version in order to eliminate build failure. + +## idn2 not found +This issue appeared in the OSX build with XCode version 8.0 above (reproduced on 9.2). Please add the following build flag in `curl.sh` if your XCode version is more than 8.0: +``` +--without-libidn2 +``` \ No newline at end of file diff --git a/tools/staticbuild/README.md b/tools/staticbuild/README.md new file mode 100644 index 000000000000..7cfeb7855a55 --- /dev/null +++ b/tools/staticbuild/README.md @@ -0,0 +1,32 @@ +# MXNet Static build + +This folder contains the core script used to build the static library. This README would bring you the information and usages of the script in here. Please be aware, all of the scripts are designed to be run under the root folder. + +## `build.sh` +This script is a wrapper around `build_lib.sh` aimed to simplify the usage of it. It would automatically identify the system version, number of cores and all environment variable settings. Here are the examples you can run this script: + +``` +tools/staticbuild/build.sh cu92 maven +``` +This would build the mxnet package based on CUDA9.2 and Maven (Scala) build setttings. +``` +tools/staticbuild/build.sh mkl pip +``` +This would build the mxnet package based on MKLDNN and and pypi configuration settings. + +As the result, users would have a complete static dependencies in `/staticdeps` in the root folder as well as a static-linked `libmxnet.so` file lives in `lib`. You can build your language binding by using the `libmxnet.so`. + +## `build_lib.sh` +This script would clone the most up-to-date master and build the MXNet backend with static library. In order to run that, you should have prepare the the following environment variable: + +- `DEPS_PATH` Path to your static dependencies +- `STATIC_BUILD_TARGET` Either `pip` or `maven` as your publish platform +- `PLATFORM` linux, darwin +- `VARIANT` cpu, cu*, cu*mkl, mkl + +It is not recommended to run this file alone since there are a bunch of variables need to be set. + +After running this script, you would have everything you need ready in the `/lib` folder. + +## `build_wheel.sh` +This script is used to build the python package as well as running a sanity test \ No newline at end of file diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh index dc6bd4e8b766..b08057321bf7 100755 --- a/tools/staticbuild/build_lib.sh +++ b/tools/staticbuild/build_lib.sh @@ -52,13 +52,6 @@ if [[ $VARIANT == *mkl ]]; then cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE fi -if [[ $VARIANT == *mkl ]]; then - >&2 echo "Copying MKL license." - rm lib/libmkldnn.{so,dylib} - rm lib/libmkldnn.0.*.dylib - rm lib/libmkldnn.so.0.* -fi - >&2 echo "Now building mxnet..." $MAKE DEPS_PATH=$DEPS_PATH From 915a4ffce147091e0678cc1379cbcc111de8138b Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Thu, 11 Apr 2019 10:09:07 -0700 Subject: [PATCH 15/16] Avoid secondary deployment of package to local (#14647) --- scala-package/deploy/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml index 4e9da8908be1..4290174e5b9b 100644 --- a/scala-package/deploy/pom.xml +++ b/scala-package/deploy/pom.xml @@ -134,6 +134,9 @@ + + true + From d463059e016f66510ff80b0daefb2b6c5688a651 Mon Sep 17 00:00:00 2001 From: Lanking Date: Thu, 28 Feb 2019 13:14:19 -0800 Subject: [PATCH 16/16] MXNet Java bug fixes and experience improvement (#14213) * improve Java user experience * add the new examples * fixed based on the comments --- .../org/apache/mxnet/javaapi/NDArray.scala | 2 + scala-package/mxnet-demo/java-demo/README.md | 22 +-- .../mxnet-demo/java-demo/bin/java_sample.sh | 4 +- .../mxnet-demo/java-demo/bin/run_od.sh | 4 +- scala-package/mxnet-demo/java-demo/pom.xml | 6 + .../main/java/mxnet/ImageClassification.java | 131 ++++++++++++++++++ .../src/main/java/mxnet/NDArrayCreation.java | 47 +++++++ ...{HelloWorld.java => NDArrayOperation.java} | 24 +++- .../src/main/java/mxnet/ObjectDetection.java | 16 +-- 9 files changed, 231 insertions(+), 25 deletions(-) create mode 100644 scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java create mode 100644 scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java rename scala-package/mxnet-demo/java-demo/src/main/java/mxnet/{HelloWorld.java => NDArrayOperation.java} (63%) diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala b/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala index 198102d2377f..cbd5a26845de 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala @@ -209,6 +209,8 @@ class NDArray private[mxnet] (val nd: org.apache.mxnet.NDArray ) { this(NDArray.array(arr, shape, ctx)) } + override def toString: String = nd.toString + def serialize(): Array[Byte] = nd.serialize() /** diff --git a/scala-package/mxnet-demo/java-demo/README.md b/scala-package/mxnet-demo/java-demo/README.md index 0f9a13b76447..5b39e15261f4 100644 --- a/scala-package/mxnet-demo/java-demo/README.md +++ b/scala-package/mxnet-demo/java-demo/README.md @@ -16,11 +16,17 @@ # MXNet Java Sample Project -This is an project created to use Maven-published Scala/Java package with two Java examples. +This is a project demonstrating how to use the Maven published Scala/Java MXNet package. +The examples provided include: +* NDArray creation +* NDArray operation +* Object Detection using the Inference API +* Image Classification using the Predictor API + ## Setup -You can use the `Makefile` to make the Java package. Simply do the following: -```Bash -make javademo +You are required to use Maven to build the package with the following commands under `java-demo`: +``` +mvn package ``` This will load the default parameter for all the environment variable. If you want to run with GPU on Linux, just simply add `USE_CUDA=1` when you run the make file @@ -41,16 +47,16 @@ The `SCALA_PKG_PROFILE` should be chosen from `osx-x86_64-cpu`, `linux-x86_64-cp ## Run -### Hello World -The Scala file is being executed using Java. You can execute the helloWorld example as follows: +### NDArrayCreation +The Scala file is being executed using Java. You can execute the `NDArrayCreation` example as follows: ```Bash bash bin/java_sample.sh ``` You can also run the following command manually: ```Bash -java -cp $CLASSPATH sample.HelloWorld +java -cp $CLASSPATH sample.NDArrayCreation ``` -However, you have to define the Classpath before you run the demo code. More information can be found in the `java_sample.sh`. +However, you have to define the Classpath before you run the demo code. More information can be found in `bin/java_sample.sh`. The `CLASSPATH` should point to the jar file you have downloaded. It will load the library automatically and run the example diff --git a/scala-package/mxnet-demo/java-demo/bin/java_sample.sh b/scala-package/mxnet-demo/java-demo/bin/java_sample.sh index 2ec9a78c3233..fb1795f20f9d 100644 --- a/scala-package/mxnet-demo/java-demo/bin/java_sample.sh +++ b/scala-package/mxnet-demo/java-demo/bin/java_sample.sh @@ -16,5 +16,5 @@ # under the License. #!/bin/bash CURR_DIR=$(cd $(dirname $0)/../; pwd) -CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/classes/lib/* -java -Xmx8G -cp $CLASSPATH mxnet.HelloWorld \ No newline at end of file +CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* +java -Xmx8G -cp $CLASSPATH mxnet.NDArrayCreation diff --git a/scala-package/mxnet-demo/java-demo/bin/run_od.sh b/scala-package/mxnet-demo/java-demo/bin/run_od.sh index e3c8fd545048..4370518dc8cd 100644 --- a/scala-package/mxnet-demo/java-demo/bin/run_od.sh +++ b/scala-package/mxnet-demo/java-demo/bin/run_od.sh @@ -16,5 +16,5 @@ # under the License. #!/bin/bash CURR_DIR=$(cd $(dirname $0)/../; pwd) -CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/classes/lib/* -java -Xmx8G -cp $CLASSPATH mxnet.ObjectDetection \ No newline at end of file +CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* +java -Xmx8G -cp $CLASSPATH mxnet.ObjectDetection diff --git a/scala-package/mxnet-demo/java-demo/pom.xml b/scala-package/mxnet-demo/java-demo/pom.xml index c12efa527c60..9dc7fcd431af 100644 --- a/scala-package/mxnet-demo/java-demo/pom.xml +++ b/scala-package/mxnet-demo/java-demo/pom.xml @@ -42,6 +42,12 @@ mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} ${mxnet.version} + + org.apache.mxnet + mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} + ${mxnet.version} + sources + commons-io commons-io diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java new file mode 100644 index 000000000000..8cb58da5c2e6 --- /dev/null +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mxnet; + +import org.apache.commons.io.FileUtils; +import org.apache.mxnet.infer.javaapi.Predictor; +import org.apache.mxnet.javaapi.*; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +public class ImageClassification { + private static String modelPath; + private static String imagePath; + + private static void downloadUrl(String url, String filePath) { + File tmpFile = new File(filePath); + if (!tmpFile.exists()) { + try { + FileUtils.copyURLToFile(new URL(url), tmpFile); + } catch (Exception exception) { + System.err.println(exception); + } + } + } + + public static void downloadModelImage() { + String tempDirPath = System.getProperty("java.io.tmpdir"); + String baseUrl = "https://s3.us-east-2.amazonaws.com/scala-infer-models"; + downloadUrl(baseUrl + "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/resnet-18/resnet-18-symbol.json", + tempDirPath + "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/resnet18/resnet-18-symbol.json"); + downloadUrl(baseUrl + "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/resnet-18/resnet-18-0000.params", + tempDirPath + "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/resnet18/resnet-18-0000.params"); + downloadUrl(baseUrl + "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/resnet-18/synset.txt", + tempDirPath + "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/resnet18/synset.txt"); + downloadUrl("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg", + tempDirPath + "/inputImages/resnet18/Pug-Cookie.jpg"); + modelPath = tempDirPath + File.separator + "resnet18/resnet-18"; + imagePath = tempDirPath + File.separator + + "inputImages/resnet18/Pug-Cookie.jpg"; + } + + /** + * Helper class to print the maximum prediction result + * @param probabilities The float array of probability + * @param modelPathPrefix model Path needs to load the synset.txt + */ + private static String printMaximumClass(float[] probabilities, + String modelPathPrefix) throws IOException { + String synsetFilePath = modelPathPrefix.substring(0, + 1 + modelPathPrefix.lastIndexOf(File.separator)) + "/synset.txt"; + BufferedReader reader = new BufferedReader(new FileReader(synsetFilePath)); + ArrayList list = new ArrayList<>(); + String line = reader.readLine(); + + while (line != null){ + list.add(line); + line = reader.readLine(); + } + reader.close(); + + int maxIdx = 0; + for (int i = 1;i probabilities[maxIdx]) { + maxIdx = i; + } + } + + return "Probability : " + probabilities[maxIdx] + " Class : " + list.get(maxIdx) ; + } + + public static void main(String[] args) { + // Download the model and Image + downloadModelImage(); + + // Prepare the model + List context = new ArrayList(); + context.add(Context.cpu()); + List inputDesc = new ArrayList<>(); + Shape inputShape = new Shape(new int[]{1, 3, 224, 224}); + inputDesc.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); + Predictor predictor = new Predictor(modelPath, inputDesc, context,0); + + // Prepare data + NDArray nd = Image.imRead(imagePath, 1, true); + nd = Image.imResize(nd, 224, 224, null); + nd = NDArray.transpose(nd, new Shape(new int[]{2, 0, 1}), null)[0]; // HWC to CHW + nd = NDArray.expand_dims(nd, 0, null)[0]; // Add N -> NCHW + nd = nd.asType(DType.Float32()); // Inference with Float32 + + // Predict directly + float[][] result = predictor.predict(new float[][]{nd.toArray()}); + try { + System.out.println("Predict with Float input"); + System.out.println(printMaximumClass(result[0], modelPath)); + } catch (IOException e) { + System.err.println(e); + } + + // predict with NDArray + List ndList = new ArrayList<>(); + ndList.add(nd); + List ndResult = predictor.predictWithNDArray(ndList); + try { + System.out.println("Predict with NDArray"); + System.out.println(printMaximumClass(ndResult.get(0).toArray(), modelPath)); + } catch (IOException e) { + System.err.println(e); + } + } +} diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java new file mode 100644 index 000000000000..32e2d84dcdbf --- /dev/null +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mxnet; + +import org.apache.mxnet.javaapi.*; + +public class NDArrayCreation { + static NDArray$ NDArray = NDArray$.MODULE$; + public static void main(String[] args) { + + // Create new NDArray + NDArray nd = new NDArray(new float[]{2.0f, 3.0f}, new Shape(new int[]{1, 2}), Context.cpu()); + System.out.println(nd); + + // create new Double NDArray + NDArray ndDouble = new NDArray(new double[]{2.0d, 3.0d}, new Shape(new int[]{2, 1}), Context.cpu()); + System.out.println(ndDouble); + + // create ones + NDArray ones = NDArray.ones(Context.cpu(), new int[] {1, 2, 3}); + System.out.println(ones); + + // random + NDArray random = NDArray.random_uniform( + NDArray.new random_uniformParam() + .setLow(0.0f) + .setHigh(2.0f) + .setShape(new Shape(new int[]{10, 10})) + )[0]; + System.out.println(random); + } +} diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/HelloWorld.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayOperation.java similarity index 63% rename from scala-package/mxnet-demo/java-demo/src/main/java/mxnet/HelloWorld.java rename to scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayOperation.java index 3f209a6c6c84..56a414307f46 100644 --- a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/HelloWorld.java +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayOperation.java @@ -14,15 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package mxnet; import org.apache.mxnet.javaapi.*; -import java.util.Arrays; -public class HelloWorld { +public class NDArrayOperation { + static NDArray$ NDArray = NDArray$.MODULE$; public static void main(String[] args) { - System.out.println("Hello World!"); NDArray nd = new NDArray(new float[]{2.0f, 3.0f}, new Shape(new int[]{1, 2}), Context.cpu()); - System.out.println(nd.shape()); + + // Transpose + NDArray ndT = nd.T(); + System.out.println(nd); + System.out.println(ndT); + + // change Data Type + NDArray ndInt = nd.asType(DType.Int32()); + System.out.println(ndInt); + + // element add + NDArray eleAdd = NDArray.elemwise_add(nd, nd, null)[0]; + System.out.println(eleAdd); + + // norm (L2 Norm) + NDArray normed = NDArray.norm(NDArray.new normParam(nd))[0]; + System.out.println(normed); } } diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java index cfe9b66c4b3f..65fe286aa2c7 100644 --- a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java @@ -68,20 +68,18 @@ public static void downloadModelImage() { public static void main(String[] args) { List context = new ArrayList(); - if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && - Integer.valueOf(System.getenv("SCALA_TEST_ON_GPU")) == 1) { - context.add(Context.gpu()); - } else { - context.add(Context.cpu()); - } + context.add(Context.cpu()); downloadModelImage(); + + List> output + = runObjectDetectionSingle(modelPath, imagePath, context); + Shape inputShape = new Shape(new int[] {1, 3, 512, 512}); Shape outputShape = new Shape(new int[] {1, 6132, 6}); int width = inputShape.get(2); int height = inputShape.get(3); - List> output - = runObjectDetectionSingle(modelPath, imagePath, context); String outputStr = "\n"; + for (List ele : output) { for (ObjectDetectorOutput i : ele) { outputStr += "Class: " + i.getClassName() + "\n"; @@ -98,4 +96,4 @@ public static void main(String[] args) { } System.out.println(outputStr); } -} \ No newline at end of file +}