From f80530ef7fb0de57686741c70f66ba39a1990385 Mon Sep 17 00:00:00 2001 From: Piyush Ghai Date: Thu, 27 Sep 2018 16:58:11 -0700 Subject: [PATCH 1/7] Adding Scala Inference Benchmark base class + an example of how to run it --- .../benchmark/run_image_inference_bm.sh | 46 +++++ .../org/apache/mxnetexamples/InferBase.scala | 30 ++++ .../apache/mxnetexamples/benchmark/README.md | 48 ++++++ .../benchmark/ScalaInferenceBenchmark.scala | 157 ++++++++++++++++++ .../ImageClassifierExample.scala | 78 ++++++++- .../objectdetector/SSDClassifierExample.scala | 2 +- .../mxnetexamples/rnn/TestCharRnn.scala | 8 +- .../ScalaInferenceBenchmarkSuite.scala | 60 +++++++ .../ImageClassifierExampleSuite.scala | 2 +- 9 files changed, 419 insertions(+), 12 deletions(-) create mode 100755 scala-package/examples/scripts/benchmark/run_image_inference_bm.sh create mode 100644 scala-package/examples/src/main/scala/org/apache/mxnetexamples/InferBase.scala create mode 100644 scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md create mode 100644 scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala create mode 100644 scala-package/examples/src/test/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmarkSuite.scala diff --git a/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh b/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh new file mode 100755 index 000000000000..37cb3034d17d --- /dev/null +++ b/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) +CLASS_PATH=$MXNET_ROOT/scala-package/assembly/osx-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/* + +MODEL_NAME=$1 + +RUNS=$2 + +BATCHSIZE=$3 + +# model dir +MODEL_PATH_PREFIX=$4 +# input image +INPUT_IMG=$5 +# which input image dir +INPUT_DIR=$6 + +java -Xmx8G -Dmxnet.traceLeakedObjects=true -cp $CLASS_PATH \ + org.apache.mxnetexamples.benchmark.ScalaInferenceBenchmark \ + --example $MODEL_NAME \ + --count $RUNS \ + --batchSize $BATCHSIZE \ + --model-path-prefix $MODEL_PATH_PREFIX \ + --input-image $INPUT_IMG \ + --input-dir $INPUT_DIR \ + diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/InferBase.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/InferBase.scala new file mode 100644 index 000000000000..85d5c85329bf --- /dev/null +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/InferBase.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.mxnetexamples + +import org.apache.mxnet._ + +trait InferBase { + + def loadModel(context: Array[Context]): Any + def loadSingleData(): Any + def loadBatchFileList(batchSize: Int): List[Any] + def loadInputBatch(source: Any): Any + def runSingleInference(loadedModel: Any, input: Any): Any + def runBatchInference(loadedModel: Any, input: Any): Any +} diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md new file mode 100644 index 000000000000..c0cfd3d65d08 --- /dev/null +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md @@ -0,0 +1,48 @@ +# Benchmarking Scala Inference APIs + +This folder contains a base class [ScalaInferenceBenchmark](https://github.com/apache/incubator-mxnet/tree/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/) and provides a mechanism for benchmarking [MXNet Inference APIs]((https://github.com/apache/incubator-mxnet/tree/master/scala-package/infer)) in Scala. +The benchmarking scripts provided runs an experiment for single inference calls and batch inference calls. It collects the time taken to perform an inference operation and emits the P99, P50 and Average values for these metrics. One can easily add/modify any new/existing examples to the ScalaInferenceBenchmark framework in order to get the benchmark numbers for inference calls. +Currently the ScalaInferenceBenchmark script supports three Scala examples : +1. [ImageClassification using ResNet-152](https://github.com/apache/incubator-mxnet/blob/master/scala-package/mxnet-demo/src/main/scala/sample/ImageClassificationExample.scala) +2. [Object Detection Example](https://github.com/apache/incubator-mxnet/blob/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/SSDClassifierExample.scala) +3. [Text Generation through RNNs](https://github.com/apache/incubator-mxnet/blob/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala) + +This script can be easily placed in an automated environment to run benchmark regressions on the Scala APIs. The script automatically picks up whether you are running it on a CPU machine or on a GPU machine and appropriately uses that. + +## Contents + +1. [Prerequisites](#prerequisites) +2. [Scripts](#scripts) + +## Prerequisites + +1. MXNet +2. MXNet Scala Package +3. [IntelliJ IDE (or alternative IDE) project setup](http://mxnet.incubator.apache.org/tutorials/scala/mxnet_scala_on_intellij.html) with the MXNet Scala Package +4. Model files and datasets for the model one will try to benchmark + +## Scripts +To help you easily run the benchmarks, a starter shell script has been provided for each of three examples mentioned above. The scripts can be found [here](https://github.com/apache/incubator-mxnet/blob/master/scala-package/examples/scripts/benchmark). +Each of the script takes some parameters as inputs, details of which can be found either in the bash scripts or in the example classes itself. + +* *ImageClassification Example* +
The following shows an example of running ImageClassifier under the benchmark script. The script takes in the number of iterations for inference calls, the batch size for batch inference calls, the model path, input file, and input directory. +For more details to run ImageClassificationExample as a standalone file, refer to the [README](https://github.com/apache/incubator-mxnet/blob/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md) for ImageClassifierExample. +You may need to run ```chmod u+x run_image_inference_bm.sh``` before running this script. + ```bash + cd /scala-package/examples/scripts/infer/imageclassifier + ./get_resnet_data.sh + cd /scala-package/examples/scripts/benchmark + ./run_image_inference_bm.sh ImageClassifierExample 100 10 ../infer/models/resnet-152/resnet-152 ../infer/images/kitten.jpg ../infer/images/ + ``` + Upon running this script, you might see an output like this : + ``` + [main] INFO org.apache.mxnetexamples.benchmark.CLIParserBase - + single_inference_latency p99 1663, single_inference_p50 729, single_inference_average 755.170000 + ... + + INFO org.apache.mxnetexamples.benchmark.CLIParserBase - + batch_inference_latency p99 4241, batch_inference_p50 4241, batch_inference_average 4241.000000 + ``` + +More examples to be added soon. \ No newline at end of file diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala new file mode 100644 index 000000000000..2d1955e96051 --- /dev/null +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.mxnetexamples.benchmark + +import org.apache.mxnetexamples.InferBase +import org.apache.mxnetexamples.infer.imageclassifier.ImageClassifierExample +import org.apache.mxnet._ +import org.kohsuke.args4j.{CmdLineParser, Option} +import org.slf4j.LoggerFactory + +import scala.collection.JavaConverters._ + +object ScalaInferenceBenchmark { + + private val logger = LoggerFactory.getLogger(classOf[CLIParserBase]) + + def loadModel(objectToRun: InferBase, context: Array[Context]): + Any = { + objectToRun.loadModel(context) + } + + def loadDataSet(objectToRun: InferBase): + Any = { + objectToRun.loadSingleData() + } + + def loadBatchDataSet(objectToRun: InferBase, batchSize: Int): + List[Any] = { + objectToRun.loadBatchFileList(batchSize) + } + + def runInference(objectToRun: InferBase, loadedModel: Any, dataSet: Any, totalRuns: Int): + List[Long] = { + var inferenceTimes: List[Long] = List() + for (i <- 1 to totalRuns) { + NDArrayCollector.auto().withScope { + val startTimeSingle = System.currentTimeMillis() + objectToRun.runSingleInference(loadedModel, dataSet) + val estimatedTimeSingle = System.currentTimeMillis() - startTimeSingle + inferenceTimes = estimatedTimeSingle :: inferenceTimes + logger.info("Inference time at iteration: %d is : %d \n".format(i, estimatedTimeSingle)) + } + } + + inferenceTimes + } + + def runBatchInference(objecToRun: InferBase, loadedModel: Any, dataSetBatches: List[Any]): + List[Long] = { + + var inferenceTimes: List[Long] = List() + for (batch <- dataSetBatches) { + NDArrayCollector.auto().withScope { + val loadedBatch = objecToRun.loadInputBatch(batch) + val startTimeSingle = System.currentTimeMillis() + objecToRun.runBatchInference(loadedModel, loadedBatch) + val estimatedTimeSingle = System.currentTimeMillis() - startTimeSingle + inferenceTimes = estimatedTimeSingle :: inferenceTimes + logger.info("Batch Inference time is : %d \n".format(estimatedTimeSingle)) + } + } + + inferenceTimes + } + + def percentile(p: Int, seq: Seq[Long]): Long = { + val sorted = seq.sorted + val k = math.ceil((seq.length - 1) * (p / 100.0)).toInt + sorted(k) + } + + def printStatistics(inferenceTimes: List[Long], metricsPrefix: String) { + + val times: Seq[Long] = inferenceTimes + val p50 = percentile(50, times) + val p99 = percentile(99, times) + val average = times.sum / (times.length * 1.0) + + logger.info("\n%s_latency p99 %d, %s_p50 %d, %s_average %f".format(metricsPrefix, + p99, metricsPrefix, p50, metricsPrefix, average)) + + } + + def main(args: Array[String]): Unit = { + + var context = Context.cpu() + if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && + System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { + context = Context.gpu() + } + var baseCLI : CLIParserBase = null + try { + val exampleName = args(1) + val exampleToBenchmark : InferBase = exampleName match { + case "ImageClassifierExample" => { + val imParser = new org.apache.mxnetexamples.infer.imageclassifier.CLIParser + baseCLI = imParser + val parsedVals = new CmdLineParser(imParser).parseArgument(args.toList.asJava) + new ImageClassifierExample(imParser) + } + case _ => throw new Exception("Invalid example name to run") + } + + logger.info("Running single inference call") + // Benchmarking single inference call + NDArrayCollector.auto().withScope { + val loadedModel = loadModel(exampleToBenchmark, context) + val dataSet = loadDataSet(exampleToBenchmark) + val inferenceTimes = runInference(exampleToBenchmark, loadedModel, dataSet, baseCLI.count) + printStatistics(inferenceTimes, "single_inference") + } + + if (baseCLI.batchSize != 0) { + logger.info("Running for batch inference call") + // Benchmarking batch inference call + NDArrayCollector.auto().withScope { + val loadedModel = loadModel(exampleToBenchmark, context) + val batchDataSet = loadBatchDataSet(exampleToBenchmark, baseCLI.batchSize) + val inferenceTimes = runBatchInference(exampleToBenchmark, loadedModel, batchDataSet) + printStatistics(inferenceTimes, "batch_inference") + } + } + + } catch { + case ex: Exception => { + logger.error(ex.getMessage, ex) + new CmdLineParser(baseCLI).printUsage(System.err) + sys.exit(1) + } + } + } + +} + +class CLIParserBase { + @Option(name = "--example", usage = "The scala example to benchmark") + val exampleName: String = "ImageClassifierExample" + @Option(name = "--count", usage = "number of times to run inference") + val count: Int = 1000 + @Option(name = "--batchSize", usage = "BatchSize to run batchinference calls", required = false) + val batchSize: Int = 0 +} \ No newline at end of file diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExample.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExample.scala index f6e4fe0941da..317c5a19d2cf 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExample.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExample.scala @@ -20,11 +20,18 @@ package org.apache.mxnetexamples.infer.imageclassifier import org.apache.mxnet._ import org.kohsuke.args4j.{CmdLineParser, Option} import org.slf4j.LoggerFactory -import org.apache.mxnet.infer.ImageClassifier +import org.apache.mxnet.infer.{Classifier, ImageClassifier} import scala.collection.JavaConverters._ import java.io.File +import org.apache.mxnetexamples.benchmark.CLIParserBase +// scalastyle:off +import java.awt.image.BufferedImage +// scalastyle:on + +import org.apache.mxnetexamples.InferBase + import scala.collection.mutable.ListBuffer // scalastyle:off @@ -108,7 +115,7 @@ object ImageClassifierExample { } def main(args: Array[String]): Unit = { - val inst = new ImageClassifierExample + val inst = new CLIParser val parser: CmdLineParser = new CmdLineParser(inst) var context = Context.cpu() @@ -157,11 +164,70 @@ object ImageClassifierExample { } } -class ImageClassifierExample { +class CLIParser extends CLIParserBase{ @Option(name = "--model-path-prefix", usage = "the input model directory") - private val modelPathPrefix: String = "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/resnet-152/resnet-152" + val modelPathPrefix: String = "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/resnet-152/resnet-152" @Option(name = "--input-image", usage = "the input image") - private val inputImagePath: String = "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/images/kitten.jpg" + val inputImagePath: String = "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/images/kitten.jpg" @Option(name = "--input-dir", usage = "the input batch of images directory") - private val inputImageDir: String = "/images/" + val inputImageDir: String = "/images/" +} + +class ImageClassifierExample(CLIParser: CLIParser) extends InferBase{ + + override def loadModel(context: Array[Context]): Classifier = { + val dType = DType.Float32 + val inputShape = Shape(1, 3, 224, 224) + + val inputDescriptor = IndexedSeq(DataDesc("data", inputShape, dType, "NCHW")) + + // Create object of ImageClassifier class + val imgClassifier: ImageClassifier = new ImageClassifier(CLIParser.modelPathPrefix, inputDescriptor, context) + imgClassifier + } + + override def loadSingleData(): Any = { + val img = ImageClassifier.loadImageFromFile(CLIParser.inputImagePath) + img + } + + override def loadBatchFileList(batchSize: Int): List[Any] = { + val dir = new File(CLIParser.inputImageDir) + require(dir.exists && dir.isDirectory, + "input image directory: %s not found".format(CLIParser.inputImageDir)) + val output = ListBuffer[List[String]]() + var batch = ListBuffer[String]() + for (imgFile: File <- dir.listFiles()){ + batch += imgFile.getPath + if (batch.length == batchSize) { + output += batch.toList + batch = ListBuffer[String]() + } + } + if (batch.length > 0) { + output += batch.toList + } + output.toList + } + + override def loadInputBatch(inputPaths: Any): Any = { + val batchFile = inputPaths.asInstanceOf[List[String]] + ImageClassifier.loadInputBatch(batchFile) + } + + override def runSingleInference(loadedModel: Any, input: Any): Any = { + // Running inference on single image + val imageModel = loadedModel.asInstanceOf[ImageClassifier] + val imgInput = input.asInstanceOf[BufferedImage] + val output = imageModel.classifyImage(imgInput, Some(5)) + output + } + + override def runBatchInference(loadedModel: Any, input: Any): Any = { + val imageModel = loadedModel.asInstanceOf[ImageClassifier] + val imgInput = input.asInstanceOf[Traversable[BufferedImage]] + val output = imageModel.classifyImageBatch(imgInput, Some(5)) + output + } + } diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/SSDClassifierExample.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/SSDClassifierExample.scala index 0edde9e6516b..53c4d3670489 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/SSDClassifierExample.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/SSDClassifierExample.scala @@ -193,4 +193,4 @@ object SSDClassifierExample { exist } -} +} \ No newline at end of file diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala index bd064dbd3518..b8d92d614228 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala @@ -23,9 +23,9 @@ import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ /** - * Follows the demo, to test the char rnn: - * https://github.com/dmlc/mxnet/blob/master/example/rnn/char-rnn.ipynb - */ + * Follows the demo, to test the char rnn: + * https://github.com/dmlc/mxnet/blob/master/example/rnn/char-rnn.ipynb + */ object TestCharRnn { private val logger = LoggerFactory.getLogger(classOf[TrainCharRnn]) @@ -106,4 +106,4 @@ class TestCharRnn { private val modelPrefix: String = "./model/obama" @Option(name = "--starter-sentence", usage = "the starter sentence") private val starterSentence: String = "The joke" -} +} \ No newline at end of file diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmarkSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmarkSuite.scala new file mode 100644 index 000000000000..8786e63efbb9 --- /dev/null +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmarkSuite.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.mxnetexamples.benchmark + +import java.io.File + +import org.apache.mxnetexamples.Util +import org.scalatest.{BeforeAndAfterAll, FunSuite} +import org.slf4j.LoggerFactory + +import scala.sys.process.Process + +class ScalaInferenceBenchmarkSuite extends FunSuite with BeforeAndAfterAll { + private val logger = LoggerFactory.getLogger(classOf[ScalaInferenceBenchmarkSuite]) + override def beforeAll(): Unit = { + } + + test("Testing Benchmark -- Image Classification") { + logger.info("Downloading resnet-18 model") + val tempDirPath = System.getProperty("java.io.tmpdir") + logger.info("tempDirPath: %s".format(tempDirPath)) + val baseUrl = "https://s3.us-east-2.amazonaws.com/scala-infer-models" + Util.downloadUrl(baseUrl + "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/resnet-18/resnet-18-symbol.json", + tempDirPath + "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/resnet18/resnet-18-symbol.json") + Util.downloadUrl(baseUrl + "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/resnet-18/resnet-18-0000.params", + tempDirPath + "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/resnet18/resnet-18-0000.params") + Util.downloadUrl(baseUrl + "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/resnet-18/synset.txt", + tempDirPath + "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/resnet18/synset.txt") + Util.downloadUrl("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg", + tempDirPath + "/inputImages/resnet18/Pug-Cookie.jpg") + val modelDirPath = tempDirPath + File.separator + "resnet18/" + val inputImagePath = tempDirPath + File.separator + + "inputImages/resnet18/Pug-Cookie.jpg" + val inputImageDir = tempDirPath + File.separator + "inputImages/resnet18/" + val args = Array( + "--example", "ImageClassifierExample", + "--count", "1", + "--batchSize", "10", + "--model-path-prefix", s"$modelDirPath/resnet-18", + "--input-image", inputImagePath, + "--input-dir", inputImageDir + ) + ScalaInferenceBenchmark.main(args) + } + +} diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala index 34d3bc97a005..d8631df54052 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala @@ -64,7 +64,7 @@ class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll { } val output = ImageClassifierExample.runInferenceOnSingleImage(modelDirPath + "resnet-18", - inputImagePath, context) + inputImagePath, context) val outputList = ImageClassifierExample.runInferenceOnBatchOfImage(modelDirPath + "resnet-18", inputImageDir, context) From ca3d4e943e0f13a4e187a3067391ef45203c981b Mon Sep 17 00:00:00 2001 From: Piyush Ghai Date: Tue, 2 Oct 2018 13:48:51 -0700 Subject: [PATCH 2/7] Fixed scalastyle issues --- .../mxnetexamples/benchmark/ScalaInferenceBenchmark.scala | 2 +- .../infer/imageclassifier/ImageClassifierExample.scala | 3 ++- .../main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala index 2d1955e96051..3b37806ae420 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala @@ -154,4 +154,4 @@ class CLIParserBase { val count: Int = 1000 @Option(name = "--batchSize", usage = "BatchSize to run batchinference calls", required = false) val batchSize: Int = 0 -} \ No newline at end of file +} diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExample.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExample.scala index 317c5a19d2cf..9b6f19a93a98 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExample.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExample.scala @@ -182,7 +182,8 @@ class ImageClassifierExample(CLIParser: CLIParser) extends InferBase{ val inputDescriptor = IndexedSeq(DataDesc("data", inputShape, dType, "NCHW")) // Create object of ImageClassifier class - val imgClassifier: ImageClassifier = new ImageClassifier(CLIParser.modelPathPrefix, inputDescriptor, context) + val imgClassifier: ImageClassifier = new ImageClassifier(CLIParser.modelPathPrefix, + inputDescriptor, context) imgClassifier } diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala index b8d92d614228..5eb3ab41805b 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/TestCharRnn.scala @@ -106,4 +106,4 @@ class TestCharRnn { private val modelPrefix: String = "./model/obama" @Option(name = "--starter-sentence", usage = "the starter sentence") private val starterSentence: String = "The joke" -} \ No newline at end of file +} From 3f1cb28a1cb9a013faeef1abd7bdeff85b7fdb81 Mon Sep 17 00:00:00 2001 From: Piyush Ghai Date: Tue, 2 Oct 2018 15:21:28 -0700 Subject: [PATCH 3/7] Added platform check to the classpath --- .../scripts/benchmark/run_image_inference_bm.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh b/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh index 37cb3034d17d..a6e4b857158e 100755 --- a/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh +++ b/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh @@ -19,8 +19,16 @@ set -e +echo $OSTYPE +platform=linux-x86_64-cpu + +if [[ "$OSTYPE" == "darwin"* ]]; then + platform=osx-x86_64-cpu +fi + + MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) -CLASS_PATH=$MXNET_ROOT/scala-package/assembly/osx-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/* +CLASS_PATH=$MXNET_ROOT/scala-package/assembly/$platform/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/* MODEL_NAME=$1 From b3b1da6c114d6d8acb6a0c639478f2b1cb725247 Mon Sep 17 00:00:00 2001 From: Piyush Ghai Date: Tue, 2 Oct 2018 15:34:06 -0700 Subject: [PATCH 4/7] Formatting the metrics to print upto 2 decimal digits in float --- .../main/scala/org/apache/mxnetexamples/benchmark/README.md | 4 ++-- .../mxnetexamples/benchmark/ScalaInferenceBenchmark.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md index c0cfd3d65d08..76f12fa97b87 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md @@ -38,11 +38,11 @@ You may need to run ```chmod u+x run_image_inference_bm.sh``` before running thi Upon running this script, you might see an output like this : ``` [main] INFO org.apache.mxnetexamples.benchmark.CLIParserBase - - single_inference_latency p99 1663, single_inference_p50 729, single_inference_average 755.170000 + single_inference_latency p99 1663, single_inference_p50 729, single_inference_average 755.17 ... INFO org.apache.mxnetexamples.benchmark.CLIParserBase - - batch_inference_latency p99 4241, batch_inference_p50 4241, batch_inference_average 4241.000000 + batch_inference_latency p99 4241, batch_inference_p50 4241, batch_inference_average 4241.00 ``` More examples to be added soon. \ No newline at end of file diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala index 3b37806ae420..9ae50dc9d12f 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/ScalaInferenceBenchmark.scala @@ -91,7 +91,7 @@ object ScalaInferenceBenchmark { val p99 = percentile(99, times) val average = times.sum / (times.length * 1.0) - logger.info("\n%s_latency p99 %d, %s_p50 %d, %s_average %f".format(metricsPrefix, + logger.info("\n%s_p99 %d, %s_p50 %d, %s_average %1.2f".format(metricsPrefix, p99, metricsPrefix, p50, metricsPrefix, average)) } From b152203718f0b18cfe8b1b936fbeed97f5eea909 Mon Sep 17 00:00:00 2001 From: Piyush Ghai Date: Tue, 2 Oct 2018 15:49:39 -0700 Subject: [PATCH 5/7] Added bash script to fetch resnet-18 data and params --- .../imageclassifier/get_resnet_18_data.sh | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100755 scala-package/examples/scripts/infer/imageclassifier/get_resnet_18_data.sh diff --git a/scala-package/examples/scripts/infer/imageclassifier/get_resnet_18_data.sh b/scala-package/examples/scripts/infer/imageclassifier/get_resnet_18_data.sh new file mode 100755 index 000000000000..4ba9fd5ac4ce --- /dev/null +++ b/scala-package/examples/scripts/infer/imageclassifier/get_resnet_18_data.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +MXNET_ROOT=$(cd "$(dirname $0)/../../.."; pwd) + +data_path=$MXNET_ROOT/scripts/infer/models/resnet-18/ + +image_path=$MXNET_ROOT/scripts/infer/images/ + +if [ ! -d "$data_path" ]; then + mkdir -p "$data_path" +fi + +if [ ! -d "$image_path" ]; then + mkdir -p "$image_path" +fi + +if [ ! -f "$data_path" ]; then + wget https://s3.us-east-2.amazonaws.com/scala-infer-models/resnet-18/resnet-18-symbol.json -P $data_path + wget https://s3.us-east-2.amazonaws.com/scala-infer-models/resnet-18/resnet-18-0000.params -P $data_path + wget https://s3.us-east-2.amazonaws.com/scala-infer-models/resnet-18/synset.txt -P $data_path + wget https://s3.amazonaws.com/model-server/inputs/kitten.jpg -P $image_path +fi From 6a5314cda31e71063e6bb1a05002c881ac9559be Mon Sep 17 00:00:00 2001 From: Piyush Ghai Date: Thu, 4 Oct 2018 13:39:47 -0700 Subject: [PATCH 6/7] Added flag for cpu/gpu for running the script --- .../benchmark/run_image_inference_bm.sh | 31 +++++++++++++------ .../apache/mxnetexamples/benchmark/README.md | 5 +-- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh b/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh index a6e4b857158e..8b7c94b0db8c 100755 --- a/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh +++ b/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh @@ -20,28 +20,41 @@ set -e echo $OSTYPE -platform=linux-x86_64-cpu + +hw_type=cpu +if [ "$1" = "gpu" ] +then + hw_type=gpu +fi + +platform=linux-x86_64 + +if [[ "$OSTYPE" == "darwin"* ]]; then + platform=osx-x86_64 +fi + +platform=linux-x86_64 if [[ "$OSTYPE" == "darwin"* ]]; then - platform=osx-x86_64-cpu + platform=osx-x86_64 fi MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) -CLASS_PATH=$MXNET_ROOT/scala-package/assembly/$platform/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/* +CLASS_PATH=$MXNET_ROOT/scala-package/assembly/$platform-$hw_type/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/* -MODEL_NAME=$1 +MODEL_NAME=$2 -RUNS=$2 +RUNS=$3 -BATCHSIZE=$3 +BATCHSIZE=$4 # model dir -MODEL_PATH_PREFIX=$4 +MODEL_PATH_PREFIX=$5 # input image -INPUT_IMG=$5 +INPUT_IMG=$6 # which input image dir -INPUT_DIR=$6 +INPUT_DIR=$7 java -Xmx8G -Dmxnet.traceLeakedObjects=true -cp $CLASS_PATH \ org.apache.mxnetexamples.benchmark.ScalaInferenceBenchmark \ diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md index 76f12fa97b87..cfff93397be8 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md @@ -26,14 +26,15 @@ To help you easily run the benchmarks, a starter shell script has been provided Each of the script takes some parameters as inputs, details of which can be found either in the bash scripts or in the example classes itself. * *ImageClassification Example* -
The following shows an example of running ImageClassifier under the benchmark script. The script takes in the number of iterations for inference calls, the batch size for batch inference calls, the model path, input file, and input directory. +
The following shows an example of running ImageClassifier under the benchmark script. The script takes as parameters, the platform type (cpu/gpu), number of iterations for inference calls, the batch size for batch inference calls, the model path, input file, and input directory. For more details to run ImageClassificationExample as a standalone file, refer to the [README](https://github.com/apache/incubator-mxnet/blob/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md) for ImageClassifierExample. You may need to run ```chmod u+x run_image_inference_bm.sh``` before running this script. ```bash cd /scala-package/examples/scripts/infer/imageclassifier ./get_resnet_data.sh cd /scala-package/examples/scripts/benchmark - ./run_image_inference_bm.sh ImageClassifierExample 100 10 ../infer/models/resnet-152/resnet-152 ../infer/images/kitten.jpg ../infer/images/ + ./run_image_inference_bm.sh gpu ImageClassifierExample 100 10 ../infer/models/resnet-152/resnet-152 ../infer/images/kitten.jpg ../infer/images/ + ./run_image_inference_bm.sh cpu ImageClassifierExample 100 10 ../infer/models/resnet-152/resnet-152 ../infer/images/kitten.jpg ../infer/images/ ``` Upon running this script, you might see an output like this : ``` From ca55904d3f39d261702812b39b0f89c35a1e4ee3 Mon Sep 17 00:00:00 2001 From: Piyush Ghai Date: Thu, 4 Oct 2018 16:17:32 -0700 Subject: [PATCH 7/7] Fixed duplicate if check --- .../scripts/benchmark/run_image_inference_bm.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh b/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh index 8b7c94b0db8c..82aa9f622349 100755 --- a/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh +++ b/scala-package/examples/scripts/benchmark/run_image_inference_bm.sh @@ -29,17 +29,11 @@ fi platform=linux-x86_64 -if [[ "$OSTYPE" == "darwin"* ]]; then - platform=osx-x86_64 -fi - -platform=linux-x86_64 - -if [[ "$OSTYPE" == "darwin"* ]]; then - platform=osx-x86_64 +if [ "$OSTYPE" == "darwin"* ] +then + platform=osx-x86_64 fi - MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/$platform-$hw_type/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/*