From da6442fb0852ba4b75de3323371ee0e5fdc2f2d5 Mon Sep 17 00:00:00 2001 From: Mario Inchiosa <4316698+inchiosa@users.noreply.github.com> Date: Mon, 3 Dec 2018 18:17:40 -0800 Subject: [PATCH 1/5] Initial commit of automl sample --- .../spark/h2o/h2o-automl-powerplant.ipynb | 426 ++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb diff --git a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb new file mode 100644 index 0000000000..1e6c2db665 --- /dev/null +++ b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb @@ -0,0 +1,426 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting Spark application\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
1application_1543381571657_0002pyspark3idleLinkLink✔
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SparkSession available as 'spark'.\n" + ] + } + ], + "source": [ + "import time\n", + "\n", + "start_time = time.time()" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting h2o_pysparkling_2.3\n", + " Downloading https://files.pythonhosted.org/packages/e2/2d/43ea2377f68a072161c0dc05a90d722e9b391bac6bb28ac0f54fb1bdafbf/h2o_pysparkling_2.3-2.3.18.tar.gz (51.2MB)\n", + "Requirement already satisfied (use --upgrade to upgrade): requests in /usr/local/lib/python3.5/dist-packages (from h2o_pysparkling_2.3)\n", + "Collecting tabulate (from h2o_pysparkling_2.3)\n", + " Downloading https://files.pythonhosted.org/packages/12/c2/11d6845db5edf1295bc08b2f488cf5937806586afe42936c3f34c097ebdc/tabulate-0.8.2.tar.gz (45kB)\n", + "Collecting future (from h2o_pysparkling_2.3)\n", + " Downloading https://files.pythonhosted.org/packages/90/52/e20466b85000a181e1e144fd8305caf2cf475e2f9674e797b222f8105f5f/future-0.17.1.tar.gz (829kB)\n", + "Collecting colorama>=0.3.8 (from h2o_pysparkling_2.3)\n", + " Downloading https://files.pythonhosted.org/packages/4f/a6/728666f39bfff1719fc94c481890b2106837da9318031f71a8424b662e12/colorama-0.4.1-py2.py3-none-any.whl\n", + "Requirement already satisfied (use --upgrade to upgrade): six in /usr/local/lib/python3.5/dist-packages (from h2o_pysparkling_2.3)\n", + "Collecting pyspark<=2.3.2,>=2.3.0 (from h2o_pysparkling_2.3)\n", + " Downloading https://files.pythonhosted.org/packages/5e/cb/d8ff49ba885e2c88b8cf2967edd84235ffa9ac301bffef657dfa5605a112/pyspark-2.3.2.tar.gz (211.9MB)\n", + "Requirement already satisfied (use --upgrade to upgrade): certifi>=2017.4.17 in /usr/local/lib/python3.5/dist-packages (from requests->h2o_pysparkling_2.3)\n", + "Requirement already satisfied (use --upgrade to upgrade): chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.5/dist-packages (from requests->h2o_pysparkling_2.3)\n", + "Requirement already satisfied (use --upgrade to upgrade): urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.5/dist-packages (from requests->h2o_pysparkling_2.3)\n", + "Requirement already satisfied (use --upgrade to upgrade): idna<2.8,>=2.5 in /usr/local/lib/python3.5/dist-packages (from requests->h2o_pysparkling_2.3)\n", + "Collecting py4j==0.10.7 (from pyspark<=2.3.2,>=2.3.0->h2o_pysparkling_2.3)\n", + " Downloading https://files.pythonhosted.org/packages/e3/53/c737818eb9a7dc32a7cd4f1396e787bd94200c3997c72c1dbe028587bd76/py4j-0.10.7-py2.py3-none-any.whl (197kB)\n", + "Building wheels for collected packages: h2o-pysparkling-2.3, tabulate, future, pyspark\n", + " Running setup.py bdist_wheel for h2o-pysparkling-2.3: started\n", + " Running setup.py bdist_wheel for h2o-pysparkling-2.3: finished with status 'done'\n", + " Stored in directory: /home/.cache/pip/wheels/0c/cd/e6/65d3a3c52525dc1ea3d13ad17e0935172db2f0dcd4a08322ba\n", + " Running setup.py bdist_wheel for tabulate: started\n", + " Running setup.py bdist_wheel for tabulate: finished with status 'done'\n", + " Stored in directory: /home/.cache/pip/wheels/2a/85/33/2f6da85d5f10614cbe5a625eab3b3aebfdf43e7b857f25f829\n", + " Running setup.py bdist_wheel for future: started\n", + " Running setup.py bdist_wheel for future: finished with status 'done'\n", + " Stored in directory: /home/.cache/pip/wheels/0c/61/d2/d6b7317325828fbb39ee6ad559dbe4664d0896da4721bf379e\n", + " Running setup.py bdist_wheel for pyspark: started\n", + " Running setup.py bdist_wheel for pyspark: finished with status 'done'\n", + " Stored in directory: /home/.cache/pip/wheels/be/7d/34/cd3cfbc75d8b6b6ae0658e5425348560b86d187fe3e53832cc\n", + "Successfully built h2o-pysparkling-2.3 tabulate future pyspark\n", + "Installing collected packages: tabulate, future, colorama, py4j, pyspark, h2o-pysparkling-2.3\n", + "Successfully installed colorama-0.4.1 future-0.17.1 h2o-pysparkling-2.3-2.3.18 py4j-0.10.7 pyspark-2.3.2 tabulate-0.8.2\n", + "You are using pip version 8.1.1, however version 18.1 is available.\n", + "You should consider upgrading via the 'pip install --upgrade pip' command." + ] + } + ], + "source": [ + "import subprocess\n", + "\n", + "# Install H2O PySparkling\n", + "stdout = subprocess.check_output(\n", + " \"pip3 install h2o_pysparkling_2.3\",\n", + " stderr=subprocess.STDOUT,\n", + " shell=True).decode(\"utf-8\")\n", + "print(stdout)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ls: `/tmp/powerplant_output.csv': No such file or directory\n", + "--2018-11-28 18:21:03-- https://raw.githubusercontent.com/h2oai/h2o-tutorials/master/h2o-world-2017/automl/data/powerplant_output.csv\n", + "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.48.133\n", + "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.48.133|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 308777 (302K) [text/plain]\n", + "Saving to: 'powerplant_output.csv'\n", + "\n", + " 0K .......... .......... .......... .......... .......... 16% 293K 1s\n", + " 50K .......... .......... .......... .......... .......... 33% 568K 1s\n", + " 100K .......... .......... .......... .......... .......... 49% 294K 0s\n", + " 150K .......... .......... .......... .......... .......... 66% 228K 0s\n", + " 200K .......... .......... .......... .......... .......... 82% 193K 0s\n", + " 250K .......... .......... .......... .......... .......... 99% 231K 0s\n", + " 300K . 100% 2937G=1.1s\n", + "\n", + "2018-11-28 18:21:05 (268 KB/s) - 'powerplant_output.csv' saved [308777/308777]" + ] + } + ], + "source": [ + "dataFileName = \"powerplant_output.csv\"\n", + "dataFileUrl = \"https://raw.githubusercontent.com/h2oai/h2o-tutorials/master/h2o-world-2017/automl/data/\" + dataFileName\n", + "\n", + "# Download data file and copy to HDFS, if not already there\n", + "cmd = 'hdfs dfs -ls /tmp/' + dataFileName + ' || ' \\\n", + " '(wget ' + dataFileUrl + ' && ' \\\n", + " 'hdfs dfs -copyFromLocal ' + dataFileName + ' /tmp && ' \\\n", + " 'rm ' + dataFileName + ')'\n", + "\n", + "stdout = subprocess.check_output(\n", + " cmd,\n", + " stderr=subprocess.STDOUT,\n", + " shell=True).decode(\"utf-8\")\n", + "print(stdout)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Connecting to H2O server at http://10.244.0.66:54323... successful.\n", + "-------------------------- ---------------------------------------------------\n", + "H2O cluster uptime: 12 secs\n", + "H2O cluster timezone: Etc/UTC\n", + "H2O data parsing timezone: UTC\n", + "H2O cluster version: 3.22.0.2\n", + "H2O cluster version age: 6 days\n", + "H2O cluster name: sparkling-water-root_application_1543381571657_0002\n", + "H2O cluster total nodes: 3\n", + "H2O cluster free memory: 3.698 Gb\n", + "H2O cluster total cores: 48\n", + "H2O cluster allowed cores: 3\n", + "H2O cluster status: accepting new members, healthy\n", + "H2O connection url: http://10.244.0.66:54323\n", + "H2O connection proxy:\n", + "H2O internal security: False\n", + "H2O API Extensions: XGBoost, Algos, AutoML, Core V3, Core V4\n", + "Python version: 3.5.2 final\n", + "-------------------------- ---------------------------------------------------\n", + "\n", + "Sparkling Water Context:\n", + " * H2O name: sparkling-water-root_application_1543381571657_0002\n", + " * cluster size: 3\n", + " * list of used nodes:\n", + " (executorId, host, port)\n", + " ------------------------\n", + " (1,mssql-storage-pool-default-1.service-storage-pool-default.test.svc.cluster.local,54321)\n", + " (2,mssql-storage-pool-default-0.service-storage-pool-default.test.svc.cluster.local,54321)\n", + " (3,mssql-storage-pool-default-1.service-storage-pool-default.test.svc.cluster.local,54323)\n", + " ------------------------\n", + "\n", + " Open H2O Flow in browser: http://10.244.0.66:54323 (CMD + click in Mac OSX)\n", + "\n", + " \n", + " * Yarn App ID of Spark application: application_1543381571657_0002" + ] + } + ], + "source": [ + "from pysparkling import H2OContext\n", + "\n", + "hc = H2OContext.getOrCreate(spark)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mssql-storage-pool-default-0" + ] + } + ], + "source": [ + "# Print the hostname\n", + "stdout = subprocess.check_output(\n", + " \"hostname\",\n", + " stderr=subprocess.STDOUT,\n", + " shell=True).decode(\"utf-8\")\n", + "print(stdout)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------------+---------------+-----------------------+----------------+--------------------+--------------------+\n", + "|TemperatureCelcius|ExhaustVacuumHg|AmbientPressureMillibar|RelativeHumidity|HourlyEnergyOutputMW| prediction_output|\n", + "+------------------+---------------+-----------------------+----------------+--------------------+--------------------+\n", + "| 10.01| 41.17| 1018.78| 86.84| 479.4|[478.98443473089264]|\n", + "| 10.02| 39.66| 1016.34| 79.98| 480.05| [478.4744401166399]|\n", + "| 10.03| 43.13| 1014.85| 70.09| 482.16|[475.98085258410845]|\n", + "| 10.04| 41.62| 1013.36| 95.17| 463.87|[468.36092209033416]|\n", + "| 10.05| 41.58| 1021.35| 95.19| 469.03|[468.67240194913506]|\n", + "| 10.06| 34.69| 1027.9| 71.73| 477.68|[477.79641646128107]|\n", + "| 10.08| 37.92| 1010.47| 66.37| 474.63| [475.2033972206058]|\n", + "| 10.08| 41.16| 1023.14| 96.03| 469.17| [470.3570481951197]|\n", + "| 10.09| 41.01| 1019.89| 96.55| 471.15| [469.6710775300654]|\n", + "| 10.1| 41.4| 1024.29| 85.94| 474.28|[477.42433033233783]|\n", + "| 10.11| 39.35| 1015.19| 90.74| 479.83| [477.792144625552]|\n", + "| 10.11| 39.72| 1019.1| 69.68| 476.8|[473.80793646542656]|\n", + "| 10.11| 42.49| 1010.22| 82.11| 483.56| [476.5521742794438]|\n", + "| 10.12| 41.55| 1005.78| 62.34| 475.46| [475.9253245215578]|\n", + "| 10.12| 41.78| 1013.43| 73.47| 477.67| [475.0193582197604]|\n", + "| 10.13| 39.18| 1024.09| 85.48| 479.42| [477.9988789711212]|\n", + "| 10.15| 39.22| 1020.09| 68.75| 474.87| [476.4446239317696]|\n", + "| 10.15| 41.46| 1019.78| 83.56| 481.31| [479.5577986068803]|\n", + "| 10.15| 43.41| 1018.4| 82.07| 473.43|[476.77545876514284]|\n", + "| 10.16| 39.3| 1019.71| 81.21| 480.74| [477.2729727910962]|\n", + "+------------------+---------------+-----------------------+----------------+--------------------+--------------------+\n", + "only showing top 20 rows" + ] + } + ], + "source": [ + "from pysparkling.ml import H2OAutoML\n", + "from pyspark.ml import Pipeline\n", + "from pyspark.ml.feature import SQLTransformer\n", + "\n", + "powerplant_df = spark.read.option(\"inferSchema\", \"true\").csv(\"/tmp/powerplant_output.csv\", header=True)\n", + "\n", + "splits = powerplant_df.randomSplit([0.8, 0.2], seed=1)\n", + "train = splits[0]\n", + "for_predictions = splits[1]\n", + "\n", + "temperatureTransformer = SQLTransformer(statement=\"SELECT * FROM __THIS__ WHERE TemperatureCelcius > 10\")\n", + "\n", + "automlEstimator = H2OAutoML(maxModels=2, predictionCol=\"HourlyEnergyOutputMW\",\n", + " ratio=0.9, seed=1)\n", + "\n", + "pipeline = Pipeline(stages=[temperatureTransformer, automlEstimator])\n", + "model = pipeline.fit(train)\n", + "predicted = model.transform(for_predictions)\n", + "\n", + "predicted.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------+----------------------+------------------+------------------+------------------+---------------------+\n", + "|model_id |mean_residual_deviance|rmse |mse |mae |rmsle |\n", + "+---------------------------------------------------+----------------------+------------------+------------------+------------------+---------------------+\n", + "|StackedEnsemble_BestOfFamily_AutoML_20181128_003911|11.00711037433692 |3.317696546451607 |11.00711037433692 |2.460978645004018 |0.0073362304526039795|\n", + "|StackedEnsemble_AllModels_AutoML_20181128_003911 |11.00711037433692 |3.317696546451607 |11.00711037433692 |2.460978645004018 |0.0073362304526039795|\n", + "|XRT_1_AutoML_20181128_003911 |11.18040320029184 |3.3437109923394757|11.18040320029184 |2.5010207855031896|0.007393792671247257 |\n", + "|DRF_1_AutoML_20181128_003911 |11.245178401907951 |3.3533831278140513|11.245178401907951|2.4970917585784322|0.007413255513108019 |\n", + "+---------------------------------------------------+----------------------+------------------+------------------+------------------+---------------------+" + ] + } + ], + "source": [ + "automlEstimator.leaderboard().show(truncate=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "from pyspark.sql.functions import *\n", + "\n", + "scores = predicted.select(predicted['HourlyEnergyOutputMW'], predicted['prediction_output']['value'].alias('prediction'))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mean absolute error: 2.4058999799874976" + ] + } + ], + "source": [ + "from pyspark.ml.evaluation import RegressionEvaluator\n", + "\n", + "evaluator = RegressionEvaluator(predictionCol=\"prediction\",\n", + " labelCol=\"HourlyEnergyOutputMW\",\n", + " metricName=\"mae\")\n", + "\n", + "mae = evaluator.evaluate(scores)\n", + "\n", + "print(\"Mean absolute error:\", mae)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "r squared: 0.9516895664346249" + ] + } + ], + "source": [ + "from pyspark.ml.evaluation import RegressionEvaluator\n", + "\n", + "evaluator = RegressionEvaluator(predictionCol=\"prediction\",\n", + " labelCol=\"HourlyEnergyOutputMW\",\n", + " metricName=\"r2\")\n", + "\n", + "r2 = evaluator.evaluate(scores)\n", + "\n", + "print(\"r squared:\", r2)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Elapsed time: 199.5 seconds." + ] + } + ], + "source": [ + "elapsed_time = time.time() - start_time\n", + "print(\"Elapsed time: {:.1f} seconds.\".format(elapsed_time))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "PySpark3", + "language": "", + "name": "pyspark3kernel" + }, + "language_info": { + "codemirror_mode": { + "name": "python", + "version": 3 + }, + "mimetype": "text/x-python", + "name": "pyspark3", + "pygments_lexer": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 799678085a0e134467743d7bb4d30ad6095c2707 Mon Sep 17 00:00:00 2001 From: Mario Inchiosa <4316698+inchiosa@users.noreply.github.com> Date: Thu, 6 Dec 2018 15:01:09 -0800 Subject: [PATCH 2/5] update notebook and add readme --- .../machine-learning/spark/h2o/README.md | 19 + .../spark/h2o/h2o-automl-powerplant.ipynb | 419 +++++++++++------- 2 files changed, 289 insertions(+), 149 deletions(-) create mode 100644 samples/features/sql-big-data-cluster/machine-learning/spark/h2o/README.md diff --git a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/README.md b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/README.md new file mode 100644 index 0000000000..98421cc220 --- /dev/null +++ b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/README.md @@ -0,0 +1,19 @@ +# Automated Machine Learning using H2O in SQL Server 2019 Big Data Cluster + +## Machine Learning using Spark + +The new built-in notebooks in Azure Data Studio enable data scientists and data engineers to run Python, R, or Scala code against the cluster. This is a great way to explore the data and build machine learning models. Notebooks facilitate collaboration between teammates working on a shared data set. + +This sample uses the automated machine learning capabilities of the third party H2O package running in Spark in a SQL Server 2019 Big Data Cluster to build a machine learning model that predicts powerplant output. + +### Instructions + +1. From Azure Data Studio, connect to the SQL Server Big Data Cluster endpoint. Information about how you connect from Azure Data Studio can be found [here](https://docs.microsoft.com/en-us/sql/azure-data-studio/sql-server-2019-extension?view=sql-server-ver15). + +1. Download and save the notebook file [h2o-automl-powerplant.ipynb](h2o-automl-powerplant.ipynb/) locally. + +1. Open the notebook file in Azure Data Studio (right click on the SQL Server big data cluster server name-> **Manage**-> Open Notebook). + +1. Wait for the “Kernel” and the target context (“Attach to”) to be populated. Set the “Kernel” to **PySpark3** and “Attach to” needs to be the IP address of your big data cluster endpoint. + +1. Run each cell in the Notebook sequentially using Azure Data Studio. Expect the first cell to take 20 sec to finish. Other cells downloading and installing H2O, downloading data, and performing automated machine learning may take several minutes to finish. diff --git a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb index 1e6c2db665..4b7dac4498 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb +++ b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb @@ -1,5 +1,23 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Powerplant Output Prediction\n", + "- This notebook is based on the power plant output prediction example presented in H2O’s [blog post on H2O AutoML in Spark](https://www.h2o.ai/blog/h2os-automl-in-spark/).\n", + "- Run this notebook in Azure Data Studio connected to a SQL Server 2019 Big Data Cluster by following the instructions [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/notebooks-guidance?view=sqlallproducts-allversions)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Spark Configuration\n", + "- We can control the Spark Driver and Executor memory, cores, and number of executors per pod using the “%%configure” cell magic\n", + "- Additional configuration settings are listed at the end of this notebook\n" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -7,18 +25,10 @@ "language": "python" }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting Spark application\n" - ] - }, { "data": { "text/html": [ - "\n", - "
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
1application_1543381571657_0002pyspark3idleLinkLink✔
" + "Current session configs: {'executorMemory': '4g', 'driverMemory': '4g', 'executorCores': 2, 'driverCores': 2, 'numExecutors': 2, 'kind': 'pyspark3'}
" ], "text/plain": [ "" @@ -28,17 +38,36 @@ "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "SparkSession available as 'spark'.\n" - ] + "data": { + "text/html": [ + "No active sessions." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "import time\n", - "\n", - "start_time = time.time()" + "%%configure -f\n", + "{\n", + " \"executorMemory\": \"4g\",\n", + " \"driverMemory\": \"4g\",\n", + " \"executorCores\": 2,\n", + " \"driverCores\": 2,\n", + " \"numExecutors\": 2\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Install H2O\n", + "- This cell downloads the h2o_pysparkling_2.3 python package and installs it on the pod where the Spark driver is currently running, if it is not already installed. Propagating the software to additional pods is handled automatically once we launch H2O.\n", + "- For an enterprise scenario where we cannot reach out to the PyPi repository on the Internet, pip3 can be pointed to a local copy." ] }, { @@ -52,40 +81,41 @@ "name": "stdout", "output_type": "stream", "text": [ + "Starting Spark application\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
24application_1543381571657_0025pyspark3idleLinkLink✔
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SparkSession available as 'spark'.\n", "Collecting h2o_pysparkling_2.3\n", - " Downloading https://files.pythonhosted.org/packages/e2/2d/43ea2377f68a072161c0dc05a90d722e9b391bac6bb28ac0f54fb1bdafbf/h2o_pysparkling_2.3-2.3.18.tar.gz (51.2MB)\n", - "Requirement already satisfied (use --upgrade to upgrade): requests in /usr/local/lib/python3.5/dist-packages (from h2o_pysparkling_2.3)\n", - "Collecting tabulate (from h2o_pysparkling_2.3)\n", - " Downloading https://files.pythonhosted.org/packages/12/c2/11d6845db5edf1295bc08b2f488cf5937806586afe42936c3f34c097ebdc/tabulate-0.8.2.tar.gz (45kB)\n", - "Collecting future (from h2o_pysparkling_2.3)\n", - " Downloading https://files.pythonhosted.org/packages/90/52/e20466b85000a181e1e144fd8305caf2cf475e2f9674e797b222f8105f5f/future-0.17.1.tar.gz (829kB)\n", - "Collecting colorama>=0.3.8 (from h2o_pysparkling_2.3)\n", - " Downloading https://files.pythonhosted.org/packages/4f/a6/728666f39bfff1719fc94c481890b2106837da9318031f71a8424b662e12/colorama-0.4.1-py2.py3-none-any.whl\n", + "Requirement already satisfied (use --upgrade to upgrade): tabulate in /usr/local/lib/python3.5/dist-packages (from h2o_pysparkling_2.3)\n", "Requirement already satisfied (use --upgrade to upgrade): six in /usr/local/lib/python3.5/dist-packages (from h2o_pysparkling_2.3)\n", - "Collecting pyspark<=2.3.2,>=2.3.0 (from h2o_pysparkling_2.3)\n", - " Downloading https://files.pythonhosted.org/packages/5e/cb/d8ff49ba885e2c88b8cf2967edd84235ffa9ac301bffef657dfa5605a112/pyspark-2.3.2.tar.gz (211.9MB)\n", + "Requirement already satisfied (use --upgrade to upgrade): future in /usr/local/lib/python3.5/dist-packages (from h2o_pysparkling_2.3)\n", + "Requirement already satisfied (use --upgrade to upgrade): requests in /usr/local/lib/python3.5/dist-packages (from h2o_pysparkling_2.3)\n", + "Requirement already satisfied (use --upgrade to upgrade): colorama>=0.3.8 in /usr/local/lib/python3.5/dist-packages (from h2o_pysparkling_2.3)\n", + "Requirement already satisfied (use --upgrade to upgrade): pyspark<=2.3.2,>=2.3.0 in /usr/local/lib/python3.5/dist-packages (from h2o_pysparkling_2.3)\n", "Requirement already satisfied (use --upgrade to upgrade): certifi>=2017.4.17 in /usr/local/lib/python3.5/dist-packages (from requests->h2o_pysparkling_2.3)\n", "Requirement already satisfied (use --upgrade to upgrade): chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.5/dist-packages (from requests->h2o_pysparkling_2.3)\n", "Requirement already satisfied (use --upgrade to upgrade): urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.5/dist-packages (from requests->h2o_pysparkling_2.3)\n", "Requirement already satisfied (use --upgrade to upgrade): idna<2.8,>=2.5 in /usr/local/lib/python3.5/dist-packages (from requests->h2o_pysparkling_2.3)\n", - "Collecting py4j==0.10.7 (from pyspark<=2.3.2,>=2.3.0->h2o_pysparkling_2.3)\n", - " Downloading https://files.pythonhosted.org/packages/e3/53/c737818eb9a7dc32a7cd4f1396e787bd94200c3997c72c1dbe028587bd76/py4j-0.10.7-py2.py3-none-any.whl (197kB)\n", - "Building wheels for collected packages: h2o-pysparkling-2.3, tabulate, future, pyspark\n", - " Running setup.py bdist_wheel for h2o-pysparkling-2.3: started\n", - " Running setup.py bdist_wheel for h2o-pysparkling-2.3: finished with status 'done'\n", - " Stored in directory: /home/.cache/pip/wheels/0c/cd/e6/65d3a3c52525dc1ea3d13ad17e0935172db2f0dcd4a08322ba\n", - " Running setup.py bdist_wheel for tabulate: started\n", - " Running setup.py bdist_wheel for tabulate: finished with status 'done'\n", - " Stored in directory: /home/.cache/pip/wheels/2a/85/33/2f6da85d5f10614cbe5a625eab3b3aebfdf43e7b857f25f829\n", - " Running setup.py bdist_wheel for future: started\n", - " Running setup.py bdist_wheel for future: finished with status 'done'\n", - " Stored in directory: /home/.cache/pip/wheels/0c/61/d2/d6b7317325828fbb39ee6ad559dbe4664d0896da4721bf379e\n", - " Running setup.py bdist_wheel for pyspark: started\n", - " Running setup.py bdist_wheel for pyspark: finished with status 'done'\n", - " Stored in directory: /home/.cache/pip/wheels/be/7d/34/cd3cfbc75d8b6b6ae0658e5425348560b86d187fe3e53832cc\n", - "Successfully built h2o-pysparkling-2.3 tabulate future pyspark\n", - "Installing collected packages: tabulate, future, colorama, py4j, pyspark, h2o-pysparkling-2.3\n", - "Successfully installed colorama-0.4.1 future-0.17.1 h2o-pysparkling-2.3-2.3.18 py4j-0.10.7 pyspark-2.3.2 tabulate-0.8.2\n", + "Requirement already satisfied (use --upgrade to upgrade): py4j==0.10.7 in /usr/local/lib/python3.5/dist-packages (from pyspark<=2.3.2,>=2.3.0->h2o_pysparkling_2.3)\n", + "Installing collected packages: h2o-pysparkling-2.3\n", + "Successfully installed h2o-pysparkling-2.3-2.3.18\n", "You are using pip version 8.1.1, however version 18.1 is available.\n", "You should consider upgrading via the 'pip install --upgrade pip' command." ] @@ -102,6 +132,13 @@ "print(stdout)\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Download and copy data to HDFS" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -114,22 +151,22 @@ "output_type": "stream", "text": [ "ls: `/tmp/powerplant_output.csv': No such file or directory\n", - "--2018-11-28 18:21:03-- https://raw.githubusercontent.com/h2oai/h2o-tutorials/master/h2o-world-2017/automl/data/powerplant_output.csv\n", + "--2018-12-06 19:29:35-- https://raw.githubusercontent.com/h2oai/h2o-tutorials/master/h2o-world-2017/automl/data/powerplant_output.csv\n", "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.48.133\n", "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.48.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 308777 (302K) [text/plain]\n", "Saving to: 'powerplant_output.csv'\n", "\n", - " 0K .......... .......... .......... .......... .......... 16% 293K 1s\n", - " 50K .......... .......... .......... .......... .......... 33% 568K 1s\n", - " 100K .......... .......... .......... .......... .......... 49% 294K 0s\n", - " 150K .......... .......... .......... .......... .......... 66% 228K 0s\n", - " 200K .......... .......... .......... .......... .......... 82% 193K 0s\n", - " 250K .......... .......... .......... .......... .......... 99% 231K 0s\n", - " 300K . 100% 2937G=1.1s\n", + " 0K .......... .......... .......... .......... .......... 16% 574K 0s\n", + " 50K .......... .......... .......... .......... .......... 33% 1.14M 0s\n", + " 100K .......... .......... .......... .......... .......... 49% 30.9M 0s\n", + " 150K .......... .......... .......... .......... .......... 66% 1.16M 0s\n", + " 200K .......... .......... .......... .......... .......... 82% 42.7M 0s\n", + " 250K .......... .......... .......... .......... .......... 99% 77.2M 0s\n", + " 300K . 100% 2937G=0.2s\n", "\n", - "2018-11-28 18:21:05 (268 KB/s) - 'powerplant_output.csv' saved [308777/308777]" + "2018-12-06 19:29:36 (1.68 MB/s) - 'powerplant_output.csv' saved [308777/308777]" ] } ], @@ -150,6 +187,13 @@ "print(stdout)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Start H2O engine" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -163,16 +207,16 @@ "text": [ "Connecting to H2O server at http://10.244.0.66:54323... successful.\n", "-------------------------- ---------------------------------------------------\n", - "H2O cluster uptime: 12 secs\n", + "H2O cluster uptime: 13 secs\n", "H2O cluster timezone: Etc/UTC\n", "H2O data parsing timezone: UTC\n", "H2O cluster version: 3.22.0.2\n", - "H2O cluster version age: 6 days\n", - "H2O cluster name: sparkling-water-root_application_1543381571657_0002\n", - "H2O cluster total nodes: 3\n", - "H2O cluster free memory: 3.698 Gb\n", - "H2O cluster total cores: 48\n", - "H2O cluster allowed cores: 3\n", + "H2O cluster version age: 14 days, 12 hours and 24 minutes\n", + "H2O cluster name: sparkling-water-root_application_1543381571657_0021\n", + "H2O cluster total nodes: 2\n", + "H2O cluster free memory: 6.928 Gb\n", + "H2O cluster total cores: 32\n", + "H2O cluster allowed cores: 4\n", "H2O cluster status: accepting new members, healthy\n", "H2O connection url: http://10.244.0.66:54323\n", "H2O connection proxy:\n", @@ -182,20 +226,19 @@ "-------------------------- ---------------------------------------------------\n", "\n", "Sparkling Water Context:\n", - " * H2O name: sparkling-water-root_application_1543381571657_0002\n", - " * cluster size: 3\n", + " * H2O name: sparkling-water-root_application_1543381571657_0021\n", + " * cluster size: 2\n", " * list of used nodes:\n", " (executorId, host, port)\n", " ------------------------\n", " (1,mssql-storage-pool-default-1.service-storage-pool-default.test.svc.cluster.local,54321)\n", " (2,mssql-storage-pool-default-0.service-storage-pool-default.test.svc.cluster.local,54321)\n", - " (3,mssql-storage-pool-default-1.service-storage-pool-default.test.svc.cluster.local,54323)\n", " ------------------------\n", "\n", " Open H2O Flow in browser: http://10.244.0.66:54323 (CMD + click in Mac OSX)\n", "\n", " \n", - " * Yarn App ID of Spark application: application_1543381571657_0002" + " * Yarn App ID of Spark application: application_1543381571657_0021" ] } ], @@ -221,7 +264,7 @@ } ], "source": [ - "# Print the hostname\n", + "# Print the hostname of the pod where the driver is running\n", "stdout = subprocess.check_output(\n", " \"hostname\",\n", " stderr=subprocess.STDOUT,\n", @@ -229,6 +272,37 @@ "print(stdout)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Read and split data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "powerplant_df = spark.read.option(\"inferSchema\", \"true\").csv(\"/tmp/powerplant_output.csv\", header=True)\n", + "\n", + "splits = powerplant_df.randomSplit([0.8, 0.2], seed=1)\n", + "train = splits[0]\n", + "for_predictions = splits[1]\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Training and Prediction\n", + "- Fit AutoML model on training data\n", + "- Generate predictions on \"for_predictions\" data" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -243,54 +317,58 @@ "+------------------+---------------+-----------------------+----------------+--------------------+--------------------+\n", "|TemperatureCelcius|ExhaustVacuumHg|AmbientPressureMillibar|RelativeHumidity|HourlyEnergyOutputMW| prediction_output|\n", "+------------------+---------------+-----------------------+----------------+--------------------+--------------------+\n", - "| 10.01| 41.17| 1018.78| 86.84| 479.4|[478.98443473089264]|\n", - "| 10.02| 39.66| 1016.34| 79.98| 480.05| [478.4744401166399]|\n", - "| 10.03| 43.13| 1014.85| 70.09| 482.16|[475.98085258410845]|\n", - "| 10.04| 41.62| 1013.36| 95.17| 463.87|[468.36092209033416]|\n", - "| 10.05| 41.58| 1021.35| 95.19| 469.03|[468.67240194913506]|\n", - "| 10.06| 34.69| 1027.9| 71.73| 477.68|[477.79641646128107]|\n", - "| 10.08| 37.92| 1010.47| 66.37| 474.63| [475.2033972206058]|\n", - "| 10.08| 41.16| 1023.14| 96.03| 469.17| [470.3570481951197]|\n", - "| 10.09| 41.01| 1019.89| 96.55| 471.15| [469.6710775300654]|\n", - "| 10.1| 41.4| 1024.29| 85.94| 474.28|[477.42433033233783]|\n", - "| 10.11| 39.35| 1015.19| 90.74| 479.83| [477.792144625552]|\n", - "| 10.11| 39.72| 1019.1| 69.68| 476.8|[473.80793646542656]|\n", - "| 10.11| 42.49| 1010.22| 82.11| 483.56| [476.5521742794438]|\n", - "| 10.12| 41.55| 1005.78| 62.34| 475.46| [475.9253245215578]|\n", - "| 10.12| 41.78| 1013.43| 73.47| 477.67| [475.0193582197604]|\n", - "| 10.13| 39.18| 1024.09| 85.48| 479.42| [477.9988789711212]|\n", - "| 10.15| 39.22| 1020.09| 68.75| 474.87| [476.4446239317696]|\n", - "| 10.15| 41.46| 1019.78| 83.56| 481.31| [479.5577986068803]|\n", - "| 10.15| 43.41| 1018.4| 82.07| 473.43|[476.77545876514284]|\n", - "| 10.16| 39.3| 1019.71| 81.21| 480.74| [477.2729727910962]|\n", + "| 10.01| 41.17| 1018.78| 86.84| 479.4|[477.07336140008584]|\n", + "| 10.02| 39.66| 1016.34| 79.98| 480.05| [476.9418514217447]|\n", + "| 10.03| 43.13| 1014.85| 70.09| 482.16| [476.1059605448468]|\n", + "| 10.04| 41.62| 1013.36| 95.17| 463.87| [470.5313939263834]|\n", + "| 10.05| 41.58| 1021.35| 95.19| 469.03| [469.2805814291723]|\n", + "| 10.06| 34.69| 1027.9| 71.73| 477.68| [477.8063480544951]|\n", + "| 10.08| 37.92| 1010.47| 66.37| 474.63| [475.4396167315097]|\n", + "| 10.08| 41.16| 1023.14| 96.03| 469.17|[471.84298416925895]|\n", + "| 10.09| 41.01| 1019.89| 96.55| 471.15| [471.4895037099167]|\n", + "| 10.1| 41.4| 1024.29| 85.94| 474.28|[477.99005354475656]|\n", + "| 10.11| 39.35| 1015.19| 90.74| 479.83| [478.0068968374228]|\n", + "| 10.11| 39.72| 1019.1| 69.68| 476.8|[474.19764922516293]|\n", + "| 10.11| 42.49| 1010.22| 82.11| 483.56|[477.02467192323263]|\n", + "| 10.12| 41.55| 1005.78| 62.34| 475.46|[475.25159731408183]|\n", + "| 10.12| 41.78| 1013.43| 73.47| 477.67| [475.3766641350098]|\n", + "| 10.13| 39.18| 1024.09| 85.48| 479.42|[478.11614912148826]|\n", + "| 10.15| 39.22| 1020.09| 68.75| 474.87| [477.1365182805492]|\n", + "| 10.15| 41.46| 1019.78| 83.56| 481.31|[479.42546663321224]|\n", + "| 10.15| 43.41| 1018.4| 82.07| 473.43|[476.34953390951085]|\n", + "| 10.16| 39.3| 1019.71| 81.21| 480.74|[476.95324226094823]|\n", "+------------------+---------------+-----------------------+----------------+--------------------+--------------------+\n", "only showing top 20 rows" ] } ], "source": [ + "from pyspark.ml.feature import SQLTransformer\n", "from pysparkling.ml import H2OAutoML\n", "from pyspark.ml import Pipeline\n", - "from pyspark.ml.feature import SQLTransformer\n", - "\n", - "powerplant_df = spark.read.option(\"inferSchema\", \"true\").csv(\"/tmp/powerplant_output.csv\", header=True)\n", - "\n", - "splits = powerplant_df.randomSplit([0.8, 0.2], seed=1)\n", - "train = splits[0]\n", - "for_predictions = splits[1]\n", "\n", "temperatureTransformer = SQLTransformer(statement=\"SELECT * FROM __THIS__ WHERE TemperatureCelcius > 10\")\n", "\n", - "automlEstimator = H2OAutoML(maxModels=2, predictionCol=\"HourlyEnergyOutputMW\",\n", - " ratio=0.9, seed=1)\n", + "automlEstimator = H2OAutoML(maxModels=2, predictionCol=\"HourlyEnergyOutputMW\", seed=1)\n", "\n", "pipeline = Pipeline(stages=[temperatureTransformer, automlEstimator])\n", + "\n", + "# Fit AutoML model\n", "model = pipeline.fit(train)\n", + "\n", + "# Generate predictions using fitted model\n", "predicted = model.transform(for_predictions)\n", "\n", "predicted.show()\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Display the leaderboard metrics" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -302,14 +380,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "+---------------------------------------------------+----------------------+------------------+------------------+------------------+---------------------+\n", - "|model_id |mean_residual_deviance|rmse |mse |mae |rmsle |\n", - "+---------------------------------------------------+----------------------+------------------+------------------+------------------+---------------------+\n", - "|StackedEnsemble_BestOfFamily_AutoML_20181128_003911|11.00711037433692 |3.317696546451607 |11.00711037433692 |2.460978645004018 |0.0073362304526039795|\n", - "|StackedEnsemble_AllModels_AutoML_20181128_003911 |11.00711037433692 |3.317696546451607 |11.00711037433692 |2.460978645004018 |0.0073362304526039795|\n", - "|XRT_1_AutoML_20181128_003911 |11.18040320029184 |3.3437109923394757|11.18040320029184 |2.5010207855031896|0.007393792671247257 |\n", - "|DRF_1_AutoML_20181128_003911 |11.245178401907951 |3.3533831278140513|11.245178401907951|2.4970917585784322|0.007413255513108019 |\n", - "+---------------------------------------------------+----------------------+------------------+------------------+------------------+---------------------+" + "+---------------------------------------------------+----------------------+------------------+------------------+------------------+--------------------+\n", + "|model_id |mean_residual_deviance|rmse |mse |mae |rmsle |\n", + "+---------------------------------------------------+----------------------+------------------+------------------+------------------+--------------------+\n", + "|StackedEnsemble_BestOfFamily_AutoML_20181206_193040|11.204658852035337 |3.3473360829225585|11.204658852035337|2.509389117612746 |0.007425043374216511|\n", + "|StackedEnsemble_AllModels_AutoML_20181206_193040 |11.204658852035337 |3.3473360829225585|11.204658852035337|2.509389117612746 |0.007425043374216511|\n", + "|DRF_1_AutoML_20181206_193040 |11.349494687812056 |3.3689011098297406|11.349494687812056|2.5426288374345605|0.007472705853530634|\n", + "|XRT_1_AutoML_20181206_193040 |11.464865035526516 |3.3859806608317355|11.464865035526516|2.545269555169042 |0.007510102104757881|\n", + "+---------------------------------------------------+----------------------+------------------+------------------+------------------+--------------------+" ] } ], @@ -318,16 +396,11 @@ ] }, { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "language": "python" - }, - "outputs": [], + "cell_type": "markdown", + "metadata": {}, "source": [ - "from pyspark.sql.functions import *\n", - "\n", - "scores = predicted.select(predicted['HourlyEnergyOutputMW'], predicted['prediction_output']['value'].alias('prediction'))\n" + "# Evaluate predictions on held-out data\n", + "- As expected, we find that the mean absolute error (mae) on the for_predictions data is similar to the leaderboard mae" ] }, { @@ -341,13 +414,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "Mean absolute error: 2.4058999799874976" + "Mean absolute error: 2.3167231443313843" ] } ], "source": [ + "from pyspark.sql.functions import *\n", "from pyspark.ml.evaluation import RegressionEvaluator\n", "\n", + "scores = predicted.select(predicted['HourlyEnergyOutputMW'], predicted['prediction_output']['value'].alias('prediction'))\n", + "\n", "evaluator = RegressionEvaluator(predictionCol=\"prediction\",\n", " labelCol=\"HourlyEnergyOutputMW\",\n", " metricName=\"mae\")\n", @@ -358,50 +434,95 @@ ] }, { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "language": "python" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "r squared: 0.9516895664346249" - ] - } - ], + "cell_type": "markdown", + "metadata": {}, "source": [ - "from pyspark.ml.evaluation import RegressionEvaluator\n", + "# Configuration settings for scaling to larger data\n", "\n", - "evaluator = RegressionEvaluator(predictionCol=\"prediction\",\n", - " labelCol=\"HourlyEnergyOutputMW\",\n", - " metricName=\"r2\")\n", + "## Number and size of nodes in our Kubernetes cluster\n", + "We can control the number and size of nodes in our Kubernetes cluster via the node-vm-size and node-count switches in our `aks create` command:\n", + "\n", + "`az aks create --name mycluster --resource-group myrg --generate-ssh-keys --node-vm-size Standard_DS14_v2 --node-count 3 --kubernetes-version 1.10.9`\n", + "\n", + "For information is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-on-aks?view=sqlallproducts-allversions#create-a-kubernetes-cluster).\n", + "\n", + "## Number of Spark pods\n", + "We can control the number of Spark pods via the CLUSTER_STORAGE_POOL_REPLICAS environment variable used by `mssqlctl create cluster`:\n", + "\n", + "SET CLUSTER_STORAGE_POOL_REPLICAS=2\n", + "\n", + "## YARN scheduler memory and cores\n", + "We can control the YARN scheduler memory and cores via the following environment variable used by `mssqlctl create cluster`:\n", + "\n", + "- YARN_SCHEDULER_MAX_MEMORY\n", + "- YARN_SCHEDULER_MAX_VCORES\n", + "- YARN_NODEMANAGER_RESOURCE_MEMORY\n", + "- YARN_NODEMANAGER_RESOURCE_VCORES\n", "\n", - "r2 = evaluator.evaluate(scores)\n", + "Further information regarding mssqlctl environtment variables is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions#define-environment-variables).\n", "\n", - "print(\"r squared:\", r2)" + "## Livy timeout\n", + "The Livy timeout sets a limit on the runtime of a cell in a PySpark3 Jupyter notebook. In SQL Server 2019 Big Data CTP 2.1, the Livy timeout defaults to 1 hour. In CTP 2.2, it defaults to 24 days. One can modify this as follows:\n", + "\n", + "- Log into the mssql-master-pool-0 pod using this command (requires permission to run kubectl):\n", + "\n", + "```\n", + "kubectl exec -it mssql-master-pool-0 -n -- /bin/bash\n", + "```\n", + "- To set the Livy timeout to 24 days, run the following command or edit /livy/conf/livy.conf accordingly:\n", + "\n", + "```\n", + "echo 'livy.server.session.timeout = 24d' | cat >> /livy/conf/livy.conf \n", + "```\n", + "- Then restart the Livy server by running the following command:\n", + "\n", + "```\n", + "supervisorctl restart livy\n", + "```" ] }, { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "language": "python" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Elapsed time: 199.5 seconds." - ] - } - ], + "cell_type": "markdown", + "metadata": {}, "source": [ - "elapsed_time = time.time() - start_time\n", - "print(\"Elapsed time: {:.1f} seconds.\".format(elapsed_time))" + "# Monitoring and Diagnostics\n", + "## YARN UI\n", + "\n", + "Access from the \"View Yarn History\" button in Azure Data Studio (ADS) or at `https://:30443/gateway/default/yarn`\n", + "\n", + "## Spark UI\n", + "\n", + "Access from the \"Spark UI\" link that appears after running the first cell in a notebook in a (Py)Spark kernel in ADS or by clicking the ApplicationMaster link of a running application in the YARN UI\n", + "\n", + "## Spark History\n", + "\n", + "Access from the \"View Spark History\" button in ADS or at `https://:30443/gateway/default/sparkhistory`\n", + "\n", + "## H2O Flow UI\n", + "- The command `H2OContext.getOrCreate(spark)` outputs the IP address and port number for connection to H2O’s Flow UI, for example:\n", + "\n", + " `H2O connection url: http://10.244.0.16:54325`\n", + "\n", + "- This connection can be forwarded to one’s workstation using this command (requires permission to run kubectl):\n", + "\n", + " `kubectl -n test port-forward `\n", + "\n", + " Here is an example:\n", + "\n", + " `kubectl -n test port-forward mssql-storage-pool-default-0 54325`\n", + "\n", + " The port number is the number after the colon in the H2O connection URL.\n", + " \n", + " To determine ``, run this command in the PySpark3 kernel in the ADS notebook:\n", + "\n", + "```python\n", + " # Print the hostname of the pod where the driver is running\n", + " stdout = subprocess.check_output( \n", + " \"hostname\", \n", + " stderr=subprocess.STDOUT, \n", + " shell=True).decode(\"utf-8\") \n", + " print(stdout) \n", + "```" ] } ], From 9bac0a1617bcc34191e28bff7783253c312a319c Mon Sep 17 00:00:00 2001 From: Mario Inchiosa <4316698+inchiosa@users.noreply.github.com> Date: Fri, 7 Dec 2018 14:21:05 -0800 Subject: [PATCH 3/5] Add local URL for FLow UI --- .../machine-learning/spark/h2o/h2o-automl-powerplant.ipynb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb index 4b7dac4498..d7f353233f 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb +++ b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb @@ -522,7 +522,9 @@ " stderr=subprocess.STDOUT, \n", " shell=True).decode(\"utf-8\") \n", " print(stdout) \n", - "```" + "```\n", + "\n", + "- After setting up port forwarding, the Flow UI can be accessed at `http://localhost:`; for example, `http://localhost:54325`" ] } ], From bd3028bdcb7208691587cad923fc201b68834361 Mon Sep 17 00:00:00 2001 From: Mario Inchiosa <4316698+inchiosa@users.noreply.github.com> Date: Fri, 7 Dec 2018 16:13:45 -0800 Subject: [PATCH 4/5] Fix typo --- .../machine-learning/spark/h2o/h2o-automl-powerplant.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb index d7f353233f..4f2aa6d349 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb +++ b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb @@ -444,7 +444,7 @@ "\n", "`az aks create --name mycluster --resource-group myrg --generate-ssh-keys --node-vm-size Standard_DS14_v2 --node-count 3 --kubernetes-version 1.10.9`\n", "\n", - "For information is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-on-aks?view=sqlallproducts-allversions#create-a-kubernetes-cluster).\n", + "More information is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-on-aks?view=sqlallproducts-allversions#create-a-kubernetes-cluster).\n", "\n", "## Number of Spark pods\n", "We can control the number of Spark pods via the CLUSTER_STORAGE_POOL_REPLICAS environment variable used by `mssqlctl create cluster`:\n", From 1b4457a7972698a8fd583cb6729a44ec69e8971b Mon Sep 17 00:00:00 2001 From: Mario Inchiosa <4316698+inchiosa@users.noreply.github.com> Date: Tue, 8 Jan 2019 15:43:28 -0800 Subject: [PATCH 5/5] merge from upstream --- .../bootstrap-sample-db.sql | 4 + .../data-pool/data-ingestion-spark.sql | 5 +- .../deployment/aks/README.md | 2 +- .../deployment/aks/deploy-sql-big-data-aks.py | 2 +- .../deployment/kubeadm/README.md | 8 +- .../ubuntu/local-storage-provisioner.yaml | 129 ++++++++++++++++++ .../kubeadm/ubuntu/setup-volumes-agent.sh | 11 ++ 7 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/local-storage-provisioner.yaml create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-volumes-agent.sh diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 8963d7776b..8ffe3873c6 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -15,6 +15,10 @@ GO USE sales; GO +-- Create database master key (required for database scoped credentials used in the samples) +IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = DB_NAME() and is_master_key_encrypted_by_server = 1) + CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'sql19bigdatacluster!'; + -- Create default data sources for SQL Big Data Cluster IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') CREATE EXTERNAL DATA SOURCE SqlDataPool diff --git a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-spark.sql b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-spark.sql index d0e1599040..9d802b537b 100644 --- a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-spark.sql +++ b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-spark.sql @@ -26,7 +26,7 @@ IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'web_clickstreams_s ---- Main class: "FileStreaming" ---- Path to jar: /jar/mssql-spark-lib-assembly-1.0.jar ---- Arguments: ----- mssql-master-pool-0.service-master-pool 1433 sa %PASSWORD% sales web_clickstreams_spark_results hdfs:///clickstream_data csv false +---- --server mssql-master-pool-0.service-master-pool --port 1433 --user sa --password !yourPassword! --database sales --table web_clickstreams_spark_results --source_dir hdfs:///clickstream_data --input_format csv --enable_checkpoint false --timeout 380000 -- The arguments to jar file are -- 1: server name - sql server to connect to read the table schema @@ -38,6 +38,7 @@ IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'web_clickstreams_s -- 7: Source directory for streaming. This must be a full URI - such as "hdfs:///clickstream_data" -- 8: Input format. This can be "csv", "parquet", "json". -- 9: enable checkpoint: true or false +-- 10: timeout - in milliseconds - how long to run for before stopping -- -- After the Spark streaming job has been sucessfully submitted, you can run below query to view the results. @@ -52,4 +53,4 @@ SELECT TOP 10 * FROM [web_clickstreams_spark_results]; GO DROP EXTERNAL TABLE [dbo].[web_clickstreams_spark_results]; -GO \ No newline at end of file +GO diff --git a/samples/features/sql-big-data-cluster/deployment/aks/README.md b/samples/features/sql-big-data-cluster/deployment/aks/README.md index 5d2fbf91c9..5d1cc06d85 100644 --- a/samples/features/sql-big-data-cluster/deployment/aks/README.md +++ b/samples/features/sql-big-data-cluster/deployment/aks/README.md @@ -17,7 +17,7 @@ Using this sample Python script, you will deploy a Kubernetes cluster in Azure u ``` - Install mssqlctl CLI latest version using . Run the command below using elevated priviledges (sudo or admin cmd window): ``` - pip3 install --index-url https://private-repo.microsoft.com/python/ctp-2.0 mssqlctl + pip3 install --extra-index-url https://private-repo.microsoft.com/python/ctp-2.2 mssqlctl ``` 1. Login into your Azure account. Run this command: ``` diff --git a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py index 76f135c153..89deee367b 100644 --- a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py +++ b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py @@ -81,7 +81,7 @@ def executeCmd (cmd): print("") print("SQL Server big data cluster connection endpoints: ") print("SQL Server master instance:") -command="kubectl get service service-master-pool-lb -o=custom-columns=""IP:.status.loadBalancer.ingress[0].ip,PORT:.spec.ports[0].port"" -n "+CLUSTER_NAME +command="kubectl get service endpoint-master-pool -o=custom-columns=""IP:.status.loadBalancer.ingress[0].ip,PORT:.spec.ports[0].port"" -n "+CLUSTER_NAME executeCmd(command) print("") print("HDFS/KNOX:") diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md index 9291ede887..539d454f4b 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md @@ -53,5 +53,9 @@ To install docker, you can follow the steps below: 1. Start a sudo shell context 1. Execute [ubuntu/setup-k8s-prereqs.sh](ubuntu/setup-k8s-prereqs.sh/) script on each machine 1. Execute [ubuntu/setup-k8s-master.sh](ubuntu/setup-k8s-master.sh/) script on the machine designated as Kubernetes master -1. After successful initialization of the Kubernetes master, follow the kubeadm join commands output by the script on each agent machine -1. Now, you can deploy SQL Server 2019 big data cluster using instructions [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) +1. After successful initialization of the Kubernetes master, follow the kubeadm join commands output by the setup script on each agent machine +1. Execute [ubuntu/setup-volumes-agent.sh](ubuntu/setup-volumes-agent.sh/) script on each agent machine to create volumes for local storage + +**NOTE: Ensure there is sufficient local storage on your agents. Each volume will use up to 6GB by default. The script creates 25 volumes. Not all of the volumes will be used since it depends on the number of pods being deployed on each agent node. It is recommended to have at least 200 GB of storage on the agent nodes** +1. Execute ***kubectl apply -f ubuntu/local-storage-provisioner.yaml*** against the Kubernetes cluster to create the local storage provisioner. You can now use persistent volumes by setting the USE_PERSISTENT_VOLUME environment variable to ***true*** and STORAGE_CLASS_NAME environment variable to ***local-storage*** during deployment +1. Now, you can deploy the SQL Server 2019 big data cluster following instructions [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/local-storage-provisioner.yaml b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/local-storage-provisioner.yaml new file mode 100644 index 0000000000..fc010145a7 --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/local-storage-provisioner.yaml @@ -0,0 +1,129 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: local-storage +--- +apiVersion: v1 +kind: Service +metadata: + name: local-volume-provisioner + namespace: local-storage + labels: + app: local-volume-provisioner +spec: + type: ClusterIP + selector: + app: local-volume-provisioner + ports: + - name: metrics + port: 8080 + protocol: TCP +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-provisioner-config + namespace: local-storage +data: + storageClassMap: | + local-storage: + hostDir: /mnt/local-storage + mountDir: /mnt/local-storage + blockCleanerCommand: + - "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/scripts/shred.sh" + - "2" +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: local-volume-provisioner + namespace: local-storage + labels: + app: local-volume-provisioner +spec: + selector: + matchLabels: + app: local-volume-provisioner + template: + metadata: + labels: + app: local-volume-provisioner + spec: + serviceAccountName: local-storage-admin + containers: + - image: "quay.io/external_storage/local-volume-provisioner:v2.1.0" + imagePullPolicy: "Always" + name: provisioner + securityContext: + privileged: true + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /etc/provisioner/config + name: provisioner-config + readOnly: true + - mountPath: /mnt/local-storage + name: local-storage + mountPropagation: "HostToContainer" + volumes: + - name: provisioner-config + configMap: + name: local-provisioner-config + - name: local-storage + hostPath: + path: /mnt/local-storage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-storage-admin + namespace: local-storage +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-storage-provisioner-pv-binding + namespace: local-storage +subjects: +- kind: ServiceAccount + name: local-storage-admin + namespace: local-storage +roleRef: + kind: ClusterRole + name: system:persistent-volume-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-storage-provisioner-node-clusterrole + namespace: local-storage +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-storage-provisioner-node-binding + namespace: local-storage +subjects: +- kind: ServiceAccount + name: local-storage-admin + namespace: local-storage +roleRef: + kind: ClusterRole + name: local-storage-provisioner-node-clusterrole + apiGroup: rbac.authorization.k8s.io diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-volumes-agent.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-volumes-agent.sh new file mode 100644 index 0000000000..ef04333642 --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-volumes-agent.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +# num of persistent volumes +PV_COUNT=25 + +for i in $(seq 1 $PV_COUNT); do + vol="vol$i" + + mkdir -p /mnt/local-storage/$vol + mount --bind /mnt/local-storage/$vol /mnt/local-storage/$vol +done