From 8dbbb01b8a084c39dffc4837ba7405f7c0898573 Mon Sep 17 00:00:00 2001 From: Roope Astala Date: Fri, 26 Apr 2019 13:44:15 -0400 Subject: [PATCH] version 1.0.33 --- NBSETUP.md | 17 +- README.md | 4 +- .../automated-machine-learning/README.md | 64 +- .../auto-ml-classification.ipynb | 9 +- .../auto-ml-forecasting-bike-share.ipynb | 493 ++++++++++++ .../forecasting-bike-share/bike-no.csv | 732 ++++++++++++++++++ .../auto-ml-forecasting-energy-demand.ipynb | 294 +++++-- ...to-ml-forecasting-orange-juice-sales.ipynb | 482 ++++++++++-- .../auto-ml-model-explanation.ipynb | 4 +- .../remote-attach/auto-ml-remote-attach.ipynb | 2 +- how-to-use-azureml/azure-databricks/README.md | 4 +- how-to-use-azureml/deploy-to-cloud/README.md | 12 + .../deploy-to-cloud/helloworld.txt | 1 + .../model-register-and-deploy.ipynb | 275 +++++++ how-to-use-azureml/deploy-to-cloud/myenv.yml | 8 + how-to-use-azureml/deploy-to-cloud/score.py | 34 + .../sklearn_regression_model.pkl | Bin 0 -> 658 bytes how-to-use-azureml/deploy-to-local/README.md | 12 + .../deploy-to-local/dockerSharedDrive.JPG | Bin 0 -> 23042 bytes .../deploy-to-local/helloworld.txt | 1 + how-to-use-azureml/deploy-to-local/myenv.yml | 8 + ...register-model-deploy-local-advanced.ipynb | 487 ++++++++++++ .../register-model-deploy-local.ipynb | 342 ++++++++ how-to-use-azureml/deploy-to-local/score.py | 34 + .../sklearn_regression_model.pkl | Bin 0 -> 658 bytes how-to-use-azureml/deployment/onnx/README.md | 2 - how-to-use-azureml/explain-model/README.md | 2 - .../machine-learning-pipelines/README.md | 2 - .../training-with-deep-learning/README.md | 2 - .../distributed-tensorflow-with-horovod.ipynb | 2 +- ...arameter-tune-deploy-with-tensorflow.ipynb | 2 +- how-to-use-azureml/training/README.md | 4 +- .../train-on-local/train-on-local.ipynb | 37 +- .../train-on-remote-vm.ipynb | 4 +- .../work-with-data/dataprep/README.md | 44 +- tutorials/regression-part1-data-prep.ipynb | 2 +- 36 files changed, 3211 insertions(+), 211 deletions(-) create mode 100644 how-to-use-azureml/automated-machine-learning/forecasting-bike-share/auto-ml-forecasting-bike-share.ipynb create mode 100644 how-to-use-azureml/automated-machine-learning/forecasting-bike-share/bike-no.csv create mode 100644 how-to-use-azureml/deploy-to-cloud/README.md create mode 100644 how-to-use-azureml/deploy-to-cloud/helloworld.txt create mode 100644 how-to-use-azureml/deploy-to-cloud/model-register-and-deploy.ipynb create mode 100644 how-to-use-azureml/deploy-to-cloud/myenv.yml create mode 100644 how-to-use-azureml/deploy-to-cloud/score.py create mode 100644 how-to-use-azureml/deploy-to-cloud/sklearn_regression_model.pkl create mode 100644 how-to-use-azureml/deploy-to-local/README.md create mode 100644 how-to-use-azureml/deploy-to-local/dockerSharedDrive.JPG create mode 100644 how-to-use-azureml/deploy-to-local/helloworld.txt create mode 100644 how-to-use-azureml/deploy-to-local/myenv.yml create mode 100644 how-to-use-azureml/deploy-to-local/register-model-deploy-local-advanced.ipynb create mode 100644 how-to-use-azureml/deploy-to-local/register-model-deploy-local.ipynb create mode 100644 how-to-use-azureml/deploy-to-local/score.py create mode 100644 how-to-use-azureml/deploy-to-local/sklearn_regression_model.pkl diff --git a/NBSETUP.md b/NBSETUP.md index 611b56bf2..c51f404f2 100644 --- a/NBSETUP.md +++ b/NBSETUP.md @@ -1,6 +1,4 @@ -# Setting up environment - ---- +# Set up your notebook environment for Azure Machine Learning To run the notebooks in this repository use one of following options. @@ -12,9 +10,7 @@ Azure Notebooks is a hosted Jupyter-based notebook service in the Azure cloud. A 1. Follow the instructions in the [Configuration](configuration.ipynb) notebook to create and connect to a workspace 1. Open one of the sample notebooks - **Make sure the Azure Notebook kernel is set to `Python 3.6`** when you open a notebook - - ![set kernel to Python 3.6](images/python36.png) + **Make sure the Azure Notebook kernel is set to `Python 3.6`** when you open a notebook by choosing Kernel > Change Kernel > Python 3.6 from the menus. ## **Option 2: Use your own notebook server** @@ -31,9 +27,6 @@ git clone https://github.com/Azure/MachineLearningNotebooks.git # install the base SDK and a Jupyter notebook server pip install azureml-sdk[notebooks] -# install the data prep component -pip install azureml-dataprep - # install model explainability component pip install azureml-sdk[explain] @@ -58,8 +51,7 @@ Please make sure you start with the [Configuration](configuration.ipynb) noteboo ### Video walkthrough: -[![Get Started video](images/yt_cover.png)](https://youtu.be/VIsXeTuW3FU) - +[!VIDEO https://youtu.be/VIsXeTuW3FU] ## **Option 3: Use Docker** @@ -90,9 +82,6 @@ Now you can point your browser to http://localhost:8887. We recommend that you s If you need additional Azure ML SDK components, you can either modify the Docker files before you build the Docker images to add additional steps, or install them through command line in the live container after you build the Docker image. For example: ```sh -# install dataprep components -pip install azureml-dataprep - # install the core SDK and automated ml components pip install azureml-sdk[automl] diff --git a/README.md b/README.md index a7ebe53a1..5edf1824c 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,4 @@ The [How to use Azure ML](./how-to-use-azureml) folder contains specific example Visit following repos to see projects contributed by Azure ML users: - [Fine tune natural language processing models using Azure Machine Learning service](https://github.com/Microsoft/AzureML-BERT) - - [Fashion MNIST with Azure ML SDK](https://github.com/amynic/azureml-sdk-fashion) - - ![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/README.png) + - [Fashion MNIST with Azure ML SDK](https://github.com/amynic/azureml-sdk-fashion) \ No newline at end of file diff --git a/how-to-use-azureml/automated-machine-learning/README.md b/how-to-use-azureml/automated-machine-learning/README.md index 66e46c073..dc667c8e9 100644 --- a/how-to-use-azureml/automated-machine-learning/README.md +++ b/how-to-use-azureml/automated-machine-learning/README.md @@ -1,8 +1,8 @@ # Table of Contents 1. [Automated ML Introduction](#introduction) -1. [Running samples in Azure Notebooks](#jupyter) -1. [Running samples in Azure Databricks](#databricks) -1. [Running samples in a Local Conda environment](#localconda) +1. [Setup using Azure Notebooks](#jupyter) +1. [Setup using Azure Databricks](#databricks) +1. [Setup using a Local Conda environment](#localconda) 1. [Automated ML SDK Sample Notebooks](#samples) 1. [Documentation](#documentation) 1. [Running using python command](#pythoncommand) @@ -13,15 +13,15 @@ Automated machine learning (automated ML) builds high quality machine learning models for you by automating model and hyperparameter selection. Bring a labelled dataset that you want to build a model for, automated ML will give you a high quality machine learning model that you can use for predictions. -If you are new to Data Science, AutoML will help you get jumpstarted by simplifying machine learning model building. It abstracts you from needing to perform model selection, hyperparameter selection and in one step creates a high quality trained model for you to use. +If you are new to Data Science, automated ML will help you get jumpstarted by simplifying machine learning model building. It abstracts you from needing to perform model selection, hyperparameter selection and in one step creates a high quality trained model for you to use. -If you are an experienced data scientist, AutoML will help increase your productivity by intelligently performing the model and hyperparameter selection for your training and generates high quality models much quicker than manually specifying several combinations of the parameters and running training jobs. AutoML provides visibility and access to all the training jobs and the performance characteristics of the models to help you further tune the pipeline if you desire. +If you are an experienced data scientist, automated ML will help increase your productivity by intelligently performing the model and hyperparameter selection for your training and generates high quality models much quicker than manually specifying several combinations of the parameters and running training jobs. Automated ML provides visibility and access to all the training jobs and the performance characteristics of the models to help you further tune the pipeline if you desire. -Below are the three execution environments supported by AutoML. +Below are the three execution environments supported by automated ML. -## Running samples in Azure Notebooks - Jupyter based notebooks in the Azure cloud +## Setup using Azure Notebooks - Jupyter based notebooks in the Azure cloud 1. [![Azure Notebooks](https://notebooks.azure.com/launch.png)](https://aka.ms/aml-clone-azure-notebooks) [Import sample notebooks ](https://aka.ms/aml-clone-azure-notebooks) into Azure Notebooks. @@ -29,7 +29,7 @@ Below are the three execution environments supported by AutoML. 1. Open one of the sample notebooks. -## Running samples in Azure Databricks +## Setup using Azure Databricks **NOTE**: Please create your Azure Databricks cluster as v4.x (high concurrency preferred) with **Python 3** (dropdown). **NOTE**: You should at least have contributor access to your Azure subcription to run the notebook. @@ -39,7 +39,7 @@ Below are the three execution environments supported by AutoML. - Attach the notebook to the cluster. -## Running samples in a Local Conda environment +## Setup using a Local Conda environment To run these notebook on your own notebook server, use these installation instructions. The instructions below will install everything you need and then start a Jupyter notebook. @@ -49,11 +49,15 @@ The instructions below will install everything you need and then start a Jupyter There's no need to install mini-conda specifically. ### 2. Downloading the sample notebooks -- Download the sample notebooks from [GitHub](https://github.com/Azure/MachineLearningNotebooks) as zip and extract the contents to a local directory. The AutoML sample notebooks are in the "automl" folder. +- Download the sample notebooks from [GitHub](https://github.com/Azure/MachineLearningNotebooks) as zip and extract the contents to a local directory. The automated ML sample notebooks are in the "automated-machine-learning" folder. ### 3. Setup a new conda environment -The **automl/automl_setup** script creates a new conda environment, installs the necessary packages, configures the widget and starts a jupyter notebook. -It takes the conda environment name as an optional parameter. The default conda environment name is azure_automl. The exact command depends on the operating system. See the specific sections below for Windows, Mac and Linux. It can take about 10 minutes to execute. +The **automl_setup** script creates a new conda environment, installs the necessary packages, configures the widget and starts a jupyter notebook. It takes the conda environment name as an optional parameter. The default conda environment name is azure_automl. The exact command depends on the operating system. See the specific sections below for Windows, Mac and Linux. It can take about 10 minutes to execute. + +Packages installed by the **automl_setup** script: + + +For more details refer to the [automl_env.yml](./automl_env.yml) ## Windows Start an **Anaconda Prompt** window, cd to the **how-to-use-azureml/automated-machine-learning** folder where the sample notebooks were extracted and then run: ``` @@ -81,7 +85,7 @@ bash automl_setup_linux.sh ### 5. Running Samples - Please make sure you use the Python [conda env:azure_automl] kernel when trying the sample Notebooks. -- Follow the instructions in the individual notebooks to explore various features in AutoML +- Follow the instructions in the individual notebooks to explore various features in automated ML. ### 6. Starting jupyter notebook manually To start your Jupyter notebook manually, use: @@ -103,22 +107,22 @@ jupyter notebook - [auto-ml-classification.ipynb](classification/auto-ml-classification.ipynb) - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits) - - Simple example of using Auto ML for classification + - Simple example of using automated ML for classification - Uses local compute for training - [auto-ml-regression.ipynb](regression/auto-ml-regression.ipynb) - Dataset: scikit learn's [diabetes dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) - - Simple example of using Auto ML for regression + - Simple example of using automated ML for regression - Uses local compute for training - [auto-ml-remote-execution.ipynb](remote-execution/auto-ml-remote-execution.ipynb) - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits) - - Example of using Auto ML for classification using a remote linux DSVM for training + - Example of using automated ML for classification using a remote linux DSVM for training - Parallel execution of iterations - Async tracking of progress - Cancelling individual iterations or entire run - Retrieving models for any iteration or logged metric - - Specify automl settings as kwargs + - Specify automated ML settings as kwargs - [auto-ml-remote-amlcompute.ipynb](remote-batchai/auto-ml-remote-amlcompute.ipynb) - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits) @@ -127,7 +131,7 @@ jupyter notebook - Async tracking of progress - Cancelling individual iterations or entire run - Retrieving models for any iteration or logged metric - - Specify automl settings as kwargs + - Specify automated ML settings as kwargs - [auto-ml-remote-attach.ipynb](remote-attach/auto-ml-remote-attach.ipynb) - Dataset: Scikit learn's [20newsgroup](http://scikit-learn.org/stable/datasets/twenty_newsgroups.html) @@ -148,8 +152,8 @@ jupyter notebook - [auto-ml-exploring-previous-runs.ipynb](exploring-previous-runs/auto-ml-exploring-previous-runs.ipynb) - List all projects for the workspace - - List all AutoML Runs for a given project - - Get details for a AutoML Run. (Automl settings, run widget & all metrics) + - List all automated ML Runs for a given project + - Get details for a automated ML Run. (automated ML settings, run widget & all metrics) - Download fitted pipeline for any iteration - [auto-ml-remote-execution-with-datastore.ipynb](remote-execution-with-datastore/auto-ml-remote-execution-with-datastore.ipynb) @@ -158,7 +162,7 @@ jupyter notebook - [auto-ml-classification-with-deployment.ipynb](classification-with-deployment/auto-ml-classification-with-deployment.ipynb) - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits) - - Simple example of using Auto ML for classification + - Simple example of using automated ML for classification - Registering the model - Creating Image and creating aci service - Testing the aci service @@ -178,20 +182,20 @@ jupyter notebook - [auto-ml-classification-with-whitelisting.ipynb](classification-with-whitelisting/auto-ml-classification-with-whitelisting.ipynb) - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits) - - Simple example of using Auto ML for classification with whitelisting tensorflow models. + - Simple example of using automated ML for classification with whitelisting tensorflow models. - Uses local compute for training - [auto-ml-forecasting-energy-demand.ipynb](forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb) - Dataset: [NYC energy demand data](forecasting-a/nyc_energy.csv) - - Example of using AutoML for training a forecasting model + - Example of using automated ML for training a forecasting model - [auto-ml-forecasting-orange-juice-sales.ipynb](forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb) - Dataset: [Dominick's grocery sales of orange juice](forecasting-b/dominicks_OJ.csv) - - Example of training an AutoML forecasting model on multiple time-series + - Example of training an automated ML forecasting model on multiple time-series - [auto-ml-classification-with-onnx.ipynb](classification-with-onnx/auto-ml-classification-with-onnx.ipynb) - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits) - - Simple example of using Auto ML for classification with ONNX models + - Simple example of using automated ML for classification with ONNX models - Uses local compute for training @@ -259,7 +263,7 @@ There are several reasons why the DsvmCompute.create can fail. The reason is us 2) `The requested VM size xxxxx is not available in the current region.` You can select a different region or vm_size. ## Remote run: Unable to establish SSH connection -AutoML uses the SSH protocol to communicate with remote DSVMs. This defaults to port 22. Possible causes for this error are: +Automated ML uses the SSH protocol to communicate with remote DSVMs. This defaults to port 22. Possible causes for this error are: 1) The DSVM is not ready for SSH connections. When DSVM creation completes, the DSVM might still not be ready to acceept SSH connections. The sample notebooks have a one minute delay to allow for this. 2) Your Azure Subscription may restrict the IP address ranges that can access the DSVM on port 22. You can check this in the Azure Portal by selecting the Virtual Machine and then clicking Networking. The Virtual Machine name is the name that you provided in the notebook plus 10 alpha numeric characters to make the name unique. The Inbound Port Rules define what can access the VM on specific ports. Note that there is a priority priority order. So, a Deny entry with a low priority number will override a Allow entry with a higher priority number. @@ -270,18 +274,16 @@ This is often an issue with the `get_data` method. 3) You can get to the error log for the setup iteration by clicking the `Click here to see the run in Azure portal` link, click `Back to Experiment`, click on the highest run number and then click on Logs. ## Remote run: disk full -AutoML creates files under /tmp/azureml_runs for each iteration that it runs. It creates a folder with the iteration id. For example: AutoML_9a038a18-77cc-48f1-80fb-65abdbc33abe_93. Under this, there is a azureml-logs folder, which contains logs. If you run too many iterations on the same DSVM, these files can fill the disk. +Automated ML creates files under /tmp/azureml_runs for each iteration that it runs. It creates a folder with the iteration id. For example: AutoML_9a038a18-77cc-48f1-80fb-65abdbc33abe_93. Under this, there is a azureml-logs folder, which contains logs. If you run too many iterations on the same DSVM, these files can fill the disk. You can delete the files under /tmp/azureml_runs or just delete the VM and create a new one. If your get_data downloads files, make sure the delete them or they can use disk space as well. When using DataStore, it is good to specify an absolute path for the files so that they are downloaded just once. If you specify a relative path, it will download a file for each iteration. ## Remote run: Iterations fail and the log contains "MemoryError" -This can be caused by insufficient memory on the DSVM. AutoML loads all training data into memory. So, the available memory should be more than the training data size. +This can be caused by insufficient memory on the DSVM. Automated ML loads all training data into memory. So, the available memory should be more than the training data size. If you are using a remote DSVM, memory is needed for each concurrent iteration. The max_concurrent_iterations setting specifies the maximum concurrent iterations. For example, if the training data size is 8Gb and max_concurrent_iterations is set to 10, the minimum memory required is at least 80Gb. To resolve this issue, allocate a DSVM with more memory or reduce the value specified for max_concurrent_iterations. ## Remote run: Iterations show as "Not Responding" in the RunDetails widget. This can be caused by too many concurrent iterations for a remote DSVM. Each concurrent iteration usually takes 100% of a core when it is running. Some iterations can use multiple cores. So, the max_concurrent_iterations setting should always be less than the number of cores of the DSVM. -To resolve this issue, try reducing the value specified for the max_concurrent_iterations setting. - -![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/README.png) +To resolve this issue, try reducing the value specified for the max_concurrent_iterations setting. \ No newline at end of file diff --git a/how-to-use-azureml/automated-machine-learning/classification/auto-ml-classification.ipynb b/how-to-use-azureml/automated-machine-learning/classification/auto-ml-classification.ipynb index 8e0a68f77..a4e891ecf 100644 --- a/how-to-use-azureml/automated-machine-learning/classification/auto-ml-classification.ipynb +++ b/how-to-use-azureml/automated-machine-learning/classification/auto-ml-classification.ipynb @@ -162,7 +162,14 @@ "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n", "|**y**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n", "|**n_cross_validations**|Number of cross validation splits.|\n", - "|Exit Criteria [optional]

iterations
experiment_timeout_minutes|An optional duration parameter that says how long AutoML should be run.
This could be either number of iterations or number of minutes AutoML is allowed to run.

iterations number of algorithm iterations to run
experiment_timeout_minutes is the number of minutes that AutoML should run

By default, this is set to stop whenever AutoML determines that progress in scores is not being made|" + "|\n", + "\n", + "Automated machine learning trains multiple machine learning pipelines. Each pipelines training is known as an iteration.\n", + "* You can specify a maximum number of iterations using the `iterations` parameter.\n", + "* You can specify a maximum time for the run using the `experiment_timeout_minutes` parameter.\n", + "* If you specify neither the `iterations` nor the `experiment_timeout_minutes`, automated ML keeps running iterations while it continues to see improvements in the scores.\n", + "\n", + "The following example doesn't specify `iterations` or `experiment_timeout_minutes` and so runs until the scores stop improving.\n" ] }, { diff --git a/how-to-use-azureml/automated-machine-learning/forecasting-bike-share/auto-ml-forecasting-bike-share.ipynb b/how-to-use-azureml/automated-machine-learning/forecasting-bike-share/auto-ml-forecasting-bike-share.ipynb new file mode 100644 index 000000000..5b45181ca --- /dev/null +++ b/how-to-use-azureml/automated-machine-learning/forecasting-bike-share/auto-ml-forecasting-bike-share.ipynb @@ -0,0 +1,493 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Automated Machine Learning\n", + "**BikeShare Demand Forecasting**\n", + "\n", + "## Contents\n", + "1. [Introduction](#Introduction)\n", + "1. [Setup](#Setup)\n", + "1. [Data](#Data)\n", + "1. [Train](#Train)\n", + "1. [Evaluate](#Evaluate)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction\n", + "In this example, we show how AutoML can be used for bike share forecasting.\n", + "\n", + "The purpose is to demonstrate how to take advantage of the built-in holiday featurization, access the feature names, and further demonstrate how to work with the `forecast` function. Please also look at the additional forecasting notebooks, which document lagging, rolling windows, forecast quantiles, other ways to use the forecast function, and forecaster deployment.\n", + "\n", + "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n", + "\n", + "In this notebook you would see\n", + "1. Creating an Experiment in an existing Workspace\n", + "2. Instantiating AutoMLConfig with new task type \"forecasting\" for timeseries data training, and other timeseries related settings: for this dataset we use the basic one: \"time_column_name\" \n", + "3. Training the Model using local compute\n", + "4. Exploring the results\n", + "5. Viewing the engineered names for featurized data and featurization summary for all raw features\n", + "6. Testing the fitted model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import azureml.core\n", + "import pandas as pd\n", + "import numpy as np\n", + "import logging\n", + "import warnings\n", + "# Squash warning messages for cleaner output in the notebook\n", + "warnings.showwarning = lambda *args, **kwargs: None\n", + "\n", + "\n", + "from azureml.core.workspace import Workspace\n", + "from azureml.core.experiment import Experiment\n", + "from azureml.train.automl import AutoMLConfig\n", + "from matplotlib import pyplot as plt\n", + "from sklearn.metrics import mean_absolute_error, mean_squared_error" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As part of the setup you have already created a Workspace. For AutoML you would need to create an Experiment. An Experiment is a named object in a Workspace, which is used to run experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ws = Workspace.from_config()\n", + "\n", + "# choose a name for the run history container in the workspace\n", + "experiment_name = 'automl-bikeshareforecasting'\n", + "# project folder\n", + "project_folder = './sample_projects/automl-local-bikeshareforecasting'\n", + "\n", + "experiment = Experiment(ws, experiment_name)\n", + "\n", + "output = {}\n", + "output['SDK version'] = azureml.core.VERSION\n", + "output['Subscription ID'] = ws.subscription_id\n", + "output['Workspace'] = ws.name\n", + "output['Resource Group'] = ws.resource_group\n", + "output['Location'] = ws.location\n", + "output['Project Directory'] = project_folder\n", + "output['Run History Name'] = experiment_name\n", + "pd.set_option('display.max_colwidth', -1)\n", + "outputDf = pd.DataFrame(data = output, index = [''])\n", + "outputDf.T" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data\n", + "Read bike share demand data from file, and preview data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = pd.read_csv('bike-no.csv', parse_dates=['date'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's set up what we know abou the dataset. \n", + "\n", + "**Target column** is what we want to forecast.\n", + "\n", + "**Time column** is the time axis along which to predict.\n", + "\n", + "**Grain** is another word for an individual time series in your dataset. Grains are identified by values of the columns listed `grain_column_names`, for example \"store\" and \"item\" if your data has multiple time series of sales, one series for each combination of store and item sold.\n", + "\n", + "This dataset has only one time series. Please see the [orange juice notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales) for an example of a multi-time series dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "target_column_name = 'cnt'\n", + "time_column_name = 'date'\n", + "grain_column_names = []" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Split the data\n", + "\n", + "The first split we make is into train and test sets. Note we are splitting on time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train = data[data[time_column_name] < '2012-09-01']\n", + "test = data[data[time_column_name] >= '2012-09-01']\n", + "\n", + "X_train = train.copy()\n", + "y_train = X_train.pop(target_column_name).values\n", + "\n", + "X_test = test.copy()\n", + "y_test = X_test.pop(target_column_name).values\n", + "\n", + "print(X_train.shape)\n", + "print(y_train.shape)\n", + "print(X_test.shape)\n", + "print(y_test.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setting forecaster maximum horizon \n", + "\n", + "Assuming your test data forms a full and regular time series(regular time intervals and no holes), \n", + "the maximum horizon you will need to forecast is the length of the longest grain in your test set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if len(grain_column_names) == 0:\n", + " max_horizon = len(X_test)\n", + "else:\n", + " max_horizon = X_test.groupby(grain_column_names)[time_column_name].count().max()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train\n", + "\n", + "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n", + "\n", + "|Property|Description|\n", + "|-|-|\n", + "|**task**|forecasting|\n", + "|**primary_metric**|This is the metric that you want to optimize.
Forecasting supports the following primary metrics
spearman_correlation
normalized_root_mean_squared_error
r2_score
normalized_mean_absolute_error\n", + "|**iterations**|Number of iterations. In each iteration, Auto ML trains a specific pipeline on the given data|\n", + "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n", + "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n", + "|**y**|(sparse) array-like, shape = [n_samples, ], targets values.|\n", + "|**n_cross_validations**|Number of cross validation splits.|\n", + "|**country**|The country used to generate holiday features. These should be ISO 3166 two-letter country codes (i.e. 'US', 'GB').|\n", + "|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "time_column_name = 'date'\n", + "automl_settings = {\n", + " \"time_column_name\": time_column_name,\n", + " # these columns are a breakdown of the total and therefore a leak\n", + " \"drop_column_names\": ['casual', 'registered'],\n", + " # knowing the country allows Automated ML to bring in holidays\n", + " \"country\" : 'US',\n", + " \"max_horizon\" : max_horizon,\n", + " \"target_lags\": 1 \n", + "}\n", + "\n", + "automl_config = AutoMLConfig(task = 'forecasting', \n", + " primary_metric='normalized_root_mean_squared_error',\n", + " iterations = 10,\n", + " iteration_timeout_minutes = 5,\n", + " X = X_train,\n", + " y = y_train,\n", + " n_cross_validations = 3, \n", + " path=project_folder,\n", + " verbosity = logging.INFO,\n", + " **automl_settings)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will now run the experiment, starting with 10 iterations of model search. Experiment can be continued for more iterations if the results are not yet good. You will see the currently running iterations printing to the console." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_run = experiment.submit(automl_config, show_output=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Displaying the run objects gives you links to the visual tools in the Azure Portal. Go try them!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_run" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Retrieve the Best Model\n", + "Below we select the best pipeline from our iterations. The get_output method on automl_classifier returns the best run and the fitted model for the last fit invocation. There are overloads on get_output that allow you to retrieve the best run and fitted model for any logged metric or a particular iteration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "best_run, fitted_model = local_run.get_output()\n", + "fitted_model.steps" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### View the engineered names for featurized data\n", + "\n", + "You can accees the engineered feature names generated in time-series featurization. Note that a number of named holiday periods are represented. We recommend that you have at least one year of data when using this feature to ensure that all yearly holidays are captured in the training featurization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fitted_model.named_steps['timeseriestransformer'].get_engineered_feature_names()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### View the featurization summary\n", + "\n", + "You can also see what featurization steps were performed on different raw features in the user data. For each raw feature in the user data, the following information is displayed:\n", + "\n", + "- Raw feature name\n", + "- Number of engineered features formed out of this raw feature\n", + "- Type detected\n", + "- If feature was dropped\n", + "- List of feature transformations for the raw feature" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fitted_model.named_steps['timeseriestransformer'].get_featurization_summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test the Best Fitted Model\n", + "\n", + "Predict on training and test set, and calculate residual values.\n", + "\n", + "We always score on the original dataset whose schema matches the scheme of the training dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "X_test.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "y_query = y_test.copy().astype(np.float)\n", + "y_query.fill(np.NaN)\n", + "y_fcst, X_trans = fitted_model.forecast(X_test, y_query)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is a good practice to always align the output explicitly to the input, as the count and order of the rows may have changed during transformations that span multiple rows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def align_outputs(y_predicted, X_trans, X_test, y_test, predicted_column_name = 'predicted'):\n", + " \"\"\"\n", + " Demonstrates how to get the output aligned to the inputs\n", + " using pandas indexes. Helps understand what happened if\n", + " the output's shape differs from the input shape, or if\n", + " the data got re-sorted by time and grain during forecasting.\n", + " \n", + " Typical causes of misalignment are:\n", + " * we predicted some periods that were missing in actuals -> drop from eval\n", + " * model was asked to predict past max_horizon -> increase max horizon\n", + " * data at start of X_test was needed for lags -> provide previous periods\n", + " \"\"\"\n", + " df_fcst = pd.DataFrame({predicted_column_name : y_predicted})\n", + " # y and X outputs are aligned by forecast() function contract\n", + " df_fcst.index = X_trans.index\n", + " \n", + " # align original X_test to y_test \n", + " X_test_full = X_test.copy()\n", + " X_test_full[target_column_name] = y_test\n", + "\n", + " # X_test_full's index does not include origin, so reset for merge\n", + " df_fcst.reset_index(inplace=True)\n", + " X_test_full = X_test_full.reset_index().drop(columns='index')\n", + " together = df_fcst.merge(X_test_full, how='right')\n", + " \n", + " # drop rows where prediction or actuals are nan \n", + " # happens because of missing actuals \n", + " # or at edges of time due to lags/rolling windows\n", + " clean = together[together[[target_column_name, predicted_column_name]].notnull().all(axis=1)]\n", + " return(clean)\n", + "\n", + "df_all = align_outputs(y_fcst, X_trans, X_test, y_test)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def MAPE(actual, pred):\n", + " \"\"\"\n", + " Calculate mean absolute percentage error.\n", + " Remove NA and values where actual is close to zero\n", + " \"\"\"\n", + " not_na = ~(np.isnan(actual) | np.isnan(pred))\n", + " not_zero = ~np.isclose(actual, 0.0)\n", + " actual_safe = actual[not_na & not_zero]\n", + " pred_safe = pred[not_na & not_zero]\n", + " APE = 100*np.abs((actual_safe - pred_safe)/actual_safe)\n", + " return np.mean(APE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Simple forecasting model\")\n", + "rmse = np.sqrt(mean_squared_error(df_all[target_column_name], df_all['predicted']))\n", + "print(\"[Test Data] \\nRoot Mean squared error: %.2f\" % rmse)\n", + "mae = mean_absolute_error(df_all[target_column_name], df_all['predicted'])\n", + "print('mean_absolute_error score: %.2f' % mae)\n", + "print('MAPE: %.2f' % MAPE(df_all[target_column_name], df_all['predicted']))\n", + "\n", + "# Plot outputs\n", + "%matplotlib notebook\n", + "test_pred = plt.scatter(df_all[target_column_name], df_all['predicted'], color='b')\n", + "test_test = plt.scatter(y_test, y_test, color='g')\n", + "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n", + "plt.show()" + ] + } + ], + "metadata": { + "authors": [ + { + "name": "xiaga@microsoft.com, tosingli@microsoft.com" + } + ], + "kernelspec": { + "display_name": "Python 3.6", + "language": "python", + "name": "python36" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/how-to-use-azureml/automated-machine-learning/forecasting-bike-share/bike-no.csv b/how-to-use-azureml/automated-machine-learning/forecasting-bike-share/bike-no.csv new file mode 100644 index 000000000..ded279501 --- /dev/null +++ b/how-to-use-azureml/automated-machine-learning/forecasting-bike-share/bike-no.csv @@ -0,0 +1,732 @@ +instant,date,season,yr,mnth,weekday,weathersit,temp,atemp,hum,windspeed,casual,registered,cnt +1,1/1/2011,1,0,1,6,2,0.344167,0.363625,0.805833,0.160446,331,654,985 +2,1/2/2011,1,0,1,0,2,0.363478,0.353739,0.696087,0.248539,131,670,801 +3,1/3/2011,1,0,1,1,1,0.196364,0.189405,0.437273,0.248309,120,1229,1349 +4,1/4/2011,1,0,1,2,1,0.2,0.212122,0.590435,0.160296,108,1454,1562 +5,1/5/2011,1,0,1,3,1,0.226957,0.22927,0.436957,0.1869,82,1518,1600 +6,1/6/2011,1,0,1,4,1,0.204348,0.233209,0.518261,0.0895652,88,1518,1606 +7,1/7/2011,1,0,1,5,2,0.196522,0.208839,0.498696,0.168726,148,1362,1510 +8,1/8/2011,1,0,1,6,2,0.165,0.162254,0.535833,0.266804,68,891,959 +9,1/9/2011,1,0,1,0,1,0.138333,0.116175,0.434167,0.36195,54,768,822 +10,1/10/2011,1,0,1,1,1,0.150833,0.150888,0.482917,0.223267,41,1280,1321 +11,1/11/2011,1,0,1,2,2,0.169091,0.191464,0.686364,0.122132,43,1220,1263 +12,1/12/2011,1,0,1,3,1,0.172727,0.160473,0.599545,0.304627,25,1137,1162 +13,1/13/2011,1,0,1,4,1,0.165,0.150883,0.470417,0.301,38,1368,1406 +14,1/14/2011,1,0,1,5,1,0.16087,0.188413,0.537826,0.126548,54,1367,1421 +15,1/15/2011,1,0,1,6,2,0.233333,0.248112,0.49875,0.157963,222,1026,1248 +16,1/16/2011,1,0,1,0,1,0.231667,0.234217,0.48375,0.188433,251,953,1204 +17,1/17/2011,1,0,1,1,2,0.175833,0.176771,0.5375,0.194017,117,883,1000 +18,1/18/2011,1,0,1,2,2,0.216667,0.232333,0.861667,0.146775,9,674,683 +19,1/19/2011,1,0,1,3,2,0.292174,0.298422,0.741739,0.208317,78,1572,1650 +20,1/20/2011,1,0,1,4,2,0.261667,0.25505,0.538333,0.195904,83,1844,1927 +21,1/21/2011,1,0,1,5,1,0.1775,0.157833,0.457083,0.353242,75,1468,1543 +22,1/22/2011,1,0,1,6,1,0.0591304,0.0790696,0.4,0.17197,93,888,981 +23,1/23/2011,1,0,1,0,1,0.0965217,0.0988391,0.436522,0.2466,150,836,986 +24,1/24/2011,1,0,1,1,1,0.0973913,0.11793,0.491739,0.15833,86,1330,1416 +25,1/25/2011,1,0,1,2,2,0.223478,0.234526,0.616957,0.129796,186,1799,1985 +26,1/26/2011,1,0,1,3,3,0.2175,0.2036,0.8625,0.29385,34,472,506 +27,1/27/2011,1,0,1,4,1,0.195,0.2197,0.6875,0.113837,15,416,431 +28,1/28/2011,1,0,1,5,2,0.203478,0.223317,0.793043,0.1233,38,1129,1167 +29,1/29/2011,1,0,1,6,1,0.196522,0.212126,0.651739,0.145365,123,975,1098 +30,1/30/2011,1,0,1,0,1,0.216522,0.250322,0.722174,0.0739826,140,956,1096 +31,1/31/2011,1,0,1,1,2,0.180833,0.18625,0.60375,0.187192,42,1459,1501 +32,2/1/2011,1,0,2,2,2,0.192174,0.23453,0.829565,0.053213,47,1313,1360 +33,2/2/2011,1,0,2,3,2,0.26,0.254417,0.775417,0.264308,72,1454,1526 +34,2/3/2011,1,0,2,4,1,0.186957,0.177878,0.437826,0.277752,61,1489,1550 +35,2/4/2011,1,0,2,5,2,0.211304,0.228587,0.585217,0.127839,88,1620,1708 +36,2/5/2011,1,0,2,6,2,0.233333,0.243058,0.929167,0.161079,100,905,1005 +37,2/6/2011,1,0,2,0,1,0.285833,0.291671,0.568333,0.1418,354,1269,1623 +38,2/7/2011,1,0,2,1,1,0.271667,0.303658,0.738333,0.0454083,120,1592,1712 +39,2/8/2011,1,0,2,2,1,0.220833,0.198246,0.537917,0.36195,64,1466,1530 +40,2/9/2011,1,0,2,3,2,0.134783,0.144283,0.494783,0.188839,53,1552,1605 +41,2/10/2011,1,0,2,4,1,0.144348,0.149548,0.437391,0.221935,47,1491,1538 +42,2/11/2011,1,0,2,5,1,0.189091,0.213509,0.506364,0.10855,149,1597,1746 +43,2/12/2011,1,0,2,6,1,0.2225,0.232954,0.544167,0.203367,288,1184,1472 +44,2/13/2011,1,0,2,0,1,0.316522,0.324113,0.457391,0.260883,397,1192,1589 +45,2/14/2011,1,0,2,1,1,0.415,0.39835,0.375833,0.417908,208,1705,1913 +46,2/15/2011,1,0,2,2,1,0.266087,0.254274,0.314348,0.291374,140,1675,1815 +47,2/16/2011,1,0,2,3,1,0.318261,0.3162,0.423478,0.251791,218,1897,2115 +48,2/17/2011,1,0,2,4,1,0.435833,0.428658,0.505,0.230104,259,2216,2475 +49,2/18/2011,1,0,2,5,1,0.521667,0.511983,0.516667,0.264925,579,2348,2927 +50,2/19/2011,1,0,2,6,1,0.399167,0.391404,0.187917,0.507463,532,1103,1635 +51,2/20/2011,1,0,2,0,1,0.285217,0.27733,0.407826,0.223235,639,1173,1812 +52,2/21/2011,1,0,2,1,2,0.303333,0.284075,0.605,0.307846,195,912,1107 +53,2/22/2011,1,0,2,2,1,0.182222,0.186033,0.577778,0.195683,74,1376,1450 +54,2/23/2011,1,0,2,3,1,0.221739,0.245717,0.423043,0.094113,139,1778,1917 +55,2/24/2011,1,0,2,4,2,0.295652,0.289191,0.697391,0.250496,100,1707,1807 +56,2/25/2011,1,0,2,5,2,0.364348,0.350461,0.712174,0.346539,120,1341,1461 +57,2/26/2011,1,0,2,6,1,0.2825,0.282192,0.537917,0.186571,424,1545,1969 +58,2/27/2011,1,0,2,0,1,0.343478,0.351109,0.68,0.125248,694,1708,2402 +59,2/28/2011,1,0,2,1,2,0.407273,0.400118,0.876364,0.289686,81,1365,1446 +60,3/1/2011,1,0,3,2,1,0.266667,0.263879,0.535,0.216425,137,1714,1851 +61,3/2/2011,1,0,3,3,1,0.335,0.320071,0.449583,0.307833,231,1903,2134 +62,3/3/2011,1,0,3,4,1,0.198333,0.200133,0.318333,0.225754,123,1562,1685 +63,3/4/2011,1,0,3,5,2,0.261667,0.255679,0.610417,0.203346,214,1730,1944 +64,3/5/2011,1,0,3,6,2,0.384167,0.378779,0.789167,0.251871,640,1437,2077 +65,3/6/2011,1,0,3,0,2,0.376522,0.366252,0.948261,0.343287,114,491,605 +66,3/7/2011,1,0,3,1,1,0.261739,0.238461,0.551304,0.341352,244,1628,1872 +67,3/8/2011,1,0,3,2,1,0.2925,0.3024,0.420833,0.12065,316,1817,2133 +68,3/9/2011,1,0,3,3,2,0.295833,0.286608,0.775417,0.22015,191,1700,1891 +69,3/10/2011,1,0,3,4,3,0.389091,0.385668,0,0.261877,46,577,623 +70,3/11/2011,1,0,3,5,2,0.316522,0.305,0.649565,0.23297,247,1730,1977 +71,3/12/2011,1,0,3,6,1,0.329167,0.32575,0.594583,0.220775,724,1408,2132 +72,3/13/2011,1,0,3,0,1,0.384348,0.380091,0.527391,0.270604,982,1435,2417 +73,3/14/2011,1,0,3,1,1,0.325217,0.332,0.496957,0.136926,359,1687,2046 +74,3/15/2011,1,0,3,2,2,0.317391,0.318178,0.655652,0.184309,289,1767,2056 +75,3/16/2011,1,0,3,3,2,0.365217,0.36693,0.776522,0.203117,321,1871,2192 +76,3/17/2011,1,0,3,4,1,0.415,0.410333,0.602917,0.209579,424,2320,2744 +77,3/18/2011,1,0,3,5,1,0.54,0.527009,0.525217,0.231017,884,2355,3239 +78,3/19/2011,1,0,3,6,1,0.4725,0.466525,0.379167,0.368167,1424,1693,3117 +79,3/20/2011,1,0,3,0,1,0.3325,0.32575,0.47375,0.207721,1047,1424,2471 +80,3/21/2011,2,0,3,1,2,0.430435,0.409735,0.737391,0.288783,401,1676,2077 +81,3/22/2011,2,0,3,2,1,0.441667,0.440642,0.624583,0.22575,460,2243,2703 +82,3/23/2011,2,0,3,3,2,0.346957,0.337939,0.839565,0.234261,203,1918,2121 +83,3/24/2011,2,0,3,4,2,0.285,0.270833,0.805833,0.243787,166,1699,1865 +84,3/25/2011,2,0,3,5,1,0.264167,0.256312,0.495,0.230725,300,1910,2210 +85,3/26/2011,2,0,3,6,1,0.265833,0.257571,0.394167,0.209571,981,1515,2496 +86,3/27/2011,2,0,3,0,2,0.253043,0.250339,0.493913,0.1843,472,1221,1693 +87,3/28/2011,2,0,3,1,1,0.264348,0.257574,0.302174,0.212204,222,1806,2028 +88,3/29/2011,2,0,3,2,1,0.3025,0.292908,0.314167,0.226996,317,2108,2425 +89,3/30/2011,2,0,3,3,2,0.3,0.29735,0.646667,0.172888,168,1368,1536 +90,3/31/2011,2,0,3,4,3,0.268333,0.257575,0.918333,0.217646,179,1506,1685 +91,4/1/2011,2,0,4,5,2,0.3,0.283454,0.68625,0.258708,307,1920,2227 +92,4/2/2011,2,0,4,6,2,0.315,0.315637,0.65375,0.197146,898,1354,2252 +93,4/3/2011,2,0,4,0,1,0.378333,0.378767,0.48,0.182213,1651,1598,3249 +94,4/4/2011,2,0,4,1,1,0.573333,0.542929,0.42625,0.385571,734,2381,3115 +95,4/5/2011,2,0,4,2,2,0.414167,0.39835,0.642083,0.388067,167,1628,1795 +96,4/6/2011,2,0,4,3,1,0.390833,0.387608,0.470833,0.263063,413,2395,2808 +97,4/7/2011,2,0,4,4,1,0.4375,0.433696,0.602917,0.162312,571,2570,3141 +98,4/8/2011,2,0,4,5,2,0.335833,0.324479,0.83625,0.226992,172,1299,1471 +99,4/9/2011,2,0,4,6,2,0.3425,0.341529,0.8775,0.133083,879,1576,2455 +100,4/10/2011,2,0,4,0,2,0.426667,0.426737,0.8575,0.146767,1188,1707,2895 +101,4/11/2011,2,0,4,1,2,0.595652,0.565217,0.716956,0.324474,855,2493,3348 +102,4/12/2011,2,0,4,2,2,0.5025,0.493054,0.739167,0.274879,257,1777,2034 +103,4/13/2011,2,0,4,3,2,0.4125,0.417283,0.819167,0.250617,209,1953,2162 +104,4/14/2011,2,0,4,4,1,0.4675,0.462742,0.540417,0.1107,529,2738,3267 +105,4/15/2011,2,0,4,5,1,0.446667,0.441913,0.67125,0.226375,642,2484,3126 +106,4/16/2011,2,0,4,6,3,0.430833,0.425492,0.888333,0.340808,121,674,795 +107,4/17/2011,2,0,4,0,1,0.456667,0.445696,0.479583,0.303496,1558,2186,3744 +108,4/18/2011,2,0,4,1,1,0.5125,0.503146,0.5425,0.163567,669,2760,3429 +109,4/19/2011,2,0,4,2,2,0.505833,0.489258,0.665833,0.157971,409,2795,3204 +110,4/20/2011,2,0,4,3,1,0.595,0.564392,0.614167,0.241925,613,3331,3944 +111,4/21/2011,2,0,4,4,1,0.459167,0.453892,0.407083,0.325258,745,3444,4189 +112,4/22/2011,2,0,4,5,2,0.336667,0.321954,0.729583,0.219521,177,1506,1683 +113,4/23/2011,2,0,4,6,2,0.46,0.450121,0.887917,0.230725,1462,2574,4036 +114,4/24/2011,2,0,4,0,2,0.581667,0.551763,0.810833,0.192175,1710,2481,4191 +115,4/25/2011,2,0,4,1,1,0.606667,0.5745,0.776667,0.185333,773,3300,4073 +116,4/26/2011,2,0,4,2,1,0.631667,0.594083,0.729167,0.3265,678,3722,4400 +117,4/27/2011,2,0,4,3,2,0.62,0.575142,0.835417,0.3122,547,3325,3872 +118,4/28/2011,2,0,4,4,2,0.6175,0.578929,0.700833,0.320908,569,3489,4058 +119,4/29/2011,2,0,4,5,1,0.51,0.497463,0.457083,0.240063,878,3717,4595 +120,4/30/2011,2,0,4,6,1,0.4725,0.464021,0.503333,0.235075,1965,3347,5312 +121,5/1/2011,2,0,5,0,2,0.451667,0.448204,0.762083,0.106354,1138,2213,3351 +122,5/2/2011,2,0,5,1,2,0.549167,0.532833,0.73,0.183454,847,3554,4401 +123,5/3/2011,2,0,5,2,2,0.616667,0.582079,0.697083,0.342667,603,3848,4451 +124,5/4/2011,2,0,5,3,2,0.414167,0.40465,0.737083,0.328996,255,2378,2633 +125,5/5/2011,2,0,5,4,1,0.459167,0.441917,0.444167,0.295392,614,3819,4433 +126,5/6/2011,2,0,5,5,1,0.479167,0.474117,0.59,0.228246,894,3714,4608 +127,5/7/2011,2,0,5,6,1,0.52,0.512621,0.54125,0.16045,1612,3102,4714 +128,5/8/2011,2,0,5,0,1,0.528333,0.518933,0.631667,0.0746375,1401,2932,4333 +129,5/9/2011,2,0,5,1,1,0.5325,0.525246,0.58875,0.176,664,3698,4362 +130,5/10/2011,2,0,5,2,1,0.5325,0.522721,0.489167,0.115671,694,4109,4803 +131,5/11/2011,2,0,5,3,1,0.5425,0.5284,0.632917,0.120642,550,3632,4182 +132,5/12/2011,2,0,5,4,1,0.535,0.523363,0.7475,0.189667,695,4169,4864 +133,5/13/2011,2,0,5,5,2,0.5125,0.4943,0.863333,0.179725,692,3413,4105 +134,5/14/2011,2,0,5,6,2,0.520833,0.500629,0.9225,0.13495,902,2507,3409 +135,5/15/2011,2,0,5,0,2,0.5625,0.536,0.867083,0.152979,1582,2971,4553 +136,5/16/2011,2,0,5,1,1,0.5775,0.550512,0.787917,0.126871,773,3185,3958 +137,5/17/2011,2,0,5,2,2,0.561667,0.538529,0.837917,0.277354,678,3445,4123 +138,5/18/2011,2,0,5,3,2,0.55,0.527158,0.87,0.201492,536,3319,3855 +139,5/19/2011,2,0,5,4,2,0.530833,0.510742,0.829583,0.108213,735,3840,4575 +140,5/20/2011,2,0,5,5,1,0.536667,0.529042,0.719583,0.125013,909,4008,4917 +141,5/21/2011,2,0,5,6,1,0.6025,0.571975,0.626667,0.12065,2258,3547,5805 +142,5/22/2011,2,0,5,0,1,0.604167,0.5745,0.749583,0.148008,1576,3084,4660 +143,5/23/2011,2,0,5,1,2,0.631667,0.590296,0.81,0.233842,836,3438,4274 +144,5/24/2011,2,0,5,2,2,0.66,0.604813,0.740833,0.207092,659,3833,4492 +145,5/25/2011,2,0,5,3,1,0.660833,0.615542,0.69625,0.154233,740,4238,4978 +146,5/26/2011,2,0,5,4,1,0.708333,0.654688,0.6775,0.199642,758,3919,4677 +147,5/27/2011,2,0,5,5,1,0.681667,0.637008,0.65375,0.240679,871,3808,4679 +148,5/28/2011,2,0,5,6,1,0.655833,0.612379,0.729583,0.230092,2001,2757,4758 +149,5/29/2011,2,0,5,0,1,0.6675,0.61555,0.81875,0.213938,2355,2433,4788 +150,5/30/2011,2,0,5,1,1,0.733333,0.671092,0.685,0.131225,1549,2549,4098 +151,5/31/2011,2,0,5,2,1,0.775,0.725383,0.636667,0.111329,673,3309,3982 +152,6/1/2011,2,0,6,3,2,0.764167,0.720967,0.677083,0.207092,513,3461,3974 +153,6/2/2011,2,0,6,4,1,0.715,0.643942,0.305,0.292287,736,4232,4968 +154,6/3/2011,2,0,6,5,1,0.62,0.587133,0.354167,0.253121,898,4414,5312 +155,6/4/2011,2,0,6,6,1,0.635,0.594696,0.45625,0.123142,1869,3473,5342 +156,6/5/2011,2,0,6,0,2,0.648333,0.616804,0.6525,0.138692,1685,3221,4906 +157,6/6/2011,2,0,6,1,1,0.678333,0.621858,0.6,0.121896,673,3875,4548 +158,6/7/2011,2,0,6,2,1,0.7075,0.65595,0.597917,0.187808,763,4070,4833 +159,6/8/2011,2,0,6,3,1,0.775833,0.727279,0.622083,0.136817,676,3725,4401 +160,6/9/2011,2,0,6,4,2,0.808333,0.757579,0.568333,0.149883,563,3352,3915 +161,6/10/2011,2,0,6,5,1,0.755,0.703292,0.605,0.140554,815,3771,4586 +162,6/11/2011,2,0,6,6,1,0.725,0.678038,0.654583,0.15485,1729,3237,4966 +163,6/12/2011,2,0,6,0,1,0.6925,0.643325,0.747917,0.163567,1467,2993,4460 +164,6/13/2011,2,0,6,1,1,0.635,0.601654,0.494583,0.30535,863,4157,5020 +165,6/14/2011,2,0,6,2,1,0.604167,0.591546,0.507083,0.269283,727,4164,4891 +166,6/15/2011,2,0,6,3,1,0.626667,0.587754,0.471667,0.167912,769,4411,5180 +167,6/16/2011,2,0,6,4,2,0.628333,0.595346,0.688333,0.206471,545,3222,3767 +168,6/17/2011,2,0,6,5,1,0.649167,0.600383,0.735833,0.143029,863,3981,4844 +169,6/18/2011,2,0,6,6,1,0.696667,0.643954,0.670417,0.119408,1807,3312,5119 +170,6/19/2011,2,0,6,0,2,0.699167,0.645846,0.666667,0.102,1639,3105,4744 +171,6/20/2011,2,0,6,1,2,0.635,0.595346,0.74625,0.155475,699,3311,4010 +172,6/21/2011,3,0,6,2,2,0.680833,0.637646,0.770417,0.171025,774,4061,4835 +173,6/22/2011,3,0,6,3,1,0.733333,0.693829,0.7075,0.172262,661,3846,4507 +174,6/23/2011,3,0,6,4,2,0.728333,0.693833,0.703333,0.238804,746,4044,4790 +175,6/24/2011,3,0,6,5,1,0.724167,0.656583,0.573333,0.222025,969,4022,4991 +176,6/25/2011,3,0,6,6,1,0.695,0.643313,0.483333,0.209571,1782,3420,5202 +177,6/26/2011,3,0,6,0,1,0.68,0.637629,0.513333,0.0945333,1920,3385,5305 +178,6/27/2011,3,0,6,1,2,0.6825,0.637004,0.658333,0.107588,854,3854,4708 +179,6/28/2011,3,0,6,2,1,0.744167,0.692558,0.634167,0.144283,732,3916,4648 +180,6/29/2011,3,0,6,3,1,0.728333,0.654688,0.497917,0.261821,848,4377,5225 +181,6/30/2011,3,0,6,4,1,0.696667,0.637008,0.434167,0.185312,1027,4488,5515 +182,7/1/2011,3,0,7,5,1,0.7225,0.652162,0.39625,0.102608,1246,4116,5362 +183,7/2/2011,3,0,7,6,1,0.738333,0.667308,0.444583,0.115062,2204,2915,5119 +184,7/3/2011,3,0,7,0,2,0.716667,0.668575,0.6825,0.228858,2282,2367,4649 +185,7/4/2011,3,0,7,1,2,0.726667,0.665417,0.637917,0.0814792,3065,2978,6043 +186,7/5/2011,3,0,7,2,1,0.746667,0.696338,0.590417,0.126258,1031,3634,4665 +187,7/6/2011,3,0,7,3,1,0.72,0.685633,0.743333,0.149883,784,3845,4629 +188,7/7/2011,3,0,7,4,1,0.75,0.686871,0.65125,0.1592,754,3838,4592 +189,7/8/2011,3,0,7,5,2,0.709167,0.670483,0.757917,0.225129,692,3348,4040 +190,7/9/2011,3,0,7,6,1,0.733333,0.664158,0.609167,0.167912,1988,3348,5336 +191,7/10/2011,3,0,7,0,1,0.7475,0.690025,0.578333,0.183471,1743,3138,4881 +192,7/11/2011,3,0,7,1,1,0.7625,0.729804,0.635833,0.282337,723,3363,4086 +193,7/12/2011,3,0,7,2,1,0.794167,0.739275,0.559167,0.200254,662,3596,4258 +194,7/13/2011,3,0,7,3,1,0.746667,0.689404,0.631667,0.146133,748,3594,4342 +195,7/14/2011,3,0,7,4,1,0.680833,0.635104,0.47625,0.240667,888,4196,5084 +196,7/15/2011,3,0,7,5,1,0.663333,0.624371,0.59125,0.182833,1318,4220,5538 +197,7/16/2011,3,0,7,6,1,0.686667,0.638263,0.585,0.208342,2418,3505,5923 +198,7/17/2011,3,0,7,0,1,0.719167,0.669833,0.604167,0.245033,2006,3296,5302 +199,7/18/2011,3,0,7,1,1,0.746667,0.703925,0.65125,0.215804,841,3617,4458 +200,7/19/2011,3,0,7,2,1,0.776667,0.747479,0.650417,0.1306,752,3789,4541 +201,7/20/2011,3,0,7,3,1,0.768333,0.74685,0.707083,0.113817,644,3688,4332 +202,7/21/2011,3,0,7,4,2,0.815,0.826371,0.69125,0.222021,632,3152,3784 +203,7/22/2011,3,0,7,5,1,0.848333,0.840896,0.580417,0.1331,562,2825,3387 +204,7/23/2011,3,0,7,6,1,0.849167,0.804287,0.5,0.131221,987,2298,3285 +205,7/24/2011,3,0,7,0,1,0.83,0.794829,0.550833,0.169171,1050,2556,3606 +206,7/25/2011,3,0,7,1,1,0.743333,0.720958,0.757083,0.0908083,568,3272,3840 +207,7/26/2011,3,0,7,2,1,0.771667,0.696979,0.540833,0.200258,750,3840,4590 +208,7/27/2011,3,0,7,3,1,0.775,0.690667,0.402917,0.183463,755,3901,4656 +209,7/28/2011,3,0,7,4,1,0.779167,0.7399,0.583333,0.178479,606,3784,4390 +210,7/29/2011,3,0,7,5,1,0.838333,0.785967,0.5425,0.174138,670,3176,3846 +211,7/30/2011,3,0,7,6,1,0.804167,0.728537,0.465833,0.168537,1559,2916,4475 +212,7/31/2011,3,0,7,0,1,0.805833,0.729796,0.480833,0.164813,1524,2778,4302 +213,8/1/2011,3,0,8,1,1,0.771667,0.703292,0.550833,0.156717,729,3537,4266 +214,8/2/2011,3,0,8,2,1,0.783333,0.707071,0.49125,0.20585,801,4044,4845 +215,8/3/2011,3,0,8,3,2,0.731667,0.679937,0.6575,0.135583,467,3107,3574 +216,8/4/2011,3,0,8,4,2,0.71,0.664788,0.7575,0.19715,799,3777,4576 +217,8/5/2011,3,0,8,5,1,0.710833,0.656567,0.630833,0.184696,1023,3843,4866 +218,8/6/2011,3,0,8,6,2,0.716667,0.676154,0.755,0.22825,1521,2773,4294 +219,8/7/2011,3,0,8,0,1,0.7425,0.715292,0.752917,0.201487,1298,2487,3785 +220,8/8/2011,3,0,8,1,1,0.765,0.703283,0.592083,0.192175,846,3480,4326 +221,8/9/2011,3,0,8,2,1,0.775,0.724121,0.570417,0.151121,907,3695,4602 +222,8/10/2011,3,0,8,3,1,0.766667,0.684983,0.424167,0.200258,884,3896,4780 +223,8/11/2011,3,0,8,4,1,0.7175,0.651521,0.42375,0.164796,812,3980,4792 +224,8/12/2011,3,0,8,5,1,0.708333,0.654042,0.415,0.125621,1051,3854,4905 +225,8/13/2011,3,0,8,6,2,0.685833,0.645858,0.729583,0.211454,1504,2646,4150 +226,8/14/2011,3,0,8,0,2,0.676667,0.624388,0.8175,0.222633,1338,2482,3820 +227,8/15/2011,3,0,8,1,1,0.665833,0.616167,0.712083,0.208954,775,3563,4338 +228,8/16/2011,3,0,8,2,1,0.700833,0.645837,0.578333,0.236329,721,4004,4725 +229,8/17/2011,3,0,8,3,1,0.723333,0.666671,0.575417,0.143667,668,4026,4694 +230,8/18/2011,3,0,8,4,1,0.711667,0.662258,0.654583,0.233208,639,3166,3805 +231,8/19/2011,3,0,8,5,2,0.685,0.633221,0.722917,0.139308,797,3356,4153 +232,8/20/2011,3,0,8,6,1,0.6975,0.648996,0.674167,0.104467,1914,3277,5191 +233,8/21/2011,3,0,8,0,1,0.710833,0.675525,0.77,0.248754,1249,2624,3873 +234,8/22/2011,3,0,8,1,1,0.691667,0.638254,0.47,0.27675,833,3925,4758 +235,8/23/2011,3,0,8,2,1,0.640833,0.606067,0.455417,0.146763,1281,4614,5895 +236,8/24/2011,3,0,8,3,1,0.673333,0.630692,0.605,0.253108,949,4181,5130 +237,8/25/2011,3,0,8,4,2,0.684167,0.645854,0.771667,0.210833,435,3107,3542 +238,8/26/2011,3,0,8,5,1,0.7,0.659733,0.76125,0.0839625,768,3893,4661 +239,8/27/2011,3,0,8,6,2,0.68,0.635556,0.85,0.375617,226,889,1115 +240,8/28/2011,3,0,8,0,1,0.707059,0.647959,0.561765,0.304659,1415,2919,4334 +241,8/29/2011,3,0,8,1,1,0.636667,0.607958,0.554583,0.159825,729,3905,4634 +242,8/30/2011,3,0,8,2,1,0.639167,0.594704,0.548333,0.125008,775,4429,5204 +243,8/31/2011,3,0,8,3,1,0.656667,0.611121,0.597917,0.0833333,688,4370,5058 +244,9/1/2011,3,0,9,4,1,0.655,0.614921,0.639167,0.141796,783,4332,5115 +245,9/2/2011,3,0,9,5,2,0.643333,0.604808,0.727083,0.139929,875,3852,4727 +246,9/3/2011,3,0,9,6,1,0.669167,0.633213,0.716667,0.185325,1935,2549,4484 +247,9/4/2011,3,0,9,0,1,0.709167,0.665429,0.742083,0.206467,2521,2419,4940 +248,9/5/2011,3,0,9,1,2,0.673333,0.625646,0.790417,0.212696,1236,2115,3351 +249,9/6/2011,3,0,9,2,3,0.54,0.5152,0.886957,0.343943,204,2506,2710 +250,9/7/2011,3,0,9,3,3,0.599167,0.544229,0.917083,0.0970208,118,1878,1996 +251,9/8/2011,3,0,9,4,3,0.633913,0.555361,0.939565,0.192748,153,1689,1842 +252,9/9/2011,3,0,9,5,2,0.65,0.578946,0.897917,0.124379,417,3127,3544 +253,9/10/2011,3,0,9,6,1,0.66,0.607962,0.75375,0.153608,1750,3595,5345 +254,9/11/2011,3,0,9,0,1,0.653333,0.609229,0.71375,0.115054,1633,3413,5046 +255,9/12/2011,3,0,9,1,1,0.644348,0.60213,0.692174,0.088913,690,4023,4713 +256,9/13/2011,3,0,9,2,1,0.650833,0.603554,0.7125,0.141804,701,4062,4763 +257,9/14/2011,3,0,9,3,1,0.673333,0.6269,0.697083,0.1673,647,4138,4785 +258,9/15/2011,3,0,9,4,2,0.5775,0.553671,0.709167,0.271146,428,3231,3659 +259,9/16/2011,3,0,9,5,2,0.469167,0.461475,0.590417,0.164183,742,4018,4760 +260,9/17/2011,3,0,9,6,2,0.491667,0.478512,0.718333,0.189675,1434,3077,4511 +261,9/18/2011,3,0,9,0,1,0.5075,0.490537,0.695,0.178483,1353,2921,4274 +262,9/19/2011,3,0,9,1,2,0.549167,0.529675,0.69,0.151742,691,3848,4539 +263,9/20/2011,3,0,9,2,2,0.561667,0.532217,0.88125,0.134954,438,3203,3641 +264,9/21/2011,3,0,9,3,2,0.595,0.550533,0.9,0.0964042,539,3813,4352 +265,9/22/2011,3,0,9,4,2,0.628333,0.554963,0.902083,0.128125,555,4240,4795 +266,9/23/2011,4,0,9,5,2,0.609167,0.522125,0.9725,0.0783667,258,2137,2395 +267,9/24/2011,4,0,9,6,2,0.606667,0.564412,0.8625,0.0783833,1776,3647,5423 +268,9/25/2011,4,0,9,0,2,0.634167,0.572637,0.845,0.0503792,1544,3466,5010 +269,9/26/2011,4,0,9,1,2,0.649167,0.589042,0.848333,0.1107,684,3946,4630 +270,9/27/2011,4,0,9,2,2,0.636667,0.574525,0.885417,0.118171,477,3643,4120 +271,9/28/2011,4,0,9,3,2,0.635,0.575158,0.84875,0.148629,480,3427,3907 +272,9/29/2011,4,0,9,4,1,0.616667,0.574512,0.699167,0.172883,653,4186,4839 +273,9/30/2011,4,0,9,5,1,0.564167,0.544829,0.6475,0.206475,830,4372,5202 +274,10/1/2011,4,0,10,6,2,0.41,0.412863,0.75375,0.292296,480,1949,2429 +275,10/2/2011,4,0,10,0,2,0.356667,0.345317,0.791667,0.222013,616,2302,2918 +276,10/3/2011,4,0,10,1,2,0.384167,0.392046,0.760833,0.0833458,330,3240,3570 +277,10/4/2011,4,0,10,2,1,0.484167,0.472858,0.71,0.205854,486,3970,4456 +278,10/5/2011,4,0,10,3,1,0.538333,0.527138,0.647917,0.17725,559,4267,4826 +279,10/6/2011,4,0,10,4,1,0.494167,0.480425,0.620833,0.134954,639,4126,4765 +280,10/7/2011,4,0,10,5,1,0.510833,0.504404,0.684167,0.0223917,949,4036,4985 +281,10/8/2011,4,0,10,6,1,0.521667,0.513242,0.70125,0.0454042,2235,3174,5409 +282,10/9/2011,4,0,10,0,1,0.540833,0.523983,0.7275,0.06345,2397,3114,5511 +283,10/10/2011,4,0,10,1,1,0.570833,0.542925,0.73375,0.0423042,1514,3603,5117 +284,10/11/2011,4,0,10,2,2,0.566667,0.546096,0.80875,0.143042,667,3896,4563 +285,10/12/2011,4,0,10,3,3,0.543333,0.517717,0.90625,0.24815,217,2199,2416 +286,10/13/2011,4,0,10,4,2,0.589167,0.551804,0.896667,0.141787,290,2623,2913 +287,10/14/2011,4,0,10,5,2,0.550833,0.529675,0.71625,0.223883,529,3115,3644 +288,10/15/2011,4,0,10,6,1,0.506667,0.498725,0.483333,0.258083,1899,3318,5217 +289,10/16/2011,4,0,10,0,1,0.511667,0.503154,0.486667,0.281717,1748,3293,5041 +290,10/17/2011,4,0,10,1,1,0.534167,0.510725,0.579583,0.175379,713,3857,4570 +291,10/18/2011,4,0,10,2,2,0.5325,0.522721,0.701667,0.110087,637,4111,4748 +292,10/19/2011,4,0,10,3,3,0.541739,0.513848,0.895217,0.243339,254,2170,2424 +293,10/20/2011,4,0,10,4,1,0.475833,0.466525,0.63625,0.422275,471,3724,4195 +294,10/21/2011,4,0,10,5,1,0.4275,0.423596,0.574167,0.221396,676,3628,4304 +295,10/22/2011,4,0,10,6,1,0.4225,0.425492,0.629167,0.0926667,1499,2809,4308 +296,10/23/2011,4,0,10,0,1,0.421667,0.422333,0.74125,0.0995125,1619,2762,4381 +297,10/24/2011,4,0,10,1,1,0.463333,0.457067,0.772083,0.118792,699,3488,4187 +298,10/25/2011,4,0,10,2,1,0.471667,0.463375,0.622917,0.166658,695,3992,4687 +299,10/26/2011,4,0,10,3,2,0.484167,0.472846,0.720417,0.148642,404,3490,3894 +300,10/27/2011,4,0,10,4,2,0.47,0.457046,0.812917,0.197763,240,2419,2659 +301,10/28/2011,4,0,10,5,2,0.330833,0.318812,0.585833,0.229479,456,3291,3747 +302,10/29/2011,4,0,10,6,3,0.254167,0.227913,0.8825,0.351371,57,570,627 +303,10/30/2011,4,0,10,0,1,0.319167,0.321329,0.62375,0.176617,885,2446,3331 +304,10/31/2011,4,0,10,1,1,0.34,0.356063,0.703333,0.10635,362,3307,3669 +305,11/1/2011,4,0,11,2,1,0.400833,0.397088,0.68375,0.135571,410,3658,4068 +306,11/2/2011,4,0,11,3,1,0.3775,0.390133,0.71875,0.0820917,370,3816,4186 +307,11/3/2011,4,0,11,4,1,0.408333,0.405921,0.702083,0.136817,318,3656,3974 +308,11/4/2011,4,0,11,5,2,0.403333,0.403392,0.6225,0.271779,470,3576,4046 +309,11/5/2011,4,0,11,6,1,0.326667,0.323854,0.519167,0.189062,1156,2770,3926 +310,11/6/2011,4,0,11,0,1,0.348333,0.362358,0.734583,0.0920542,952,2697,3649 +311,11/7/2011,4,0,11,1,1,0.395,0.400871,0.75875,0.057225,373,3662,4035 +312,11/8/2011,4,0,11,2,1,0.408333,0.412246,0.721667,0.0690375,376,3829,4205 +313,11/9/2011,4,0,11,3,1,0.4,0.409079,0.758333,0.0621958,305,3804,4109 +314,11/10/2011,4,0,11,4,2,0.38,0.373721,0.813333,0.189067,190,2743,2933 +315,11/11/2011,4,0,11,5,1,0.324167,0.306817,0.44625,0.314675,440,2928,3368 +316,11/12/2011,4,0,11,6,1,0.356667,0.357942,0.552917,0.212062,1275,2792,4067 +317,11/13/2011,4,0,11,0,1,0.440833,0.43055,0.458333,0.281721,1004,2713,3717 +318,11/14/2011,4,0,11,1,1,0.53,0.524612,0.587083,0.306596,595,3891,4486 +319,11/15/2011,4,0,11,2,2,0.53,0.507579,0.68875,0.199633,449,3746,4195 +320,11/16/2011,4,0,11,3,3,0.456667,0.451988,0.93,0.136829,145,1672,1817 +321,11/17/2011,4,0,11,4,2,0.341667,0.323221,0.575833,0.305362,139,2914,3053 +322,11/18/2011,4,0,11,5,1,0.274167,0.272721,0.41,0.168533,245,3147,3392 +323,11/19/2011,4,0,11,6,1,0.329167,0.324483,0.502083,0.224496,943,2720,3663 +324,11/20/2011,4,0,11,0,2,0.463333,0.457058,0.684583,0.18595,787,2733,3520 +325,11/21/2011,4,0,11,1,3,0.4475,0.445062,0.91,0.138054,220,2545,2765 +326,11/22/2011,4,0,11,2,3,0.416667,0.421696,0.9625,0.118792,69,1538,1607 +327,11/23/2011,4,0,11,3,2,0.440833,0.430537,0.757917,0.335825,112,2454,2566 +328,11/24/2011,4,0,11,4,1,0.373333,0.372471,0.549167,0.167304,560,935,1495 +329,11/25/2011,4,0,11,5,1,0.375,0.380671,0.64375,0.0988958,1095,1697,2792 +330,11/26/2011,4,0,11,6,1,0.375833,0.385087,0.681667,0.0684208,1249,1819,3068 +331,11/27/2011,4,0,11,0,1,0.459167,0.4558,0.698333,0.208954,810,2261,3071 +332,11/28/2011,4,0,11,1,1,0.503478,0.490122,0.743043,0.142122,253,3614,3867 +333,11/29/2011,4,0,11,2,2,0.458333,0.451375,0.830833,0.258092,96,2818,2914 +334,11/30/2011,4,0,11,3,1,0.325,0.311221,0.613333,0.271158,188,3425,3613 +335,12/1/2011,4,0,12,4,1,0.3125,0.305554,0.524583,0.220158,182,3545,3727 +336,12/2/2011,4,0,12,5,1,0.314167,0.331433,0.625833,0.100754,268,3672,3940 +337,12/3/2011,4,0,12,6,1,0.299167,0.310604,0.612917,0.0957833,706,2908,3614 +338,12/4/2011,4,0,12,0,1,0.330833,0.3491,0.775833,0.0839583,634,2851,3485 +339,12/5/2011,4,0,12,1,2,0.385833,0.393925,0.827083,0.0622083,233,3578,3811 +340,12/6/2011,4,0,12,2,3,0.4625,0.4564,0.949583,0.232583,126,2468,2594 +341,12/7/2011,4,0,12,3,3,0.41,0.400246,0.970417,0.266175,50,655,705 +342,12/8/2011,4,0,12,4,1,0.265833,0.256938,0.58,0.240058,150,3172,3322 +343,12/9/2011,4,0,12,5,1,0.290833,0.317542,0.695833,0.0827167,261,3359,3620 +344,12/10/2011,4,0,12,6,1,0.275,0.266412,0.5075,0.233221,502,2688,3190 +345,12/11/2011,4,0,12,0,1,0.220833,0.253154,0.49,0.0665417,377,2366,2743 +346,12/12/2011,4,0,12,1,1,0.238333,0.270196,0.670833,0.06345,143,3167,3310 +347,12/13/2011,4,0,12,2,1,0.2825,0.301138,0.59,0.14055,155,3368,3523 +348,12/14/2011,4,0,12,3,2,0.3175,0.338362,0.66375,0.0609583,178,3562,3740 +349,12/15/2011,4,0,12,4,2,0.4225,0.412237,0.634167,0.268042,181,3528,3709 +350,12/16/2011,4,0,12,5,2,0.375,0.359825,0.500417,0.260575,178,3399,3577 +351,12/17/2011,4,0,12,6,2,0.258333,0.249371,0.560833,0.243167,275,2464,2739 +352,12/18/2011,4,0,12,0,1,0.238333,0.245579,0.58625,0.169779,220,2211,2431 +353,12/19/2011,4,0,12,1,1,0.276667,0.280933,0.6375,0.172896,260,3143,3403 +354,12/20/2011,4,0,12,2,2,0.385833,0.396454,0.595417,0.0615708,216,3534,3750 +355,12/21/2011,1,0,12,3,2,0.428333,0.428017,0.858333,0.2214,107,2553,2660 +356,12/22/2011,1,0,12,4,2,0.423333,0.426121,0.7575,0.047275,227,2841,3068 +357,12/23/2011,1,0,12,5,1,0.373333,0.377513,0.68625,0.274246,163,2046,2209 +358,12/24/2011,1,0,12,6,1,0.3025,0.299242,0.5425,0.190304,155,856,1011 +359,12/25/2011,1,0,12,0,1,0.274783,0.279961,0.681304,0.155091,303,451,754 +360,12/26/2011,1,0,12,1,1,0.321739,0.315535,0.506957,0.239465,430,887,1317 +361,12/27/2011,1,0,12,2,2,0.325,0.327633,0.7625,0.18845,103,1059,1162 +362,12/28/2011,1,0,12,3,1,0.29913,0.279974,0.503913,0.293961,255,2047,2302 +363,12/29/2011,1,0,12,4,1,0.248333,0.263892,0.574167,0.119412,254,2169,2423 +364,12/30/2011,1,0,12,5,1,0.311667,0.318812,0.636667,0.134337,491,2508,2999 +365,12/31/2011,1,0,12,6,1,0.41,0.414121,0.615833,0.220154,665,1820,2485 +366,1/1/2012,1,1,1,0,1,0.37,0.375621,0.6925,0.192167,686,1608,2294 +367,1/2/2012,1,1,1,1,1,0.273043,0.252304,0.381304,0.329665,244,1707,1951 +368,1/3/2012,1,1,1,2,1,0.15,0.126275,0.44125,0.365671,89,2147,2236 +369,1/4/2012,1,1,1,3,2,0.1075,0.119337,0.414583,0.1847,95,2273,2368 +370,1/5/2012,1,1,1,4,1,0.265833,0.278412,0.524167,0.129987,140,3132,3272 +371,1/6/2012,1,1,1,5,1,0.334167,0.340267,0.542083,0.167908,307,3791,4098 +372,1/7/2012,1,1,1,6,1,0.393333,0.390779,0.531667,0.174758,1070,3451,4521 +373,1/8/2012,1,1,1,0,1,0.3375,0.340258,0.465,0.191542,599,2826,3425 +374,1/9/2012,1,1,1,1,2,0.224167,0.247479,0.701667,0.0989,106,2270,2376 +375,1/10/2012,1,1,1,2,1,0.308696,0.318826,0.646522,0.187552,173,3425,3598 +376,1/11/2012,1,1,1,3,2,0.274167,0.282821,0.8475,0.131221,92,2085,2177 +377,1/12/2012,1,1,1,4,2,0.3825,0.381938,0.802917,0.180967,269,3828,4097 +378,1/13/2012,1,1,1,5,1,0.274167,0.249362,0.5075,0.378108,174,3040,3214 +379,1/14/2012,1,1,1,6,1,0.18,0.183087,0.4575,0.187183,333,2160,2493 +380,1/15/2012,1,1,1,0,1,0.166667,0.161625,0.419167,0.251258,284,2027,2311 +381,1/16/2012,1,1,1,1,1,0.19,0.190663,0.5225,0.231358,217,2081,2298 +382,1/17/2012,1,1,1,2,2,0.373043,0.364278,0.716087,0.34913,127,2808,2935 +383,1/18/2012,1,1,1,3,1,0.303333,0.275254,0.443333,0.415429,109,3267,3376 +384,1/19/2012,1,1,1,4,1,0.19,0.190038,0.4975,0.220158,130,3162,3292 +385,1/20/2012,1,1,1,5,2,0.2175,0.220958,0.45,0.20275,115,3048,3163 +386,1/21/2012,1,1,1,6,2,0.173333,0.174875,0.83125,0.222642,67,1234,1301 +387,1/22/2012,1,1,1,0,2,0.1625,0.16225,0.79625,0.199638,196,1781,1977 +388,1/23/2012,1,1,1,1,2,0.218333,0.243058,0.91125,0.110708,145,2287,2432 +389,1/24/2012,1,1,1,2,1,0.3425,0.349108,0.835833,0.123767,439,3900,4339 +390,1/25/2012,1,1,1,3,1,0.294167,0.294821,0.64375,0.161071,467,3803,4270 +391,1/26/2012,1,1,1,4,2,0.341667,0.35605,0.769583,0.0733958,244,3831,4075 +392,1/27/2012,1,1,1,5,2,0.425,0.415383,0.74125,0.342667,269,3187,3456 +393,1/28/2012,1,1,1,6,1,0.315833,0.326379,0.543333,0.210829,775,3248,4023 +394,1/29/2012,1,1,1,0,1,0.2825,0.272721,0.31125,0.24005,558,2685,3243 +395,1/30/2012,1,1,1,1,1,0.269167,0.262625,0.400833,0.215792,126,3498,3624 +396,1/31/2012,1,1,1,2,1,0.39,0.381317,0.416667,0.261817,324,4185,4509 +397,2/1/2012,1,1,2,3,1,0.469167,0.466538,0.507917,0.189067,304,4275,4579 +398,2/2/2012,1,1,2,4,2,0.399167,0.398971,0.672917,0.187187,190,3571,3761 +399,2/3/2012,1,1,2,5,1,0.313333,0.309346,0.526667,0.178496,310,3841,4151 +400,2/4/2012,1,1,2,6,2,0.264167,0.272725,0.779583,0.121896,384,2448,2832 +401,2/5/2012,1,1,2,0,2,0.265833,0.264521,0.687917,0.175996,318,2629,2947 +402,2/6/2012,1,1,2,1,1,0.282609,0.296426,0.622174,0.1538,206,3578,3784 +403,2/7/2012,1,1,2,2,1,0.354167,0.361104,0.49625,0.147379,199,4176,4375 +404,2/8/2012,1,1,2,3,2,0.256667,0.266421,0.722917,0.133721,109,2693,2802 +405,2/9/2012,1,1,2,4,1,0.265,0.261988,0.562083,0.194037,163,3667,3830 +406,2/10/2012,1,1,2,5,2,0.280833,0.293558,0.54,0.116929,227,3604,3831 +407,2/11/2012,1,1,2,6,3,0.224167,0.210867,0.73125,0.289796,192,1977,2169 +408,2/12/2012,1,1,2,0,1,0.1275,0.101658,0.464583,0.409212,73,1456,1529 +409,2/13/2012,1,1,2,1,1,0.2225,0.227913,0.41125,0.167283,94,3328,3422 +410,2/14/2012,1,1,2,2,2,0.319167,0.333946,0.50875,0.141179,135,3787,3922 +411,2/15/2012,1,1,2,3,1,0.348333,0.351629,0.53125,0.1816,141,4028,4169 +412,2/16/2012,1,1,2,4,2,0.316667,0.330162,0.752917,0.091425,74,2931,3005 +413,2/17/2012,1,1,2,5,1,0.343333,0.351629,0.634583,0.205846,349,3805,4154 +414,2/18/2012,1,1,2,6,1,0.346667,0.355425,0.534583,0.190929,1435,2883,4318 +415,2/19/2012,1,1,2,0,2,0.28,0.265788,0.515833,0.253112,618,2071,2689 +416,2/20/2012,1,1,2,1,1,0.28,0.273391,0.507826,0.229083,502,2627,3129 +417,2/21/2012,1,1,2,2,1,0.287826,0.295113,0.594348,0.205717,163,3614,3777 +418,2/22/2012,1,1,2,3,1,0.395833,0.392667,0.567917,0.234471,394,4379,4773 +419,2/23/2012,1,1,2,4,1,0.454167,0.444446,0.554583,0.190913,516,4546,5062 +420,2/24/2012,1,1,2,5,2,0.4075,0.410971,0.7375,0.237567,246,3241,3487 +421,2/25/2012,1,1,2,6,1,0.290833,0.255675,0.395833,0.421642,317,2415,2732 +422,2/26/2012,1,1,2,0,1,0.279167,0.268308,0.41,0.205229,515,2874,3389 +423,2/27/2012,1,1,2,1,1,0.366667,0.357954,0.490833,0.268033,253,4069,4322 +424,2/28/2012,1,1,2,2,1,0.359167,0.353525,0.395833,0.193417,229,4134,4363 +425,2/29/2012,1,1,2,3,2,0.344348,0.34847,0.804783,0.179117,65,1769,1834 +426,3/1/2012,1,1,3,4,1,0.485833,0.475371,0.615417,0.226987,325,4665,4990 +427,3/2/2012,1,1,3,5,2,0.353333,0.359842,0.657083,0.144904,246,2948,3194 +428,3/3/2012,1,1,3,6,2,0.414167,0.413492,0.62125,0.161079,956,3110,4066 +429,3/4/2012,1,1,3,0,1,0.325833,0.303021,0.403333,0.334571,710,2713,3423 +430,3/5/2012,1,1,3,1,1,0.243333,0.241171,0.50625,0.228858,203,3130,3333 +431,3/6/2012,1,1,3,2,1,0.258333,0.255042,0.456667,0.200875,221,3735,3956 +432,3/7/2012,1,1,3,3,1,0.404167,0.3851,0.513333,0.345779,432,4484,4916 +433,3/8/2012,1,1,3,4,1,0.5275,0.524604,0.5675,0.441563,486,4896,5382 +434,3/9/2012,1,1,3,5,2,0.410833,0.397083,0.407083,0.4148,447,4122,4569 +435,3/10/2012,1,1,3,6,1,0.2875,0.277767,0.350417,0.22575,968,3150,4118 +436,3/11/2012,1,1,3,0,1,0.361739,0.35967,0.476957,0.222587,1658,3253,4911 +437,3/12/2012,1,1,3,1,1,0.466667,0.459592,0.489167,0.207713,838,4460,5298 +438,3/13/2012,1,1,3,2,1,0.565,0.542929,0.6175,0.23695,762,5085,5847 +439,3/14/2012,1,1,3,3,1,0.5725,0.548617,0.507083,0.115062,997,5315,6312 +440,3/15/2012,1,1,3,4,1,0.5575,0.532825,0.579583,0.149883,1005,5187,6192 +441,3/16/2012,1,1,3,5,2,0.435833,0.436229,0.842083,0.113192,548,3830,4378 +442,3/17/2012,1,1,3,6,2,0.514167,0.505046,0.755833,0.110704,3155,4681,7836 +443,3/18/2012,1,1,3,0,2,0.4725,0.464,0.81,0.126883,2207,3685,5892 +444,3/19/2012,1,1,3,1,1,0.545,0.532821,0.72875,0.162317,982,5171,6153 +445,3/20/2012,1,1,3,2,1,0.560833,0.538533,0.807917,0.121271,1051,5042,6093 +446,3/21/2012,2,1,3,3,2,0.531667,0.513258,0.82125,0.0895583,1122,5108,6230 +447,3/22/2012,2,1,3,4,1,0.554167,0.531567,0.83125,0.117562,1334,5537,6871 +448,3/23/2012,2,1,3,5,2,0.601667,0.570067,0.694167,0.1163,2469,5893,8362 +449,3/24/2012,2,1,3,6,2,0.5025,0.486733,0.885417,0.192783,1033,2339,3372 +450,3/25/2012,2,1,3,0,2,0.4375,0.437488,0.880833,0.220775,1532,3464,4996 +451,3/26/2012,2,1,3,1,1,0.445833,0.43875,0.477917,0.386821,795,4763,5558 +452,3/27/2012,2,1,3,2,1,0.323333,0.315654,0.29,0.187192,531,4571,5102 +453,3/28/2012,2,1,3,3,1,0.484167,0.47095,0.48125,0.291671,674,5024,5698 +454,3/29/2012,2,1,3,4,1,0.494167,0.482304,0.439167,0.31965,834,5299,6133 +455,3/30/2012,2,1,3,5,2,0.37,0.375621,0.580833,0.138067,796,4663,5459 +456,3/31/2012,2,1,3,6,2,0.424167,0.421708,0.738333,0.250617,2301,3934,6235 +457,4/1/2012,2,1,4,0,2,0.425833,0.417287,0.67625,0.172267,2347,3694,6041 +458,4/2/2012,2,1,4,1,1,0.433913,0.427513,0.504348,0.312139,1208,4728,5936 +459,4/3/2012,2,1,4,2,1,0.466667,0.461483,0.396667,0.100133,1348,5424,6772 +460,4/4/2012,2,1,4,3,1,0.541667,0.53345,0.469583,0.180975,1058,5378,6436 +461,4/5/2012,2,1,4,4,1,0.435,0.431163,0.374167,0.219529,1192,5265,6457 +462,4/6/2012,2,1,4,5,1,0.403333,0.390767,0.377083,0.300388,1807,4653,6460 +463,4/7/2012,2,1,4,6,1,0.4375,0.426129,0.254167,0.274871,3252,3605,6857 +464,4/8/2012,2,1,4,0,1,0.5,0.492425,0.275833,0.232596,2230,2939,5169 +465,4/9/2012,2,1,4,1,1,0.489167,0.476638,0.3175,0.358196,905,4680,5585 +466,4/10/2012,2,1,4,2,1,0.446667,0.436233,0.435,0.249375,819,5099,5918 +467,4/11/2012,2,1,4,3,1,0.348696,0.337274,0.469565,0.295274,482,4380,4862 +468,4/12/2012,2,1,4,4,1,0.3975,0.387604,0.46625,0.290429,663,4746,5409 +469,4/13/2012,2,1,4,5,1,0.4425,0.431808,0.408333,0.155471,1252,5146,6398 +470,4/14/2012,2,1,4,6,1,0.495,0.487996,0.502917,0.190917,2795,4665,7460 +471,4/15/2012,2,1,4,0,1,0.606667,0.573875,0.507917,0.225129,2846,4286,7132 +472,4/16/2012,2,1,4,1,1,0.664167,0.614925,0.561667,0.284829,1198,5172,6370 +473,4/17/2012,2,1,4,2,1,0.608333,0.598487,0.390417,0.273629,989,5702,6691 +474,4/18/2012,2,1,4,3,2,0.463333,0.457038,0.569167,0.167912,347,4020,4367 +475,4/19/2012,2,1,4,4,1,0.498333,0.493046,0.6125,0.0659292,846,5719,6565 +476,4/20/2012,2,1,4,5,1,0.526667,0.515775,0.694583,0.149871,1340,5950,7290 +477,4/21/2012,2,1,4,6,1,0.57,0.542921,0.682917,0.283587,2541,4083,6624 +478,4/22/2012,2,1,4,0,3,0.396667,0.389504,0.835417,0.344546,120,907,1027 +479,4/23/2012,2,1,4,1,2,0.321667,0.301125,0.766667,0.303496,195,3019,3214 +480,4/24/2012,2,1,4,2,1,0.413333,0.405283,0.454167,0.249383,518,5115,5633 +481,4/25/2012,2,1,4,3,1,0.476667,0.470317,0.427917,0.118792,655,5541,6196 +482,4/26/2012,2,1,4,4,2,0.498333,0.483583,0.756667,0.176625,475,4551,5026 +483,4/27/2012,2,1,4,5,1,0.4575,0.452637,0.400833,0.347633,1014,5219,6233 +484,4/28/2012,2,1,4,6,2,0.376667,0.377504,0.489583,0.129975,1120,3100,4220 +485,4/29/2012,2,1,4,0,1,0.458333,0.450121,0.587083,0.116908,2229,4075,6304 +486,4/30/2012,2,1,4,1,2,0.464167,0.457696,0.57,0.171638,665,4907,5572 +487,5/1/2012,2,1,5,2,2,0.613333,0.577021,0.659583,0.156096,653,5087,5740 +488,5/2/2012,2,1,5,3,1,0.564167,0.537896,0.797083,0.138058,667,5502,6169 +489,5/3/2012,2,1,5,4,2,0.56,0.537242,0.768333,0.133696,764,5657,6421 +490,5/4/2012,2,1,5,5,1,0.6275,0.590917,0.735417,0.162938,1069,5227,6296 +491,5/5/2012,2,1,5,6,2,0.621667,0.584608,0.756667,0.152992,2496,4387,6883 +492,5/6/2012,2,1,5,0,2,0.5625,0.546737,0.74,0.149879,2135,4224,6359 +493,5/7/2012,2,1,5,1,2,0.5375,0.527142,0.664167,0.230721,1008,5265,6273 +494,5/8/2012,2,1,5,2,2,0.581667,0.557471,0.685833,0.296029,738,4990,5728 +495,5/9/2012,2,1,5,3,2,0.575,0.553025,0.744167,0.216412,620,4097,4717 +496,5/10/2012,2,1,5,4,1,0.505833,0.491783,0.552083,0.314063,1026,5546,6572 +497,5/11/2012,2,1,5,5,1,0.533333,0.520833,0.360417,0.236937,1319,5711,7030 +498,5/12/2012,2,1,5,6,1,0.564167,0.544817,0.480417,0.123133,2622,4807,7429 +499,5/13/2012,2,1,5,0,1,0.6125,0.585238,0.57625,0.225117,2172,3946,6118 +500,5/14/2012,2,1,5,1,2,0.573333,0.5499,0.789583,0.212692,342,2501,2843 +501,5/15/2012,2,1,5,2,2,0.611667,0.576404,0.794583,0.147392,625,4490,5115 +502,5/16/2012,2,1,5,3,1,0.636667,0.595975,0.697917,0.122512,991,6433,7424 +503,5/17/2012,2,1,5,4,1,0.593333,0.572613,0.52,0.229475,1242,6142,7384 +504,5/18/2012,2,1,5,5,1,0.564167,0.551121,0.523333,0.136817,1521,6118,7639 +505,5/19/2012,2,1,5,6,1,0.6,0.566908,0.45625,0.083975,3410,4884,8294 +506,5/20/2012,2,1,5,0,1,0.620833,0.583967,0.530417,0.254367,2704,4425,7129 +507,5/21/2012,2,1,5,1,2,0.598333,0.565667,0.81125,0.233204,630,3729,4359 +508,5/22/2012,2,1,5,2,2,0.615,0.580825,0.765833,0.118167,819,5254,6073 +509,5/23/2012,2,1,5,3,2,0.621667,0.584612,0.774583,0.102,766,4494,5260 +510,5/24/2012,2,1,5,4,1,0.655,0.6067,0.716667,0.172896,1059,5711,6770 +511,5/25/2012,2,1,5,5,1,0.68,0.627529,0.747083,0.14055,1417,5317,6734 +512,5/26/2012,2,1,5,6,1,0.6925,0.642696,0.7325,0.198992,2855,3681,6536 +513,5/27/2012,2,1,5,0,1,0.69,0.641425,0.697083,0.215171,3283,3308,6591 +514,5/28/2012,2,1,5,1,1,0.7125,0.6793,0.67625,0.196521,2557,3486,6043 +515,5/29/2012,2,1,5,2,1,0.7225,0.672992,0.684583,0.2954,880,4863,5743 +516,5/30/2012,2,1,5,3,2,0.656667,0.611129,0.67,0.134329,745,6110,6855 +517,5/31/2012,2,1,5,4,1,0.68,0.631329,0.492917,0.195279,1100,6238,7338 +518,6/1/2012,2,1,6,5,2,0.654167,0.607962,0.755417,0.237563,533,3594,4127 +519,6/2/2012,2,1,6,6,1,0.583333,0.566288,0.549167,0.186562,2795,5325,8120 +520,6/3/2012,2,1,6,0,1,0.6025,0.575133,0.493333,0.184087,2494,5147,7641 +521,6/4/2012,2,1,6,1,1,0.5975,0.578283,0.487083,0.284833,1071,5927,6998 +522,6/5/2012,2,1,6,2,2,0.540833,0.525892,0.613333,0.209575,968,6033,7001 +523,6/6/2012,2,1,6,3,1,0.554167,0.542292,0.61125,0.077125,1027,6028,7055 +524,6/7/2012,2,1,6,4,1,0.6025,0.569442,0.567083,0.15735,1038,6456,7494 +525,6/8/2012,2,1,6,5,1,0.649167,0.597862,0.467917,0.175383,1488,6248,7736 +526,6/9/2012,2,1,6,6,1,0.710833,0.648367,0.437083,0.144287,2708,4790,7498 +527,6/10/2012,2,1,6,0,1,0.726667,0.663517,0.538333,0.133721,2224,4374,6598 +528,6/11/2012,2,1,6,1,2,0.720833,0.659721,0.587917,0.207713,1017,5647,6664 +529,6/12/2012,2,1,6,2,2,0.653333,0.597875,0.833333,0.214546,477,4495,4972 +530,6/13/2012,2,1,6,3,1,0.655833,0.611117,0.582083,0.343279,1173,6248,7421 +531,6/14/2012,2,1,6,4,1,0.648333,0.624383,0.569583,0.253733,1180,6183,7363 +532,6/15/2012,2,1,6,5,1,0.639167,0.599754,0.589583,0.176617,1563,6102,7665 +533,6/16/2012,2,1,6,6,1,0.631667,0.594708,0.504167,0.166667,2963,4739,7702 +534,6/17/2012,2,1,6,0,1,0.5925,0.571975,0.59875,0.144904,2634,4344,6978 +535,6/18/2012,2,1,6,1,2,0.568333,0.544842,0.777917,0.174746,653,4446,5099 +536,6/19/2012,2,1,6,2,1,0.688333,0.654692,0.69,0.148017,968,5857,6825 +537,6/20/2012,2,1,6,3,1,0.7825,0.720975,0.592083,0.113812,872,5339,6211 +538,6/21/2012,3,1,6,4,1,0.805833,0.752542,0.567917,0.118787,778,5127,5905 +539,6/22/2012,3,1,6,5,1,0.7775,0.724121,0.57375,0.182842,964,4859,5823 +540,6/23/2012,3,1,6,6,1,0.731667,0.652792,0.534583,0.179721,2657,4801,7458 +541,6/24/2012,3,1,6,0,1,0.743333,0.674254,0.479167,0.145525,2551,4340,6891 +542,6/25/2012,3,1,6,1,1,0.715833,0.654042,0.504167,0.300383,1139,5640,6779 +543,6/26/2012,3,1,6,2,1,0.630833,0.594704,0.373333,0.347642,1077,6365,7442 +544,6/27/2012,3,1,6,3,1,0.6975,0.640792,0.36,0.271775,1077,6258,7335 +545,6/28/2012,3,1,6,4,1,0.749167,0.675512,0.4225,0.17165,921,5958,6879 +546,6/29/2012,3,1,6,5,1,0.834167,0.786613,0.48875,0.165417,829,4634,5463 +547,6/30/2012,3,1,6,6,1,0.765,0.687508,0.60125,0.161071,1455,4232,5687 +548,7/1/2012,3,1,7,0,1,0.815833,0.750629,0.51875,0.168529,1421,4110,5531 +549,7/2/2012,3,1,7,1,1,0.781667,0.702038,0.447083,0.195267,904,5323,6227 +550,7/3/2012,3,1,7,2,1,0.780833,0.70265,0.492083,0.126237,1052,5608,6660 +551,7/4/2012,3,1,7,3,1,0.789167,0.732337,0.53875,0.13495,2562,4841,7403 +552,7/5/2012,3,1,7,4,1,0.8275,0.761367,0.457917,0.194029,1405,4836,6241 +553,7/6/2012,3,1,7,5,1,0.828333,0.752533,0.450833,0.146142,1366,4841,6207 +554,7/7/2012,3,1,7,6,1,0.861667,0.804913,0.492083,0.163554,1448,3392,4840 +555,7/8/2012,3,1,7,0,1,0.8225,0.790396,0.57375,0.125629,1203,3469,4672 +556,7/9/2012,3,1,7,1,2,0.710833,0.654054,0.683333,0.180975,998,5571,6569 +557,7/10/2012,3,1,7,2,2,0.720833,0.664796,0.6675,0.151737,954,5336,6290 +558,7/11/2012,3,1,7,3,1,0.716667,0.650271,0.633333,0.151733,975,6289,7264 +559,7/12/2012,3,1,7,4,1,0.715833,0.654683,0.529583,0.146775,1032,6414,7446 +560,7/13/2012,3,1,7,5,2,0.731667,0.667933,0.485833,0.08085,1511,5988,7499 +561,7/14/2012,3,1,7,6,2,0.703333,0.666042,0.699167,0.143679,2355,4614,6969 +562,7/15/2012,3,1,7,0,1,0.745833,0.705196,0.717917,0.166667,1920,4111,6031 +563,7/16/2012,3,1,7,1,1,0.763333,0.724125,0.645,0.164187,1088,5742,6830 +564,7/17/2012,3,1,7,2,1,0.818333,0.755683,0.505833,0.114429,921,5865,6786 +565,7/18/2012,3,1,7,3,1,0.793333,0.745583,0.577083,0.137442,799,4914,5713 +566,7/19/2012,3,1,7,4,1,0.77,0.714642,0.600417,0.165429,888,5703,6591 +567,7/20/2012,3,1,7,5,2,0.665833,0.613025,0.844167,0.208967,747,5123,5870 +568,7/21/2012,3,1,7,6,3,0.595833,0.549912,0.865417,0.2133,1264,3195,4459 +569,7/22/2012,3,1,7,0,2,0.6675,0.623125,0.7625,0.0939208,2544,4866,7410 +570,7/23/2012,3,1,7,1,1,0.741667,0.690017,0.694167,0.138683,1135,5831,6966 +571,7/24/2012,3,1,7,2,1,0.750833,0.70645,0.655,0.211454,1140,6452,7592 +572,7/25/2012,3,1,7,3,1,0.724167,0.654054,0.45,0.1648,1383,6790,8173 +573,7/26/2012,3,1,7,4,1,0.776667,0.739263,0.596667,0.284813,1036,5825,6861 +574,7/27/2012,3,1,7,5,1,0.781667,0.734217,0.594583,0.152992,1259,5645,6904 +575,7/28/2012,3,1,7,6,1,0.755833,0.697604,0.613333,0.15735,2234,4451,6685 +576,7/29/2012,3,1,7,0,1,0.721667,0.667933,0.62375,0.170396,2153,4444,6597 +577,7/30/2012,3,1,7,1,1,0.730833,0.684987,0.66875,0.153617,1040,6065,7105 +578,7/31/2012,3,1,7,2,1,0.713333,0.662896,0.704167,0.165425,968,6248,7216 +579,8/1/2012,3,1,8,3,1,0.7175,0.667308,0.6775,0.141179,1074,6506,7580 +580,8/2/2012,3,1,8,4,1,0.7525,0.707088,0.659583,0.129354,983,6278,7261 +581,8/3/2012,3,1,8,5,2,0.765833,0.722867,0.6425,0.215792,1328,5847,7175 +582,8/4/2012,3,1,8,6,1,0.793333,0.751267,0.613333,0.257458,2345,4479,6824 +583,8/5/2012,3,1,8,0,1,0.769167,0.731079,0.6525,0.290421,1707,3757,5464 +584,8/6/2012,3,1,8,1,2,0.7525,0.710246,0.654167,0.129354,1233,5780,7013 +585,8/7/2012,3,1,8,2,2,0.735833,0.697621,0.70375,0.116908,1278,5995,7273 +586,8/8/2012,3,1,8,3,2,0.75,0.707717,0.672917,0.1107,1263,6271,7534 +587,8/9/2012,3,1,8,4,1,0.755833,0.699508,0.620417,0.1561,1196,6090,7286 +588,8/10/2012,3,1,8,5,2,0.715833,0.667942,0.715833,0.238813,1065,4721,5786 +589,8/11/2012,3,1,8,6,2,0.6925,0.638267,0.732917,0.206479,2247,4052,6299 +590,8/12/2012,3,1,8,0,1,0.700833,0.644579,0.530417,0.122512,2182,4362,6544 +591,8/13/2012,3,1,8,1,1,0.720833,0.662254,0.545417,0.136212,1207,5676,6883 +592,8/14/2012,3,1,8,2,1,0.726667,0.676779,0.686667,0.169158,1128,5656,6784 +593,8/15/2012,3,1,8,3,1,0.706667,0.654037,0.619583,0.169771,1198,6149,7347 +594,8/16/2012,3,1,8,4,1,0.719167,0.654688,0.519167,0.141796,1338,6267,7605 +595,8/17/2012,3,1,8,5,1,0.723333,0.2424,0.570833,0.231354,1483,5665,7148 +596,8/18/2012,3,1,8,6,1,0.678333,0.618071,0.603333,0.177867,2827,5038,7865 +597,8/19/2012,3,1,8,0,2,0.635833,0.603554,0.711667,0.08645,1208,3341,4549 +598,8/20/2012,3,1,8,1,2,0.635833,0.595967,0.734167,0.129979,1026,5504,6530 +599,8/21/2012,3,1,8,2,1,0.649167,0.601025,0.67375,0.0727708,1081,5925,7006 +600,8/22/2012,3,1,8,3,1,0.6675,0.621854,0.677083,0.0702833,1094,6281,7375 +601,8/23/2012,3,1,8,4,1,0.695833,0.637008,0.635833,0.0845958,1363,6402,7765 +602,8/24/2012,3,1,8,5,2,0.7025,0.6471,0.615,0.0721458,1325,6257,7582 +603,8/25/2012,3,1,8,6,2,0.661667,0.618696,0.712917,0.244408,1829,4224,6053 +604,8/26/2012,3,1,8,0,2,0.653333,0.595996,0.845833,0.228858,1483,3772,5255 +605,8/27/2012,3,1,8,1,1,0.703333,0.654688,0.730417,0.128733,989,5928,6917 +606,8/28/2012,3,1,8,2,1,0.728333,0.66605,0.62,0.190925,935,6105,7040 +607,8/29/2012,3,1,8,3,1,0.685,0.635733,0.552083,0.112562,1177,6520,7697 +608,8/30/2012,3,1,8,4,1,0.706667,0.652779,0.590417,0.0771167,1172,6541,7713 +609,8/31/2012,3,1,8,5,1,0.764167,0.6894,0.5875,0.168533,1433,5917,7350 +610,9/1/2012,3,1,9,6,2,0.753333,0.702654,0.638333,0.113187,2352,3788,6140 +611,9/2/2012,3,1,9,0,2,0.696667,0.649,0.815,0.0640708,2613,3197,5810 +612,9/3/2012,3,1,9,1,1,0.7075,0.661629,0.790833,0.151121,1965,4069,6034 +613,9/4/2012,3,1,9,2,1,0.725833,0.686888,0.755,0.236321,867,5997,6864 +614,9/5/2012,3,1,9,3,1,0.736667,0.708983,0.74125,0.187808,832,6280,7112 +615,9/6/2012,3,1,9,4,2,0.696667,0.655329,0.810417,0.142421,611,5592,6203 +616,9/7/2012,3,1,9,5,1,0.703333,0.657204,0.73625,0.171646,1045,6459,7504 +617,9/8/2012,3,1,9,6,2,0.659167,0.611121,0.799167,0.281104,1557,4419,5976 +618,9/9/2012,3,1,9,0,1,0.61,0.578925,0.5475,0.224496,2570,5657,8227 +619,9/10/2012,3,1,9,1,1,0.583333,0.565654,0.50375,0.258713,1118,6407,7525 +620,9/11/2012,3,1,9,2,1,0.5775,0.554292,0.52,0.0920542,1070,6697,7767 +621,9/12/2012,3,1,9,3,1,0.599167,0.570075,0.577083,0.131846,1050,6820,7870 +622,9/13/2012,3,1,9,4,1,0.6125,0.579558,0.637083,0.0827208,1054,6750,7804 +623,9/14/2012,3,1,9,5,1,0.633333,0.594083,0.6725,0.103863,1379,6630,8009 +624,9/15/2012,3,1,9,6,1,0.608333,0.585867,0.501667,0.247521,3160,5554,8714 +625,9/16/2012,3,1,9,0,1,0.58,0.563125,0.57,0.0901833,2166,5167,7333 +626,9/17/2012,3,1,9,1,2,0.580833,0.55305,0.734583,0.151742,1022,5847,6869 +627,9/18/2012,3,1,9,2,2,0.623333,0.565067,0.8725,0.357587,371,3702,4073 +628,9/19/2012,3,1,9,3,1,0.5525,0.540404,0.536667,0.215175,788,6803,7591 +629,9/20/2012,3,1,9,4,1,0.546667,0.532192,0.618333,0.118167,939,6781,7720 +630,9/21/2012,3,1,9,5,1,0.599167,0.571971,0.66875,0.154229,1250,6917,8167 +631,9/22/2012,3,1,9,6,1,0.65,0.610488,0.646667,0.283583,2512,5883,8395 +632,9/23/2012,4,1,9,0,1,0.529167,0.518933,0.467083,0.223258,2454,5453,7907 +633,9/24/2012,4,1,9,1,1,0.514167,0.502513,0.492917,0.142404,1001,6435,7436 +634,9/25/2012,4,1,9,2,1,0.55,0.544179,0.57,0.236321,845,6693,7538 +635,9/26/2012,4,1,9,3,1,0.635,0.596613,0.630833,0.2444,787,6946,7733 +636,9/27/2012,4,1,9,4,2,0.65,0.607975,0.690833,0.134342,751,6642,7393 +637,9/28/2012,4,1,9,5,2,0.619167,0.585863,0.69,0.164179,1045,6370,7415 +638,9/29/2012,4,1,9,6,1,0.5425,0.530296,0.542917,0.227604,2589,5966,8555 +639,9/30/2012,4,1,9,0,1,0.526667,0.517663,0.583333,0.134958,2015,4874,6889 +640,10/1/2012,4,1,10,1,2,0.520833,0.512,0.649167,0.0908042,763,6015,6778 +641,10/2/2012,4,1,10,2,3,0.590833,0.542333,0.871667,0.104475,315,4324,4639 +642,10/3/2012,4,1,10,3,2,0.6575,0.599133,0.79375,0.0665458,728,6844,7572 +643,10/4/2012,4,1,10,4,2,0.6575,0.607975,0.722917,0.117546,891,6437,7328 +644,10/5/2012,4,1,10,5,1,0.615,0.580187,0.6275,0.10635,1516,6640,8156 +645,10/6/2012,4,1,10,6,1,0.554167,0.538521,0.664167,0.268025,3031,4934,7965 +646,10/7/2012,4,1,10,0,2,0.415833,0.419813,0.708333,0.141162,781,2729,3510 +647,10/8/2012,4,1,10,1,2,0.383333,0.387608,0.709583,0.189679,874,4604,5478 +648,10/9/2012,4,1,10,2,2,0.446667,0.438112,0.761667,0.1903,601,5791,6392 +649,10/10/2012,4,1,10,3,1,0.514167,0.503142,0.630833,0.187821,780,6911,7691 +650,10/11/2012,4,1,10,4,1,0.435,0.431167,0.463333,0.181596,834,6736,7570 +651,10/12/2012,4,1,10,5,1,0.4375,0.433071,0.539167,0.235092,1060,6222,7282 +652,10/13/2012,4,1,10,6,1,0.393333,0.391396,0.494583,0.146142,2252,4857,7109 +653,10/14/2012,4,1,10,0,1,0.521667,0.508204,0.640417,0.278612,2080,4559,6639 +654,10/15/2012,4,1,10,1,2,0.561667,0.53915,0.7075,0.296037,760,5115,5875 +655,10/16/2012,4,1,10,2,1,0.468333,0.460846,0.558333,0.182221,922,6612,7534 +656,10/17/2012,4,1,10,3,1,0.455833,0.450108,0.692917,0.101371,979,6482,7461 +657,10/18/2012,4,1,10,4,2,0.5225,0.512625,0.728333,0.236937,1008,6501,7509 +658,10/19/2012,4,1,10,5,2,0.563333,0.537896,0.815,0.134954,753,4671,5424 +659,10/20/2012,4,1,10,6,1,0.484167,0.472842,0.572917,0.117537,2806,5284,8090 +660,10/21/2012,4,1,10,0,1,0.464167,0.456429,0.51,0.166054,2132,4692,6824 +661,10/22/2012,4,1,10,1,1,0.4875,0.482942,0.568333,0.0814833,830,6228,7058 +662,10/23/2012,4,1,10,2,1,0.544167,0.530304,0.641667,0.0945458,841,6625,7466 +663,10/24/2012,4,1,10,3,1,0.5875,0.558721,0.63625,0.0727792,795,6898,7693 +664,10/25/2012,4,1,10,4,2,0.55,0.529688,0.800417,0.124375,875,6484,7359 +665,10/26/2012,4,1,10,5,2,0.545833,0.52275,0.807083,0.132467,1182,6262,7444 +666,10/27/2012,4,1,10,6,2,0.53,0.515133,0.72,0.235692,2643,5209,7852 +667,10/28/2012,4,1,10,0,2,0.4775,0.467771,0.694583,0.398008,998,3461,4459 +668,10/29/2012,4,1,10,1,3,0.44,0.4394,0.88,0.3582,2,20,22 +669,10/30/2012,4,1,10,2,2,0.318182,0.309909,0.825455,0.213009,87,1009,1096 +670,10/31/2012,4,1,10,3,2,0.3575,0.3611,0.666667,0.166667,419,5147,5566 +671,11/1/2012,4,1,11,4,2,0.365833,0.369942,0.581667,0.157346,466,5520,5986 +672,11/2/2012,4,1,11,5,1,0.355,0.356042,0.522083,0.266175,618,5229,5847 +673,11/3/2012,4,1,11,6,2,0.343333,0.323846,0.49125,0.270529,1029,4109,5138 +674,11/4/2012,4,1,11,0,1,0.325833,0.329538,0.532917,0.179108,1201,3906,5107 +675,11/5/2012,4,1,11,1,1,0.319167,0.308075,0.494167,0.236325,378,4881,5259 +676,11/6/2012,4,1,11,2,1,0.280833,0.281567,0.567083,0.173513,466,5220,5686 +677,11/7/2012,4,1,11,3,2,0.295833,0.274621,0.5475,0.304108,326,4709,5035 +678,11/8/2012,4,1,11,4,1,0.352174,0.341891,0.333478,0.347835,340,4975,5315 +679,11/9/2012,4,1,11,5,1,0.361667,0.355413,0.540833,0.214558,709,5283,5992 +680,11/10/2012,4,1,11,6,1,0.389167,0.393937,0.645417,0.0578458,2090,4446,6536 +681,11/11/2012,4,1,11,0,1,0.420833,0.421713,0.659167,0.1275,2290,4562,6852 +682,11/12/2012,4,1,11,1,1,0.485,0.475383,0.741667,0.173517,1097,5172,6269 +683,11/13/2012,4,1,11,2,2,0.343333,0.323225,0.662917,0.342046,327,3767,4094 +684,11/14/2012,4,1,11,3,1,0.289167,0.281563,0.552083,0.199625,373,5122,5495 +685,11/15/2012,4,1,11,4,2,0.321667,0.324492,0.620417,0.152987,320,5125,5445 +686,11/16/2012,4,1,11,5,1,0.345,0.347204,0.524583,0.171025,484,5214,5698 +687,11/17/2012,4,1,11,6,1,0.325,0.326383,0.545417,0.179729,1313,4316,5629 +688,11/18/2012,4,1,11,0,1,0.3425,0.337746,0.692917,0.227612,922,3747,4669 +689,11/19/2012,4,1,11,1,2,0.380833,0.375621,0.623333,0.235067,449,5050,5499 +690,11/20/2012,4,1,11,2,2,0.374167,0.380667,0.685,0.082725,534,5100,5634 +691,11/21/2012,4,1,11,3,1,0.353333,0.364892,0.61375,0.103246,615,4531,5146 +692,11/22/2012,4,1,11,4,1,0.34,0.350371,0.580417,0.0528708,955,1470,2425 +693,11/23/2012,4,1,11,5,1,0.368333,0.378779,0.56875,0.148021,1603,2307,3910 +694,11/24/2012,4,1,11,6,1,0.278333,0.248742,0.404583,0.376871,532,1745,2277 +695,11/25/2012,4,1,11,0,1,0.245833,0.257583,0.468333,0.1505,309,2115,2424 +696,11/26/2012,4,1,11,1,1,0.313333,0.339004,0.535417,0.04665,337,4750,5087 +697,11/27/2012,4,1,11,2,2,0.291667,0.281558,0.786667,0.237562,123,3836,3959 +698,11/28/2012,4,1,11,3,1,0.296667,0.289762,0.50625,0.210821,198,5062,5260 +699,11/29/2012,4,1,11,4,1,0.28087,0.298422,0.555652,0.115522,243,5080,5323 +700,11/30/2012,4,1,11,5,1,0.298333,0.323867,0.649583,0.0584708,362,5306,5668 +701,12/1/2012,4,1,12,6,2,0.298333,0.316904,0.806667,0.0597042,951,4240,5191 +702,12/2/2012,4,1,12,0,2,0.3475,0.359208,0.823333,0.124379,892,3757,4649 +703,12/3/2012,4,1,12,1,1,0.4525,0.455796,0.7675,0.0827208,555,5679,6234 +704,12/4/2012,4,1,12,2,1,0.475833,0.469054,0.73375,0.174129,551,6055,6606 +705,12/5/2012,4,1,12,3,1,0.438333,0.428012,0.485,0.324021,331,5398,5729 +706,12/6/2012,4,1,12,4,1,0.255833,0.258204,0.50875,0.174754,340,5035,5375 +707,12/7/2012,4,1,12,5,2,0.320833,0.321958,0.764167,0.1306,349,4659,5008 +708,12/8/2012,4,1,12,6,2,0.381667,0.389508,0.91125,0.101379,1153,4429,5582 +709,12/9/2012,4,1,12,0,2,0.384167,0.390146,0.905417,0.157975,441,2787,3228 +710,12/10/2012,4,1,12,1,2,0.435833,0.435575,0.925,0.190308,329,4841,5170 +711,12/11/2012,4,1,12,2,2,0.353333,0.338363,0.596667,0.296037,282,5219,5501 +712,12/12/2012,4,1,12,3,2,0.2975,0.297338,0.538333,0.162937,310,5009,5319 +713,12/13/2012,4,1,12,4,1,0.295833,0.294188,0.485833,0.174129,425,5107,5532 +714,12/14/2012,4,1,12,5,1,0.281667,0.294192,0.642917,0.131229,429,5182,5611 +715,12/15/2012,4,1,12,6,1,0.324167,0.338383,0.650417,0.10635,767,4280,5047 +716,12/16/2012,4,1,12,0,2,0.3625,0.369938,0.83875,0.100742,538,3248,3786 +717,12/17/2012,4,1,12,1,2,0.393333,0.4015,0.907083,0.0982583,212,4373,4585 +718,12/18/2012,4,1,12,2,1,0.410833,0.409708,0.66625,0.221404,433,5124,5557 +719,12/19/2012,4,1,12,3,1,0.3325,0.342162,0.625417,0.184092,333,4934,5267 +720,12/20/2012,4,1,12,4,2,0.33,0.335217,0.667917,0.132463,314,3814,4128 +721,12/21/2012,1,1,12,5,2,0.326667,0.301767,0.556667,0.374383,221,3402,3623 +722,12/22/2012,1,1,12,6,1,0.265833,0.236113,0.44125,0.407346,205,1544,1749 +723,12/23/2012,1,1,12,0,1,0.245833,0.259471,0.515417,0.133083,408,1379,1787 +724,12/24/2012,1,1,12,1,2,0.231304,0.2589,0.791304,0.0772304,174,746,920 +725,12/25/2012,1,1,12,2,2,0.291304,0.294465,0.734783,0.168726,440,573,1013 +726,12/26/2012,1,1,12,3,3,0.243333,0.220333,0.823333,0.316546,9,432,441 +727,12/27/2012,1,1,12,4,2,0.254167,0.226642,0.652917,0.350133,247,1867,2114 +728,12/28/2012,1,1,12,5,2,0.253333,0.255046,0.59,0.155471,644,2451,3095 +729,12/29/2012,1,1,12,6,2,0.253333,0.2424,0.752917,0.124383,159,1182,1341 +730,12/30/2012,1,1,12,0,1,0.255833,0.2317,0.483333,0.350754,364,1432,1796 +731,12/31/2012,1,1,12,1,2,0.215833,0.223487,0.5775,0.154846,439,2290,2729 diff --git a/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb b/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb index 97b1f1268..bbfbc6134 100644 --- a/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb +++ b/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb @@ -123,48 +123,22 @@ "data.head()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Get the train data\n", - "\n" - ] - }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "train = data[data['timeStamp'] < '2017-02-01']\n", - "test = data[data['timeStamp'] >= '2017-02-01']\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Prepare the test data, we will feed X_test to the fitted model and get prediction" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "y_test = test.pop('demand').values\n", - "X_test = test" + "# let's take note of what columns means what in the data\n", + "time_column_name = 'timeStamp'\n", + "target_column_name = 'demand'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Split the train data to train and valid\n", - "\n", - "Use one month's data as valid data\n" + "### Split the data into train and test sets\n" ] }, { @@ -173,10 +147,10 @@ "metadata": {}, "outputs": [], "source": [ - "X_train = train\n", - "y_train = X_train.pop('demand').values\n", - "print(X_train.shape)\n", - "print(y_train.shape)" + "X_train = data[data[time_column_name] < '2017-02-01']\n", + "X_test = data[data[time_column_name] >= '2017-02-01']\n", + "y_train = X_train.pop(target_column_name).values\n", + "y_test = X_test.pop(target_column_name).values" ] }, { @@ -205,9 +179,8 @@ "metadata": {}, "outputs": [], "source": [ - "time_column_name = 'timeStamp'\n", "automl_settings = {\n", - " \"time_column_name\": time_column_name,\n", + " \"time_column_name\": time_column_name \n", "}\n", "\n", "\n", @@ -218,7 +191,7 @@ " iteration_timeout_minutes = 5,\n", " X = X_train,\n", " y = y_train,\n", - " n_cross_validations = 2,\n", + " n_cross_validations = 3,\n", " path=project_folder,\n", " verbosity = logging.INFO,\n", " **automl_settings)" @@ -228,7 +201,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You can call the submit method on the experiment object and pass the run configuration. For Local runs the execution is synchronous. Depending on the data and number of iterations this can run for while.\n", + "Submitting the configuration will start a new run in this experiment. For local runs, the execution is synchronous. Depending on the data and number of iterations, this can run for a while. Parameters controlling concurrency may speed up the process, depending on your hardware.\n", + "\n", "You will see the currently running iterations printing to the console." ] }, @@ -289,13 +263,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### View the featurization summary\n", - "Below we display the featurization that was performed on different raw features in the user data. For each raw feature in the user data, the following information is displayed:-\n", - "- Raw feature name\n", - "- Number of engineered features formed out of this raw feature\n", - "- Type detected\n", - "- If feature was dropped\n", - "- List of feature transformations for the raw feature" + "### Test the Best Fitted Model\n", + "\n", + "For forecasting, we will use the `forecast` function instead of the `predict` function. There are two reasons for this.\n", + "\n", + "We need to pass the recent values of the target variable `y`, whereas the scikit-compatible `predict` function only takes the non-target variables `X`. In our case, the test data immediately follows the training data, and we fill the `y` variable with `NaN`. The `NaN` serves as a question mark for the forecaster to fill with the actuals. Using the forecast function will produce forecasts using the shortest possible forecast horizon. The last time at which a definite (non-NaN) value is seen is the _forecast origin_ - the last time when the value of the target is known. \n", + "\n", + "Using the `predict` method would result in getting predictions for EVERY horizon the forecaster can predict at. This is useful when training and evaluating the performance of the forecaster at various horizons, but the level of detail is excessive for normal use." ] }, { @@ -304,16 +278,64 @@ "metadata": {}, "outputs": [], "source": [ - "fitted_model.named_steps['timeseriestransformer'].get_featurization_summary()" + "# Replace ALL values in y_pred by NaN. \n", + "# The forecast origin will be at the beginning of the first forecast period\n", + "# (which is the same time as the end of the last training period).\n", + "y_query = y_test.copy().astype(np.float)\n", + "y_query.fill(np.nan)\n", + "# The featurized data, aligned to y, will also be returned.\n", + "# This contains the assumptions that were made in the forecast\n", + "# and helps align the forecast to the original data\n", + "y_fcst, X_trans = fitted_model.forecast(X_test, y_query)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "### Test the Best Fitted Model\n", + "# limit the evaluation to data where y_test has actuals\n", + "def align_outputs(y_predicted, X_trans, X_test, y_test, predicted_column_name = 'predicted'):\n", + " \"\"\"\n", + " Demonstrates how to get the output aligned to the inputs\n", + " using pandas indexes. Helps understand what happened if\n", + " the output's shape differs from the input shape, or if\n", + " the data got re-sorted by time and grain during forecasting.\n", + " \n", + " Typical causes of misalignment are:\n", + " * we predicted some periods that were missing in actuals -> drop from eval\n", + " * model was asked to predict past max_horizon -> increase max horizon\n", + " * data at start of X_test was needed for lags -> provide previous periods\n", + " \"\"\"\n", + " df_fcst = pd.DataFrame({predicted_column_name : y_predicted})\n", + " # y and X outputs are aligned by forecast() function contract\n", + " df_fcst.index = X_trans.index\n", + " \n", + " # align original X_test to y_test \n", + " X_test_full = X_test.copy()\n", + " X_test_full[target_column_name] = y_test\n", "\n", - "Predict on training and test set, and calculate residual values." + " # X_test_full's does not include origin, so reset for merge\n", + " df_fcst.reset_index(inplace=True)\n", + " X_test_full = X_test_full.reset_index().drop(columns='index')\n", + " together = df_fcst.merge(X_test_full, how='right')\n", + " \n", + " # drop rows where prediction or actuals are nan \n", + " # happens because of missing actuals \n", + " # or at edges of time due to lags/rolling windows\n", + " clean = together[together[[target_column_name, predicted_column_name]].notnull().all(axis=1)]\n", + " return(clean)\n", + "\n", + "df_all = align_outputs(y_fcst, X_trans, X_test, y_test)\n", + "df_all.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Looking at `X_trans` is also useful to see what featurization happened to the data." ] }, { @@ -322,15 +344,14 @@ "metadata": {}, "outputs": [], "source": [ - "y_pred = fitted_model.predict(X_test)\n", - "y_pred" + "X_trans" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Use the Check Data Function to remove the nan values from y_test to avoid error when calculate metrics " + "### Calculate accuracy metrics\n" ] }, { @@ -339,29 +360,61 @@ "metadata": {}, "outputs": [], "source": [ - "if len(y_test) != len(y_pred):\n", - " raise ValueError(\n", - " 'the true values and prediction values do not have equal length.')\n", - "elif len(y_test) == 0:\n", - " raise ValueError(\n", - " 'y_true and y_pred are empty.')\n", - "\n", - "# if there is any non-numeric element in the y_true or y_pred,\n", - "# the ValueError exception will be thrown.\n", - "y_test_f = np.array(y_test).astype(float)\n", - "y_pred_f = np.array(y_pred).astype(float)\n", + "def MAPE(actual, pred):\n", + " \"\"\"\n", + " Calculate mean absolute percentage error.\n", + " Remove NA and values where actual is close to zero\n", + " \"\"\"\n", + " not_na = ~(np.isnan(actual) | np.isnan(pred))\n", + " not_zero = ~np.isclose(actual, 0.0)\n", + " actual_safe = actual[not_na & not_zero]\n", + " pred_safe = pred[not_na & not_zero]\n", + " APE = 100*np.abs((actual_safe - pred_safe)/actual_safe)\n", + " return np.mean(APE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Simple forecasting model\")\n", + "rmse = np.sqrt(mean_squared_error(df_all[target_column_name], df_all['predicted']))\n", + "print(\"[Test Data] \\nRoot Mean squared error: %.2f\" % rmse)\n", + "mae = mean_absolute_error(df_all[target_column_name], df_all['predicted'])\n", + "print('mean_absolute_error score: %.2f' % mae)\n", + "print('MAPE: %.2f' % MAPE(df_all[target_column_name], df_all['predicted']))\n", "\n", - "# remove entries both in y_true and y_pred where at least\n", - "# one element in y_true or y_pred is missing\n", - "y_test = y_test_f[~(np.isnan(y_test_f) | np.isnan(y_pred_f))]\n", - "y_pred = y_pred_f[~(np.isnan(y_test_f) | np.isnan(y_pred_f))]" + "# Plot outputs\n", + "%matplotlib notebook\n", + "test_pred = plt.scatter(df_all[target_column_name], df_all['predicted'], color='b')\n", + "test_test = plt.scatter(y_test, y_test, color='g')\n", + "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The distribution looks a little heavy tailed: we underestimate the excursions of the extremes. A normal-quantile transform of the target might help, but let's first try using some past data with the lags and rolling window transforms.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using lags and rolling window features to improve the forecast" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Calculate metrics for the prediction\n" + "We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, grain and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data.\n", + "\n", + "Now that we configured target lags, that is the previous values of the target variables, and the prediction is no longer horizon-less. We therefore must specify the `max_horizon` that the model will learn to forecast. The `target_lags` keyword specifies how far back we will construct the lags of the target variable, and the `target_rolling_window_size` specifies the size of the rolling window over which we will generate the `max`, `min` and `sum` features." ] }, { @@ -370,26 +423,117 @@ "metadata": {}, "outputs": [], "source": [ - "print(\"[Test Data] \\nRoot Mean squared error: %.2f\" % np.sqrt(mean_squared_error(y_test, y_pred)))\n", - "# Explained variance score: 1 is perfect prediction\n", - "print('mean_absolute_error score: %.2f' % mean_absolute_error(y_test, y_pred))\n", - "print('R2 score: %.2f' % r2_score(y_test, y_pred))\n", + "automl_settings_lags = {\n", + " 'time_column_name': time_column_name,\n", + " 'target_lags': 1,\n", + " 'target_rolling_window_size': 5,\n", + " # you MUST set the max_horizon when using lags and rolling windows\n", + " # it is optional when looking-back features are not used \n", + " 'max_horizon': len(y_test), # only one grain\n", + "}\n", "\n", "\n", + "automl_config_lags = AutoMLConfig(task = 'forecasting',\n", + " debug_log = 'automl_nyc_energy_errors.log',\n", + " primary_metric='normalized_root_mean_squared_error',\n", + " iterations = 10,\n", + " iteration_timeout_minutes = 5,\n", + " X = X_train,\n", + " y = y_train,\n", + " n_cross_validations = 3,\n", + " path=project_folder,\n", + " verbosity = logging.INFO,\n", + " **automl_settings_lags)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_run_lags = experiment.submit(automl_config_lags, show_output=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "best_run_lags, fitted_model_lags = local_run_lags.get_output()\n", + "y_fcst_lags, X_trans_lags = fitted_model_lags.forecast(X_test, y_query)\n", + "df_lags = align_outputs(y_fcst_lags, X_trans_lags, X_test, y_test)\n", + "df_lags.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "X_trans_lags" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Forecasting model with lags\")\n", + "rmse = np.sqrt(mean_squared_error(df_lags[target_column_name], df_lags['predicted']))\n", + "print(\"[Test Data] \\nRoot Mean squared error: %.2f\" % rmse)\n", + "mae = mean_absolute_error(df_lags[target_column_name], df_lags['predicted'])\n", + "print('mean_absolute_error score: %.2f' % mae)\n", + "print('MAPE: %.2f' % MAPE(df_lags[target_column_name], df_lags['predicted']))\n", "\n", "# Plot outputs\n", "%matplotlib notebook\n", - "test_pred = plt.scatter(y_test, y_pred, color='b')\n", + "test_pred = plt.scatter(df_lags[target_column_name], df_lags['predicted'], color='b')\n", "test_test = plt.scatter(y_test, y_test, color='g')\n", "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n", "plt.show()" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### What features matter for the forecast?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.train.automl.automlexplainer import explain_model\n", + "\n", + "# feature names are everything in the transformed data except the target\n", + "features = X_trans.columns[:-1]\n", + "expl = explain_model(fitted_model, X_train, X_test, features = features, best_run=best_run_lags, y_train = y_train)\n", + "# unpack the tuple\n", + "shap_values, expected_values, feat_overall_imp, feat_names, per_class_summary, per_class_imp = expl\n", + "best_run_lags" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Please go to the Azure Portal's best run to see the top features chart.\n", + "\n", + "The informative features make all sorts of intuitive sense. Temperature is a strong driver of heating and cooling demand in NYC. Apart from that, the daily life cycle, expressed by `hour`, and the weekly cycle, expressed by `wday` drives people's energy use habits." + ] } ], "metadata": { "authors": [ { - "name": "xiaga" + "name": "xiaga, tosingli" } ], "kernelspec": { @@ -407,7 +551,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.8" + "version": "3.6.7" } }, "nbformat": 4, diff --git a/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb b/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb index 52beb9197..e0da28d03 100644 --- a/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb +++ b/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb @@ -20,7 +20,9 @@ "1. [Introduction](#Introduction)\n", "1. [Setup](#Setup)\n", "1. [Data](#Data)\n", - "1. [Train](#Train)" + "1. [Train](#Train)\n", + "1. [Predict](#Predict)\n", + "1. [Operationalize](#Operationalize)" ] }, { @@ -36,8 +38,7 @@ "1. Create an Experiment in an existing Workspace\n", "2. Instantiate an AutoMLConfig \n", "3. Find and train a forecasting model using local compute\n", - "4. Viewing the engineered names for featurized data and featurization summary for all raw features\n", - "5. Evaluate the performance of the model\n", + "4. Evaluate the performance of the model\n", "\n", "The examples in the follow code samples use the University of Chicago's Dominick's Finer Foods dataset to forecast orange juice sales. Dominick's was a grocery chain in the Chicago metropolitan area." ] @@ -86,9 +87,9 @@ "ws = Workspace.from_config()\n", "\n", "# choose a name for the run history container in the workspace\n", - "experiment_name = 'automl-ojsalesforecasting'\n", + "experiment_name = 'automl-ojforecasting'\n", "# project folder\n", - "project_folder = './sample_projects/automl-local-ojsalesforecasting'\n", + "project_folder = './sample_projects/automl-local-ojforecasting'\n", "\n", "experiment = Experiment(ws, experiment_name)\n", "\n", @@ -261,12 +262,12 @@ " 'time_column_name': time_column_name,\n", " 'grain_column_names': grain_column_names,\n", " 'drop_column_names': ['logQuantity'],\n", - " 'max_horizon': n_test_periods\n", + " 'max_horizon': n_test_periods # optional\n", "}\n", "\n", "automl_config = AutoMLConfig(task='forecasting',\n", " debug_log='automl_oj_sales_errors.log',\n", - " primary_metric='normalized_root_mean_squared_error',\n", + " primary_metric='normalized_mean_absolute_error',\n", " iterations=10,\n", " X=X_train,\n", " y=y_train,\n", @@ -294,21 +295,30 @@ "local_run = experiment.submit(automl_config, show_output=True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Retrieve the Best Model\n", + "Each run within an Experiment stores serialized (i.e. pickled) pipelines from the AutoML iterations. We can now retrieve the pipeline with the best performance on the validation dataset:" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "local_run" + "best_run, fitted_pipeline = local_run.get_output()\n", + "fitted_pipeline.steps" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Retrieve the Best Model\n", - "Each run within an Experiment stores serialized (i.e. pickled) pipelines from the AutoML iterations. We can now retrieve the pipeline with the best performance on the validation dataset:" + "# Predict\n", + "Now that we have retrieved the best pipeline/model, it can be used to make predictions on test data. First, we remove the target values from the test set:" ] }, { @@ -317,16 +327,25 @@ "metadata": {}, "outputs": [], "source": [ - "best_run, fitted_pipeline = local_run.get_output()\n", - "fitted_pipeline.steps" + "y_test = X_test.pop(target_column_name).values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "X_test.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### View the engineered names for featurized data\n", - "Below we display the engineered feature names generated for the featurized data using the time-series featurization." + "To produce predictions on the test set, we need to know the feature values at all dates in the test set. This requirement is somewhat reasonable for the OJ sales data since the features mainly consist of price, which is usually set in advance, and customer demographics which are approximately constant for each store over the 20 week forecast horizon in the testing data. \n", + "\n", + "We will first create a query `y_query`, which is aligned index-for-index to `X_test`. This is a vector of target values where each `NaN` serves the function of the question mark to be replaced by forecast. Passing definite values in the `y` argument allows the `forecast` function to make predictions on data that does not immediately follow the train data which contains `y`. In each grain, the last time point where the model sees a definite value of `y` is that grain's _forecast origin_." ] }, { @@ -335,20 +354,35 @@ "metadata": {}, "outputs": [], "source": [ - "fitted_pipeline.named_steps['timeseriestransformer'].get_engineered_feature_names()" + "# Replace ALL values in y_pred by NaN.\n", + "# The forecast origin will be at the beginning of the first forecast period.\n", + "# (Which is the same time as the end of the last training period.)\n", + "y_query = y_test.copy().astype(np.float)\n", + "y_query.fill(np.nan)\n", + "# The featurized data, aligned to y, will also be returned.\n", + "# This contains the assumptions that were made in the forecast\n", + "# and helps align the forecast to the original data\n", + "y_pred, X_trans = fitted_pipeline.forecast(X_test, y_query)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### View the featurization summary\n", - "Below we display the featurization that was performed on different raw features in the user data. For each raw feature in the user data, the following information is displayed:-\n", - "- Raw feature name\n", - "- Number of engineered features formed out of this raw feature\n", - "- Type detected\n", - "- If feature was dropped\n", - "- List of feature transformations for the raw feature" + "If you are used to scikit pipelines, perhaps you expected `predict(X_test)`. However, forecasting requires a more general interface that also supplies the past target `y` values. Please use `forecast(X,y)` as `predict(X)` is reserved for internal purposes on forecasting models.\n", + "\n", + "The [energy demand forecasting notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand) demonstrates the use of the forecast function in more detail in the context of using lags and rolling window features. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Evaluate\n", + "\n", + "To evaluate the accuracy of the forecast, we'll compare against the actual sales quantities for some select metrics, included the mean absolute percentage error (MAPE). \n", + "\n", + "It is a good practice to always align the output explicitly to the input, as the count and order of the rows may have changed during transformations that span multiple rows." ] }, { @@ -357,15 +391,58 @@ "metadata": {}, "outputs": [], "source": [ - "fitted_pipeline.named_steps['timeseriestransformer'].get_featurization_summary()" + "def align_outputs(y_predicted, X_trans, X_test, y_test, predicted_column_name = 'predicted'):\n", + " \"\"\"\n", + " Demonstrates how to get the output aligned to the inputs\n", + " using pandas indexes. Helps understand what happened if\n", + " the output's shape differs from the input shape, or if\n", + " the data got re-sorted by time and grain during forecasting.\n", + " \n", + " Typical causes of misalignment are:\n", + " * we predicted some periods that were missing in actuals -> drop from eval\n", + " * model was asked to predict past max_horizon -> increase max horizon\n", + " * data at start of X_test was needed for lags -> provide previous periods in y\n", + " \"\"\"\n", + " \n", + " df_fcst = pd.DataFrame({predicted_column_name : y_predicted})\n", + " # y and X outputs are aligned by forecast() function contract\n", + " df_fcst.index = X_trans.index\n", + " \n", + " # align original X_test to y_test \n", + " X_test_full = X_test.copy()\n", + " X_test_full[target_column_name] = y_test\n", + "\n", + " # X_test_full's index does not include origin, so reset for merge\n", + " df_fcst.reset_index(inplace=True)\n", + " X_test_full = X_test_full.reset_index().drop(columns='index')\n", + " together = df_fcst.merge(X_test_full, how='right')\n", + " \n", + " # drop rows where prediction or actuals are nan \n", + " # happens because of missing actuals \n", + " # or at edges of time due to lags/rolling windows\n", + " clean = together[together[[target_column_name, predicted_column_name]].notnull().all(axis=1)]\n", + " return(clean)\n", + "\n", + "df_all = align_outputs(y_pred, X_trans, X_test, y_test)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "### Make Predictions from the Best Fitted Model\n", - "Now that we have retrieved the best pipeline/model, it can be used to make predictions on test data. First, we remove the target values from the test set:" + "def MAPE(actual, pred):\n", + " \"\"\"\n", + " Calculate mean absolute percentage error.\n", + " Remove NA and values where actual is close to zero\n", + " \"\"\"\n", + " not_na = ~(np.isnan(actual) | np.isnan(pred))\n", + " not_zero = ~np.isclose(actual, 0.0)\n", + " actual_safe = actual[not_na & not_zero]\n", + " pred_safe = pred[not_na & not_zero]\n", + " APE = 100*np.abs((actual_safe - pred_safe)/actual_safe)\n", + " return np.mean(APE)" ] }, { @@ -374,7 +451,35 @@ "metadata": {}, "outputs": [], "source": [ - "y_test = X_test.pop(target_column_name).values" + "print(\"Simple forecasting model\")\n", + "rmse = np.sqrt(mean_squared_error(df_all[target_column_name], df_all['predicted']))\n", + "print(\"[Test Data] \\nRoot Mean squared error: %.2f\" % rmse)\n", + "mae = mean_absolute_error(df_all[target_column_name], df_all['predicted'])\n", + "print('mean_absolute_error score: %.2f' % mae)\n", + "print('MAPE: %.2f' % MAPE(df_all[target_column_name], df_all['predicted']))\n", + "\n", + "# Plot outputs\n", + "import matplotlib.pyplot as plt\n", + "\n", + "%matplotlib notebook\n", + "test_pred = plt.scatter(df_all[target_column_name], df_all['predicted'], color='b')\n", + "test_test = plt.scatter(y_test, y_test, color='g')\n", + "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Operationalize" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_Operationalization_ means getting the model into the cloud so that other can run it after you close the notebook. We will create a docker running on Azure Container Instances with the model." ] }, { @@ -383,16 +488,65 @@ "metadata": {}, "outputs": [], "source": [ - "X_test.head()" + "description = 'AutoML OJ forecaster'\n", + "tags = None\n", + "model = local_run.register_model(description = description, tags = tags)\n", + "\n", + "print(local_run.model_id)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "To produce predictions on the test set, we need to know the feature values at all dates in the test set. This requirement is somewhat reasonable for the OJ sales data since the features mainly consist of price, which is usually set in advance, and customer demographics which are approximately constant for each store over the 20 week forecast horizon in the testing data. \n", + "### Develop the scoring script\n", + "\n", + "Serializing and deserializing complex data frames may be tricky. We first develop the `run()` function of the scoring script locally, then write it into a scoring script. It is much easier to debug any quirks of the scoring function without crossing two compute environments. For this exercise, we handle a common quirk of how pandas dataframes serialize time stamp values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this is where we test the run function of the scoring script interactively\n", + "# before putting it in the scoring script\n", + "\n", + "timestamp_columns = ['WeekStarting']\n", "\n", - "The target predictions can be retrieved by calling the `predict` method on the best model:" + "def run(rawdata, test_model = None):\n", + " \"\"\"\n", + " Intended to process 'rawdata' string produced by\n", + " \n", + " {'X': X_test.to_json(), y' : y_test.to_json()}\n", + " \n", + " Don't convert the X payload to numpy.array, use it as pandas.DataFrame\n", + " \"\"\"\n", + " try:\n", + " # unpack the data frame with timestamp \n", + " rawobj = json.loads(rawdata) # rawobj is now a dict of strings \n", + " X_pred = pd.read_json(rawobj['X'], convert_dates=False) # load the pandas DF from a json string\n", + " for col in timestamp_columns: # fix timestamps\n", + " X_pred[col] = pd.to_datetime(X_pred[col], unit='ms') \n", + " \n", + " y_pred = np.array(rawobj['y']) # reconstitute numpy array from serialized list\n", + " \n", + " if test_model is None:\n", + " result = model.forecast(X_pred, y_pred) # use the global model from init function\n", + " else:\n", + " result = test_model.forecast(X_pred, y_pred) # use the model on which we are testing\n", + " \n", + " except Exception as e:\n", + " result = str(e)\n", + " return json.dumps({\"error\": result})\n", + " \n", + " forecast_as_list = result[0].tolist()\n", + " index_as_df = result[1].index.to_frame().reset_index(drop=True)\n", + " \n", + " return json.dumps({\"forecast\": forecast_as_list, # return the minimum over the wire: \n", + " \"index\": index_as_df.to_json() # no forecast and its featurized values\n", + " })" ] }, { @@ -401,15 +555,25 @@ "metadata": {}, "outputs": [], "source": [ - "y_pred = fitted_pipeline.predict(X_test)" + "# test the run function here before putting in the scoring script\n", + "import json\n", + "\n", + "test_sample = json.dumps({'X': X_test.to_json(), 'y' : y_query.tolist()})\n", + "response = run(test_sample, fitted_pipeline)\n", + "\n", + "# unpack the response, dealing with the timestamp serialization again\n", + "res_dict = json.loads(response)\n", + "y_fcst_all = pd.read_json(res_dict['index'])\n", + "y_fcst_all[time_column_name] = pd.to_datetime(y_fcst_all[time_column_name], unit = 'ms')\n", + "y_fcst_all['forecast'] = res_dict['forecast']\n", + "y_fcst_all.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Calculate evaluation metrics for the prediction\n", - "To evaluate the accuracy of the forecast, we'll compare against the actual sales quantities for some select metrics, included the mean absolute percentage error (MAPE)." + "Now that the function works locally in the notebook, let's write it down into the scoring script. The scoring script is authored by the data scientist. Adjust it to taste, adding inputs, outputs and processing as needed." ] }, { @@ -418,28 +582,252 @@ "metadata": {}, "outputs": [], "source": [ - "def MAPE(actual, pred):\n", + "%%writefile score_fcast.py\n", + "import pickle\n", + "import json\n", + "import numpy as np\n", + "import pandas as pd\n", + "import azureml.train.automl\n", + "from sklearn.externals import joblib\n", + "from azureml.core.model import Model\n", + "\n", + "\n", + "def init():\n", + " global model\n", + " model_path = Model.get_model_path(model_name = '<>') # this name is model.id of model that we want to deploy\n", + " # deserialize the model file back into a sklearn model\n", + " model = joblib.load(model_path)\n", + "\n", + "timestamp_columns = ['WeekStarting']\n", + "\n", + "def run(rawdata, test_model = None):\n", " \"\"\"\n", - " Calculate mean absolute percentage error.\n", - " Remove NA and values where actual is close to zero\n", + " Intended to process 'rawdata' string produced by\n", + " \n", + " {'X': X_test.to_json(), y' : y_test.to_json()}\n", + " \n", + " Don't convert the X payload to numpy.array, use it as pandas.DataFrame\n", " \"\"\"\n", - " not_na = ~(np.isnan(actual) | np.isnan(pred))\n", - " not_zero = ~np.isclose(actual, 0.0)\n", - " actual_safe = actual[not_na & not_zero]\n", - " pred_safe = pred[not_na & not_zero]\n", - " APE = 100*np.abs((actual_safe - pred_safe)/actual_safe)\n", - " return np.mean(APE)\n", + " try:\n", + " # unpack the data frame with timestamp \n", + " rawobj = json.loads(rawdata) # rawobj is now a dict of strings \n", + " X_pred = pd.read_json(rawobj['X'], convert_dates=False) # load the pandas DF from a json string\n", + " for col in timestamp_columns: # fix timestamps\n", + " X_pred[col] = pd.to_datetime(X_pred[col], unit='ms') \n", + " \n", + " y_pred = np.array(rawobj['y']) # reconstitute numpy array from serialized list\n", + " \n", + " if test_model is None:\n", + " result = model.forecast(X_pred, y_pred) # use the global model from init function\n", + " else:\n", + " result = test_model.forecast(X_pred, y_pred) # use the model on which we are testing\n", + " \n", + " except Exception as e:\n", + " result = str(e)\n", + " return json.dumps({\"error\": result})\n", + " \n", + " # prepare to send over wire as json\n", + " forecast_as_list = result[0].tolist()\n", + " index_as_df = result[1].index.to_frame().reset_index(drop=True)\n", + " \n", + " return json.dumps({\"forecast\": forecast_as_list, # return the minimum over the wire: \n", + " \"index\": index_as_df.to_json() # no forecast and its featurized values\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get the model\n", + "from azureml.train.automl.run import AutoMLRun\n", + "\n", + "experiment = Experiment(ws, experiment_name)\n", + "ml_run = AutoMLRun(experiment = experiment, run_id = local_run.id)\n", + "best_iteration = int(str.split(best_run.id,'_')[-1]) # the iteration number is a postfix of the run ID." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get the best model's dependencies and write them into this file\n", + "from azureml.core.conda_dependencies import CondaDependencies\n", + "\n", + "conda_env_file_name = 'fcast_env.yml'\n", + "\n", + "dependencies = ml_run.get_run_sdk_dependencies(iteration = best_iteration)\n", + "for p in ['azureml-train-automl', 'azureml-sdk', 'azureml-core']:\n", + " print('{}\\t{}'.format(p, dependencies[p]))\n", + "\n", + "myenv = CondaDependencies.create(conda_packages=['numpy','scikit-learn'], pip_packages=['azureml-sdk[automl]'])\n", + "\n", + "myenv.save_to_file('.', conda_env_file_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this is the script file name we wrote a few cells above\n", + "script_file_name = 'score_fcast.py'\n", + "\n", + "# Substitute the actual version number in the environment file.\n", + "# This is not strictly needed in this notebook because the model should have been generated using the current SDK version.\n", + "# However, we include this in case this code is used on an experiment from a previous SDK version.\n", + "\n", + "with open(conda_env_file_name, 'r') as cefr:\n", + " content = cefr.read()\n", + "\n", + "with open(conda_env_file_name, 'w') as cefw:\n", + " cefw.write(content.replace(azureml.core.VERSION, dependencies['azureml-sdk']))\n", + "\n", + "# Substitute the actual model id in the script file.\n", + "\n", + "with open(script_file_name, 'r') as cefr:\n", + " content = cefr.read()\n", + "\n", + "with open(script_file_name, 'w') as cefw:\n", + " cefw.write(content.replace('<>', local_run.model_id))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a Container Image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.image import Image, ContainerImage\n", + "\n", + "image_config = ContainerImage.image_configuration(runtime= \"python\",\n", + " execution_script = script_file_name,\n", + " conda_file = conda_env_file_name,\n", + " tags = {'type': \"automl-forecasting\"},\n", + " description = \"Image for automl forecasting sample\")\n", + "\n", + "image = Image.create(name = \"automl-fcast-image\",\n", + " # this is the model object \n", + " models = [model],\n", + " image_config = image_config, \n", + " workspace = ws)\n", + "\n", + "image.wait_for_creation(show_output = True)\n", + "\n", + "if image.creation_state == 'Failed':\n", + " print(\"Image build log at: \" + image.image_build_log_uri)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Deploy the Image as a Web Service on Azure Container Instance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.webservice import AciWebservice\n", + "\n", + "aciconfig = AciWebservice.deploy_configuration(cpu_cores = 1, \n", + " memory_gb = 2, \n", + " tags = {'type': \"automl-forecasting\"},\n", + " description = \"Automl forecasting sample service\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.webservice import Webservice\n", + "\n", + "aci_service_name = 'automl-forecast-01'\n", + "print(aci_service_name)\n", + "\n", + "aci_service = Webservice.deploy_from_image(deployment_config = aciconfig,\n", + " image = image,\n", + " name = aci_service_name,\n", + " workspace = ws)\n", + "aci_service.wait_for_deployment(True)\n", + "print(aci_service.state)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Call the service" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# we send the data to the service serialized into a json string\n", + "test_sample = json.dumps({'X':X_test.to_json(), 'y' : y_query.tolist()})\n", + "response = aci_service.run(input_data = test_sample)\n", "\n", - "print(\"[Test Data] \\nRoot Mean squared error: %.2f\" % np.sqrt(mean_squared_error(y_test, y_pred)))\n", - "print('mean_absolute_error score: %.2f' % mean_absolute_error(y_test, y_pred))\n", - "print('MAPE: %.2f' % MAPE(y_test, y_pred))" + "# translate from networkese to datascientese\n", + "try: \n", + " res_dict = json.loads(response)\n", + " y_fcst_all = pd.read_json(res_dict['index'])\n", + " y_fcst_all[time_column_name] = pd.to_datetime(y_fcst_all[time_column_name], unit = 'ms')\n", + " y_fcst_all['forecast'] = res_dict['forecast'] \n", + "except:\n", + " print(res_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "y_fcst_all.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Delete the web service if desired" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "serv = Webservice(ws, 'automl-forecast-01')\n", + "# serv.delete() # don't do it accidentally" ] } ], "metadata": { "authors": [ { - "name": "erwright" + "name": "erwright, tosingli" } ], "kernelspec": { @@ -457,7 +845,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.8" + "version": "3.6.7" } }, "nbformat": 4, diff --git a/how-to-use-azureml/automated-machine-learning/model-explanation/auto-ml-model-explanation.ipynb b/how-to-use-azureml/automated-machine-learning/model-explanation/auto-ml-model-explanation.ipynb index 103aa4bc1..3d473cc76 100644 --- a/how-to-use-azureml/automated-machine-learning/model-explanation/auto-ml-model-explanation.ipynb +++ b/how-to-use-azureml/automated-machine-learning/model-explanation/auto-ml-model-explanation.ipynb @@ -254,7 +254,9 @@ "3.\toverall_summary: The model level feature importance values sorted in descending order\n", "4.\toverall_imp: The feature names sorted in the same order as in overall_summary\n", "5.\tper_class_summary: The class level feature importance values sorted in descending order. Only available for the classification case\n", - "6.\tper_class_imp: The feature names sorted in the same order as in per_class_summary. Only available for the classification case" + "6.\tper_class_imp: The feature names sorted in the same order as in per_class_summary. Only available for the classification case\n", + "\n", + "Note:- The **retrieve_model_explanation()** API only works in case AutoML has been configured with **'model_explainability'** flag set to **True**. " ] }, { diff --git a/how-to-use-azureml/automated-machine-learning/remote-attach/auto-ml-remote-attach.ipynb b/how-to-use-azureml/automated-machine-learning/remote-attach/auto-ml-remote-attach.ipynb index 35ef1bbb2..75a10d437 100644 --- a/how-to-use-azureml/automated-machine-learning/remote-attach/auto-ml-remote-attach.ipynb +++ b/how-to-use-azureml/automated-machine-learning/remote-attach/auto-ml-remote-attach.ipynb @@ -111,7 +111,7 @@ "source": [ "### Attach a Remote Linux DSVM\n", "To use a remote Docker compute target:\n", - "1. Create a Linux DSVM in Azure, following these [quick instructions](https://docs.microsoft.com/en-us/azure/machine-learning/desktop-workbench/how-to-create-dsvm-hdi). Make sure you use the Ubuntu flavor (not CentOS). Make sure that disk space is available under `/tmp` because AutoML creates files under `/tmp/azureml_run`s. The DSVM should have more cores than the number of parallel runs that you plan to enable. It should also have at least 4GB per core.\n", + "1. Create a Linux DSVM in Azure, following these [instructions](https://docs.microsoft.com/en-us/azure/machine-learning/data-science-virtual-machine/dsvm-ubuntu-intro). Make sure you use the Ubuntu flavor (not CentOS). Make sure that disk space is available under `/tmp` because AutoML creates files under `/tmp/azureml_run`s. The DSVM should have more cores than the number of parallel runs that you plan to enable. It should also have at least 4GB per core.\n", "2. Enter the IP address, user name and password below.\n", "\n", "**Note:** By default, SSH runs on port 22 and you don't need to change the port number below. If you've configured SSH to use a different port, change `dsvm_ssh_port` accordinglyaddress. [Read more](https://docs.microsoft.com/en-us/azure/virtual-machines/troubleshooting/detailed-troubleshoot-ssh-connection) on changing SSH ports for security reasons." diff --git a/how-to-use-azureml/azure-databricks/README.md b/how-to-use-azureml/azure-databricks/README.md index 300c27cda..cef512746 100644 --- a/how-to-use-azureml/azure-databricks/README.md +++ b/how-to-use-azureml/azure-databricks/README.md @@ -26,6 +26,4 @@ You can use Azure Databricks as a compute target from [Azure Machine Learning Pi For more on SDK concepts, please refer to [notebooks](https://github.com/Azure/MachineLearningNotebooks). -**Please let us know your feedback.** - -![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/azure-databricks/README.png) +**Please let us know your feedback.** \ No newline at end of file diff --git a/how-to-use-azureml/deploy-to-cloud/README.md b/how-to-use-azureml/deploy-to-cloud/README.md new file mode 100644 index 000000000..46288c6eb --- /dev/null +++ b/how-to-use-azureml/deploy-to-cloud/README.md @@ -0,0 +1,12 @@ +# Model Deployment with Azure ML service +You can use Azure Machine Learning to package, debug, validate and deploy inference containers to a variety of compute targets. This process is known as "MLOps" (ML operationalization). +For more information please check out this article: https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-and-where + +## Get Started +To begin, you will need an ML workspace. +For more information please check out this article: https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-manage-workspace + +## Deploy to the cloud +You can deploy to the cloud using the Azure ML CLI or the Azure ML SDK. +- CLI example: https://aka.ms/azmlcli +- Notebook example: [model-register-and-deploy](./model-register-and-deploy.ipynb). \ No newline at end of file diff --git a/how-to-use-azureml/deploy-to-cloud/helloworld.txt b/how-to-use-azureml/deploy-to-cloud/helloworld.txt new file mode 100644 index 000000000..a12521d86 --- /dev/null +++ b/how-to-use-azureml/deploy-to-cloud/helloworld.txt @@ -0,0 +1 @@ +RUN echo "this is test" \ No newline at end of file diff --git a/how-to-use-azureml/deploy-to-cloud/model-register-and-deploy.ipynb b/how-to-use-azureml/deploy-to-cloud/model-register-and-deploy.ipynb new file mode 100644 index 000000000..8acdb4a26 --- /dev/null +++ b/how-to-use-azureml/deploy-to-cloud/model-register-and-deploy.ipynb @@ -0,0 +1,275 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Register Model and deploy as Webservice\n", + "\n", + "This example shows how to deploy a Webservice in step-by-step fashion:\n", + "\n", + " 1. Register Model\n", + " 2. Deploy Model as Webservice" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "Make sure you go through the [configuration](../../../configuration.ipynb) Notebook first if you haven't." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check core SDK version number\n", + "import azureml.core\n", + "\n", + "print(\"SDK version:\", azureml.core.VERSION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialize Workspace\n", + "\n", + "Initialize a workspace object from persisted configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "create workspace" + ] + }, + "outputs": [], + "source": [ + "from azureml.core import Workspace\n", + "\n", + "ws = Workspace.from_config()\n", + "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Register Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can add tags and descriptions to your Models. Note you need to have a `sklearn_regression_model.pkl` file in the current directory. This file is generated by the 01 notebook. The below call registers that file as a Model with the same name `sklearn_regression_model.pkl` in the workspace.\n", + "\n", + "Using tags, you can track useful information such as the name and version of the machine learning library used to train the model. Note that tags must be alphanumeric." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "register model from file" + ] + }, + "outputs": [], + "source": [ + "from azureml.core.model import Model\n", + "\n", + "model = Model.register(model_path = \"sklearn_regression_model.pkl\",\n", + " model_name = \"sklearn_regression_model.pkl\",\n", + " tags = {'area': \"diabetes\", 'type': \"regression\"},\n", + " description = \"Ridge regression model to predict diabetes\",\n", + " workspace = ws)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create Inference Configuration\n", + "\n", + "There is now support for a source directory, you can upload an entire folder from your local machine as dependencies for the Webservice.\n", + "Note: in that case, your entry_script, conda_file, and extra_docker_file_steps paths are relative paths to the source_directory path.\n", + "\n", + "Sample code for using a source directory:\n", + "\n", + "```python\n", + "inference_config = InferenceConfig(source_directory=\"C:/abc\",\n", + " runtime= \"python\", \n", + " entry_script=\"x/y/score.py\",\n", + " conda_file=\"env/myenv.yml\", \n", + " extra_docker_file_steps=\"helloworld.txt\")\n", + "```\n", + "\n", + " - source_directory = holds source path as string, this entire folder gets added in image so its really easy to access any files within this folder or subfolder\n", + " - runtime = Which runtime to use for the image. Current supported runtimes are 'spark-py' and 'python\n", + " - entry_script = contains logic specific to initializing your model and running predictions\n", + " - conda_file = manages conda and python package dependencies.\n", + " - extra_docker_file_steps = optional: any extra steps you want to inject into docker file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "create image" + ] + }, + "outputs": [], + "source": [ + "from azureml.core.model import InferenceConfig\n", + "\n", + "inference_config = InferenceConfig(runtime= \"python\", \n", + " entry_script=\"score.py\",\n", + " conda_file=\"myenv.yml\", \n", + " extra_docker_file_steps=\"helloworld.txt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Deploy Model as Webservice on Azure Container Instance\n", + "\n", + "Note that the service creation can take few minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.webservice import AciWebservice, Webservice\n", + "from azureml.exceptions import WebserviceException\n", + "\n", + "deployment_config = AciWebservice.deploy_configuration(cpu_cores = 1, memory_gb = 1)\n", + "aci_service_name = 'aciservice1'\n", + "\n", + "try:\n", + " # if you want to get existing service below is the command\n", + " # since aci name needs to be unique in subscription deleting existing aci if any\n", + " # we use aci_service_name to create azure aci\n", + " service = Webservice(ws, name=aci_service_name)\n", + " if service:\n", + " service.delete()\n", + "except WebserviceException as e:\n", + " print()\n", + "\n", + "service = Model.deploy(ws, aci_service_name, [model], inference_config, deployment_config)\n", + "\n", + "service.wait_for_deployment(True)\n", + "print(service.state)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Test web service" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "test_sample = json.dumps({'data': [\n", + " [1,2,3,4,5,6,7,8,9,10], \n", + " [10,9,8,7,6,5,4,3,2,1]\n", + "]})\n", + "\n", + "test_sample_encoded = bytes(test_sample,encoding = 'utf8')\n", + "prediction = service.run(input_data=test_sample_encoded)\n", + "print(prediction)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Delete ACI to clean up" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "deploy service", + "aci" + ] + }, + "outputs": [], + "source": [ + "service.delete()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Model Profiling\n", + "\n", + "you can also take advantage of profiling feature for model\n", + "\n", + "```python\n", + "\n", + "profile = model.profile(ws, \"profilename\", [model], inference_config, test_sample)\n", + "profile.wait_for_profiling(True)\n", + "profiling_results = profile.get_results()\n", + "print(profiling_results)\n", + "\n", + "```" + ] + } + ], + "metadata": { + "authors": [ + { + "name": "aashishb" + } + ], + "kernelspec": { + "display_name": "Python 3.6", + "language": "python", + "name": "python36" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/how-to-use-azureml/deploy-to-cloud/myenv.yml b/how-to-use-azureml/deploy-to-cloud/myenv.yml new file mode 100644 index 000000000..36ee6703a --- /dev/null +++ b/how-to-use-azureml/deploy-to-cloud/myenv.yml @@ -0,0 +1,8 @@ +name: project_environment +dependencies: + - python=3.6.2 + - pip: + - azureml-defaults + - scikit-learn + - numpy + - inference-schema[numpy-support] diff --git a/how-to-use-azureml/deploy-to-cloud/score.py b/how-to-use-azureml/deploy-to-cloud/score.py new file mode 100644 index 000000000..0086d27bc --- /dev/null +++ b/how-to-use-azureml/deploy-to-cloud/score.py @@ -0,0 +1,34 @@ +import pickle +import json +import numpy as np +from sklearn.externals import joblib +from sklearn.linear_model import Ridge +from azureml.core.model import Model + +from inference_schema.schema_decorators import input_schema, output_schema +from inference_schema.parameter_types.numpy_parameter_type import NumpyParameterType + + +def init(): + global model + # note here "sklearn_regression_model.pkl" is the name of the model registered under + # this is a different behavior than before when the code is run locally, even though the code is the same. + model_path = Model.get_model_path('sklearn_regression_model.pkl') + # deserialize the model file back into a sklearn model + model = joblib.load(model_path) + + +input_sample = np.array([[10, 9, 8, 7, 6, 5, 4, 3, 2, 1]]) +output_sample = np.array([3726.995]) + + +@input_schema('data', NumpyParameterType(input_sample)) +@output_schema(NumpyParameterType(output_sample)) +def run(data): + try: + result = model.predict(data) + # you can return any datatype as long as it is JSON-serializable + return result.tolist() + except Exception as e: + error = str(e) + return error diff --git a/how-to-use-azureml/deploy-to-cloud/sklearn_regression_model.pkl b/how-to-use-azureml/deploy-to-cloud/sklearn_regression_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d10309b6cf4c8e87846850edfe8b8d54a1aa64f2 GIT binary patch literal 658 zcmX|<&rcLF6vt=T1%{DT5XE00;?FfP6EJauKjMzrgOLdnGx0~`rRmHRcC<6?n;*&= zN%R0L{srpQlL>dP>%oH_y>T~@%bJ)Nz2J{-A-uzD`+a?Vub=jL(N7;SN|M-QVJt@+ z@qjWj34;Y{xXOce{sk14pr?X*HBBQ-Gzb*^IFCfr^m#(fC}&wnl7uvk)F+H229&nr zMvyfHHJ}&u$kh26=(9DuunPSy=oPz&3R1lW1CHa&{*$Jhtz}?%b^Xoju5Hv{&k78> zP)6nM5n+bIIHQSAMFx9YXh4cFPa?v?rxf!-0Au_Kjv^vpvXy(kXKN2W-78z0 z>vNZ`Pm(rkKN~H7ZCY%7rZV=8Sr`9mTen_aZBJFl-Q~M?+P^-#owpkc?F;ts_YsV( zD-%xj=Voq+)eoJuDzZ?&tPv}u7*0B>J1k_wTEIk~|7}d>O?QN|r zt&kX?{HZTsJN|~rFMZ+v#gXw#=3fZ>Lf{tyzYzF^z~3UE25<#<04$No0bmAT1gIlF zT>+bF>&dclQ#$#sbWNyV{;RNFKHgo3X<9W;rkd*OuHnXs| za%VKRvH?0uu^lwR*cgG9QfzucYLC^N<*jUi%D%2v+P>;9Eqv`Q#4OolqzP~(y~Vvj z&LAsyGe&QagQJ_cx735*YZpi6e=FvFAT8-?X)UhxRPhfAGX8%e?K=HuvY=FRQs#{5SGPp#Z6T!GH+Kqp7W-zqdSck*zTdf?#! zv=p~CvlcS9v=rhtHxsbr=Hs(8<2JJt;N!Lw5U>;x;uGb!<}-ir2YXA4zjAl>aCP{d zx1|NIm4g+?%F*2oi3T6<1Kz(h`u{3VX-P{9aSK-~Gj}J~zp&S~a{2c#a{x;JXJt~7 z_c!p6Kl~+wWP$8O92^`;-oNGkJJH`r{|7a)6Op;Ub~*At8UHUlznJ_>Ape5v7hL}m z0{;^6uhI1ju73%Ee~I|l==%Q|Tz{&KR*pzr(F>^*-oXL#04xj)Obm1^ObkqHEG%qX zVmzddflEeo51*KljGBs)jFN(eo{foyj+Ksr@&V5SR`y4nT%6R5yaK!&{A?VY9KTh9 zf`yHZi-SvwheygmOG(S|FF$v!0Ag$u29$m@lm`G*ViYuDl)Fv<1Jay^fpp;g=EM8r zK|w`B$H2tG#=*ry64c%UprW9mp`xQbdin;2 z7M51lHnu=JH+K(DFK-{;H*dqjBi_A_Oh`;hPD%ZkmY$biP*_x4Qd;)4x~3LVSKk0_ z@96C6?&Dl?k<<)O;p#aeSBGz9e`%iKa zBjrLxM@K`)`b{nrR4-&gBSyzy;=?47)5J1!x&MGa1e^3pd~Q`64zqyPA(^@B2rfB` z;4ItUT^D;^~_k}C+dh78H`;y3y#9mf_zp)V@lo?qrZ3uAE66^_2&zi@q6bv4EP=-hiv#;P8DmN-R%`+8 zF--vd%EqE%<9MZ{N{KwMT#0K0oKg1G(y!E_x%9$-UYdTJ#2 zbN<+8R-6E`o3R6d{tdvJTPCltSreDo{B_FQa)}_5pk9!H>D3m(=8LAmE6NmuCf?#c z7aJEVX4wMx+z`tsTeW1c1jBi&(+eW9a(P`(R=dzh_JYTGk?er~LkU%EFBkf*%MIU6 z(TDRvI758dBvd-nN-TZy8}y{%j$eFk)OHgmJ@_F(jbw8lYT(tc~5!poC2#QBy| zHrqMunXFM>na^>M-_QJ^#OuXEfPOk_77ml^vZYo*e$s2~9{hrP0d2jX_h>rO9fPNc zik0|MwvvmpwzrMaEN36*^l=KjCVV4GB}$uK{Wn}t93xBrhhAf`c-~g^4XVgukZS&0 ziIhF>E3c%{@}P{CxW^sfonWKUl8)Nvsj7kohu02ll4}-If+&>=q%Tl@Ph?tS+SM)4 zW=L(uKzg?(LHi!FMUoyGi`((nEP=^j?hyr2faX6=D0c8$gRAjcvbwA4(^A`icf_Ej z9a&B6rM~fq0<-fqhkpYy&Njv#zKtf_lq%02=gB+F?(lZ3p`-73Im%pfQt|X$Q8-N> z)%4xmv!Mk2Dk{mfjYi*%wL`23_Me$~i`4ReY+PPrnA<0rm)4;3pyP&JKwNR4zeL(tl`k{8yP`P|~OIY`+usatWg{Q(S7RQ+HGJ(GfQy?T}Z`kWf>sol5 zVQS51j{flxA)aM+kjKMVNT9tK=p%z2D3>2c6yNowC1KsvizAPTp0Juh z#ceh6{Yxi|qgZx&UHz){`a6Jr&D+ z@*Uu33U#f=k_OSM1yvU}8-cW9qtvLqL~Q@{*X?rm0z!^+_Hl7GrPMvZg8dRSn{0VygRP*$SHzcqjFIHLBi8r+#Z@NECItN zvcal1A?A0Op<1F9vT{pMq7d90? z#P?!zi*DqP2zE8U73~k{W$*C~(Fa-?-*&`CU>)u}^_aTaB^gsM_Tniub1Ic99tr@F z?R~43b2wp7cmeY+OH4s)5!Wd@J|GY3>-N5nXmSeriF@GSubE?){$?7OzLDYML+=HL z&-buI#WuW9A>}sD_CpX<4$~*tj~yu%l93HgG$&Wjz0#m-B93nQTCbzIt+J#;$Jq%b zZ#sS$0S3+{D)oaiXP;~D)O>n@nMtM^yrw+wqf{5zPy@I3uh*R$0Y#;|gLyR#xL06=72w;X*|U#Abnw7#C+3st zt!lBMWy5iwU>!9How3s>>-tfrJ*Qkfz8c6FS0tr}aVazNXqnPVC6Pv1CJ%w%^Hfdr z(rVS-f!5gk*J8JvLw5k1pi4Hr#WN}>!v3iT$j9_Z^%N`b5`O6PQ_3(hb}6|wsk9AS znYoT^DH~6cqU_?N^|3uPyHkb=KCdP?l2kV_t%%xy7K*dCqhdF^E0Wi6@KY~xhdLq# zL0~*#YkIWwv$a=fXG**~&4w<2!Hyi#@Asg~!8%QQ z$n&}^O-6ve)lCNqWwWt1&ihi!c?Tf!exp7qs3KIZ{dHy)X5`mNGMlS~Crqe~CYfHo zoO0Onwuw0pXc*Y&aOD?*nJqQA>)_ZYiS7`*$rcQgeZ;N!Ju6blBUadl@&P+*S}cIR zs9fJJg}Ndub=Ksxb}wTS)Ya6_0($c%TuZJE{RKuDCr3xuEBXnitbWxg_kMWI_Z>g` z-nDp_DUO18Lkd05GvP`%KS|v~MgHS~Wiq8wwh*O{BqIjt8t>h?eF{v7498BHDf^=& z3Nmb_fTCb=0DE5KUsJAvTlQhx{2gFOH0U4^AsEyy?{x=|T)qReXk1Jp+HGVeA7#>! zeg6;TBX^PQ&qvibUj7Xw|^izR9^Ie{zml6QTyyXUVkQ=;aH!r zcwq)8fy9qKBNfhgMVx3{k-h3fae>$A8~El(iu;s6Ml)mgOq!|MB)C`OTjE1ewpdxp zJAgfUY*ZICqBDTAQ`Kden?h~M-6l-lu-}!jUIj&r36572)9iMkCsg9?){}kTBY?@@ zqCo$`;>j2<$?2X&BZVFX37cF);ESeg`5|8>=w3jO=@vTV1d=^<8F%x1M7xuL&Y!r* zSdB|}Y}O_Adt* ztxS+rFN?$>`i(}Sdlyc+Ef*8ZQofU8foa9F^GY5vAK&!b7_-l+uO|K=kXU61KQ$%u zmr8s-3<{j`o^n=ZJ|ULYan@gdM@!_(oEK4vm`BKQTk&e#)_8R`+z`-b7KwD~#VV6* z4%XCI`QRV#G2a@Z8WYU(#VXqO-N=_-_%5#Kg`5(hU#MywuJ!1~P8t=Ds>L*qYuzhU zQX35QbAQ-O)6+tA#P59~dsO`P=QYS0#J`{ltm%K>$(np}qPTY)zP6SvH~FPWGv`L( z2-2`jr>!{*a_=`0FbH2!6Xu-WNMOOo>t}ge_6a4p1ZQniQoHy_;SvG4+`0pxYh4%~ z-^viENONrARQlUp1Qcm-#kaZ89S^NnX&qg@hFZF!v2(agr#Bygz%BIt9MBJKh7QzE zXOqWu!)p4iVl}cuWnb`L%K=U|!4BTV9fpwnBP7XsW^r0E&}9Ly@1~PpGeN5KII({1 z3hlt~4sfrDjv@Aj!Yi+~tv2mjKgb!=zx2hr(N-2wl@QSd&}V0B5TS4f=@wzuH#$G0{_z>DCbI;$jwnk0(p!LdXZ-qDYwnE5V)k_NPV-;*J(aA#)&5dgs)1>V}VZIr8L|YH{uk&{MFuFLGSTf14?@7Gn@J zHPM5J7TP)~_46T9RAmALk}boCl)7X97vkKg4BM9~C@XvAsQ=KwpuQei2k4J|MaTQa;>mXINp z6e{+v9U!!G{_IEg!qg4w zLg_LC(S>QN`0)>Fsm!QM4(|n9GfkEp#l?W5h7bM;_CxRv*NYElFGkW^8TRP-M}+W% zHBWFJn9uno&&d$KYFn`vJ0-%|p7aE67#ka?k*dUr2rvOG55=T**p@4z{BmyihkwRc z&&m1N+c9U6Y+tKuP-)5C5GYor!?&DLAOBRBk+=h_6*H}x(n(hz^0e70SUA1 zsk0LLk9$QCKL`c%O-Yj_ALU#yw34UHW-i1#KQlGj;yY+)j6@op%0hAG^i$T{%NKAj ztSYt`NKoy*updI+xOjNvWs4d4eb+;KEk=X6pX2IX0b9`vJ!QBrUuc&_4Gkabq^291 zb$I3M-yjft)FRdakm2<$wgDO`j@4Pg*UL{x9kY2Yr-ood8uEQjuid%jJEZmA1QK5~ z%1?1qv5p-qAr;QJk=_D^%#ot8yO$)7P;zKjLp+9qI65! zDSUI$`I`O9Ahb52I*NDF+;+AQ&ZQUug^$aYCE{a1*_sB&EPytwv1G;t!)yvU*2Bg_~-FII30*uu@_-a}=x zdL?(j(z;x4Tcz$6J$Hs6dkL2l_*6<75B^9J?)R=UUV2pI5fv6z{Kfds*59?6vXe!) z;ruLISMLv|mFsV>60W=Eg{c8ngiy*MzL>63Udvom%Kf%(R?VtH#+OP22!d+Xy}HgCz8JP`)S5H!oD5 zzYZTs(0?<87Hx0`cpLRX9Fgs2;ZiaW^s2i9ST72aZ`r*JzXRxvI)uV}z80g$Vpo(& z;i*SO%=mty`K^@dalVox$GlW11PG6`7<=Cgs)eX5$PSue^AffD&LZ|`n2Apx2Z%-*{ zPf`j%7M)Pmtb8v&Pyp*EVQ!B8avFt4qKxbeHcs_&ENhmaok5S%eBjZv@eJNed;7Ws zm6b%EuYzw=zq^em_B;#i???wlYd?)2l5|sA7k_1<>MHE@g4McJs&%K^S4%UwZtG&& z(FL?MbtraSPIJvmabdAHI{^KV#EF@r0|@2EGM95i(Tilzy7fucaNiSdf8>#Wh@rMc*5PD+m6x#rlK6GDX4~j z{G@k@IIg$#r@8ofvCT#Cxu6C=tgmvQZ-Ob7m)^FHJhI%2OS`b|xz!S0oY6QAuElWG z>E{3eoyj}EGSTp<{h*=Bz7S{m%XrCmDT0dr_i}h#m9$hgix!e6x1g-yNu$l5G&Tl5 z2>TTI6grEeF(-5qc8F4~#0%01OYskBs5%(Vh)XpETKI<%zT6ZOm2SvYC=SvXda@13!wkR5;$Goa2ZVABq&>OyhO$SCbpz%_0j9h&H55 z(2A+AzJ!0l>Fp+z))GH6ozXRg%gnaOUOOeO`80l*`|c9GTIp|H7St|K7^8BK@F+id z{l2)82glQ6eE-3FucN)OSbd{MHGS=Gfw%rWgLLi)SY>Zt9c(=R$SeNy;=;`P3KI}_ zSd$vikp_TF1^5%b1U4_Y=Q` zjK^-&a^$7!TJ{y~BdrK!EPs1``YpdFiQH@$m0Q3!gTP6fh@dqbiV0X6Sp0D=o5E}1Da~v+@+HIFSoN(`-yd5tRG)=Rj?O;m+tnp{3t~2;n38{jex>~rQ zHDfOe?rlNNg}(?lx)t^@q*JCn>$vapk&%aTPYdR?!BW0;>vIQ4$R+_>Ok~9qtiR*;`y(hmr*}rHZV7N!}8`!n8Z{p294rD z;T-_;4$w=|^cf02Hr?yo{B-87sYwhbX7XmpHl4uVQXq*n)cv3fW`^igrsUQ&`=VZ zsB%{Fpg~pdnv*j*zEB^MC5vIuK-JrJ&}FapzU4hZb_Wr0ltlTDjUbXsl#S+;AT9z0 z;V%XTsh-%ZRVmvu6%mFFutUf4%C6j3rp6)5eID2=u*Q@hEJqC}sRLZHObwbn-yTa@ z30Yp>0gffR;JQtF7t=qX-?NP%bw_krkz0=%n(9(kL;wzWRI;s~Hf5S9S0|AMIc(qp zW8IVkuVrH$wbevM(>H&4mr|jb_M~KT{UoL6PrFRctLw`yGEqD{4hSY-oiLxz)QcbD zpW=F=51e0f@a?;(zuHFxg$AuX54dfD$wTv+M)CE5S>#iKFyd|Tm@}}C6YT=%XF`|% zS2uQj6~3MtN0%x++vo1RA~zEFhVTV@-|pHH@NH2g7v=)@6J;|9@gPL>Ic^(`>#mG= zij%;bZ_BmTV_UFggn9GClqqs)S@TNH%(eGiU>q|%8)Q{ZS+RFz1qlBrw4FZGppyUK z;J)?B-VD5`x^jSLcZRtQS3AO3D=YT__e<72{%Gl+@dRHJ@U4In&iMMC$UX7J@StNh z733OeqI3sPeV8|E5^D&&MJ|}key=9MH&VHC+jXU!Z4M|NZ&7==umEuqR}*KT5+n2_ z8-S~Qx4-<_HVoH81}W_N#kf_It!2dCR~F(g_1N4k-!)Pm1E}Cl*%@MD6)m$k2M8AL z&!bX{f5`DA{;^RO4I zZ^GcNy2-E#Sl*3J2XaDHWS8Fz>E<=nX+e$%AR%TRm5XZSa3DlmIG^zS=jPhh%kO<1 zmC2i+ryTO6RQR$MU7pa|+V53M!23^{>m#>~bW*;r4sfiu;%B_;K@X<%LVq)v@?Hdf zbt?mwFEhCVRF2Q*HTAzdQa}_J*}Y;NIe1uI^JR?b9m)xpz;n%w_*F>(1*#_kpGYpr zipk2n{kjkgkP(j<#WQa8M=cW=3%T4&34$M7=g^pi2gGzPg-cJcXz__OcE>p5BL);6-QPPIJN3Ne6Hah~z%p)<5v!Biq>Ye-W22|_?<>p{e3%IF@*tiG z&9V|kSS>__K5B<8rnug9oi;`jE%SwT=iKJy0S~%Jlpd@IP)eAnV>7k2ic*62n3W(5 z)TcWg+&_}t;g6ElZV(1sQ@Ar90w*;?!WhFOgK-`vd8nN0-SCJM-oU>1xN*-n-S0p@ zCsyL_APX09rhbB{{K7X5MbeXu+c{8lWY2?v?m}qas`*2}_)$>);t*hEL6P!!-r0Lw zjG6JM%f;|K#*yiWzK}5Yi0(#8P{g%UaD<@YIA4#G>TI@NtPzwsvW7&FB49ZGKF(?Ovznq5Y48xR2ubT zf%uR&S&unPOgnt(Tg=7xV~V$}X)jGdwJJ?btD2XiFjaNpTFQM1Z zjGk0qK4B14M03Dm5}0!I>3LRI=xzUG*eXShNMWpI<{{wOr008_IKqs`kgVqI^V!$e zrZfIO5F^zC$6hhl0x{D&$a810%)1&@BRLPZ&-(I?hEMOSxJQQNe?LlaNuHhg**5e{ zvZGg(f#8R))oIs$Mn3S&OV8l?>DgH+&0u0;LvX+4$&hch+Gx!l^?;%6899W!EeacC zrk}`GI-7%^p7m%YYo3QGnI-%wVFEMi=K{-nzQnPonZ*~y9al9X0zkw4G(WPugNEPi z@cMd*sT}GfZ96|~r$xr&n!KD!GCu1be`bb%xZCNLh zZGSCk3Ov54lzF{aEe3C_5ctlKlD(ovq+6ABt~%*{4OexzJb|0A#CL@xHYCM~Zf1?c zo$mjjNYx{sOkc>5>Z-9eSc%*h_Kpj>4{*(o?^{h59>y5G5z4hqy>Dy_<;g4_6z%+K zXvm_*CUzMatRY?N0p`v+mZ+SXTteM>;n-31IwOVc_Qk!@6Zz=5$rz}@(3Ht6{aDcLA?Yrzx|iAT zYUO~Gf=by%gu-w%v+=~P;Fz=W+gHW~(q@#sCY?4^bW#d*ZyM+-!(ICsIWD#OxVwR zDUUgl5GcL;T9dAp%bc}LoZ-^GbpN|Dqi?!Z-MgQv^M)XfA^ov^AxeRbnJ)JjL2prs zINE3u@kq%y;-O_zBI9iVI|^~aG@;!ch*28XDv*WNc|cKl$>S$kPKxKqoQ9<55!V#T z(@}JkLfpq9^u>Tr)2Cyz22)aqIo4Z0R(AEGpV@@A=_nt67z$?>Q5|h$NW6>~=yqwerRvr&)i!*ojmE)? z=N*W5>69~EGrhl&q{iBA(9Pn5((Br42~iQ-NgsyR)V0*4OrBv^fHt%eI{2}aw8zOL ziFfS`AGv?=XfWP71NjiZ4P_l)$K;1wP%csn^OsHcVxckm;uc79FCUfmmBiqwQH<6n zs8&|>Y&4=$^7&iF$%+lV;M9#2pOcJ?DFhw>UF^=t!A`CNC+%ih3DvG$sg&k^iIWK+ zX0`h9y;IH@=}Ds<9Zrm~RxlUi;gbfc7p}har>>EJ(o&$fXSE%7Es_F(;ldcXO_nTA z7!!_e={sI&1L90gn5D27uX&xQt*O$bVvt)j24&Y&kf+=GJe6pu;D$E`FRK~;9Nc79 zUGw!6u(T@W41AX!8GgKvhevh5v)CWHAe^GtCZVIK(zFr7Nh-K2ghf*@!*JlXNon__QaP{k># zf}|XL%cN4DBrX@mW+POT)=m)pdF?CY;rf}TdzJ!{6R(zqfK5ET|B6&u+P(wAVm zTZTE2vZ{@%0oRMf7n4FD8dX><#^K&Xbb|48KZtb>93De1MrHyDzS--xY-QP`cra2> z4ZZM&_`1jSSdhl6t+&gD5R_sVeMq9fYulDRFj*(%7Brm%BXUj=%@jX;K$+jc1>VvD z{0r?@G#LS&o=w@;97f@xIF8owHX)wLY2n&5UVGmS)E~R8jMl3BTqK6VcSJ&WI>oIz z1x%;kS!W3eUl&Zr=KwIoF6B@H9q!prm*j~fH-8hEUMIbC1Z=~na3Mr6ov7;};#>T_ z$0JtqJHU^%+R-TXc866Sw0$(H%j$)4J7E#X0(*VK1Ts3!#=ZooJ6r&Lm6FadG(9}!f<-U}%&~}m*Bqbo>I?=Sm^fbs%f9=#2E&7>C@HrH1WS)Wpcbw>%BOXK(E7UEJ19e;mtsSuruaU1;57nE2W&BLj!xL%sU&%IB37Nzwg2L zEQn`=GDYmem9V8d_A}DR$MOs<)0KMTKj9`8s`X>Flmn1Dc8iq9yBXuBR9b{YlNqHz zVM9nVPeu??8jvH*b|TUln;AoyF%$*;o+7*9Go&#kdx1#DX{UGDW9AeT)FI6?&OBw< zJ=tn9t7JjTC;7aJ3EzsV@tm)Z#zdCJ_awDz1vx*6fYn_RpLZTcIk^=){f22ZTP0|K z_Nn(U%QeW`55~u=R5vBEp8h)irxnC-N}7-NTQ@_Zrs(NL zD`3_B*Rpqj8Atjdp2bGS*It)ULK?|n-(a8u&}RMO)xi;LiC`zlbhQ7ls>8eZ)s$c} z|1L1S2pmvq>+79xF*9q3=feIW&<6MJ#AL8qS16RqPM``Mt%0G@C+pcCh%Pvxj>M_#e zphx4=V+wND*yE|m9e|f++*q4D2d<|#MEgmN@5hRWNcn7Jox@IHHz>O`2;{OA>XI(= zPNn)`lk7@#^FyXt`_{^!ZC->fqg8R!1b7X_`E9Vm;LX1 zU-Pj5Yio|1%@p+bBRpEef+CtKK&nX2D8I|)Y%?({pk_xsi{(Dj-~7rJ=@lVgoGLy& zdFgDSItmGVk#`G$GAj-BgS|?fn(uA z?iLgyWA?{cxjlNK;6P+&=&ox|f^Rzc`RUErP0jroUX5>yX@N)=%0HS780Ai-k zq-3exb#0Q7M!ccDR0IG?MW6p|@(!?h)_(^8An88*7T%P$ZXa}VZFL6-=0^HkOO9({ zm$zT<0IkUNaCID|Si#!IaPbB)6SRaEbhQ1y@qShJDFD(hbGq_B!25qv?tg5tLLWU2H>NMVNA8e&$WTA_~b)7qrq9>lWo{vXLCdfc^vR=YMHM ze@_dtu?;#VN=3kUU_bqn^9`Vnx-)ri15JW{Tyh|`0YgXR4MY)wk4_~%F6cz(aoNi9 z>@VN5jKegDU<;Y7@^HF$k)d>0)L6Q0D4b!>8un%mE=Y0Uo*-UMSAnldJnW8i59xky zcCjIhaQ0Be@xT*7#w+L5*eCqxd~`b+s*k@GOawT3g^XnM;SG~CD!Oo7G|rYyRPf&8 zu4QK<>2do~nT$G+XZ@s6o$w_s#y;wl>CP1*72G|gcW_m?GyF4vtx(E=Gsj^c_XHA4Hua6(xfR-DP7BonqzC(r9I*(vb zmEm;t17iHw-IaCBXl|mHN&{+QQYKK-Q=~nJ{pJx;=O9ve(|JWOuE%d=@9RD?`;@KT zdwMR#PJ^=4p>~g1dyHkgW?r{@zngo(YMkN`i%N~v*QKS`u0AVGqXcL# z>ModfbbEc*LMJO`&s(ett+Fa z;A8C>tF;ej=R~VyyV_4u$&cmv_hS{SK1o9WXjW1k}`yP%y$8$Qfv1T?8 zCUyM28;wu9$t*Ug8neo}uvmGSmspc`9n^~K%g9v2sfYDr?rPNB2Ih4mpo)T26>>#g zo4Fi@+Q%ku`eocnRJQp}1PyntB4Tc6Tq;aWwtS<2=h?Q9IwMaV_7R&h*dn?kO6+6d z_&%xR^SAB+va^C;jODpMNk6@B_(~1L(bLmw88Z=iNQlGCEM;f(0e{O&rI;&jAVN*7 z*4xqQ{aExp6MX@+Sa}kf7JJ15&P&+T&A`g%gyUP{_IdA~PV&jg$t~wfw*or>NX;JZ zhN@_x&Js$np_t||-#JmUZBK>0Z|!MwCZzBT+XJMyvwNP8NFGCj(Q^%GHqcFNo#SR9&VcF+uHD6jqer1Y}zhtFgEm zB~fZ6aAIl4^3$-fu_bCj5U`5uDDHz>+)D*uDru-<}<% z!d)IeNhrg3qDZq}cJo={eAg zlq?j<{fccF9NVfbe&C{|O(<}5#X7}@B?U;FoSda=zAvY+-4lkE(`D2!;(k3Z=Txz$ z1)Q8!ePNtX^awZg3CulI(>HcuBP?HI3K_d`9<^J0>vU_YQUo_P1b%7r3F65W=RM#l ze5x3CwI@sJ2N3r?E!WXd83hSS@z6=A8E8$SKv+_|l1g^b#Msbss9I?WIo3t$RrdL9G;{Bf@E(Krmer2ck=KYm7^A9)0#H=|_ti{(1ZRMz zZcd^Q2nNK%m^#yw@4VL+K^N)eh1=)UVV%SCKhk4UM#{d31gdfcCrIl?Dp66c(b6jb zhsT~9B}3+ehD2cp$aviqTR74)Cb@D4fJNK^-j@q%Q5pT+imMd>y8nSR|4*{dBeG=w zq{K~j|e1I+%{p8j9`{x9tPvkW(H){)Mw{>obhR@>pS~828KkT6qH14j~p}ws8 zF1zaFh;mlOj)tWO{Q^CrJ3o7Vc4E|JVNZE2Q2xMd^D^)iCZ;HO`;VIk4H_ih1S`pH z)7Ovd&a9gB$dLm~ccNfgSFvZ_G(M?n*~?XS0}AEHyF|L*Dq2q!9aC4h8H^KNj(A9z z$oBzyCllW52z)~Ak0Zj0-l!?bORW7#0(+4c`3V0%ha~*%d#(aAw1-AAnwXSipa(#t zWzVaMG_#)eN4$ADcRe9XJ%Mu=wD-qGHo|zg*TSvlX(KheUE&l8lq{^Ainh+qgf%a9Cr7w6AFg=j zMWZD4EqUdM3*XA%-2pyIflh85eD*WI7cnk_t#q zv|>O~;629MeYcHPm1@iDM=a-<=g(8NXRlTkh{Q+r=PBai%lPS*mWl#(H;Y7u!reU& zSAX!DR~CuBer;{N-Sw67)0_IWd#;4o-59zW7`FBnezu3k-&2p?Ran)xWC07JgZgEJ z=IYfozR}QTiHWHg7+t|{P8qUKVV6LQsHh(5h03K5_q-$V!p7k0wfg2YPn8DXI zIPCC<)al{@w|Y=B*glI5cJvTzkt1Kuu?3`l>i)Cp5mzN;qCh}yNAhKo`Qv7q4VIW@ zlf}305*rO7kPDNNAs> z$#8M)IgkJTtg&S7Nw`UAY-IxHFemwDJGIb8?pfnI9?Eq*S>A`_`!4{6IYjFa;n*@? zqGz*3EXaL|>7Rwwk^kj(kLF*jq`zBG1!`n2v5==m(^S(d>1QLK_lOX3NZ0h#`V`_r eLd}2savJJCYX;JY^}mhqUro!4WJkN3`Tqcqb1f+V literal 0 HcmV?d00001 diff --git a/how-to-use-azureml/deploy-to-local/helloworld.txt b/how-to-use-azureml/deploy-to-local/helloworld.txt new file mode 100644 index 000000000..a12521d86 --- /dev/null +++ b/how-to-use-azureml/deploy-to-local/helloworld.txt @@ -0,0 +1 @@ +RUN echo "this is test" \ No newline at end of file diff --git a/how-to-use-azureml/deploy-to-local/myenv.yml b/how-to-use-azureml/deploy-to-local/myenv.yml new file mode 100644 index 000000000..36ee6703a --- /dev/null +++ b/how-to-use-azureml/deploy-to-local/myenv.yml @@ -0,0 +1,8 @@ +name: project_environment +dependencies: + - python=3.6.2 + - pip: + - azureml-defaults + - scikit-learn + - numpy + - inference-schema[numpy-support] diff --git a/how-to-use-azureml/deploy-to-local/register-model-deploy-local-advanced.ipynb b/how-to-use-azureml/deploy-to-local/register-model-deploy-local-advanced.ipynb new file mode 100644 index 000000000..7fab47301 --- /dev/null +++ b/how-to-use-azureml/deploy-to-local/register-model-deploy-local-advanced.ipynb @@ -0,0 +1,487 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Register model and deploy locally with advanced usages\n", + "\n", + "This example shows how to deploy a web service in step-by-step fashion:\n", + "\n", + " 1. Register model\n", + " 2. Deploy the image as a web service in a local Docker container.\n", + " 3. Quickly test changes to your entry script by reloading the local service.\n", + " 4. Optionally, you can also make changes to model, conda or extra_docker_file_steps and update local service" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "Make sure you go through the [configuration](../../../configuration.ipynb) Notebook first if you haven't." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check core SDK version number\n", + "import azureml.core\n", + "\n", + "print(\"SDK version:\", azureml.core.VERSION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialize Workspace\n", + "\n", + "Initialize a workspace object from persisted configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "create workspace" + ] + }, + "outputs": [], + "source": [ + "from azureml.core import Workspace\n", + "\n", + "ws = Workspace.from_config()\n", + "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Register Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can add tags and descriptions to your models. we are using `sklearn_regression_model.pkl` file in the current directory as a model with the same name `sklearn_regression_model.pkl` in the workspace.\n", + "\n", + "Using tags, you can track useful information such as the name and version of the machine learning library used to train the model, framework, category, target customer etc. Note that tags must be alphanumeric." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "register model from file" + ] + }, + "outputs": [], + "source": [ + "from azureml.core.model import Model\n", + "\n", + "model = Model.register(model_path = \"sklearn_regression_model.pkl\",\n", + " model_name = \"sklearn_regression_model.pkl\",\n", + " tags = {'area': \"diabetes\", 'type': \"regression\"},\n", + " description = \"Ridge regression model to predict diabetes\",\n", + " workspace = ws)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Manage your dependencies in a folder" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "source_directory = \"C:/abc\"\n", + "\n", + "os.makedirs(source_directory, exist_ok = True)\n", + "os.makedirs(\"C:/abc/x/y\", exist_ok = True)\n", + "os.makedirs(\"C:/abc/env\", exist_ok = True)\n", + "os.makedirs(\"C:/abc/dockerstep\", exist_ok = True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Show `score.py`. Note that the `sklearn_regression_model.pkl` in the `get_model_path` call is referring to a model named `sklearn_regression_model.pkl` registered under the workspace. It is NOT referencing the local file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile C:/abc/x/y/score.py\n", + "import pickle\n", + "import json\n", + "import numpy as np\n", + "from sklearn.externals import joblib\n", + "from sklearn.linear_model import Ridge\n", + "from azureml.core.model import Model\n", + "\n", + "from inference_schema.schema_decorators import input_schema, output_schema\n", + "from inference_schema.parameter_types.numpy_parameter_type import NumpyParameterType\n", + "\n", + "def init():\n", + " global model\n", + " # note here \"sklearn_regression_model.pkl\" is the name of the model registered under\n", + " # this is a different behavior than before when the code is run locally, even though the code is the same.\n", + " model_path = Model.get_model_path('sklearn_regression_model.pkl')\n", + " # deserialize the model file back into a sklearn model\n", + " model = joblib.load(model_path)\n", + " global name\n", + " # note here, entire source directory on inference config gets added into image\n", + " # bellow is the example how you can use any extra files in image\n", + " with open('./abc/extradata.json') as json_file: \n", + " data = json.load(json_file)\n", + " name = data[\"people\"][0][\"name\"]\n", + "\n", + "input_sample = np.array([[10,9,8,7,6,5,4,3,2,1]])\n", + "output_sample = np.array([3726.995])\n", + "\n", + "@input_schema('data', NumpyParameterType(input_sample))\n", + "@output_schema(NumpyParameterType(output_sample))\n", + "def run(data):\n", + " try:\n", + " result = model.predict(data)\n", + " # you can return any datatype as long as it is JSON-serializable\n", + " return \"Hello \" + name + \" here is your result = \" + str(result)\n", + " except Exception as e:\n", + " error = str(e)\n", + " return error" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile C:/abc/env/myenv.yml\n", + "name: project_environment\n", + "dependencies:\n", + " - python=3.6.2\n", + " - pip:\n", + " - azureml-defaults\n", + " - scikit-learn\n", + " - numpy\n", + " - inference-schema[numpy-support]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile C:/abc/dockerstep/customDockerStep.txt\n", + "RUN echo \"this is test\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile C:/abc/extradata.json\n", + "{\n", + " \"people\": [\n", + " {\n", + " \"website\": \"microsoft.com\", \n", + " \"from\": \"Seattle\", \n", + " \"name\": \"Mrudula\"\n", + " }\n", + " ]\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create Inference Configuration\n", + "\n", + " - source_directory = holds source path as string, this entire folder gets added in image so its really easy to access any files within this folder or subfolder\n", + " - runtime = Which runtime to use for the image. Current supported runtimes are 'spark-py' and 'python\n", + " - entry_script = contains logic specific to initializing your model and running predictions\n", + " - conda_file = manages conda and python package dependencies.\n", + " - extra_docker_file_steps = optional: any extra steps you want to inject into docker file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.model import InferenceConfig\n", + "\n", + "inference_config = InferenceConfig(source_directory=\"C:/abc\",\n", + " runtime= \"python\", \n", + " entry_script=\"x/y/score.py\",\n", + " conda_file=\"env/myenv.yml\", \n", + " extra_docker_file_steps=\"dockerstep/customDockerStep.txt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy Model as a Local Docker Web Service\n", + "\n", + "*Make sure you have Docker installed and running.*\n", + "\n", + "Note that the service creation can take few minutes.\n", + "\n", + "NOTE:\n", + "\n", + "we require docker running with linux container. If you are running Docker for Windows, you need to ensure the Linux Engine is running\n", + "\n", + " powershell command to switch to linux engine\n", + " & 'C:\\Program Files\\Docker\\Docker\\DockerCli.exe' -SwitchLinuxEngine\n", + "\n", + "and c drive is shared https://docs.docker.com/docker-for-windows/#shared-drives\n", + "sometimes you have to reshare c drive as docker \n", + "\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "deploy service", + "aci" + ] + }, + "outputs": [], + "source": [ + "from azureml.core.webservice import LocalWebservice\n", + "\n", + "#this is optional, if not provided we choose random port\n", + "deployment_config = LocalWebservice.deploy_configuration(port=6789)\n", + "\n", + "local_service = Model.deploy(ws, \"test\", [model], inference_config, deployment_config)\n", + "\n", + "local_service.wait_for_deployment()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('Local service port: {}'.format(local_service.port))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check Status and Get Container Logs\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(local_service.get_logs())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test Web Service" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Call the web service with some input data to get a prediction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "sample_input = json.dumps({\n", + " 'data': [\n", + " [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],\n", + " [10, 9, 8, 7, 6, 5, 4, 3, 2, 1]\n", + " ]\n", + "})\n", + "\n", + "sample_input = bytes(sample_input, encoding='utf-8')\n", + "\n", + "print(local_service.run(input_data=sample_input))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reload Service\n", + "\n", + "You can update your score.py file and then call `reload()` to quickly restart the service. This will only reload your execution script and dependency files, it will not rebuild the underlying Docker image. As a result, `reload()` is fast, but if you do need to rebuild the image -- to add a new Conda or pip package, for instance -- you will have to call `update()`, instead (see below)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile C:/abc/x/y/score.py\n", + "import pickle\n", + "import json\n", + "import numpy as np\n", + "from sklearn.externals import joblib\n", + "from sklearn.linear_model import Ridge\n", + "from azureml.core.model import Model\n", + "\n", + "from inference_schema.schema_decorators import input_schema, output_schema\n", + "from inference_schema.parameter_types.numpy_parameter_type import NumpyParameterType\n", + "\n", + "def init():\n", + " global model\n", + " # note here \"sklearn_regression_model.pkl\" is the name of the model registered under\n", + " # this is a different behavior than before when the code is run locally, even though the code is the same.\n", + " model_path = Model.get_model_path('sklearn_regression_model.pkl')\n", + " # deserialize the model file back into a sklearn model\n", + " model = joblib.load(model_path)\n", + " global name, from_location\n", + " # note here, entire source directory on inference config gets added into image\n", + " # bellow is the example how you can use any extra files in image\n", + " with open('./abc/extradata.json') as json_file: \n", + " data = json.load(json_file)\n", + " name = data[\"people\"][0][\"name\"]\n", + " from_location = data[\"people\"][0][\"from\"]\n", + "\n", + "input_sample = np.array([[10,9,8,7,6,5,4,3,2,1]])\n", + "output_sample = np.array([3726.995])\n", + "\n", + "@input_schema('data', NumpyParameterType(input_sample))\n", + "@output_schema(NumpyParameterType(output_sample))\n", + "def run(data):\n", + " try:\n", + " result = model.predict(data)\n", + " # you can return any datatype as long as it is JSON-serializable\n", + " return \"Hello \" + name + \" from \" + from_location + \" here is your result = \" + str(result)\n", + " except Exception as e:\n", + " error = str(e)\n", + " return error" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_service.reload()\n", + "print(\"--------------------------------------------------------------\")\n", + "\n", + "# after reload now if you call run this will return updated return message\n", + "\n", + "print(local_service.run(input_data=sample_input))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Update Service\n", + "\n", + "If you want to change your model(s), Conda dependencies, or deployment configuration, call `update()` to rebuild the Docker image.\n", + "\n", + "```python\n", + "\n", + "local_service.update(models = [SomeOtherModelObject],\n", + " deployment_config = local_config,\n", + " inference_config = inference_config)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Delete Service" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_service.delete()" + ] + } + ], + "metadata": { + "authors": [ + { + "name": "raymondl" + } + ], + "kernelspec": { + "display_name": "Python 3.6", + "language": "python", + "name": "python36" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/how-to-use-azureml/deploy-to-local/register-model-deploy-local.ipynb b/how-to-use-azureml/deploy-to-local/register-model-deploy-local.ipynb new file mode 100644 index 000000000..640bd4848 --- /dev/null +++ b/how-to-use-azureml/deploy-to-local/register-model-deploy-local.ipynb @@ -0,0 +1,342 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Register model and deploy locally\n", + "\n", + "This example shows how to deploy a web service in step-by-step fashion:\n", + "\n", + " 1. Register model\n", + " 2. Deploy the image as a web service in a local Docker container.\n", + " 3. Quickly test changes to your entry script by reloading the local service.\n", + " 4. Optionally, you can also make changes to model, conda or extra_docker_file_steps and update local service" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "Make sure you go through the [configuration](../../../configuration.ipynb) Notebook first if you haven't." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check core SDK version number\n", + "import azureml.core\n", + "\n", + "print(\"SDK version:\", azureml.core.VERSION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialize Workspace\n", + "\n", + "Initialize a workspace object from persisted configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core import Workspace\n", + "\n", + "ws = Workspace.from_config()\n", + "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Register Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can add tags and descriptions to your models. we are using `sklearn_regression_model.pkl` file in the current directory as a model with the same name `sklearn_regression_model.pkl` in the workspace.\n", + "\n", + "Using tags, you can track useful information such as the name and version of the machine learning library used to train the model, framework, category, target customer etc. Note that tags must be alphanumeric." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "register model from file" + ] + }, + "outputs": [], + "source": [ + "from azureml.core.model import Model\n", + "\n", + "model = Model.register(model_path = \"sklearn_regression_model.pkl\",\n", + " model_name = \"sklearn_regression_model.pkl\",\n", + " tags = {'area': \"diabetes\", 'type': \"regression\"},\n", + " description = \"Ridge regression model to predict diabetes\",\n", + " workspace = ws)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create Inference Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.model import InferenceConfig\n", + "\n", + "inference_config = InferenceConfig(runtime= \"python\", \n", + " entry_script=\"score.py\",\n", + " conda_file=\"myenv.yml\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy Model as a Local Docker Web Service\n", + "\n", + "*Make sure you have Docker installed and running.*\n", + "\n", + "Note that the service creation can take few minutes.\n", + "\n", + "NOTE:\n", + "\n", + "we require docker running with linux container. If you are running Docker for Windows, you need to ensure the Linux Engine is running\n", + "\n", + " powershell command to switch to linux engine\n", + " & 'C:\\Program Files\\Docker\\Docker\\DockerCli.exe' -SwitchLinuxEngine\n", + "\n", + "and c drive is shared https://docs.docker.com/docker-for-windows/#shared-drives\n", + "sometimes you have to reshare c drive as docker \n", + "\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.webservice import LocalWebservice\n", + "\n", + "#this is optional, if not provided we choose random port\n", + "deployment_config = LocalWebservice.deploy_configuration(port=6789)\n", + "\n", + "local_service = Model.deploy(ws, \"test\", [model], inference_config, deployment_config)\n", + "\n", + "local_service.wait_for_deployment()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('Local service port: {}'.format(local_service.port))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check Status and Get Container Logs\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(local_service.get_logs())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test Web Service" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Call the web service with some input data to get a prediction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "sample_input = json.dumps({\n", + " 'data': [\n", + " [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],\n", + " [10, 9, 8, 7, 6, 5, 4, 3, 2, 1]\n", + " ]\n", + "})\n", + "\n", + "sample_input = bytes(sample_input, encoding='utf-8')\n", + "\n", + "print(local_service.run(input_data=sample_input))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reload Service\n", + "\n", + "You can update your score.py file and then call `reload()` to quickly restart the service. This will only reload your execution script and dependency files, it will not rebuild the underlying Docker image. As a result, `reload()` is fast, but if you do need to rebuild the image -- to add a new Conda or pip package, for instance -- you will have to call `update()`, instead (see below)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile score.py\n", + "import pickle\n", + "import json\n", + "import numpy as np\n", + "from sklearn.externals import joblib\n", + "from sklearn.linear_model import Ridge\n", + "from azureml.core.model import Model\n", + "\n", + "from inference_schema.schema_decorators import input_schema, output_schema\n", + "from inference_schema.parameter_types.numpy_parameter_type import NumpyParameterType\n", + "\n", + "def init():\n", + " global model\n", + " # note here \"sklearn_regression_model.pkl\" is the name of the model registered under\n", + " # this is a different behavior than before when the code is run locally, even though the code is the same.\n", + " model_path = Model.get_model_path('sklearn_regression_model.pkl')\n", + " # deserialize the model file back into a sklearn model\n", + " model = joblib.load(model_path)\n", + "\n", + "input_sample = np.array([[10,9,8,7,6,5,4,3,2,1]])\n", + "output_sample = np.array([3726.995])\n", + "\n", + "@input_schema('data', NumpyParameterType(input_sample))\n", + "@output_schema(NumpyParameterType(output_sample))\n", + "def run(data):\n", + " try:\n", + " result = model.predict(data)\n", + " # you can return any datatype as long as it is JSON-serializable\n", + " return 'hello from updated score.py'\n", + " except Exception as e:\n", + " error = str(e)\n", + " return error" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_service.reload()\n", + "print(\"--------------------------------------------------------------\")\n", + "\n", + "# after reload now if you call run this will return updated return message\n", + "\n", + "print(local_service.run(input_data=sample_input))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Update Service\n", + "\n", + "If you want to change your model(s), Conda dependencies, or deployment configuration, call `update()` to rebuild the Docker image.\n", + "\n", + "```python\n", + "\n", + "local_service.update(models = [SomeOtherModelObject],\n", + " deployment_config = local_config,\n", + " inference_config = inference_config)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Delete Service" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_service.delete()" + ] + } + ], + "metadata": { + "authors": [ + { + "name": "raymondl" + } + ], + "kernelspec": { + "display_name": "Python 3.6", + "language": "python", + "name": "python36" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/how-to-use-azureml/deploy-to-local/score.py b/how-to-use-azureml/deploy-to-local/score.py new file mode 100644 index 000000000..0086d27bc --- /dev/null +++ b/how-to-use-azureml/deploy-to-local/score.py @@ -0,0 +1,34 @@ +import pickle +import json +import numpy as np +from sklearn.externals import joblib +from sklearn.linear_model import Ridge +from azureml.core.model import Model + +from inference_schema.schema_decorators import input_schema, output_schema +from inference_schema.parameter_types.numpy_parameter_type import NumpyParameterType + + +def init(): + global model + # note here "sklearn_regression_model.pkl" is the name of the model registered under + # this is a different behavior than before when the code is run locally, even though the code is the same. + model_path = Model.get_model_path('sklearn_regression_model.pkl') + # deserialize the model file back into a sklearn model + model = joblib.load(model_path) + + +input_sample = np.array([[10, 9, 8, 7, 6, 5, 4, 3, 2, 1]]) +output_sample = np.array([3726.995]) + + +@input_schema('data', NumpyParameterType(input_sample)) +@output_schema(NumpyParameterType(output_sample)) +def run(data): + try: + result = model.predict(data) + # you can return any datatype as long as it is JSON-serializable + return result.tolist() + except Exception as e: + error = str(e) + return error diff --git a/how-to-use-azureml/deploy-to-local/sklearn_regression_model.pkl b/how-to-use-azureml/deploy-to-local/sklearn_regression_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d10309b6cf4c8e87846850edfe8b8d54a1aa64f2 GIT binary patch literal 658 zcmX|<&rcLF6vt=T1%{DT5XE00;?FfP6EJauKjMzrgOLdnGx0~`rRmHRcC<6?n;*&= zN%R0L{srpQlL>dP>%oH_y>T~@%bJ)Nz2J{-A-uzD`+a?Vub=jL(N7;SN|M-QVJt@+ z@qjWj34;Y{xXOce{sk14pr?X*HBBQ-Gzb*^IFCfr^m#(fC}&wnl7uvk)F+H229&nr zMvyfHHJ}&u$kh26=(9DuunPSy=oPz&3R1lW1CHa&{*$Jhtz}?%b^Xoju5Hv{&k78> zP)6nM5n+bIIHQSAMFx9YXh4cFPa?v?rxf!-0Au_Kjv^vpvXy(kXKN2W-78z0 z>vNZ`Pm(rkKN~H7ZCY%7rZV=8Sr`9mTen_aZBJFl-Q~M?+P^-#owpkc?F;ts_YsV( zD-%xj=Voq+)eoJuDzZ?&tPv}u7*0B>=1.1.0,<1.2.0\"\n", + "pip install \"azureml-dataprep[pandas]>=1.1.2,<1.2.0\"\n", "```\n", "\n", "Import the SDK."