diff --git a/NBSETUP.md b/NBSETUP.md index e24c8852b..8cebb5f8d 100644 --- a/NBSETUP.md +++ b/NBSETUP.md @@ -1,11 +1,10 @@ -# Setting up environment +# Notebook setup --- -To run the notebooks in this repository use one of the two options. +To run the notebooks in this repository use one of these methods: -## Option 1: Use Azure Notebooks -Azure Notebooks is a hosted Jupyter-based notebook service in the Azure cloud. Azure Machine Learning Python SDK is already pre-installed in the Azure Notebooks `Python 3.6` kernel. +## Use Azure Notebooks - Jupyter based notebooks in the Azure cloud 1. [![Azure Notebooks](https://notebooks.azure.com/launch.png)](https://aka.ms/aml-clone-azure-notebooks) [Import sample notebooks ](https://aka.ms/aml-clone-azure-notebooks) into Azure Notebooks @@ -16,40 +15,20 @@ Azure Notebooks is a hosted Jupyter-based notebook service in the Azure cloud. A ![set kernel to Python 3.6](images/python36.png) -## **Option 2: Use your own notebook server** +## **Use your own notebook server** -### Quick installation -We recommend you create a Python virtual environment ([Miniconda](https://conda.io/miniconda.html) preferred but [virtualenv](https://virtualenv.pypa.io/en/latest/) works too) and install the SDK in it. -```sh -# install just the base SDK -pip install azureml-sdk - -# clone the sample repoistory -git clone https://github.com/Azure/MachineLearningNotebooks.git - -# below steps are optional -# install the base SDK and a Jupyter notebook server -pip install azureml-sdk[notebooks] - -# install the data prep component -pip install azureml-dataprep - -# install model explainability component -pip install azureml-sdk[explain] - -# install automated ml components -pip install azureml-sdk[automl] - -# install experimental features (not ready for production use) -pip install azureml-sdk[contrib] -``` - -### Full instructions -[Install the Azure Machine Learning SDK](https://docs.microsoft.com/en-us/azure/machine-learning/service/quickstart-create-workspace-with-python) - -Please make sure you start with the [Configuration](configuration.ipynb) notebook to create and connect to a workspace. +Video walkthrough: +[![Get Started video](images/yt_cover.png)](https://youtu.be/VIsXeTuW3FU) -### Video walkthrough: +1. Setup a Jupyter Notebook server and [install the Azure Machine Learning SDK](https://docs.microsoft.com/en-us/azure/machine-learning/service/quickstart-create-workspace-with-python) +1. Clone [this repository](https://aka.ms/aml-notebooks) +1. You may need to install other packages for specific notebook + - For example, to run the Azure Machine Learning Data Prep notebooks, install the extra dataprep SDK: + ```bash + pip install azureml-dataprep + ``` -[![Get Started video](images/yt_cover.png)](https://youtu.be/VIsXeTuW3FU) +1. Start your notebook server +1. Follow the instructions in the [Configuration](configuration.ipynb) notebook to create and connect to a workspace +1. Open one of the sample notebooks diff --git a/README.md b/README.md index 9985d584c..ffc3c474e 100644 --- a/README.md +++ b/README.md @@ -55,5 +55,4 @@ The [How to use Azure ML](./how-to-use-azureml) folder contains specific example Visit following repos to see projects contributed by Azure ML users: - [Fine tune natural language processing models using Azure Machine Learning service](https://github.com/Microsoft/AzureML-BERT) - - [Fashion MNIST with Azure ML SDK](https://github.com/amynic/azureml-sdk-fashion) - + - [Fashion MNIST with Azure ML SDK](https://github.com/amynic/azureml-sdk-fashion) \ No newline at end of file diff --git a/configuration.ipynb b/configuration.ipynb index ab287a259..6fc45686c 100644 --- a/configuration.ipynb +++ b/configuration.ipynb @@ -96,7 +96,7 @@ "source": [ "import azureml.core\n", "\n", - "print(\"This notebook was created using version 1.0.6 of the Azure ML SDK\")\n", + "print(\"This notebook was created using version 1.0.10 of the Azure ML SDK\")\n", "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")" ] }, @@ -373,4 +373,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/how-to-use-azureml/automated-machine-learning/automl_setup.cmd b/how-to-use-azureml/automated-machine-learning/automl_setup.cmd index 0f2478eaa..2ef804201 100644 --- a/how-to-use-azureml/automated-machine-learning/automl_setup.cmd +++ b/how-to-use-azureml/automated-machine-learning/automl_setup.cmd @@ -1,6 +1,7 @@ @echo off set conda_env_name=%1 set automl_env_file=%2 +set options=%3 set PIP_NO_WARN_SCRIPT_LOCATION=0 IF "%conda_env_name%"=="" SET conda_env_name="azure_automl" @@ -32,10 +33,12 @@ echo. echo *************************************** echo * AutoML setup completed successfully * echo *************************************** -echo. -echo Starting jupyter notebook - please run the configuration notebook -echo. -jupyter notebook --log-level=50 --notebook-dir='..\..' +IF NOT "%options%"=="nolaunch" ( + echo. + echo Starting jupyter notebook - please run the configuration notebook + echo. + jupyter notebook --log-level=50 --notebook-dir='..\..' +) goto End diff --git a/how-to-use-azureml/automated-machine-learning/automl_setup_linux.sh b/how-to-use-azureml/automated-machine-learning/automl_setup_linux.sh index 2f2e96cc2..db8a357c6 100644 --- a/how-to-use-azureml/automated-machine-learning/automl_setup_linux.sh +++ b/how-to-use-azureml/automated-machine-learning/automl_setup_linux.sh @@ -2,6 +2,7 @@ CONDA_ENV_NAME=$1 AUTOML_ENV_FILE=$2 +OPTIONS=$3 PIP_NO_WARN_SCRIPT_LOCATION=0 if [ "$CONDA_ENV_NAME" == "" ] @@ -34,10 +35,13 @@ else echo "***************************************" && echo "* AutoML setup completed successfully *" && echo "***************************************" && - echo "" && - echo "Starting jupyter notebook - please run the configuration notebook" && - echo "" && - jupyter notebook --log-level=50 --notebook-dir '../..' + if [ "$OPTIONS" != "nolaunch" ] + then + echo "" && + echo "Starting jupyter notebook - please run the configuration notebook" && + echo "" && + jupyter notebook --log-level=50 --notebook-dir '../..' + fi fi if [ $? -gt 0 ] diff --git a/how-to-use-azureml/automated-machine-learning/automl_setup_mac.sh b/how-to-use-azureml/automated-machine-learning/automl_setup_mac.sh index 298fc67db..84a45e155 100644 --- a/how-to-use-azureml/automated-machine-learning/automl_setup_mac.sh +++ b/how-to-use-azureml/automated-machine-learning/automl_setup_mac.sh @@ -2,6 +2,7 @@ CONDA_ENV_NAME=$1 AUTOML_ENV_FILE=$2 +OPTIONS=$3 PIP_NO_WARN_SCRIPT_LOCATION=0 if [ "$CONDA_ENV_NAME" == "" ] @@ -36,10 +37,13 @@ else echo "***************************************" && echo "* AutoML setup completed successfully *" && echo "***************************************" && - echo "" && - echo "Starting jupyter notebook - please run the configuration notebook" && - echo "" && - jupyter notebook --log-level=50 --notebook-dir '../..' + if [ "$OPTIONS" != "nolaunch" ] + then + echo "" && + echo "Starting jupyter notebook - please run the configuration notebook" && + echo "" && + jupyter notebook --log-level=50 --notebook-dir '../..' + fi fi if [ $? -gt 0 ] diff --git a/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb b/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb index 72bc6dc24..1688b9821 100644 --- a/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb +++ b/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb @@ -44,9 +44,31 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Setup\n", - "\n", - "As part of the setup you have already created a Workspace. For AutoML you would need to create an Experiment. An Experiment is a named object in a Workspace, which is used to run experiments." + "## Setup\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use the *forecasting* task in AutoML, you need to have the **azuremlftk** package installed in your environment. The following cell tests whether this package is installed locally and, if not, gives you instructions for installing it. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " import ftk\n", + " print('Using FTK version ' + ftk.__version__)\n", + "except ImportError:\n", + " print(\"Unable to import forecasting package. This notebook does not work without this package.\\n\"\n", + " + \"Please open a command prompt and run `pip install azuremlftk` to install the package. \\n\"\n", + " + \"Make sure you install the package into AutoML's Python environment.\\n\\n\"\n", + " + \"For instance, if AutoML is installed in a conda environment called `python36`, run:\\n\"\n", + " + \"> activate python36\\n> pip install azuremlftk\")" ] }, { @@ -71,6 +93,13 @@ "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As part of the setup you have already created a Workspace. For AutoML you would need to create an Experiment. An Experiment is a named object in a Workspace, which is used to run experiments." + ] + }, { "cell_type": "code", "execution_count": null, @@ -368,7 +397,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb b/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb index 45cdc2912..29912d06b 100644 --- a/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb +++ b/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb @@ -45,9 +45,31 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Setup\n", - "\n", - "As part of the setup you have already created a Workspace. To run AutoML, you also need to create an Experiment. An Experiment is a named object in a Workspace which represents a predictive task, the output of which is a trained model and a set of evaluation metrics for the model. " + "## Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use the *forecasting* task in AutoML, you need to have the **azuremlftk** package installed in your environment. The following cell tests whether this package is installed locally and, if not, gives you instructions for installing it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " import ftk\n", + " print('Using FTK version ' + ftk.__version__)\n", + "except ImportError:\n", + " print(\"Unable to import forecasting package. This notebook does not work without this package.\\n\"\n", + " + \"Please open a command prompt and run `pip install azuremlftk` to install the package. \\n\"\n", + " + \"Make sure you install the package into AutoML's Python environment.\\n\\n\"\n", + " + \"For instance, if AutoML is installed in a conda environment called `python36`, run:\\n\"\n", + " + \"> activate python36\\n> pip install azuremlftk\")" ] }, { @@ -71,6 +93,13 @@ "from sklearn.metrics import mean_absolute_error, mean_squared_error" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As part of the setup you have already created a Workspace. To run AutoML, you also need to create an Experiment. An Experiment is a named object in a Workspace which represents a predictive task, the output of which is a trained model and a set of evaluation metrics for the model. " + ] + }, { "cell_type": "code", "execution_count": null, @@ -404,7 +433,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/how-to-use-azureml/automated-machine-learning/remote-execution/auto-ml-remote-execution.ipynb b/how-to-use-azureml/automated-machine-learning/remote-execution/auto-ml-remote-execution.ipynb index b6f4ceb56..5dca304a2 100644 --- a/how-to-use-azureml/automated-machine-learning/remote-execution/auto-ml-remote-execution.ipynb +++ b/how-to-use-azureml/automated-machine-learning/remote-execution/auto-ml-remote-execution.ipynb @@ -150,7 +150,7 @@ " dsvm_compute = DsvmCompute.create(ws, name = dsvm_name, provisioning_configuration = dsvm_config)\n", " dsvm_compute.wait_for_completion(show_output = True)\n", " print(\"Waiting one minute for ssh to be accessible\")\n", - " time.sleep(60) # Wait for ssh to be accessible" + " time.sleep(90) # Wait for ssh to be accessible" ] }, { diff --git a/how-to-use-azureml/azure-databricks/README.md b/how-to-use-azureml/azure-databricks/README.md index 2a48316c4..a545ad0a4 100644 --- a/how-to-use-azureml/azure-databricks/README.md +++ b/how-to-use-azureml/azure-databricks/README.md @@ -10,52 +10,7 @@ In this section, you will find sample notebooks on how to use Azure Machine Lear - Every run (including the best run) is available as a pipeline, which you can tune further if needed. - The model trained using Azure Databricks can be registered in Azure ML SDK workspace and then deployed to Azure managed compute (ACI or AKS) using the Azure Machine learning SDK. - -**Create Azure Databricks Cluster:** - -Select New Cluster and fill in following detail: - - Cluster name: _yourclustername_ - - Databricks Runtime: Any **non ML** runtime (non ML 4.x, 5.x) - - Python version: **3** - - Workers: 2 or higher. - -These settings are only for using Automated Machine Learning on Databricks. - - Max. number of **concurrent iterations** in Automated ML settings is **<=** to the number of **worker nodes** in your Databricks cluster. - - Worker node VM types: **Memory optimized VM** preferred. - - Uncheck _Enable Autoscaling_ - - -It will take few minutes to create the cluster. Please ensure that the cluster state is running before proceeding further. - -**Install Azure ML SDK without Automated ML capability on your Azure Databricks cluster** - -- Select Import library - -- Source: Upload Python Egg or PyPI - -- PyPi Name: **azureml-sdk[databricks]** - -**Install Azure ML with Automated ML SDK on your Azure Databricks cluster** - -- Select Import library - -- Source: Upload Python Egg or PyPI - -- PyPi Name: **azureml-sdk[automl_databricks]** - -**For installation with or without Automated ML** - -- Click Install Library - -- Do not select _Attach automatically to all clusters_. In case you selected this earlier, please go to your Home folder and deselect it. - -- Select the check box _Attach_ next to your cluster name - -(More details on how to attach and detach libs are here - [https://docs.databricks.com/user-guide/libraries.html#attach-a-library-to-a-cluster](https://docs.databricks.com/user-guide/libraries.html#attach-a-library-to-a-cluster) ) - -- Ensure that there are no errors until Status changes to _Attached_. It may take a couple of minutes. - -**Note** - If you have an old SDK version, please deselect it from cluster’s installed libs > move to trash. Install the new SDK verdion and restart the cluster. If there is an issue after this, please detach and reattach your cluster. +Please follow our [Azure doc](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-environment#azure-databricks) to install the sdk in your Azure Databricks cluster before trying any of the sample notebooks. **Single file** - The following archive contains all the sample notebooks. You can the run notebooks after importing [DBC](Databricks_AMLSDK_1-4_6.dbc) in your Databricks workspace instead of downloading individually. diff --git a/how-to-use-azureml/deployment/enable-app-insights-in-production-service/enable-app-insights-in-production-service.ipynb b/how-to-use-azureml/deployment/enable-app-insights-in-production-service/enable-app-insights-in-production-service.ipynb index fb35d6546..48108ff67 100644 --- a/how-to-use-azureml/deployment/enable-app-insights-in-production-service/enable-app-insights-in-production-service.ipynb +++ b/how-to-use-azureml/deployment/enable-app-insights-in-production-service/enable-app-insights-in-production-service.ipynb @@ -38,13 +38,11 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.core import Workspace, Run\n", + "from azureml.core import Workspace\n", "from azureml.core.compute import AksCompute, ComputeTarget\n", - "from azureml.core.webservice import Webservice, AksWebservice\n", - "from azureml.core.image import Image\n", - "from azureml.core.model import Model\n", - "\n", + "from azureml.core.webservice import AksWebservice\n", "import azureml.core\n", + "import json\n", "print(azureml.core.VERSION)" ] }, @@ -247,7 +245,6 @@ "outputs": [], "source": [ "%%time\n", - "import json\n", "\n", "test_sample = json.dumps({'data': [\n", " [1,28,13,45,54,6,57,8,8,10], \n", @@ -401,7 +398,6 @@ "outputs": [], "source": [ "%%time\n", - "import json\n", "\n", "test_sample = json.dumps({'data': [\n", " [1,28,13,45,54,6,57,8,8,10], \n", @@ -413,7 +409,7 @@ " prediction = aks_service.run(input_data=test_sample)\n", " print(prediction)\n", "else:\n", - " raise ValueError(\"Service deployment isn't healthy, can't call the service\")" + " raise ValueError(\"Service deployment isn't healthy, can't call the service\")" ] }, { @@ -469,7 +465,7 @@ "metadata": { "authors": [ { - "name": "marthalc" + "name": "jocier" } ], "kernelspec": { @@ -487,7 +483,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.3" } }, "nbformat": 4, diff --git a/how-to-use-azureml/deployment/enable-data-collection-for-models-in-aks/enable-data-collection-for-models-in-aks.ipynb b/how-to-use-azureml/deployment/enable-data-collection-for-models-in-aks/enable-data-collection-for-models-in-aks.ipynb index 3a076523f..30a106d1c 100644 --- a/how-to-use-azureml/deployment/enable-data-collection-for-models-in-aks/enable-data-collection-for-models-in-aks.ipynb +++ b/how-to-use-azureml/deployment/enable-data-collection-for-models-in-aks/enable-data-collection-for-models-in-aks.ipynb @@ -37,12 +37,9 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.core import Workspace, Run\n", + "from azureml.core import Workspace\n", "from azureml.core.compute import AksCompute, ComputeTarget\n", "from azureml.core.webservice import Webservice, AksWebservice\n", - "from azureml.core.image import Image\n", - "from azureml.core.model import Model\n", - "\n", "import azureml.core\n", "print(azureml.core.VERSION)" ] @@ -51,8 +48,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Set up your configuration and create a workspace\n", - "Follow Notebook 00 instructions to do this.\n" + "## 2. Set up your configuration and create a workspace" ] }, { @@ -277,9 +273,7 @@ }, { "cell_type": "markdown", - "metadata": { - "scrolled": true - }, + "metadata": {}, "source": [ "```python \n", " %%time\n", @@ -451,7 +445,7 @@ "metadata": { "authors": [ { - "name": "marthalc" + "name": "jocier" } ], "kernelspec": { @@ -469,7 +463,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.3" } }, "nbformat": 4, diff --git a/how-to-use-azureml/deployment/onnx/onnx-convert-aml-deploy-tinyyolo.ipynb b/how-to-use-azureml/deployment/onnx/onnx-convert-aml-deploy-tinyyolo.ipynb index 4603e8ad4..6cc710683 100644 --- a/how-to-use-azureml/deployment/onnx/onnx-convert-aml-deploy-tinyyolo.ipynb +++ b/how-to-use-azureml/deployment/onnx/onnx-convert-aml-deploy-tinyyolo.ipynb @@ -409,7 +409,7 @@ "metadata": { "authors": [ { - "name": "onnx" + "name": "viswamy" } ], "kernelspec": { @@ -427,7 +427,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.6" + "version": "3.6.5" } }, "nbformat": 4, diff --git a/how-to-use-azureml/deployment/onnx/onnx-inference-facial-expression-recognition-deploy.ipynb b/how-to-use-azureml/deployment/onnx/onnx-inference-facial-expression-recognition-deploy.ipynb index 69936f1fe..fc6377b0c 100644 --- a/how-to-use-azureml/deployment/onnx/onnx-inference-facial-expression-recognition-deploy.ipynb +++ b/how-to-use-azureml/deployment/onnx/onnx-inference-facial-expression-recognition-deploy.ipynb @@ -197,7 +197,6 @@ "source": [ "# for images and plots in this notebook\n", "import matplotlib.pyplot as plt \n", - "from IPython.display import Image\n", "\n", "# display images inline\n", "%matplotlib inline" @@ -481,8 +480,8 @@ " \n", " emotion_keys = list(emotion_table.keys())\n", " emotions = []\n", - " for i in range(N):\n", - " emotions.append(emotion_keys[classes[i]])\n", + " for c in range(N):\n", + " emotions.append(emotion_keys[classes[c]])\n", " return emotions\n", "\n", "def softmax(x):\n", @@ -534,9 +533,9 @@ "# read in 3 testing images from .pb files\n", "test_data_size = 3\n", "\n", - "for i in np.arange(test_data_size):\n", - " input_test_data = os.path.join(model_dir, 'test_data_set_{0}'.format(i), 'input_0.pb')\n", - " output_test_data = os.path.join(model_dir, 'test_data_set_{0}'.format(i), 'output_0.pb')\n", + "for num in np.arange(test_data_size):\n", + " input_test_data = os.path.join(model_dir, 'test_data_set_{0}'.format(num), 'input_0.pb')\n", + " output_test_data = os.path.join(model_dir, 'test_data_set_{0}'.format(num), 'output_0.pb')\n", " \n", " # convert protobuf tensors to np arrays using the TensorProto reader from ONNX\n", " tensor = onnx.TensorProto()\n", @@ -671,19 +670,19 @@ " \"\"\"Convert the input image into grayscale\"\"\"\n", " return np.dot(rgb[...,:3], [0.299, 0.587, 0.114])\n", "\n", - "def resize_img(img):\n", - " \"\"\"Resize image to MNIST model input dimensions\"\"\"\n", - " img = cv2.resize(img, dsize=(64, 64), interpolation=cv2.INTER_AREA)\n", - " img.resize((1, 1, 64, 64))\n", - " return img\n", + "def resize_img(img_to_resize):\n", + " \"\"\"Resize image to FER+ model input dimensions\"\"\"\n", + " r_img = cv2.resize(img_to_resize, dsize=(64, 64), interpolation=cv2.INTER_AREA)\n", + " r_img.resize((1, 1, 64, 64))\n", + " return r_img\n", "\n", - "def preprocess(img):\n", + "def preprocess(img_to_preprocess):\n", " \"\"\"Resize input images and convert them to grayscale.\"\"\"\n", - " if img.shape == (64, 64):\n", - " img.resize((1, 1, 64, 64))\n", - " return img\n", + " if img_to_preprocess.shape == (64, 64):\n", + " img_to_preprocess.resize((1, 1, 64, 64))\n", + " return img_to_preprocess\n", " \n", - " grayscale = rgb2gray(img)\n", + " grayscale = rgb2gray(img_to_preprocess)\n", " processed_img = resize_img(grayscale)\n", " return processed_img" ] @@ -732,7 +731,7 @@ " r = json.loads(aci_service.run(input_data))\n", " result = r['result'][0]\n", " time_ms = np.round(r['time_in_sec'][0] * 1000, 2)\n", - " except Exception as e:\n", + " except KeyError as e:\n", " print(str(e))\n", "\n", " plt.figure(figsize = (16, 6))\n", @@ -800,7 +799,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.6.5" }, "msauthor": "vinitra.swamy" }, diff --git a/how-to-use-azureml/deployment/onnx/onnx-inference-mnist-deploy.ipynb b/how-to-use-azureml/deployment/onnx/onnx-inference-mnist-deploy.ipynb index 68e7f80d7..6bd83c2fa 100644 --- a/how-to-use-azureml/deployment/onnx/onnx-inference-mnist-deploy.ipynb +++ b/how-to-use-azureml/deployment/onnx/onnx-inference-mnist-deploy.ipynb @@ -621,19 +621,19 @@ " \"\"\"Convert the input image into grayscale\"\"\"\n", " return np.dot(rgb[...,:3], [0.299, 0.587, 0.114])\n", "\n", - "def resize_img(img):\n", + "def resize_img(img_to_resize):\n", " \"\"\"Resize image to MNIST model input dimensions\"\"\"\n", - " img = cv2.resize(img, dsize=(28, 28), interpolation=cv2.INTER_AREA)\n", - " img.resize((1, 1, 28, 28))\n", - " return img\n", + " r_img = cv2.resize(img_to_resize, dsize=(28, 28), interpolation=cv2.INTER_AREA)\n", + " r_img.resize((1, 1, 28, 28))\n", + " return r_img\n", "\n", - "def preprocess(img):\n", + "def preprocess(img_to_preprocess):\n", " \"\"\"Resize input images and convert them to grayscale.\"\"\"\n", - " if img.shape == (28, 28):\n", - " img.resize((1, 1, 28, 28))\n", - " return img\n", + " if img_to_preprocess.shape == (28, 28):\n", + " img_to_preprocess.resize((1, 1, 28, 28))\n", + " return img_to_preprocess\n", " \n", - " grayscale = rgb2gray(img)\n", + " grayscale = rgb2gray(img_to_preprocess)\n", " processed_img = resize_img(grayscale)\n", " return processed_img" ] @@ -681,7 +681,7 @@ " r = aci_service.run(input_data)\n", " result = r['result']\n", " time_ms = np.round(r['time_in_sec'] * 1000, 2)\n", - " except Exception as e:\n", + " except KeyError as e:\n", " print(str(e))\n", "\n", " plt.figure(figsize = (16, 6))\n", diff --git a/how-to-use-azureml/deployment/onnx/onnx-modelzoo-aml-deploy-resnet50.ipynb b/how-to-use-azureml/deployment/onnx/onnx-modelzoo-aml-deploy-resnet50.ipynb index f1c5858ac..099948272 100644 --- a/how-to-use-azureml/deployment/onnx/onnx-modelzoo-aml-deploy-resnet50.ipynb +++ b/how-to-use-azureml/deployment/onnx/onnx-modelzoo-aml-deploy-resnet50.ipynb @@ -393,7 +393,7 @@ "metadata": { "authors": [ { - "name": "onnx" + "name": "viswamy" } ], "kernelspec": { @@ -411,7 +411,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.6" + "version": "3.6.5" } }, "nbformat": 4, diff --git a/how-to-use-azureml/deployment/register-model-create-image-deploy-service/register-model-create-image-deploy-service.ipynb b/how-to-use-azureml/deployment/register-model-create-image-deploy-service/register-model-create-image-deploy-service.ipynb index 0afe9a3a7..e1bebf46f 100644 --- a/how-to-use-azureml/deployment/register-model-create-image-deploy-service/register-model-create-image-deploy-service.ipynb +++ b/how-to-use-azureml/deployment/register-model-create-image-deploy-service/register-model-create-image-deploy-service.ipynb @@ -241,7 +241,8 @@ " description = \"Image with ridge regression model\")\n", "\n", "image = Image.create(name = \"myimage1\",\n", - " # this is the model object \n", + " # this is the model object. note you can pass in 0-n models via this list-type parameter\n", + " # in case you need to reference multiple models, or none at all, in your scoring script.\n", " models = [model],\n", " image_config = image_config, \n", " workspace = ws)" diff --git a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-data-transfer.ipynb b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-data-transfer.ipynb index 640e879dc..2a690382c 100644 --- a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-data-transfer.ipynb +++ b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-data-transfer.ipynb @@ -35,16 +35,12 @@ "source": [ "import os\n", "import azureml.core\n", - "from azureml.core.compute import ComputeTarget, DatabricksCompute, DataFactoryCompute\n", + "from azureml.core.compute import ComputeTarget, DataFactoryCompute\n", "from azureml.exceptions import ComputeTargetException\n", - "from azureml.core import Workspace, Run, Experiment\n", - "from azureml.pipeline.core import Pipeline, PipelineData\n", - "from azureml.pipeline.steps import AdlaStep\n", + "from azureml.core import Workspace, Experiment\n", + "from azureml.pipeline.core import Pipeline\n", "from azureml.core.datastore import Datastore\n", "from azureml.data.data_reference import DataReference\n", - "from azureml.data.sql_data_reference import SqlDataReference\n", - "from azureml.core import attach_legacy_compute_target\n", - "from azureml.data.stored_procedure_parameter import StoredProcedureParameter, StoredProcedureParameterType\n", "from azureml.pipeline.steps import DataTransferStep\n", "\n", "# Check core SDK version number\n", @@ -98,8 +94,8 @@ "metadata": {}, "outputs": [], "source": [ + "from msrest.exceptions import HttpOperationError\n", "\n", - "workspace = ws.name\n", "datastore_name='MyAdlsDatastore'\n", "subscription_id=os.getenv(\"ADL_SUBSCRIPTION_62\", \"\") # subscription id of ADLS account\n", "resource_group=os.getenv(\"ADL_RESOURCE_GROUP_62\", \"\") # resource group of ADLS account\n", @@ -111,7 +107,7 @@ "try:\n", " adls_datastore = Datastore.get(ws, datastore_name)\n", " print(\"found datastore with name: %s\" % datastore_name)\n", - "except:\n", + "except HttpOperationError:\n", " adls_datastore = Datastore.register_azure_data_lake(\n", " workspace=ws,\n", " datastore_name=datastore_name,\n", @@ -133,7 +129,7 @@ "try:\n", " blob_datastore = Datastore.get(ws, blob_datastore_name)\n", " print(\"found blob datastore with name: %s\" % blob_datastore_name)\n", - "except:\n", + "except HttpOperationError:\n", " blob_datastore = Datastore.register_azure_blob_container(\n", " workspace=ws,\n", " datastore_name=blob_datastore_name,\n", diff --git a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-getting-started.ipynb b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-getting-started.ipynb index 4bb7fd688..05002f2eb 100644 --- a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-getting-started.ipynb +++ b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-getting-started.ipynb @@ -46,10 +46,9 @@ "outputs": [], "source": [ "import azureml.core\n", - "from azureml.core import Workspace, Run, Experiment, Datastore\n", + "from azureml.core import Workspace, Experiment, Datastore\n", "from azureml.core.compute import AmlCompute\n", "from azureml.core.compute import ComputeTarget\n", - "from azureml.core.compute import DataFactoryCompute\n", "from azureml.widgets import RunDetails\n", "\n", "# Check core SDK version number\n", @@ -71,12 +70,8 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.data.data_reference import DataReference\n", - "from azureml.pipeline.core import Pipeline, PipelineData, StepSequence\n", + "from azureml.pipeline.core import Pipeline\n", "from azureml.pipeline.steps import PythonScriptStep\n", - "from azureml.pipeline.steps import DataTransferStep\n", - "from azureml.pipeline.core import PublishedPipeline\n", - "from azureml.pipeline.core.graph import PipelineParameter\n", "\n", "print(\"Pipeline SDK-specific imports completed\")" ] @@ -237,12 +232,13 @@ "metadata": {}, "outputs": [], "source": [ + "from azureml.core.compute_target import ComputeTargetException\n", "\n", "aml_compute_target = \"aml-compute\"\n", "try:\n", " aml_compute = AmlCompute(ws, aml_compute_target)\n", " print(\"found existing compute target.\")\n", - "except:\n", + "except ComputeTargetException:\n", " print(\"creating new compute target\")\n", " \n", " provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\",\n", diff --git a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-publish-and-run-using-rest-endpoint.ipynb b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-publish-and-run-using-rest-endpoint.ipynb index b9bba0f7d..65e8610c7 100644 --- a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-publish-and-run-using-rest-endpoint.ipynb +++ b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-publish-and-run-using-rest-endpoint.ipynb @@ -33,20 +33,16 @@ "outputs": [], "source": [ "import azureml.core\n", - "from azureml.core import Workspace, Run, Experiment, Datastore\n", + "from azureml.core import Workspace, Datastore\n", "from azureml.core.compute import AmlCompute\n", "from azureml.core.compute import ComputeTarget\n", - "from azureml.core.compute import DataFactoryCompute\n", - "from azureml.widgets import RunDetails\n", "\n", "# Check core SDK version number\n", "print(\"SDK version:\", azureml.core.VERSION)\n", "\n", "from azureml.data.data_reference import DataReference\n", - "from azureml.pipeline.core import Pipeline, PipelineData, StepSequence\n", + "from azureml.pipeline.core import Pipeline, PipelineData\n", "from azureml.pipeline.steps import PythonScriptStep\n", - "from azureml.pipeline.steps import DataTransferStep\n", - "from azureml.pipeline.core import PublishedPipeline\n", "from azureml.pipeline.core.graph import PipelineParameter\n", "\n", "print(\"Pipeline SDK-specific imports completed\")\n", @@ -79,12 +75,13 @@ "metadata": {}, "outputs": [], "source": [ + "from azureml.core.compute_target import ComputeTargetException\n", "\n", "aml_compute_target = \"aml-compute\"\n", "try:\n", " aml_compute = AmlCompute(ws, aml_compute_target)\n", " print(\"found existing compute target.\")\n", - "except:\n", + "except ComputeTargetException:\n", " print(\"creating new compute target\")\n", " \n", " provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\",\n", diff --git a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-adla-as-compute-target.ipynb b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-adla-as-compute-target.ipynb index 410664448..328c87de8 100644 --- a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-adla-as-compute-target.ipynb +++ b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-adla-as-compute-target.ipynb @@ -31,14 +31,12 @@ "source": [ "import os\n", "import azureml.core\n", - "from azureml.core.compute import ComputeTarget, DatabricksCompute\n", "from azureml.exceptions import ComputeTargetException\n", - "from azureml.core import Workspace, Run, Experiment\n", + "from azureml.core import Workspace, Experiment\n", "from azureml.pipeline.core import Pipeline, PipelineData\n", "from azureml.pipeline.steps import AdlaStep\n", "from azureml.core.datastore import Datastore\n", "from azureml.data.data_reference import DataReference\n", - "from azureml.core import attach_legacy_compute_target\n", "\n", "# Check core SDK version number\n", "print(\"SDK version:\", azureml.core.VERSION)" @@ -91,8 +89,8 @@ "metadata": {}, "outputs": [], "source": [ + "from msrest.exceptions import HttpOperationError\n", "\n", - "workspace = ws.name\n", "datastore_name='MyAdlsDatastore'\n", "subscription_id=os.getenv(\"ADL_SUBSCRIPTION_62\", \"\") # subscription id of ADLS account\n", "resource_group=os.getenv(\"ADL_RESOURCE_GROUP_62\", \"\") # resource group of ADLS account\n", @@ -104,7 +102,7 @@ "try:\n", " adls_datastore = Datastore.get(ws, datastore_name)\n", " print(\"found datastore with name: %s\" % datastore_name)\n", - "except:\n", + "except HttpOperationError:\n", " adls_datastore = Datastore.register_azure_data_lake(\n", " workspace=ws,\n", " datastore_name=datastore_name,\n", @@ -171,9 +169,9 @@ " if 'ComputeTargetNotFound' in e.message:\n", " print('adla compute not found, creating...')\n", " provisioning_config = AdlaCompute.provisioning_configuration()\n", - " adla_compute = ComputeTarget.create(workspace, compute_name, provisioning_config)\n", - " adla_compute.wait_for_completion()\n", - " return adla_compute\n", + " new_adla_compute = ComputeTarget.create(workspace, compute_name, provisioning_config)\n", + " new_adla_compute.wait_for_completion()\n", + " return new_adla_compute\n", " else:\n", " raise e\n", " \n", @@ -284,7 +282,7 @@ " steps=[adla_step],\n", " default_source_directory=script_folder)\n", "\n", - "pipeline_run = Experiment(workspace, experiment_name).submit(pipeline)\n", + "pipeline_run = Experiment(ws, experiment_name).submit(pipeline)\n", "pipeline_run.wait_for_completion()" ] }, diff --git a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-databricks-as-compute-target.ipynb b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-databricks-as-compute-target.ipynb index b08445a77..9097ecf16 100644 --- a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-databricks-as-compute-target.ipynb +++ b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-databricks-as-compute-target.ipynb @@ -89,7 +89,7 @@ "from azureml.core.runconfig import JarLibrary\n", "from azureml.core.compute import ComputeTarget, DatabricksCompute\n", "from azureml.exceptions import ComputeTargetException\n", - "from azureml.core import Workspace, Run, Experiment\n", + "from azureml.core import Workspace, Experiment\n", "from azureml.pipeline.core import Pipeline, PipelineData\n", "from azureml.pipeline.steps import DatabricksStep\n", "from azureml.core.datastore import Datastore\n", @@ -146,7 +146,7 @@ "db_access_token=os.getenv(\"DATABRICKS_ACCESS_TOKEN\", \"\") # Databricks access token\n", " \n", "try:\n", - " databricks_compute = ComputeTarget(workspace=ws, name=db_compute_name)\n", + " databricks_compute = DatabricksCompute(workspace=ws, name=db_compute_name)\n", " print('Compute target {} already exists'.format(db_compute_name))\n", "except ComputeTargetException:\n", " print('Compute not found, will use below parameters to attach new one')\n", @@ -297,7 +297,7 @@ "print('Datastore {} will be used'.format(def_blob_store.name))\n", "\n", "# We are uploading a sample file in the local directory to be used as a datasource\n", - "def_blob_store.upload_files([\"./testdata.txt\"], target_path=\"dbtest\", overwrite=False)\n", + "def_blob_store.upload_files(files=[\"./testdata.txt\"], target_path=\"dbtest\", overwrite=False)\n", "\n", "step_1_input = DataReference(datastore=def_blob_store, path_on_datastore=\"dbtest\",\n", " data_reference_name=\"input\")\n", diff --git a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-data-dependency-steps.ipynb b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-data-dependency-steps.ipynb index e2a883b92..efe28d05c 100644 --- a/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-data-dependency-steps.ipynb +++ b/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-data-dependency-steps.ipynb @@ -33,22 +33,17 @@ "outputs": [], "source": [ "import azureml.core\n", - "from azureml.core import Workspace, Run, Experiment, Datastore\n", + "from azureml.core import Workspace, Experiment, Datastore\n", "from azureml.core.compute import AmlCompute\n", "from azureml.core.compute import ComputeTarget\n", - "from azureml.core.compute import DataFactoryCompute\n", "from azureml.widgets import RunDetails\n", "\n", "# Check core SDK version number\n", "print(\"SDK version:\", azureml.core.VERSION)\n", "\n", "from azureml.data.data_reference import DataReference\n", - "from azureml.pipeline.core import Pipeline, PipelineData, StepSequence\n", + "from azureml.pipeline.core import Pipeline, PipelineData\n", "from azureml.pipeline.steps import PythonScriptStep\n", - "from azureml.pipeline.steps import DataTransferStep\n", - "from azureml.pipeline.core import PublishedPipeline\n", - "from azureml.pipeline.core.graph import PipelineParameter\n", - "\n", "print(\"Pipeline SDK-specific imports completed\")" ] }, @@ -135,12 +130,13 @@ "metadata": {}, "outputs": [], "source": [ + "from azureml.core.compute_target import ComputeTargetException\n", "\n", "aml_compute_target = \"aml-compute\"\n", "try:\n", " aml_compute = AmlCompute(ws, aml_compute_target)\n", " print(\"found existing compute target.\")\n", - "except:\n", + "except ComputeTargetException:\n", " print(\"creating new compute target\")\n", " \n", " provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\",\n", diff --git a/how-to-use-azureml/machine-learning-pipelines/pipeline-batch-scoring/pipeline-batch-scoring.ipynb b/how-to-use-azureml/machine-learning-pipelines/pipeline-batch-scoring/pipeline-batch-scoring.ipynb index 89ec7f327..d8798a4b7 100644 --- a/how-to-use-azureml/machine-learning-pipelines/pipeline-batch-scoring/pipeline-batch-scoring.ipynb +++ b/how-to-use-azureml/machine-learning-pipelines/pipeline-batch-scoring/pipeline-batch-scoring.ipynb @@ -37,10 +37,8 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.core import Datastore\n", "from azureml.core import Experiment\n", "from azureml.core.compute import AmlCompute, ComputeTarget\n", - "from azureml.core.conda_dependencies import CondaDependencies\n", "from azureml.core.datastore import Datastore\n", "from azureml.core.runconfig import CondaDependencies, RunConfiguration\n", "from azureml.data.data_reference import DataReference\n", @@ -55,7 +53,7 @@ "outputs": [], "source": [ "import os\n", - "from azureml.core import Workspace, Run, Experiment\n", + "from azureml.core import Workspace\n", "\n", "ws = Workspace.from_config()\n", "print('Workspace name: ' + ws.name, \n", @@ -166,8 +164,6 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "\n", "# choose a name for your cluster\n", "aml_compute_name = os.environ.get(\"AML_COMPUTE_NAME\", \"gpu-cluster\")\n", "cluster_min_nodes = os.environ.get(\"AML_COMPUTE_MIN_NODES\", 0)\n", @@ -513,8 +509,6 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.pipeline.core import PublishedPipeline\n", - "\n", "rest_endpoint = published_pipeline.endpoint\n", "# specify batch size when running the pipeline\n", "response = requests.post(rest_endpoint, \n", diff --git a/how-to-use-azureml/machine-learning-pipelines/pipeline-style-transfer/pipeline-style-transfer.ipynb b/how-to-use-azureml/machine-learning-pipelines/pipeline-style-transfer/pipeline-style-transfer.ipynb index 8aa6966ea..9fdec1846 100644 --- a/how-to-use-azureml/machine-learning-pipelines/pipeline-style-transfer/pipeline-style-transfer.ipynb +++ b/how-to-use-azureml/machine-learning-pipelines/pipeline-style-transfer/pipeline-style-transfer.ipynb @@ -44,7 +44,7 @@ "outputs": [], "source": [ "import os\n", - "from azureml.core import Workspace, Run, Experiment\n", + "from azureml.core import Workspace, Experiment\n", "\n", "ws = Workspace.from_config()\n", "print('Workspace name: ' + ws.name, \n", @@ -69,7 +69,8 @@ "from azureml.data.data_reference import DataReference\n", "from azureml.pipeline.core import Pipeline, PipelineData\n", "from azureml.pipeline.steps import PythonScriptStep, MpiStep\n", - "from azureml.core.runconfig import CondaDependencies, RunConfiguration" + "from azureml.core.runconfig import CondaDependencies, RunConfiguration\n", + "from azureml.core.compute_target import ComputeTargetException" ] }, { @@ -90,7 +91,7 @@ "try:\n", " cpu_cluster = AmlCompute(ws, cpu_cluster_name)\n", " print(\"found existing cluster.\")\n", - "except:\n", + "except ComputeTargetException:\n", " print(\"creating new cluster\")\n", " provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_v2\",\n", " max_nodes = 1)\n", @@ -104,7 +105,7 @@ "try:\n", " gpu_cluster = AmlCompute(ws, gpu_cluster_name)\n", " print(\"found existing cluster.\")\n", - "except:\n", + "except ComputeTargetException:\n", " print(\"creating new cluster\")\n", " provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_NC6\",\n", " max_nodes = 3)\n", @@ -526,7 +527,6 @@ " \"ParameterAssignments\": {\"style\": \"rain_princess\", \"nodecount\": 3}}) \n", "run_id = response.json()[\"Id\"]\n", "\n", - "from azureml.pipeline.core.run import PipelineRun\n", "published_pipeline_run_rain = PipelineRun(ws.experiments[\"style_transfer\"], run_id)\n", "\n", "RunDetails(published_pipeline_run_rain).show()" @@ -545,7 +545,6 @@ " \"ParameterAssignments\": {\"style\": \"udnie\", \"nodecount\": 4}}) \n", "run_id = response.json()[\"Id\"]\n", "\n", - "from azureml.pipeline.core.run import PipelineRun\n", "published_pipeline_run_udnie = PipelineRun(ws.experiments[\"style_transfer\"], run_id)\n", "\n", "RunDetails(published_pipeline_run_udnie).show()" diff --git a/how-to-use-azureml/training-with-deep-learning/README.md b/how-to-use-azureml/training-with-deep-learning/README.md index a3d235f11..da774b76e 100644 --- a/how-to-use-azureml/training-with-deep-learning/README.md +++ b/how-to-use-azureml/training-with-deep-learning/README.md @@ -1,8 +1,15 @@ ## Azure Machine Learning service training examples These examples show you: - * Distributed training of models on Machine Learning Compute cluster - * Hyperparameter tuning at scale - * Using Tensorboard with Azure ML Python SDK. + +1. [How to use the Estimator pattern in Azure ML](how-to-use-estimator) +2. [Train using TensorFlow Estimator and tune hyperparameters using Hyperdrive](train-hyperparameter-tune-deploy-with-tensorflow) +3. [Train using Pytorch Estimator and tune hyperparameters using Hyperdrive](train-hyperparameter-tune-deploy-with-pytorch) +4. [Distributed training using TensorFlow and Parameter Server](distributed-tensorflow-with-parameter-server) +5. [Distributed training using TensorFlow and Horovod](distributed-tensorflow-with-horovod) +6. [Distributed training using Pytorch and Horovod](distributed-pytorch-with-horovod) +7. [Distributed training using CNTK and custom Docker image](distributed-cntk-with-custom-docker) +8. [Export run history records to Tensorboard](export-run-history-to-tensorboard) +9. [Use TensorBoard to monitor training execution](tensorboard) Learn more about how to use `Estimator` class to [train deep neural networks with Azure Machine Learning](https://docs.microsoft.com/azure/machine-learning/service/how-to-train-ml-models). diff --git a/how-to-use-azureml/training-with-deep-learning/distributed-cntk-with-custom-docker/distributed-cntk-with-custom-docker.ipynb b/how-to-use-azureml/training-with-deep-learning/distributed-cntk-with-custom-docker/distributed-cntk-with-custom-docker.ipynb index 1d8eb6457..4a753dabf 100644 --- a/how-to-use-azureml/training-with-deep-learning/distributed-cntk-with-custom-docker/distributed-cntk-with-custom-docker.ipynb +++ b/how-to-use-azureml/training-with-deep-learning/distributed-cntk-with-custom-docker/distributed-cntk-with-custom-docker.ipynb @@ -69,7 +69,7 @@ "source": [ "## Initialize workspace\n", "\n", - "Initialize a [Workspace](https://review.docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture?branch=release-ignite-aml#workspace) object from the existing workspace you created in the Prerequisites step. `Workspace.from_config()` creates a workspace object from the details stored in `config.json`." + "Initialize a [Workspace](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#workspace) object from the existing workspace you created in the Prerequisites step. `Workspace.from_config()` creates a workspace object from the details stored in `config.json`." ] }, { @@ -81,10 +81,10 @@ "from azureml.core.workspace import Workspace\n", "\n", "ws = Workspace.from_config()\n", - "print('Workspace name: ' + ws.name, \n", + "print('Workspace name: ' + ws.name,\n", " 'Azure region: ' + ws.location, \n", " 'Subscription id: ' + ws.subscription_id, \n", - " 'Resource group: ' + ws.resource_group, sep = '\\n')" + " 'Resource group: ' + ws.resource_group, sep='\\n')" ] }, { @@ -124,7 +124,7 @@ "\n", " compute_target.wait_for_completion(show_output=True)\n", "\n", - "# use get_status() to get a detailed status for the current AmlCompute. \n", + "# use get_status() to get a detailed status for the current AmlCompute\n", "print(compute_target.get_status().serialize())" ] }, @@ -282,7 +282,7 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.train.estimator import *\n", + "from azureml.train.estimator import Estimator\n", "\n", "script_params = {\n", " '--num_epochs': 20,\n", @@ -296,7 +296,7 @@ " script_params=script_params,\n", " node_count=2,\n", " process_count_per_node=1,\n", - " distributed_backend='mpi', \n", + " distributed_backend='mpi',\n", " pip_packages=['cntk-gpu==2.6'],\n", " custom_docker_base_image='microsoft/mmlspark:gpu-0.12',\n", " use_gpu=True)" diff --git a/how-to-use-azureml/training-with-deep-learning/distributed-pytorch-with-horovod/distributed-pytorch-with-horovod.ipynb b/how-to-use-azureml/training-with-deep-learning/distributed-pytorch-with-horovod/distributed-pytorch-with-horovod.ipynb index 5581ff44e..9f135baf4 100644 --- a/how-to-use-azureml/training-with-deep-learning/distributed-pytorch-with-horovod/distributed-pytorch-with-horovod.ipynb +++ b/how-to-use-azureml/training-with-deep-learning/distributed-pytorch-with-horovod/distributed-pytorch-with-horovod.ipynb @@ -23,7 +23,7 @@ "source": [ "## Prerequisites\n", "* Go through the [Configuration](../../../configuration.ipynb) notebook to install the Azure Machine Learning Python SDK and create an Azure ML `Workspace`\n", - "* Review the [tutorial](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.ipynb) on single-node PyTorch training using Azure Machine Learning" + "* Review the [tutorial](../train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.ipynb) on single-node PyTorch training using Azure Machine Learning" ] }, { @@ -82,7 +82,7 @@ "print('Workspace name: ' + ws.name, \n", " 'Azure region: ' + ws.location, \n", " 'Subscription id: ' + ws.subscription_id, \n", - " 'Resource group: ' + ws.resource_group, sep = '\\n')" + " 'Resource group: ' + ws.resource_group, sep='\\n')" ] }, { diff --git a/how-to-use-azureml/training-with-deep-learning/distributed-pytorch-with-horovod/pytorch_horovod_mnist.py b/how-to-use-azureml/training-with-deep-learning/distributed-pytorch-with-horovod/pytorch_horovod_mnist.py index 87bffa5ef..83562526f 100644 --- a/how-to-use-azureml/training-with-deep-learning/distributed-pytorch-with-horovod/pytorch_horovod_mnist.py +++ b/how-to-use-azureml/training-with-deep-learning/distributed-pytorch-with-horovod/pytorch_horovod_mnist.py @@ -50,7 +50,7 @@ torch.cuda.manual_seed(args.seed) -kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} +kwargs = {} train_dataset = \ datasets.MNIST('data-%d' % hvd.rank(), train=True, download=True, transform=transforms.Compose([ diff --git a/how-to-use-azureml/training-with-deep-learning/distributed-tensorflow-with-horovod/distributed-tensorflow-with-horovod.ipynb b/how-to-use-azureml/training-with-deep-learning/distributed-tensorflow-with-horovod/distributed-tensorflow-with-horovod.ipynb index 8ba590f8e..c5cc399ee 100644 --- a/how-to-use-azureml/training-with-deep-learning/distributed-tensorflow-with-horovod/distributed-tensorflow-with-horovod.ipynb +++ b/how-to-use-azureml/training-with-deep-learning/distributed-tensorflow-with-horovod/distributed-tensorflow-with-horovod.ipynb @@ -26,7 +26,7 @@ "* Go through the [configuration notebook](../../../configuration.ipynb) to:\n", " * install the AML SDK\n", " * create a workspace and its configuration file (`config.json`)\n", - "* Review the [tutorial](https://aka.ms/aml-notebook-hyperdrive) on single-node TensorFlow training using the SDK" + "* Review the [tutorial](../train-hyperparameter-tune-deploy-with-tensorflow/train-hyperparameter-tune-deploy-with-tensorflow.ipynb) on single-node TensorFlow training using the SDK" ] }, { @@ -84,7 +84,7 @@ "print('Workspace name: ' + ws.name, \n", " 'Azure region: ' + ws.location, \n", " 'Subscription id: ' + ws.subscription_id, \n", - " 'Resource group: ' + ws.resource_group, sep = '\\n')" + " 'Resource group: ' + ws.resource_group, sep='\\n')" ] }, { @@ -96,7 +96,7 @@ "\n", "**Creation of AmlCompute takes approximately 5 minutes.** If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n", "\n", - "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read [this article](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-manage-quotas) on the default limits and how to request more quota." + "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read [this article](https://docs.microsoft.com/azure/machine-learning/service/how-to-manage-quotas) on the default limits and how to request more quota." ] }, { @@ -238,8 +238,6 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "\n", "project_folder = './tf-distr-hvd'\n", "os.makedirs(project_folder, exist_ok=True)" ] diff --git a/how-to-use-azureml/training-with-deep-learning/export-run-history-to-tensorboard/export-run-history-to-tensorboard.ipynb b/how-to-use-azureml/training-with-deep-learning/export-run-history-to-tensorboard/export-run-history-to-tensorboard.ipynb index 7a1cdf1a1..2911aeac7 100644 --- a/how-to-use-azureml/training-with-deep-learning/export-run-history-to-tensorboard/export-run-history-to-tensorboard.ipynb +++ b/how-to-use-azureml/training-with-deep-learning/export-run-history-to-tensorboard/export-run-history-to-tensorboard.ipynb @@ -74,14 +74,13 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.core import Workspace, Run, Experiment\n", - "\n", + "from azureml.core import Workspace, Experiment\n", "\n", "ws = Workspace.from_config()\n", "print('Workspace name: ' + ws.name, \n", " 'Azure region: ' + ws.location, \n", " 'Subscription id: ' + ws.subscription_id, \n", - " 'Resource group: ' + ws.resource_group, sep = '\\n')" + " 'Resource group: ' + ws.resource_group, sep='\\n')" ] }, { @@ -143,7 +142,7 @@ " # More data science stuff\n", " reg = Ridge(alpha=alpha)\n", " reg.fit(data[\"train\"][\"x\"], data[\"train\"][\"y\"])\n", - " # TODO save model\n", + " \n", " preds = reg.predict(data[\"test\"][\"x\"])\n", " mse = mean_squared_error(preds, data[\"test\"][\"y\"])\n", " # End train and eval\n", @@ -169,7 +168,6 @@ "# Export Run History to Tensorboard logs\n", "from azureml.contrib.tensorboard.export import export_to_tensorboard\n", "import os\n", - "import tensorflow as tf\n", "\n", "logdir = 'exportedTBlogs'\n", "log_path = os.path.join(os.getcwd(), logdir)\n", diff --git a/how-to-use-azureml/training-with-deep-learning/how-to-use-estimator/dummy_train.py b/how-to-use-azureml/training-with-deep-learning/how-to-use-estimator/dummy_train.py new file mode 100644 index 000000000..50db4e1da --- /dev/null +++ b/how-to-use-azureml/training-with-deep-learning/how-to-use-estimator/dummy_train.py @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +print("*********************************************************") +print("Hello Azure ML!") + +try: + from azureml.core import Run + run = Run.get_context() + print("Log Fibonacci numbers.") + run.log_list('Fibonacci numbers', [0, 1, 1, 2, 3, 5, 8, 13, 21, 34]) + run.complete() +except: + print("Warning: you need to install Azure ML SDK in order to log metrics.") + +print("*********************************************************") diff --git a/how-to-use-azureml/training-with-deep-learning/how-to-use-estimator/how-to-use-estimator.ipynb b/how-to-use-azureml/training-with-deep-learning/how-to-use-estimator/how-to-use-estimator.ipynb new file mode 100644 index 000000000..ba8df12aa --- /dev/null +++ b/how-to-use-azureml/training-with-deep-learning/how-to-use-estimator/how-to-use-estimator.ipynb @@ -0,0 +1,365 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nbpresent": { + "id": "bf74d2e9-2708-49b1-934b-e0ede342f475" + } + }, + "source": [ + "# How to use Estimator in Azure ML\n", + "\n", + "## Introduction\n", + "This tutorial shows how to use the Estimator pattern in Azure Machine Learning SDK. Estimator is a convenient object in Azure Machine Learning that wraps run configuration information to help simplify the tasks of specifying how a script is executed.\n", + "\n", + "\n", + "## Prerequisite:\n", + "* Understand the [architecture and terms](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture) introduced by Azure Machine Learning\n", + "* Go through the [configuration notebook](../../../configuration.ipynb) to:\n", + " * install the AML SDK\n", + " * create a workspace and its configuration file (`config.json`)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's get started. First let's import some Python libraries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "nbpresent": { + "id": "edaa7f2f-2439-4148-b57a-8c794c0945ec" + } + }, + "outputs": [], + "source": [ + "import azureml.core\n", + "from azureml.core import Workspace\n", + "\n", + "# check core SDK version number\n", + "print(\"Azure ML SDK Version: \", azureml.core.VERSION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialize workspace\n", + "Initialize a [Workspace](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#workspace) object from the existing workspace you created in the Prerequisites step. `Workspace.from_config()` creates a workspace object from the details stored in `config.json`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ws = Workspace.from_config()\n", + "print('Workspace name: ' + ws.name, \n", + " 'Azure region: ' + ws.location, \n", + " 'Subscription id: ' + ws.subscription_id, \n", + " 'Resource group: ' + ws.resource_group, sep = '\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nbpresent": { + "id": "59f52294-4a25-4c92-bab8-3b07f0f44d15" + } + }, + "source": [ + "## Create an Azure ML experiment\n", + "Let's create an experiment named \"estimator-test\". The script runs will be recorded under this experiment in Azure." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "nbpresent": { + "id": "bc70f780-c240-4779-96f3-bc5ef9a37d59" + } + }, + "outputs": [], + "source": [ + "from azureml.core import Experiment\n", + "\n", + "exp = Experiment(workspace=ws, name='estimator-test')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create or Attach existing AmlCompute\n", + "You will need to create a [compute target](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#compute-target) for training your model. In this tutorial, you create `AmlCompute` as your training compute resource." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If we could not find the cluster with the given name, then we will create a new cluster here. We will create an `AmlCompute` cluster of `STANDARD_NC6` GPU VMs. This process is broken down into 3 steps:\n", + "1. create the configuration (this step is local and only takes a second)\n", + "2. create the cluster (this step will take about **20 seconds**)\n", + "3. provision the VMs to bring the cluster to the initial size (of 1 in this case). This step will take about **3-5 minutes** and is providing only sparse output in the process. Please make sure to wait until the call returns before moving to the next cell" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.compute import ComputeTarget, AmlCompute\n", + "from azureml.core.compute_target import ComputeTargetException\n", + "\n", + "# choose a name for your cluster\n", + "cluster_name = \"cpucluster\"\n", + "\n", + "try:\n", + " cpu_cluster = ComputeTarget(workspace=ws, name=cluster_name)\n", + " print('Found existing compute target')\n", + "except ComputeTargetException:\n", + " print('Creating a new compute target...')\n", + " compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_NC6', max_nodes=4)\n", + "\n", + " # create the cluster\n", + " cpu_cluster = ComputeTarget.create(ws, cluster_name, compute_config)\n", + "\n", + " # can poll for a minimum number of nodes and for a specific timeout. \n", + " # if no min node count is provided it uses the scale settings for the cluster\n", + " cpu_cluster.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n", + "\n", + "# use get_status() to get a detailed status for the current cluster. \n", + "print(cpu_cluster.get_status().serialize())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that you have created the compute target, let's see what the workspace's `compute_targets` property returns. You should now see one entry named 'cpucluster' of type `AmlCompute`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "compute_targets = ws.compute_targets\n", + "for name, ct in compute_targets.items():\n", + " print(name, ct.type, ct.provisioning_state)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nbpresent": { + "id": "2039d2d5-aca6-4f25-a12f-df9ae6529cae" + } + }, + "source": [ + "## Use a simple script\n", + "We have already created a simple \"hello world\" script. This is the script that we will submit through the estimator pattern. It prints a hello-world message, and if Azure ML SDK is installed, it will also logs an array of values ([Fibonacci numbers](https://en.wikipedia.org/wiki/Fibonacci_number))." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open('./dummy_train.py', 'r') as f:\n", + " print(f.read())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create A Generic Estimator" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we import the Estimator class and also a widget to visualize a run." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.train.estimator import Estimator\n", + "from azureml.widgets import RunDetails" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The simplest estimator is to submit the current folder to the local computer. Estimator by default will attempt to use Docker-based execution. Let's turn that off for now. It then builds a conda environment locally, installs Azure ML SDK in it, and runs your script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# use a conda environment, don't use Docker, on local computer\n", + "est = Estimator(source_directory='.', compute_target='local', entry_script='dummy_train.py', use_docker=False)\n", + "run = exp.submit(est)\n", + "RunDetails(run).show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also enable Docker and let estimator pick the default CPU image supplied by Azure ML for execution. You can target an AmlCompute cluster (or any other supported compute target types)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# use a conda environment on default Docker image in an AmlCompute cluster\n", + "est = Estimator(source_directory='.', compute_target=cpu_cluster, entry_script='dummy_train.py', use_docker=True)\n", + "run = exp.submit(est)\n", + "RunDetails(run).show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can customize the conda environment by adding conda and/or pip packages." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# add a conda package\n", + "est = Estimator(source_directory='.', \n", + " compute_target='local', \n", + " entry_script='dummy_train.py', \n", + " use_docker=False, \n", + " conda_packages=['scikit-learn'])\n", + "run = exp.submit(est)\n", + "RunDetails(run).show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also specify a custom Docker image for exeution. In this case, you probably want to tell the system not to build a new conda environment for you. Instead, you can specify the path to an existing Python environment in the custom Docker image.\n", + "\n", + "**Note**: since the below example points to the preinstalled Python environment in the miniconda3 image maintained by continuum.io on Docker Hub where Azure ML SDK is not present, the logging metric code is not triggered. But a run history record is still recorded. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# use a custom Docker image\n", + "from azureml.core import RunConfiguration\n", + "\n", + "rc = RunConfiguration()\n", + "rc.environment.docker.enabled = True\n", + "\n", + "# this is an image available in Docker Hub\n", + "rc.environment.docker.base_image = 'continuumio/miniconda3'\n", + "\n", + "# you can also point to an image in a private ACR\n", + "#rc.environment.docker.base_image = \"mycustomimage:1.0\"\n", + "#rc.environment.docker.base_image_registry.address = \"myregistry.azurecr.io\"\n", + "#rc.environment.docker.base_image_registry.username = \"username\"\n", + "#rc.environment.docker.base_image_registry.password = \"password\"\n", + "\n", + "# don't let the system build a new conda environment\n", + "rc.environment.python.user_managed_dependencies = True\n", + "# point to an existing python environment instead\n", + "rc.environment.python.interpreter_path = '/opt/conda/bin/python'\n", + "\n", + "# submit to a local Docker container. if you don't have Docker engine running locally, you can set compute_target to cpu_cluster.\n", + "est = Estimator(source_directory='.', compute_target='local', \n", + " entry_script='dummy_train.py',\n", + " environment_definition=rc.environment)\n", + "\n", + "run = exp.submit(est)\n", + "RunDetails(run).show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: if you need to cancel a run, you can follow [these instructions](https://aka.ms/aml-docs-cancel-run)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps\n", + "Now you can proceed to explore the other types of estimators, such as TensorFlow estimator, PyTorch estimator, etc. in the sample folder." + ] + } + ], + "metadata": { + "authors": [ + { + "name": "minxia" + } + ], + "kernelspec": { + "display_name": "Python 3.6", + "language": "python", + "name": "python36" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + }, + "msauthor": "haining" + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/how-to-use-azureml/training-with-deep-learning/tensorboard/tensorboard.ipynb b/how-to-use-azureml/training-with-deep-learning/tensorboard/tensorboard.ipynb index 44c71353c..ea4f4a5bd 100644 --- a/how-to-use-azureml/training-with-deep-learning/tensorboard/tensorboard.ipynb +++ b/how-to-use-azureml/training-with-deep-learning/tensorboard/tensorboard.ipynb @@ -104,7 +104,7 @@ "print('Workspace name: ' + ws.name, \n", " 'Azure region: ' + ws.location, \n", " 'Subscription id: ' + ws.subscription_id, \n", - " 'Resource group: ' + ws.resource_group, sep = '\\n')" + " 'Resource group: ' + ws.resource_group, sep='\\n')" ] }, { @@ -153,7 +153,7 @@ "source": [ "import requests\n", "import os\n", - "import tempfile\n", + "\n", "tf_code = requests.get(\"https://raw.githubusercontent.com/tensorflow/tensorflow/r1.8/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py\")\n", "with open(os.path.join(exp_dir, \"mnist_with_summaries.py\"), \"w\") as file:\n", " file.write(tf_code.text)" @@ -192,9 +192,8 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.core import Experiment, Run\n", + "from azureml.core import Experiment\n", "from azureml.core.script_run_config import ScriptRunConfig\n", - "import tensorflow as tf\n", "\n", "logs_dir = os.path.join(os.curdir, \"logs\")\n", "data_dir = os.path.abspath(os.path.join(os.curdir, \"mnist_data\"))\n", @@ -276,7 +275,7 @@ "Tensorboard uploading works with all compute targets. Here we demonstrate it from a DSVM.\n", "Note that the Tensorboard instance itself will be run by the notebook kernel. Again, this means this notebook's kernel must have access to the Tensorboard module.\n", "\n", - "If you are unfamiliar with DSVM configuration, check [04. Train in a remote VM](04.train-on-remote-vm.ipynb) for a more detailed breakdown.\n", + "If you are unfamiliar with DSVM configuration, check [Train in a remote VM](../../training/train-on-remote-vm/train-on-remote-vm.ipynb) for a more detailed breakdown.\n", "\n", "**Note**: To streamline the compute that Azure Machine Learning creates, we are making updates to support creating only single to multi-node `AmlCompute`. The `DSVMCompute` class will be deprecated in a later release, but the DSVM can be created using the below single line command and then attached(like any VM) using the sample code below. Also note, that we only support Linux VMs for remote execution from AML and the commands below will spin a Linux VM only.\n", "\n", @@ -296,7 +295,6 @@ "source": [ "from azureml.core.compute import RemoteCompute\n", "from azureml.core.compute_target import ComputeTargetException\n", - "import os\n", "\n", "username = os.getenv('AZUREML_DSVM_USERNAME', default='')\n", "address = os.getenv('AZUREML_DSVM_ADDRESS', default='')\n", @@ -405,7 +403,6 @@ "outputs": [], "source": [ "from azureml.core.compute import ComputeTarget, AmlCompute\n", - "from azureml.core.compute_target import ComputeTargetException\n", "\n", "# choose a name for your cluster\n", "cluster_name = \"cpucluster\"\n", diff --git a/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.ipynb b/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.ipynb index e5d0d3bb3..359eadaa9 100644 --- a/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.ipynb +++ b/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.ipynb @@ -83,7 +83,7 @@ "print('Workspace name: ' + ws.name, \n", " 'Azure region: ' + ws.location, \n", " 'Subscription id: ' + ws.subscription_id, \n", - " 'Resource group: ' + ws.resource_group, sep = '\\n')" + " 'Resource group: ' + ws.resource_group, sep='\\n')" ] }, { @@ -359,7 +359,7 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.train.hyperdrive import *\n", + "from azureml.train.hyperdrive import RandomParameterSampling, BanditPolicy, HyperDriveRunConfig, uniform, PrimaryMetricGoal\n", "\n", "param_sampling = RandomParameterSampling( {\n", " 'learning_rate': uniform(0.0005, 0.005),\n", @@ -409,8 +409,6 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.widgets import RunDetails\n", - "\n", "RunDetails(hyperdrive_run).show()" ] }, @@ -649,7 +647,7 @@ "metadata": {}, "outputs": [], "source": [ - "import os, json\n", + "import json\n", "from PIL import Image\n", "import matplotlib.pyplot as plt\n", "\n", diff --git a/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-tensorflow/train-hyperparameter-tune-deploy-with-tensorflow.ipynb b/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-tensorflow/train-hyperparameter-tune-deploy-with-tensorflow.ipynb index 816d0a095..78a30c1e4 100644 --- a/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-tensorflow/train-hyperparameter-tune-deploy-with-tensorflow.ipynb +++ b/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-tensorflow/train-hyperparameter-tune-deploy-with-tensorflow.ipynb @@ -51,7 +51,6 @@ "%matplotlib inline\n", "import numpy as np\n", "import os\n", - "import matplotlib\n", "import matplotlib.pyplot as plt" ] }, @@ -66,7 +65,7 @@ "outputs": [], "source": [ "import azureml\n", - "from azureml.core import Workspace, Run\n", + "from azureml.core import Workspace\n", "\n", "# check core SDK version number\n", "print(\"Azure ML SDK Version: \", azureml.core.VERSION)" @@ -109,8 +108,6 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.core.workspace import Workspace\n", - "\n", "ws = Workspace.from_config()\n", "print('Workspace name: ' + ws.name, \n", " 'Azure region: ' + ws.location, \n", @@ -166,7 +163,6 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", "import urllib\n", "\n", "os.makedirs('./data/mnist', exist_ok=True)\n", @@ -431,7 +427,7 @@ "metadata": {}, "source": [ "## Submit job to run\n", - "Calling the `fit` function on the estimator submits the job to Azure ML for execution. Submitting the job should only take a few seconds." + "Submit the estimator to an Azure ML experiment to kick off the execution." ] }, { @@ -552,7 +548,6 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", "\n", "os.makedirs('./imgs', exist_ok=True)\n", "metrics = run.get_metrics()\n", @@ -685,7 +680,8 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.train.hyperdrive import *\n", + "from azureml.train.hyperdrive import RandomParameterSampling, BanditPolicy, HyperDriveRunConfig, PrimaryMetricGoal\n", + "from azureml.train.hyperdrive import choice, loguniform\n", "\n", "ps = RandomParameterSampling(\n", " {\n", @@ -1079,7 +1075,6 @@ "outputs": [], "source": [ "import requests\n", - "import json\n", "\n", "# send a random row from the test set to score\n", "random_index = np.random.randint(0, len(X_test)-1)\n", @@ -1163,7 +1158,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.6.8" }, "msauthor": "minxia" }, diff --git a/how-to-use-azureml/training/README.md b/how-to-use-azureml/training/README.md index a70ddb054..40f256011 100644 --- a/how-to-use-azureml/training/README.md +++ b/how-to-use-azureml/training/README.md @@ -6,4 +6,5 @@ Follow these sample notebooks to learn: 2. [Train on local](train-on-local): train a model using local computer as compute target. 3. [Train on remote VM](train-on-remote-vm): train a model using a remote Azure VM as compute target. 4. [Train on AmlCompute](train-on-amlcompute): train a model using an AmlCompute cluster as compute target. -5. [Logging API](logging-api): experiment with various logging functions to create runs and automatically generate graphs. \ No newline at end of file +5. [Train in an HDI Spark cluster](train-in-spark): train a Spark ML model using an HDInsight Spark cluster as compute target. +6. [Logging API](logging-api): experiment with various logging functions to create runs and automatically generate graphs. \ No newline at end of file diff --git a/how-to-use-azureml/training/logging-api/logging-api.ipynb b/how-to-use-azureml/training/logging-api/logging-api.ipynb index 4c8041f3e..13a81ee18 100644 --- a/how-to-use-azureml/training/logging-api/logging-api.ipynb +++ b/how-to-use-azureml/training/logging-api/logging-api.ipynb @@ -46,7 +46,7 @@ }, "outputs": [], "source": [ - "from azureml.core import Experiment, Run, Workspace\n", + "from azureml.core import Experiment, Workspace\n", "import azureml.core\n", "import numpy as np\n", "\n", diff --git a/how-to-use-azureml/training/train-on-amlcompute/train-on-amlcompute.ipynb b/how-to-use-azureml/training/train-on-amlcompute/train-on-amlcompute.ipynb index 1e4a09cc4..ea33a1b07 100644 --- a/how-to-use-azureml/training/train-on-amlcompute/train-on-amlcompute.ipynb +++ b/how-to-use-azureml/training/train-on-amlcompute/train-on-amlcompute.ipynb @@ -214,6 +214,13 @@ "run" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: if you need to cancel a run, you can follow [these instructions](https://aka.ms/aml-docs-cancel-run)." + ] + }, { "cell_type": "code", "execution_count": null, @@ -449,8 +456,7 @@ "outputs": [], "source": [ "#Get_status () gets the latest status of the AmlCompute target\n", - "cpu_cluster.get_status()\n", - "cpu_cluster.serialize()" + "cpu_cluster.get_status().serialize()\n" ] }, { diff --git a/how-to-use-azureml/training/train-on-local/train-on-local.ipynb b/how-to-use-azureml/training/train-on-local/train-on-local.ipynb index b4b0ffa83..9048a42d0 100644 --- a/how-to-use-azureml/training/train-on-local/train-on-local.ipynb +++ b/how-to-use-azureml/training/train-on-local/train-on-local.ipynb @@ -143,7 +143,7 @@ "run_config_user_managed.environment.python.user_managed_dependencies = True\n", "\n", "# You can choose a specific Python environment by pointing to a Python path \n", - "#run_config.environment.python.interpreter_path = '/home/johndoe/miniconda3/envs/sdk2/bin/python'" + "#run_config.environment.python.interpreter_path = '/home/johndoe/miniconda3/envs/myenv/bin/python'" ] }, { @@ -182,6 +182,13 @@ "run" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: if you need to cancel a run, you can follow [these instructions](https://aka.ms/aml-docs-cancel-run)." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -212,7 +219,6 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.core.runconfig import RunConfiguration\n", "from azureml.core.conda_dependencies import CondaDependencies\n", "\n", "run_config_system_managed = RunConfiguration()\n", @@ -281,6 +287,7 @@ "source": [ "### Docker-based execution\n", "**IMPORTANT**: You must have Docker engine installed locally in order to use this execution mode. If your kernel is already running in a Docker container, such as **Azure Notebooks**, this mode will **NOT** work.\n", + "\n", "NOTE: The GPU base image must be used on Microsoft Azure Services only such as ACI, AML Compute, Azure VMs, and AKS.\n", "\n", "You can also ask the system to pull down a Docker image and execute your scripts in it." @@ -296,6 +303,8 @@ "run_config_docker.environment.python.user_managed_dependencies = False\n", "run_config_docker.auto_prepare_environment = True\n", "run_config_docker.environment.docker.enabled = True\n", + "\n", + "# use the default CPU-based Docker image from Azure ML\n", "run_config_docker.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n", "\n", "# Specify conda dependencies with scikit-learn\n", @@ -309,7 +318,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Submit script to run in the system-managed environment\n", + "### Submit script to run in the system-managed environment\n", "A new conda environment is built based on the conda dependencies object. If you are running this for the first time, this might take up to 5 mninutes. But this conda environment is reused so long as you don't change the conda dependencies.\n", "\n", "\n" @@ -353,6 +362,33 @@ "run.wait_for_completion(show_output=True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Use a custom Docker image\n", + "\n", + "You can also specify a custom Docker image if you don't want to use the default image provided by Azure ML.\n", + "\n", + "```python\n", + "# use an image available in Docker Hub without authentication\n", + "run_config_docker.environment.docker.base_image = \"continuumio/miniconda3\"\n", + "\n", + "# or, use an image available in a private Azure Container Registry\n", + "run_config_docker.environment.docker.base_image = \"mycustomimage:1.0\"\n", + "run_config_docker.environment.docker.base_image_registry.address = \"myregistry.azurecr.io\"\n", + "run_config_docker.environment.docker.base_image_registry.username = \"username\"\n", + "run_config_docker.environment.docker.base_image_registry.password = \"password\"\n", + "```\n", + "\n", + "When you are using a custom Docker image, you might already have your environment setup properly in a Python environment in the Docker image. In that case, you can skip specifying conda dependencies, and just use `user_managed_dependencies` option instead:\n", + "```python\n", + "run_config_docker.environment.python.user_managed_dependencies = True\n", + "# path to the Python environment in the custom Docker image\n", + "run_config.environment.python.interpreter_path = '/opt/conda/bin/python'\n", + "```" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -452,7 +488,7 @@ "metadata": { "authors": [ { - "name": "roastala" + "name": "haining" } ], "kernelspec": { diff --git a/how-to-use-azureml/training/train-on-remote-vm/train-on-remote-vm.ipynb b/how-to-use-azureml/training/train-on-remote-vm/train-on-remote-vm.ipynb index 7c455d987..57073ef3d 100644 --- a/how-to-use-azureml/training/train-on-remote-vm/train-on-remote-vm.ipynb +++ b/how-to-use-azureml/training/train-on-remote-vm/train-on-remote-vm.ipynb @@ -306,6 +306,13 @@ "run = exp.submit(config=src)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: if you need to cancel a run, you can follow [these instructions](https://aka.ms/aml-docs-cancel-run)." + ] + }, { "cell_type": "code", "execution_count": null, @@ -501,6 +508,33 @@ "run.wait_for_completion(show_output=True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Use a custom Docker image instead\n", + "\n", + "You can also specify a custom Docker image if you don't want to use the default image provided by Azure ML.\n", + "\n", + "```python\n", + "# use an image available in Docker Hub without authentication\n", + "run_config_docker.environment.docker.base_image = \"continuumio/miniconda3\"\n", + "\n", + "# or, use an image available in a private Azure Container Registry\n", + "run_config_docker.environment.docker.base_image = \"mycustomimage:1.0\"\n", + "run_config_docker.environment.docker.base_image_registry.address = \"myregistry.azurecr.io\"\n", + "run_config_docker.environment.docker.base_image_registry.username = \"username\"\n", + "run_config_docker.environment.docker.base_image_registry.password = \"password\"\n", + "```\n", + "\n", + "When you are using a custom Docker image, you might already have your environment setup properly in a Python environment in the Docker image. In that case, you can skip specifying conda dependencies, and just use `user_managed_dependencies` option instead:\n", + "```python\n", + "run_config_docker.environment.python.user_managed_dependencies = True\n", + "# path to the Python environment in the custom Docker image\n", + "run_config.environment.python.interpreter_path = '/opt/conda/bin/python'\n", + "```" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb b/how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb index 33a1ab748..6690ff432 100644 --- a/how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb +++ b/how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb @@ -81,7 +81,7 @@ "from azureml.core import Experiment, Workspace\n", "\n", "# Check core SDK version number\n", - "print(\"This notebook was created using version 1.0.2 of the Azure ML SDK\")\n", + "print(\"This notebook was created using version 1.0.10 of the Azure ML SDK\")\n", "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")\n", "print(\"\")\n", "\n", @@ -157,7 +157,8 @@ "experiment = Experiment(workspace=ws, name=\"train-within-notebook\")\n", "\n", "# Create a run object in the experiment\n", - "run = experiment.start_logging()# Log the algorithm parameter alpha to the run\n", + "run = experiment.start_logging()\n", + "# Log the algorithm parameter alpha to the run\n", "run.log('alpha', 0.03)\n", "\n", "# Create, fit, and test the scikit-learn Ridge regression model\n", @@ -215,7 +216,6 @@ "outputs": [], "source": [ "import numpy as np\n", - "import os\n", "from tqdm import tqdm\n", "\n", "model_name = \"model.pkl\"\n", @@ -680,7 +680,7 @@ "metadata": { "authors": [ { - "name": "roastala" + "name": "haining" } ], "kernelspec": { @@ -703,4 +703,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/tutorials/img-classification-part1-training.ipynb b/tutorials/img-classification-part1-training.ipynb index 0975cb89d..0b3ec94f8 100644 --- a/tutorials/img-classification-part1-training.ipynb +++ b/tutorials/img-classification-part1-training.ipynb @@ -570,6 +570,13 @@ "RunDetails(run).show()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you need to cancel a run, you can follow [these instructions](https://aka.ms/aml-docs-cancel-run)." + ] + }, { "cell_type": "markdown", "metadata": {},