diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/build/doctrees/contents/apis.doctree b/docs/build/doctrees/contents/apis.doctree new file mode 100644 index 0000000..bdff103 Binary files /dev/null and b/docs/build/doctrees/contents/apis.doctree differ diff --git a/docs/build/doctrees/contents/cls_tutorial.doctree b/docs/build/doctrees/contents/cls_tutorial.doctree new file mode 100644 index 0000000..cf31b4b Binary files /dev/null and b/docs/build/doctrees/contents/cls_tutorial.doctree differ diff --git a/docs/build/doctrees/contents/det_tutorial.doctree b/docs/build/doctrees/contents/det_tutorial.doctree new file mode 100644 index 0000000..33b36f7 Binary files /dev/null and b/docs/build/doctrees/contents/det_tutorial.doctree differ diff --git a/docs/build/doctrees/contents/flowchart.doctree b/docs/build/doctrees/contents/flowchart.doctree new file mode 100644 index 0000000..c3d257f Binary files /dev/null and b/docs/build/doctrees/contents/flowchart.doctree differ diff --git a/docs/build/doctrees/contents/flowcharts/index.doctree b/docs/build/doctrees/contents/flowcharts/index.doctree new file mode 100644 index 0000000..3bae237 Binary files /dev/null and b/docs/build/doctrees/contents/flowcharts/index.doctree differ diff --git a/docs/build/doctrees/contents/flowcharts/pause.doctree b/docs/build/doctrees/contents/flowcharts/pause.doctree new file mode 100644 index 0000000..013ce16 Binary files /dev/null and b/docs/build/doctrees/contents/flowcharts/pause.doctree differ diff --git a/docs/build/doctrees/contents/flowcharts/standard.doctree b/docs/build/doctrees/contents/flowcharts/standard.doctree new file mode 100644 index 0000000..d570bb7 Binary files /dev/null and b/docs/build/doctrees/contents/flowcharts/standard.doctree differ diff --git a/docs/build/doctrees/contents/getting_started.doctree b/docs/build/doctrees/contents/getting_started.doctree new file mode 100644 index 0000000..939839b Binary files /dev/null and b/docs/build/doctrees/contents/getting_started.doctree differ diff --git a/docs/build/doctrees/contents/hpo_config.doctree b/docs/build/doctrees/contents/hpo_config.doctree new file mode 100644 index 0000000..1b3befc Binary files /dev/null and b/docs/build/doctrees/contents/hpo_config.doctree differ diff --git a/docs/build/doctrees/contents/hpo_ote.doctree b/docs/build/doctrees/contents/hpo_ote.doctree new file mode 100644 index 0000000..968fd9e Binary files /dev/null and b/docs/build/doctrees/contents/hpo_ote.doctree differ diff --git a/docs/build/doctrees/contents/index.doctree b/docs/build/doctrees/contents/index.doctree new file mode 100644 index 0000000..0668601 Binary files /dev/null and b/docs/build/doctrees/contents/index.doctree differ diff --git a/docs/build/doctrees/contents/install.doctree b/docs/build/doctrees/contents/install.doctree new file mode 100644 index 0000000..53bc592 Binary files /dev/null and b/docs/build/doctrees/contents/install.doctree differ diff --git a/docs/build/doctrees/contents/intro.doctree b/docs/build/doctrees/contents/intro.doctree new file mode 100644 index 0000000..234b5e2 Binary files /dev/null and b/docs/build/doctrees/contents/intro.doctree differ diff --git a/docs/build/doctrees/contents/pause.doctree b/docs/build/doctrees/contents/pause.doctree new file mode 100644 index 0000000..0499bed Binary files /dev/null and b/docs/build/doctrees/contents/pause.doctree differ diff --git a/docs/build/doctrees/contents/standard.doctree b/docs/build/doctrees/contents/standard.doctree new file mode 100644 index 0000000..286bd83 Binary files /dev/null and b/docs/build/doctrees/contents/standard.doctree differ diff --git a/docs/build/doctrees/contents/using.doctree b/docs/build/doctrees/contents/using.doctree new file mode 100644 index 0000000..bc3bfa2 Binary files /dev/null and b/docs/build/doctrees/contents/using.doctree differ diff --git a/docs/build/doctrees/environment.pickle b/docs/build/doctrees/environment.pickle new file mode 100644 index 0000000..029427f Binary files /dev/null and b/docs/build/doctrees/environment.pickle differ diff --git a/docs/build/doctrees/flowcharts/flowchart.doctree b/docs/build/doctrees/flowcharts/flowchart.doctree new file mode 100644 index 0000000..bab7ccc Binary files /dev/null and b/docs/build/doctrees/flowcharts/flowchart.doctree differ diff --git a/docs/build/doctrees/flowcharts/index.doctree b/docs/build/doctrees/flowcharts/index.doctree new file mode 100644 index 0000000..151b353 Binary files /dev/null and b/docs/build/doctrees/flowcharts/index.doctree differ diff --git a/docs/build/doctrees/flowcharts/pause.doctree b/docs/build/doctrees/flowcharts/pause.doctree new file mode 100644 index 0000000..90e758d Binary files /dev/null and b/docs/build/doctrees/flowcharts/pause.doctree differ diff --git a/docs/build/doctrees/flowcharts/standard.doctree b/docs/build/doctrees/flowcharts/standard.doctree new file mode 100644 index 0000000..3dfedf8 Binary files /dev/null and b/docs/build/doctrees/flowcharts/standard.doctree differ diff --git a/docs/build/doctrees/hpo_ote/cls_tutorial.doctree b/docs/build/doctrees/hpo_ote/cls_tutorial.doctree new file mode 100644 index 0000000..b657c8e Binary files /dev/null and b/docs/build/doctrees/hpo_ote/cls_tutorial.doctree differ diff --git a/docs/build/doctrees/hpo_ote/det_tutorial.doctree b/docs/build/doctrees/hpo_ote/det_tutorial.doctree new file mode 100644 index 0000000..7280d62 Binary files /dev/null and b/docs/build/doctrees/hpo_ote/det_tutorial.doctree differ diff --git a/docs/build/doctrees/hpo_ote/hpo_config.doctree b/docs/build/doctrees/hpo_ote/hpo_config.doctree new file mode 100644 index 0000000..f735c6a Binary files /dev/null and b/docs/build/doctrees/hpo_ote/hpo_config.doctree differ diff --git a/docs/build/doctrees/hpo_ote/hpo_ote.doctree b/docs/build/doctrees/hpo_ote/hpo_ote.doctree new file mode 100644 index 0000000..bfdb49a Binary files /dev/null and b/docs/build/doctrees/hpo_ote/hpo_ote.doctree differ diff --git a/docs/build/doctrees/hpo_ote/index.doctree b/docs/build/doctrees/hpo_ote/index.doctree new file mode 100644 index 0000000..1d193ef Binary files /dev/null and b/docs/build/doctrees/hpo_ote/index.doctree differ diff --git a/docs/build/doctrees/index.doctree b/docs/build/doctrees/index.doctree new file mode 100644 index 0000000..b61c9e7 Binary files /dev/null and b/docs/build/doctrees/index.doctree differ diff --git a/docs/build/doctrees/nbsphinx/contents/getting_started.ipynb b/docs/build/doctrees/nbsphinx/contents/getting_started.ipynb new file mode 100644 index 0000000..2d967ad --- /dev/null +++ b/docs/build/doctrees/nbsphinx/contents/getting_started.ipynb @@ -0,0 +1,220 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "demographic-bulletin", + "metadata": {}, + "source": [ + "# Getting Started with hpopt" + ] + }, + { + "cell_type": "markdown", + "id": "81460b40", + "metadata": {}, + "source": [ + "This is quick start for hpoopt. You can learn how to use hpopt standalone with your own traninig module." + ] + }, + { + "cell_type": "markdown", + "id": "duplicate-plane", + "metadata": {}, + "source": [ + "## 1. Installation" + ] + }, + { + "cell_type": "markdown", + "id": "2d6ef719", + "metadata": {}, + "source": [ + "\n", + "You only need to run following code to install hpopt.\n", + "\n", + "``` bash\n", + "git clone https://github.com/openvinotoolkit/hyper_parameter_optimization\n", + "pip install -v -e ./hyper_parameter_optimization\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "22ebd406", + "metadata": {}, + "source": [ + "## 2. Run HPO" + ] + }, + { + "cell_type": "markdown", + "id": "aca6bb13", + "metadata": {}, + "source": [ + "First of all, you need to import hpopt to run HPO." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "4be4967c", + "metadata": {}, + "outputs": [], + "source": [ + "import hpopt" + ] + }, + { + "cell_type": "markdown", + "id": "c1ce246b", + "metadata": {}, + "source": [ + "Then, make you own models. In this code, I implement very simple black-box function which has two hyper parameters 'a' and 'b'." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "430cd36c", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import random\n", + "\n", + "def my_model(a, b, iter):\n", + " return a**2 + (b - 10)**2 + iter * 0.1 + random.random() * 0.1\n", + "\n", + "def my_trainer(config):\n", + " for iter in range(config[\"iterations\"]):\n", + " score = my_model(**config[\"params\"], iter=iter)\n", + " if hpopt.report(config=config, score=score) == hpopt.Status.STOP:\n", + " break" + ] + }, + { + "cell_type": "markdown", + "id": "b354b760", + "metadata": {}, + "source": [ + "You should define hyper parameter search space to run HPO. Hyper parameter search space is dictionary which has hyper parameter name as key and search space sampling option as value.\n", + "Please refer [](./apis.md) for more details." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "e7b54790", + "metadata": {}, + "outputs": [], + "source": [ + "# make hyper parameter space\n", + "search_space = {\"a\": hpopt.search_space(\"uniform\", [-5, 10]),\n", + " \"b\": hpopt.search_space(\"uniform\", [0, 15])}" + ] + }, + { + "cell_type": "markdown", + "id": "f1acf21f", + "metadata": {}, + "source": [ + "Let's initiate hpopt class with hyper parameter search space we made. In this case, we run simple bayesian optimization(SMBO)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ahead-visibility", + "metadata": {}, + "outputs": [], + "source": [ + "my_hpo = hpopt.create(save_path='./tmp/my_hpo', # where HPO progress is saved\n", + " search_alg=\"bayes_opt\", # HPO algorithm\n", + " search_space=search_space,\n", + " early_stop=\"median_stop\", # early stopping method\n", + " num_init_trials=5, num_trials=10, max_iterations=2,\n", + " resume=False,\n", + " num_full_iterations=2, # train iteration\n", + " full_dataset_size=1) # train dataset size. In this case, you can ignore it." + ] + }, + { + "cell_type": "markdown", + "id": "61fcc5f8", + "metadata": {}, + "source": [ + "Finally, we can run HPO and print output." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e54d2cf6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| # | a | b | score |\n", + "| 1 | 9.629164679496109 | 9.152868582262837 | 93.56057080971651 |\n", + "| 2 | 9.503930895864768 | 10.477054966570938 | 90.6920277461489 |\n", + "| 3 | 6.5275823645277775 | 14.59670633381528 | 63.90848097562949 |\n", + "| 4 | 2.8808311670306797 | 9.058790892076829 | 9.29099328793063 |\n", + "| 5 | 7.95029121913541 | 13.178755341124605 | 73.50752311230228 |\n", + "| 6 | 10.0 | 0.0 | 200.10740027553825 |\n", + "| 7 | 6.025026543316109 | 0.0 | 136.4928912794813 |\n", + "| 8 | 10.0 | 1.6328830252328062 | 170.14644018670234 |\n", + "| 9 | -5.0 | 0.0 | 125.12112966688557 |\n", + "| 10 | 9.571587105736812 |0.004128397224773539 | 191.68849629266415 |\n", + "best hp: {'a': 10.0, 'b': 0.0}\n" + ] + } + ], + "source": [ + "while True:\n", + " config = my_hpo.get_next_sample()\n", + "\n", + " if config is None:\n", + " break\n", + "\n", + " my_trainer(config)\n", + "\n", + "best_config = my_hpo.get_best_config()\n", + "\n", + "my_hpo.print_results()\n", + "\n", + "print(\"best hp: \", best_config)" + ] + }, + { + "cell_type": "markdown", + "id": "fa943338", + "metadata": {}, + "source": [ + "As you can see, hpopt found best hyper parameters for black box function." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/make_venv.sh b/docs/make_venv.sh new file mode 100644 index 0000000..1ca393f --- /dev/null +++ b/docs/make_venv.sh @@ -0,0 +1,4 @@ +python -m venv venv +source venv/bin/activate +pip install -r requirements.txt +sudo apt-get install pandoc diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..dd1c8e2 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,81 @@ +alabaster==0.7.12 +attrs==21.4.0 +Babel==2.10.2 +bayesian-optimization==1.2.0 +beautifulsoup4==4.11.1 +bleach==5.0.0 +certifi==2022.6.15 +charset-normalizer==2.0.12 +commonmark==0.9.1 +defusedxml==0.7.1 +docutils==0.17.1 +entrypoints==0.4 +fastjsonschema==2.15.3 +idna==3.3 +imagesize==1.3.0 +importlib-metadata==4.11.4 +importlib-resources==5.8.0 +Jinja2==3.1.2 +joblib==1.1.0 +jsonschema==4.6.0 +jupyter-client==7.3.4 +jupyter-core==4.10.0 +jupyterlab-pygments==0.2.2 +Markdown==3.3.7 +markdown-it-py==2.1.0 +MarkupSafe==2.1.1 +mdit-py-plugins==0.3.0 +mdurl==0.1.1 +mistune==0.8.4 +myst-parser==0.18.0 +nbclient==0.6.4 +nbconvert==6.5.0 +nbformat==5.4.0 +nbsphinx==0.8.9 +nest-asyncio==1.5.5 +numpy==1.22.4 +packaging==21.3 +pandoc==2.2 +pandocfilters==1.5.0 +Pillow==9.1.1 +plumbum==1.7.2 +ply==3.11 +pydash==5.1.0 +Pygments==2.12.0 +pymdown-extensions==9.5 +pyparsing==3.0.9 +pyrsistent==0.18.1 +python-dateutil==2.8.2 +pytz==2022.1 +PyYAML==6.0 +pyzmq==23.1.0 +recommonmark==0.7.1 +requests==2.28.0 +scikit-learn==1.1.1 +scipy==1.8.1 +six==1.16.0 +snowballstemmer==2.2.0 +soupsieve==2.3.2.post1 +Sphinx==5.0.1 +sphinx-markdown-parser==0.2.4 +sphinx-markdown-tables==0.0.15 +sphinx-rtd-theme==1.0.0 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +threadpoolctl==3.1.0 +tinycss2==1.1.1 +torch==1.11.0 +torchvision==0.12.0 +tornado==6.1 +traitlets==5.2.2.post1 +typing-extensions==4.2.0 +unify==0.5 +untokenize==0.1.1 +urllib3==1.26.9 +webencodings==0.5.1 +yapf==0.32.0 +zipp==3.8.0 diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..368c8ec --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,70 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../../hpopt')) + + +# -- Project information ----------------------------------------------------- + +project = 'hpopt' +copyright = '2022, intel' +author = 'intel' + +# The full version, including alpha/beta/rc tags +release = '0.1' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +from recommonmark.parser import CommonMarkParser + +source_parsers = { + '.md': CommonMarkParser, +} + +source_suffix = ['.rst', '.md'] + +extensions = ['recommonmark', 'sphinx_markdown_tables','sphinx.ext.autodoc','nbsphinx'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'hpopt' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['myst_parser'] + +source_suffix = ['.rst', '.md'] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] diff --git a/docs/source/contents/_config.yml b/docs/source/contents/_config.yml new file mode 100644 index 0000000..3358720 --- /dev/null +++ b/docs/source/contents/_config.yml @@ -0,0 +1,29 @@ +# Book settings +# Learn more at https://jupyterbook.org/customize/config.html + +title: hpopt +author: Intel +logo: logo.png + +# Force re-execution of notebooks on each build. +# See https://jupyterbook.org/content/execute.html +execute: + execute_notebooks: force + timeout: 100 + +# Define the name of the latex output file for PDF builds +latex: + latex_documents: + targetname: book.tex + +# Information about where the book exists on the web +repository: + url: https://github.com/openvinotoolkit/hyper_parameter_optimization + path_to_book: docs # Optional path to your book, relative to the repository root + branch: master # Which branch of the repository should be used when creating links (optional) + +# Add GitHub buttons to your book +# See https://jupyterbook.org/customize/config.html#add-a-link-to-your-repository +html: + use_issues_button: true + use_repository_button: true diff --git a/docs/source/contents/_toc.yml b/docs/source/contents/_toc.yml new file mode 100644 index 0000000..35a880d --- /dev/null +++ b/docs/source/contents/_toc.yml @@ -0,0 +1,17 @@ +# Table of content +# Learn more at https://jupyterbook.org/customize/toc.html +# +- file: intro +- file: install +- file: getting_started +- file: using +- file: flowchart + sections: + - file: standard + - file: pause +- file: apis +- file: hpo_ote + sections: + - file: hpo_config + - file: cls_tutorial + - file: det_tutorial diff --git a/docs/source/contents/apis.md b/docs/source/contents/apis.md new file mode 100644 index 0000000..d31b9a5 --- /dev/null +++ b/docs/source/contents/apis.md @@ -0,0 +1,234 @@ +# HpOpt API Reference + + +| `hpopt.create` (***full_dataset_size, num_full_iterations, search_space, save_path='hpo', search_alg='bayes_opt', early_stop=None, mode='max', num_init_trials=5, num_trials=None, max_iterations=None, min_iterations=None, reduction_factor=2, num_brackets=None, subset_ratio=None, batch_size_name=None, image_resize=[0, 0], metric='mAP', resume=False, expected_time_ratio=4, non_pure_train_ratio=0.2, num_workers=1, kappa=2.576, kappa_decay=1, kappa_decay_delay***) | +|--------------| + +Initiates a HPO task. If 'num_trials' or 'max_iterations' or 'subset_ratio' are not set, hpopt can find proper values for them which are limited by 'expected_time_ratio'. + +- Parameters + + - ***full_dataset_size*** (*int*, `required`) - Train dataset size. + + - ***num_full_iterations*** (*int*, `required`) - Epoch for traninig after HPO. + + - ***search_space*** (*Dict[search_space]*, `required`) – Hyper parameter search space to find. + + - ***save_path*** (*str*, *default="hpo"*)) - Path where result of HPO is saved. + + - ***search_alg*** (*str*, *default=baye_opt*)) - Must be one of ['baye_opt', 'asha']. + Search algorithm to use for optimizing hyper parmaeters. + + - ***early_stop*** (*str*, *default=None*) - Only for SMBO. Choice of early stopping methods. Currently *'median_stop'* is supported. + + - ***mode*** (*str*, *default='max'*) - Must be one of ['min', 'max']. + Determines whether objective is minimizing or maximizing the metric attribute. + + - ***num_init_trials*** (*int*, *default=5*) - Only for SMBO. How many trials to use to init SMBO. + + - ***num_trials*** (*int*, *default=None*) – Number of times to sample from the hyperparameter space. + It should be greater than or equal to 1. + If set to None, it's set automatically. + + - ***max_iterations*** (*int*, *default=None*) - Max training epoch for each trials. + hpopt will stop training after iterating 'max_iterations' times. + + - ***min_iterations*** (*int*, *default=None*) - Only for ASHA. hpopt will run training at least 'min_iterations' times. + + - ***reduction_factor*** (*int*, *default=2*) - Only for ASHA. Used to set halving rate and amount. + + - ***num_brackets*** (*int*, *default=None*) - Only for ASHA. Number of brackets. + Each bracket has a different halving rate, specified by the reduction factor. + + - ***subset_ratio*** (*Union[int, float]*, *default=None*) - The ratio of dataset size for HPO task. + If this value is greater than or equal to 1.0, + full dataset is used for HPO task. + When it makes dataset size lower than 500, Dataset size is set to 500. + + - ***batch_size_name*** (*str*, *default=None*) - This is used for CUDA out of memory situation. + If it is set and CUDA out of memory occurs, + hpopt automatically decreases batch size to avoid same situation. + + - ***image_resize*** (*List[int]*, *default=[0, 0]*) - The size of image for this HPO task. It has two numbers for width and height respectively. + + - ***metric*** (*str*, *default='mAP'*) - Metric name for HPO. + + - ***resume*** (*bool*, *default=False*) - If True, HPO task resumes or reuses from the results in ***save_path***. + If False, new HPO task is created and old results in ***save_path*** are deleted. + + - ***expected_time_ratio*** (*Union[int, float]*, *default=4*) - The expected ratio of running time for HPO to full + fine-tuning time. If this is 4, it means that HPO takes + four times longer than full fine-tuning phase. + hpopt refers it when configuring HPO automatically. + + - ***non_pure_train_ratio*** (*float*, *default=0.2*) - The ratio of time excluding training over full fine tuning time. + It's reffered when HPO is automatically configured. + + - ***num_workers*** (*int*, *default=1*) - The number of parallel workers for a trainning. + + - ***kappa*** (*Union[float, int]*, *default=2.576*) - Only for SMBO. Kappa vlaue for ucb used in bayesian optimization. + + - ***kappa_decay*** (*Union[float, int]*, *default=1*) - Only for SMBO. Multiply kappa by kappa_decay every trials. + + - ***kappa_decay_delay*** (*int*, *default=0*) - Only for SMBO. Kappa isn't multiplied to kappa_decay + from first trials to kappa_decay_delay trials. + +- Returns + + - `HpOpt` class instance + + +| `hpopt.report` (***config, score***) | +|--------------| + +Updates a HPO task state and decides to early-stop or not. + +- Parameters + + - ***config*** (*Dict[str, Any]*, `required`) - Train confiuration(e.g. hyper parameter, epoch, etc.) for a trial + + - ***score*** (*float*, `required`) - Score of every iteration during trial. + +- Returns + + - `hpo.status` - Must be one of [RUNNING, STOP]. + + +| `hpopt.reportOOM` (***config***) | +|--------------| + +Report if trial raise out of CUDA memory. + +- Parameters + + - ***config*** (*Dict[str, Any]*, `required`) - Train confiuration(e.g. hyper parameter, epoch, etc.) for a trial + + +| `hpopt.finalize_trial` (***config***) | +|--------------| + +Handles the status of trials that have terminated by unexpected causes. + +- Parameters + + - ***config*** (*Dict[str, Any]*, `required`) - Train confiuration(e.g. hyper parameter, epoch, etc.) for a trial + + +| `hpopt.createHpoDataset` (***dataset, config***) | +|--------------| + +Creates a proxy dataset. + +- Parameters + + - ***dataset*** (*torch.utils.data.Dataset*, `required`) - Full dataset. + + - ***config*** (*Dict[str, Any]*, `required`) - Train configuration for a trial. + It also has a key 'subset_ratio', 'resize_height', and 'resize_width' + to change the number of data and resolution of images. + + +| class `HpOpt` | +|--------------| + +An abstract class that searches next samples using underlying hyper-parameter search algorithms. + +- Functions + + - `get_next_sample`() --> dict + + Returns the next sample to try. It returs immediately until ***num_init_trials***. + After then, it waits for the completion of the previous trial. + + - `get_next_samples`() --> list of dict + + Returns all the available next sample to try. + As like `get_next_sample`(), it returs immediately until ***num_init_trials***. + After then, it waits for the completion of the previous trial. + + - `get_best_config` () --> dict + + Retrieve the best config. + + - `print_results` () --> None + + Print out all trial configurations and scores to stdout. + + +| class `hpopt.search_space` (***type, range***) | +|--------------| + +Class that implements search space used for HPO. +It supports uniform and quantized uniform with normal and log scale in addition to categorical type. +Quantized type has step which is unit for change. + +- Parameters + + - ***type*** (*str*, `required`) - Type of hyper parameter search space used for sampling + + - ***range*** (*List[Union[float, int]]*, `required`) - Range of hyper parameter search space. + Please refer bellow for a detail description. + +- Supported Type of Sampling + + - uniform - List[lower: Union[float, int], upper: Union[float, int]] + - Sample a float value uniformly between lower and upper. + + - quniform - List[lower: Union[float, int], upper: Union[float, int], q: Union[float, int]] + - Sample a quantized float value uniformly between lower and upper. + The value will be quantized, i.e. rounded to an integer increment of q. + + - loguniform List[lower: Union[float, int], upper: Union[float, int], base: Union[float, int] = 10) + - Sample a float value in different orders of magnitude. + + - qloguniform List[lower: Union[float, int], upper: Union[float, int], q: Union[float, int], base: Union[float, int] = 10] + - Sample a quantized float value in different orders of magnitude. + + - choice [categories: List[Any]] + - Sample a categorical value. diff --git a/docs/source/contents/cls_tutorial.md b/docs/source/contents/cls_tutorial.md new file mode 100644 index 0000000..5cd202a --- /dev/null +++ b/docs/source/contents/cls_tutorial.md @@ -0,0 +1,155 @@ +# Classification with OTE + +This tutorial provide you how to use HPO for classification. +We'll optimize learning rate and batch size in this tutorial using SMBO which is common method for HPO. + +Let's take a look at how HPO can be executed in OTE step by step. + +## 1. Set hpo_config.yaml +Before running HPO, you should configure HPO using **hpo_config.yaml**. +It configures everything HPO module needs from hyper parameters you'll optimize to which algorithm to use for HPO. +For more information, you can refer [](hpo_config). +Actually, there is already hpo_config.yaml file with default value in directory where *template.yaml* resides. + +Here is default hpo_config.yaml + +``` +metric: mAP +search_algorithm: smbo +early_stop: None +hp_space: + learning_parameters.learning_rate: + param_type: quniform + range: + - 0.001 + - 0.1 + - 0.001 + learning_parameters.batch_size: + param_type: qloguniform + range: + - 4 + - 8 + - 2 +``` + +We can just use this for HPO, but It seems that there is no need to set learning rate search space such that widely. +So, let's modify configuration slightly now. +``` +... +... +... + learning_parameters.learning_rate: + param_type: quniform + range: + - 0.001 + - 0.01 + - 0.001 +... +... +... +``` +It looks good. As you see, you can easily change search space or even hyper parameter to optimize by just modifying hpo_config.yaml. + +## 2. Run OTE +Now it's time to run OTE. You can enable HPO by just adding an argument **--enable-hpo** literally. +Oops! I forget that we don't have much time to use for HPO. +It would be enough to use same time as training to execute HPO (default hpo-time-ratio is 4). +In this case, you can just add one argument **--hpo-time-ratio** and set it 2. +It means that twice as much time is used as only training time. +```{note} +You should install OTE classification before running OTE. +``` +``` +ote train \ + training_extensions/external/deep-object-reid/configs/ote_custom_classification/efficientnet_b0/template.yaml \ + --train-ann-files '' \ + --train-data-roots training_extensions/data/classification/train \ + --val-ann-files '' \ + --val-data-roots training_extensions/data/classification/val \ + --save-model-to training_extensions/best.pth \ + --enable-hpo \ + --hpo-time-ratio 2 +``` +That's it. Now HPO is automatically set to use twice time. +As you can see, you can just add *--hpo-time-ratio* argument to set how much time to use for HPO. + +After HPO, HPO result is printed as bellow. You can see which parameter is chosen by this. + +``` +| # | learning_parameters.learning_rate | learning_parameters.batch_size | score | +| 1 | 0.005 | 8 | 0.6179835230685988 | +| 2 | 0.005 | 10 | 0.5103616463460058 | +| 3 | 0.01 | 48 | 0.6613380273927504 | +| 4 | 0.006 | 16 | 0.4305718659552452 | +| 5 | 0.002 | 36 | 0.5250386319610906 | +| 6 | 0.009000000000000001 | 30 | 0.7129637650433698 | +| 7 | 0.003 | 44 | 0.9165404011276648 | +| 8 | 0.01 | 42 | 0.8234693648419807 | +| 9 | 0.006 | 8 | 0.6856121908185698 | +| 10 | 0.005 | 30 | 0.8736238451806234 | +| 11 | 0.005 | 44 | 0.7295280110191358 | +| 12 | 0.004 | 28 | 0.9178114717365546 | +| 13 | 0.002 | 44 | 0.8877622967705454 | +| 14 | 0.003 | 28 | 0.6154036330149992 | +| 15 | 0.002 | 18 | 0.6622151844713957 | +| 16 | 0.007 | 24 | 0.7678530317224548 | +| 17 | 0.003 | 22 | 0.8609778838120822 | +| 18 | 0.004 | 44 | 0.8966648842324036 | +| 19 | 0.005 | 28 | 0.8302362663067738 | +| 20 | 0.005 | 36 | 0.5476845047852292 | +| 21 | 0.003 | 48 | 0.7321795851958944 | +| 22 | 0.01 | 34 | 0.9057167695385367 | +| 23 | 0.008 | 34 | 0.5073170938766568 | +| 24 | 0.006 | 30 | 0.8442392588571863 | +| 25 | 0.009000000000000001 | 12 | 0.9121951985354356 | +| 26 | 0.01 | 12 | 0.6068545610227464 | +| 27 | 0.002 | 10 | 0.701241262917842 | +| 28 | 0.002 | 20 | 0.9062640968219959 | +| 29 | 0.01 | 60 | 0.5049643461876376 | +Best Hyper-parameters +{'learning_parameters.batch_size': 28, 'learning_parameters.learning_rate': 0.004} +``` + +Then, model is trained with these parameters HPO found. +Now what you need to do is just waiting until all tasks is done. + +## 3. Resume & Reuse HPO +You may want to resume HPO after HPO stopped in the middle or reuse hyper parameters HPO found before. +In this case, you can just run OTE same as above. +Then, HPO module automatically searches HPO progress directory(named HPO) used before, +and HPO asks +1. resume in case that HPO is stopped in the middle +2. reuse optimized hyper parameter in case that HPO is finished before +3. rerun HPO from scratch + +This is convenient feature to save time by skipping unnecessary HPO. +```{warning} +HPO module asks to you only if HPO configurations are set equivalently as before. +When configurations are changed, HPO is excuted without asking. +``` + +## 4. (Optional) Hyper parameterse analysis +HPO module saves progress of each trials. So you can use it to analyze trend of hyper parameters by yourself. +It's saved in directory named **hpo** in same path where final model weight is saved. + +You can see several files like "hpopt_status.json" and "hpopt_trial_#.json" in the directory. +Someone who is clever may already have notices, +first one conatins overall information of HPO progress and second one contains each trials' infomation. + +Detail explanations are as bellow. +- hpopt_status.json + - search_algorithm : Which algorithm used for HPO. + - search_space : Which hyper parameters to optmize and scale and range of each hyper parameters. + - metric : Metric used during HPO. + - subset_ratio : Ratio of train dataset size used for HPO. + - image_resize : Ratio of trainning image used for HPO. + - early_stop : Early stop methods used for SMBO. + - max_iterations : Maximum iterations for HPO trials. + - full_dataset_size : Original train dataset size. + - config_list : List of each trials' information. Each contains trial id, hyper parameters, status and score. + - num_gen_config : Total number of trials. + - best_config_id : Trial id containing best score. +- hpopt_trial_#.json : This file contains status and each iteration's score. + +By these information, you can manually decide which hyper parameters to use. +Because these files have json format, you can easily load these files from other application. diff --git a/docs/source/contents/det_tutorial.md b/docs/source/contents/det_tutorial.md new file mode 100644 index 0000000..94d1cb1 --- /dev/null +++ b/docs/source/contents/det_tutorial.md @@ -0,0 +1,155 @@ +# Detection with OTE + +This tutorial provide you how to use HPO for detection. +We'll optimize learning rate and batch size in this tutorial using SMBO which is common method for HPO. + +Let's take a look at how HPO can be executed in OTE step by step. + +## 1. Set hpo_config.yaml +Before running HPO, you should configure HPO using **hpo_config.yaml**. +It configures everything HPO module needs from hyper parameters you'll optimize to which algorithm to use for HPO. +For more information, you can refer [](hpo_config). +Actually, there is already hpo_config.yaml file with default value in directory where *template.yaml* resides. + +Here is default hpo_config.yaml + +``` +metric: mAP +search_algorithm: smbo +early_stop: None +hp_space: + learning_parameters.learning_rate: + param_type: quniform + range: + - 0.001 + - 0.1 + - 0.001 + learning_parameters.batch_size: + param_type: qloguniform + range: + - 4 + - 8 + - 2 +``` + +We can just use this for HPO, but It seems that there is no need to set learning rate search space such that widely. +So, let's modify configuration slightly now. +``` +... +... +... + learning_parameters.learning_rate: + param_type: quniform + range: + - 0.001 + - 0.01 + - 0.001 +... +... +... +``` +It looks good. As you see, you can easily change search space or even hyper parameter to optimize by just modifying hpo_config.yaml. + +## 2. Run OTE +Now it's time to run OTE. You can enable HPO by just adding an argument **--enable-hpo** literally. +Oops! I forget that we don't have much time to use for HPO. +It would be enough to use same time as training to execute HPO (default hpo-time-ratio is 4). +In this case, you can just add one argument **--hpo-time-ratio** and set it 2. +It means that twice as much time is used as only training time. +```{note} +You should install OTE detection before running OTE. +``` +``` +ote train \ + training_extensions/external/mmdetection/configs/ote/custom-object-detection/gen3_mobilenetV2_ATSS/template.yaml \ + --train-ann-files training_extensions/data/airport/annotation_example_train.json \ + --train-data-roots training_extensions/data/airport/train \ + --val-ann-files training_extensions/data/airport/annotation_example_val.json \ + --val-data-roots training_extensions/data/airport/val \ + --save-model-to training_extensions/best.pth \ + --enable-hpo \ + --hpo-time-ratio 2 +``` +That's it. Now HPO is automatically set to use twice time. +As you can see, you can just add *--hpo-time-ratio* argument to set how much time to use for HPO. + +After HPO, HPO result is printed as bellow. You can see which parameter is chosen by this. + +``` +| # | learning_parameters.learning_rate | learning_parameters.batch_size | score | +| 1 | 0.005 | 8 | 0.6179835230685988 | +| 2 | 0.005 | 10 | 0.5103616463460058 | +| 3 | 0.01 | 48 | 0.6613380273927504 | +| 4 | 0.006 | 16 | 0.4305718659552452 | +| 5 | 0.002 | 36 | 0.5250386319610906 | +| 6 | 0.009000000000000001 | 30 | 0.7129637650433698 | +| 7 | 0.003 | 44 | 0.9165404011276648 | +| 8 | 0.01 | 42 | 0.8234693648419807 | +| 9 | 0.006 | 8 | 0.6856121908185698 | +| 10 | 0.005 | 30 | 0.8736238451806234 | +| 11 | 0.005 | 44 | 0.7295280110191358 | +| 12 | 0.004 | 28 | 0.9178114717365546 | +| 13 | 0.002 | 44 | 0.8877622967705454 | +| 14 | 0.003 | 28 | 0.6154036330149992 | +| 15 | 0.002 | 18 | 0.6622151844713957 | +| 16 | 0.007 | 24 | 0.7678530317224548 | +| 17 | 0.003 | 22 | 0.8609778838120822 | +| 18 | 0.004 | 44 | 0.8966648842324036 | +| 19 | 0.005 | 28 | 0.8302362663067738 | +| 20 | 0.005 | 36 | 0.5476845047852292 | +| 21 | 0.003 | 48 | 0.7321795851958944 | +| 22 | 0.01 | 34 | 0.9057167695385367 | +| 23 | 0.008 | 34 | 0.5073170938766568 | +| 24 | 0.006 | 30 | 0.8442392588571863 | +| 25 | 0.009000000000000001 | 12 | 0.9121951985354356 | +| 26 | 0.01 | 12 | 0.6068545610227464 | +| 27 | 0.002 | 10 | 0.701241262917842 | +| 28 | 0.002 | 20 | 0.9062640968219959 | +| 29 | 0.01 | 60 | 0.5049643461876376 | +Best Hyper-parameters +{'learning_parameters.batch_size': 28, 'learning_parameters.learning_rate': 0.004} +``` + +Then, model is trained with these parameters HPO found. +Now what you need to do is just waiting until all tasks is done. + +## 3. Resume & Reuse HPO +You may want to resume HPO after HPO stopped in the middle or reuse hyper parameters HPO found before. +In this case, you can just run OTE same as above. +Then, HPO module automatically searches HPO progress directory(named HPO) used before, +and HPO asks +1. resume in case that HPO is stopped in the middle +2. reuse optimized hyper parameter in case that HPO is finished before +3. rerun HPO from scratch + +This is convenient feature to save time by skipping unnecessary HPO. +```{warning} +HPO module asks to you only if HPO configurations are set equivalently as before. +When configurations is changed, HPO is excuted without asking. +``` + +## 4. (Optional) Hyper parameterse analysis +HPO module saves progress of each trials. So you can use it to analyze trend of hyper parameters by yourself. +It's saved in directory named **hpo** in same path where final model weight is saved. + +You can see several files like "hpopt_status.json" and "hpopt_trial_#.json" in the directory. +Someone who are clever may already have notices, +first one conatins overall information of HPO progress and second one contains each trials' infomation. + +Detail explanations are as bellow. +- hpopt_status.json + - search_algorithm : Which algorithm used for HPO. + - search_space : Which hyper parameters to optmize and scale and range of each hyper parameters. + - metric : Metric used during HPO. + - subset_ratio : Ratio of train dataset size used for HPO. + - image_resize : Ratio of trainning image used for HPO. + - early_stop : Early stop methods used for SMBO. + - max_iterations : Maximum iterations for HPO trials. + - full_dataset_size : Original train dataset size. + - config_list : List of each trials' information. Each contains trial id, hyper parameters, status and score. + - num_gen_config : Total number of trials. + - best_config_id : Trial id containing best score. +- hpopt_trial_#.json : This file contains status and each iteration's score. + +By these information, you can manually decide which hyper parameters to use. +Because these files have json format, you can easily load these files from other application. diff --git a/docs/source/contents/flowchart.md b/docs/source/contents/flowchart.md new file mode 100644 index 0000000..c55e99a --- /dev/null +++ b/docs/source/contents/flowchart.md @@ -0,0 +1,9 @@ +# Flowchart + +In this section, we introduces flowcharts of standard use case and pause/resume usecase. +You can understand overall flow of hpopt and how pause/resume are executed. + +Please see the sections below to get started. + +- [standard](standard) +- [pause](pause) diff --git a/docs/source/contents/getting_started.ipynb b/docs/source/contents/getting_started.ipynb new file mode 100644 index 0000000..2d967ad --- /dev/null +++ b/docs/source/contents/getting_started.ipynb @@ -0,0 +1,220 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "demographic-bulletin", + "metadata": {}, + "source": [ + "# Getting Started with hpopt" + ] + }, + { + "cell_type": "markdown", + "id": "81460b40", + "metadata": {}, + "source": [ + "This is quick start for hpoopt. You can learn how to use hpopt standalone with your own traninig module." + ] + }, + { + "cell_type": "markdown", + "id": "duplicate-plane", + "metadata": {}, + "source": [ + "## 1. Installation" + ] + }, + { + "cell_type": "markdown", + "id": "2d6ef719", + "metadata": {}, + "source": [ + "\n", + "You only need to run following code to install hpopt.\n", + "\n", + "``` bash\n", + "git clone https://github.com/openvinotoolkit/hyper_parameter_optimization\n", + "pip install -v -e ./hyper_parameter_optimization\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "22ebd406", + "metadata": {}, + "source": [ + "## 2. Run HPO" + ] + }, + { + "cell_type": "markdown", + "id": "aca6bb13", + "metadata": {}, + "source": [ + "First of all, you need to import hpopt to run HPO." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "4be4967c", + "metadata": {}, + "outputs": [], + "source": [ + "import hpopt" + ] + }, + { + "cell_type": "markdown", + "id": "c1ce246b", + "metadata": {}, + "source": [ + "Then, make you own models. In this code, I implement very simple black-box function which has two hyper parameters 'a' and 'b'." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "430cd36c", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import random\n", + "\n", + "def my_model(a, b, iter):\n", + " return a**2 + (b - 10)**2 + iter * 0.1 + random.random() * 0.1\n", + "\n", + "def my_trainer(config):\n", + " for iter in range(config[\"iterations\"]):\n", + " score = my_model(**config[\"params\"], iter=iter)\n", + " if hpopt.report(config=config, score=score) == hpopt.Status.STOP:\n", + " break" + ] + }, + { + "cell_type": "markdown", + "id": "b354b760", + "metadata": {}, + "source": [ + "You should define hyper parameter search space to run HPO. Hyper parameter search space is dictionary which has hyper parameter name as key and search space sampling option as value.\n", + "Please refer [](./apis.md) for more details." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "e7b54790", + "metadata": {}, + "outputs": [], + "source": [ + "# make hyper parameter space\n", + "search_space = {\"a\": hpopt.search_space(\"uniform\", [-5, 10]),\n", + " \"b\": hpopt.search_space(\"uniform\", [0, 15])}" + ] + }, + { + "cell_type": "markdown", + "id": "f1acf21f", + "metadata": {}, + "source": [ + "Let's initiate hpopt class with hyper parameter search space we made. In this case, we run simple bayesian optimization(SMBO)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ahead-visibility", + "metadata": {}, + "outputs": [], + "source": [ + "my_hpo = hpopt.create(save_path='./tmp/my_hpo', # where HPO progress is saved\n", + " search_alg=\"bayes_opt\", # HPO algorithm\n", + " search_space=search_space,\n", + " early_stop=\"median_stop\", # early stopping method\n", + " num_init_trials=5, num_trials=10, max_iterations=2,\n", + " resume=False,\n", + " num_full_iterations=2, # train iteration\n", + " full_dataset_size=1) # train dataset size. In this case, you can ignore it." + ] + }, + { + "cell_type": "markdown", + "id": "61fcc5f8", + "metadata": {}, + "source": [ + "Finally, we can run HPO and print output." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e54d2cf6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| # | a | b | score |\n", + "| 1 | 9.629164679496109 | 9.152868582262837 | 93.56057080971651 |\n", + "| 2 | 9.503930895864768 | 10.477054966570938 | 90.6920277461489 |\n", + "| 3 | 6.5275823645277775 | 14.59670633381528 | 63.90848097562949 |\n", + "| 4 | 2.8808311670306797 | 9.058790892076829 | 9.29099328793063 |\n", + "| 5 | 7.95029121913541 | 13.178755341124605 | 73.50752311230228 |\n", + "| 6 | 10.0 | 0.0 | 200.10740027553825 |\n", + "| 7 | 6.025026543316109 | 0.0 | 136.4928912794813 |\n", + "| 8 | 10.0 | 1.6328830252328062 | 170.14644018670234 |\n", + "| 9 | -5.0 | 0.0 | 125.12112966688557 |\n", + "| 10 | 9.571587105736812 |0.004128397224773539 | 191.68849629266415 |\n", + "best hp: {'a': 10.0, 'b': 0.0}\n" + ] + } + ], + "source": [ + "while True:\n", + " config = my_hpo.get_next_sample()\n", + "\n", + " if config is None:\n", + " break\n", + "\n", + " my_trainer(config)\n", + "\n", + "best_config = my_hpo.get_best_config()\n", + "\n", + "my_hpo.print_results()\n", + "\n", + "print(\"best hp: \", best_config)" + ] + }, + { + "cell_type": "markdown", + "id": "fa943338", + "metadata": {}, + "source": [ + "As you can see, hpopt found best hyper parameters for black box function." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/source/contents/hpo_config.md b/docs/source/contents/hpo_config.md new file mode 100644 index 0000000..5a697b4 --- /dev/null +++ b/docs/source/contents/hpo_config.md @@ -0,0 +1,75 @@ +# How to write hpo_config.yaml + +HPO is configured by reading **hpo_config.yaml** +-- a yaml file that contains everything related to HPO including hyper prameters to optimize, HPO algorithm, etc.. +*hpo_config.yaml* already exists with default value in same directory where *template.yaml* which is essential to run ote resides. +So, you can modify how HPO will run based on this file. + +Here is *hpo_config.yaml* provided for classification. + +``` +metric: mAP +search_algorithm: smbo +early_stop: median_stop +hp_space: + learning_parameters.learning_rate: + param_type: quniform + range: + - 0.001 + - 0.01 + - 0.001 + learning_parameters.batch_size: + param_type: qloguniform + range: + - 8 + - 64 + - 2 +``` +As you can see, There are some attributes needed to run HPO. +Fortunately, there is not much attribute. It's not difficulty to write your own file. +More detailed description is as bellow. + +## Attribute +- **hp_space** (*List[Dict[str, Any]]*, `required`) - Hyper parameter search space to find. It should be list of dictionary. Each has hyper parameter name as key and param_type and range as vaule. + - ***available hyper parameters*** + - avaiable to both classification and detection + - learning_parameters.learninig_rate + - learning_parameters.batch_size + - available to detection + - learning_parameters.learning_rate_warmup_iters + - ***Keys of each hyper parameter*** + - ***param_type*** (*str*, `required`) : Hyper parameter search space type. Must be one of bellows. + - uniform : Sample a float value uniformly between lower and upper. + - quniform : Sample a quantized float value uniformly between lower and upper. + - loguniform : Sample a float value after scaling search space by log scale. + - qloguniform : Sample a quantized float value after scaling search space by log scale. + - choice : Sample a categorical value. + - ***range*** (*List[Any]*, `required`) : Each *param_type* has a respective format. + - uniform : List[Union[float, int]] + - lower (*Union[float, int]*, `required`) : lower bound of search space. + - upper (*Union[float, int]*, `required`) : upper bound of search space. + - quniform : List[Union[float, int]] + - lower (*Union[float, int]*, `required`) : lower bound of search space. + - upper (*Union[float, int]*, `required`) : upper bound of search space. + - q (*Union[float, int]*, `required`) : unit value of search space. + - loguniform : List[Union[float, int]) + - lower (*Union[float, int]*, `required`) : lower bound of search space. + - upper (*Union[float, int]*, `required`) : upper bound of search space. + - base (*Union[float, int]*, *default=10*) : The logarithm base. + - qloguniform : List[Union[float, int]] + - lower (*Union[float, int]*, `required`) : lower bound of search space + - upper (*Union[float, int]*, `required`) : upper bound of search space + - q (*Union[float, int]*, `required`) : unit value of search space + - base (*Union[float, int]*, *default=10*) : The logarithm base. + - choice : List[Any] + - vaule : value to be chosen from candidates. +- **metric** (*str*, *default='mAP*') - Metric name for HPO. +- **serach_algorhtim** (*str*, *default=‘baye_opt’*) - Must be one of [‘baye_opt’, ‘asha’]. Search algorithm used for optimizing hyper parmaeters. +- **early_stop** (*str*, *default=None*) - Choice of early stopping methods. Currently ‘median_stop’ is supported. +- **max_iterations** (*int*, *default=None*) - Max training epoch for each trials. hpopt will stop training after iterating ‘max_iterations’ times. +- **subset_ratio** (*Union[float, int]*, *default=None*) - The ratio of dataset size for HPO task. If this value is greater than or equal to 1.0, full dataset is used for HPO task. When it makes dataset size lower than 500, Dataset size is set to 500. +- **num_init_trials** (*int*, *default=5*) - Only for SMBO. How many trials to use to init SMBO. +- **num_trials** (*int*, *default=None*) - Number of times to sample from the hyperparameter space. It should be greater than or equal to 1. If set to None, it’s set automatically. +- **num_brackets** (*int*, *default=None*) - Only for ASHA. Number of brackets. Each bracket has a different halving rate, specified by the reduction factor. +- **min_iterations** (*int*, *default=None*) - Only for ASHA. hpopt will run training at least ‘min_iterations’ times. +- **reduction_factor** (*float*, *default=2*) - Only for ASHA. Used to set halving rate and amount. \ No newline at end of file diff --git a/docs/source/contents/hpo_ote.md b/docs/source/contents/hpo_ote.md new file mode 100644 index 0000000..4168ca1 --- /dev/null +++ b/docs/source/contents/hpo_ote.md @@ -0,0 +1,11 @@ +# HPO with OTE + +In this section, We'll cover how to use hpopt with OTE. +With OTE, HPO can be activated more easily. +Only things to do HPO, you just add one argument when using OTE. + +Please see the sections below to get started. + +- [hpo_config](hpo_config) +- [cls_tutorial](cls_tutorial) +- [det_tutorial](det_tutorial) \ No newline at end of file diff --git a/docs/source/contents/image/hpopt.png b/docs/source/contents/image/hpopt.png new file mode 100644 index 0000000..34b6c14 Binary files /dev/null and b/docs/source/contents/image/hpopt.png differ diff --git a/docs/source/contents/image/hpopt_overall.png b/docs/source/contents/image/hpopt_overall.png new file mode 100755 index 0000000..a67764d Binary files /dev/null and b/docs/source/contents/image/hpopt_overall.png differ diff --git a/docs/source/contents/image/hpopt_resume.png b/docs/source/contents/image/hpopt_resume.png new file mode 100644 index 0000000..6e7fa13 Binary files /dev/null and b/docs/source/contents/image/hpopt_resume.png differ diff --git a/docs/source/contents/image/smbo.png b/docs/source/contents/image/smbo.png new file mode 100644 index 0000000..2aa0246 Binary files /dev/null and b/docs/source/contents/image/smbo.png differ diff --git a/docs/source/contents/install.md b/docs/source/contents/install.md new file mode 100644 index 0000000..1f8d6e2 --- /dev/null +++ b/docs/source/contents/install.md @@ -0,0 +1,14 @@ +# Install + +Please, run below code to install hpopt. + +``` bash +git clone https://github.com/openvinotoolkit/hyper_parameter_optimization +pip install -v -e ./hyper_parameter_optimization +``` + +Verify the install. + +```bash +python -c "import hpopt; print(hpopt.__version__)" +``` \ No newline at end of file diff --git a/docs/source/contents/intro.md b/docs/source/contents/intro.md new file mode 100644 index 0000000..c5d2689 --- /dev/null +++ b/docs/source/contents/intro.md @@ -0,0 +1,36 @@ +# What is hpopt? + +![hpopt](./image/hpopt.png) + +hpopt is a Python library for automatic hyper-parameter optimization, which finds optimized hyper-parameters work best for given tasks. hpopt aims fast hyper-parameter optimization with some convenient features like auto-config. + +Hyper-parameter optimization is time-consuming process even with state-of-the-art off-the-shelf libraries. hpopt adopts proxy task, early stopping, and multi-band optimization in a single framework, resulting in several times faster than conventional hyper-parameter optimization. Key features include: + + + +- **Easy of usability** : hpopt takes only time as control parameter, which we believe the most intuitive to most users. auto-config sets internal control parameters automatically with the given time constraint, and guarantees HPO will finish in the constraint. + +- **Free from interrupt** : hpopt communicates via persistent storage. Using this, hpopt supports pause/resume/reuse. hpopt always checks that there are previous successful HPO runs and ask you about reusing optimized hyper parameters. + +- **Scalability** : hpopt provides both sequential and parallel methods. You can select one depending on your training environment. If you have multiple GPUs, you can accelerate HPO utilizing all of GPU resources. + + + diff --git a/docs/source/contents/pause.md b/docs/source/contents/pause.md new file mode 100644 index 0000000..8fd11f5 --- /dev/null +++ b/docs/source/contents/pause.md @@ -0,0 +1,13 @@ +Pause / Resume +========================= + + +![hpopt_resume](./image/hpopt_resume.png) + +- hpopt supports pause, resume and restart features. + +- If there is a completed result, the full fine-tuning can be performed without re-running HPO using the hyper-parameters already found. + +- If there are unfinished HPO results, an user can resume HPO from the previous results. + +- However, if any configuration of current HPO is different from the previous one, HPO will unconditionally be started over from the beginning. diff --git a/docs/source/contents/standard.md b/docs/source/contents/standard.md new file mode 100644 index 0000000..b25ad0f --- /dev/null +++ b/docs/source/contents/standard.md @@ -0,0 +1,12 @@ +# Standard + + +![hpopt_overall](./image/hpopt_overall.png) + +- A user specifies the hyper-parameter search space. + +- hpopt create a new HpOpt object with the user-provided configurations. + +- hpopt tries to find the hyper-parameters that maximizes or minimizes the metric score by training a model with various combinations of hyper-parameters. + +- The best hyper-parameters is selected and is used by the following full fine-tuning phase. diff --git a/docs/source/contents/using.md b/docs/source/contents/using.md new file mode 100644 index 0000000..ac0fd24 --- /dev/null +++ b/docs/source/contents/using.md @@ -0,0 +1,298 @@ +# Using hpopt + +We'll introduce detail of hpopt in this page. +After reading this page, you can get some tips to use hpopt more elegantly. + +## Search space + +If you want to run HPO, you should have hyper parameters to optmimize. +And you should make hyper parameter search space with that hyper parameters and use it as hpopt aurgment. +Fortunately, it's not hard to make search space. + +Search space should be dictionary which has pairs of key and value. +Key is hyper parameter name you can set freely and value is **hpopt.search_space** class. +*search_space* class get two arguments when initiated. +First one is a *type* and second one is a *range*. +*type* is used when sampling hyper parameter from search space. +you can not only sample from normal uniform space, but also sample from quantized search space for some hyper parameter. +For example batch size is generally set as even integer number. In this case, you can use quantized search space. +Another type is log scale search space. +You may want to sample hyper parameter from log scale search space. +For this case, we provide log scale with any base you want. +Of course, you can use log scale quantized search space. +You can also use categorical hyper parameters. +For example if you want to optimize optmizer, you can set this by categorical search space. +With this various search space, you can make any search space you want. +For pratical implementation, refer [](apis) + +## hpopt class + +After you make search space, you should initiate hpopt class. + +Arguments you can set are slightly different depending on whether using baye_opt or ASHA. +But don't worry. They are just advanced options so you can just leave them as default value. +First of all, let's take a look at some important arguments used for initiating class. + +There are three required arguments. +- full_dataset_size : It's a just train dataset size. +- num_full_iterations : Epoch for training after HPO. +- search_space : hyper parameter search space to optimize. + +You may think that "search_space is ok, but why outhers are required?". +Actually, one magic hides in this. +**The Auto Config.** +HPO finds the optimal hyper-parameters by trying to train the model with various hyper-parameters. +Because of this behavior, it usually takes a huge amount of time to run HPO, which was the biggest obstacle to using HPO practically. +In order to get good results in appropriate time, the user needs to configure the HPO well. +hpopt has a feature called **Auto Config** that configures parameters of HPO automaticaly. +This feature makes hpopt done in expeceted time. +Auto configurated hyper parameters are as below. +- num_trials : Number of times to sample from the hyperparameter search space. It should be greater than or equal to 1. +- max_iterations : Max training epoch for each trials. hpopt will stop training after iterating ‘max_iterations’ times. +- subset_ratio : The ratio of dataset size for HPO task. If this value is greater than or equal to 1.0, full dataset is used for HPO task. When it makes dataset size lower than 500, Dataset size is set to 500. + +As you know intuitively, the lower these values are, the faster HPO is. +Auto config is activated when you set *expected_time_ratio* and don't set those arguments. +If you set some of these arugments, then arguments except what you set are configured automatically. +*non_pure_train_ratio* is also used to estimate HPO time more accurately. +Those two things are used for autu configuration to estimate HPO time. +You understand why first two are required now. + +```{note} +Expected time ratio means how many times you use for the sum of HPO and finetuning compared to just finetuning time. +In other words, if you set expected time ratio to 2, you'll use same time as training time for HPO. +``` + +*search_alg* is another important argument. It should be either *baye_opt* or *asha*. +*baye_opt* is sequentail model based optimization(SMBO). +As you can see in the name, *baye_opt* executes each trial sequentailly. +*asha* executes each trial in parallel contrariwise. +So, *baye_opt* is recommeneded if you have few training resources. +Otherwise, *asha* is recommended. + +*resume* is convenient feature of hpopt. If you stopped in the middle of HPO for some reason, +and then you want to resume HPO from stopped point, you can do that by set resume to True. +In this case, you should set same *save_path* as you set previously. +Then, hpopt automatically succeeds to previous one. +Another conveninent point is decreasing batch size adaptively. +If you set *batch_size_name* and implement code to report cuda out of memory using hpopt.reportOOM, +hpopt adaptively decrease batch size upper bound and make a new trial. + + +### Baye_opt (SMBO) + +```{image} ./image/smbo.png +:alt: smbo +:scale: 60% +:align: center +:target: https://arxiv.org/pdf/1012.2599v1.pdf +``` +*Image by [[Brochu et al., 2010](https://arxiv.org/pdf/1012.2599v1.pdf)]* + +Bayesian optimizaiton is one of HPO methods commoly used. +hpopt uses [BayesianOptimization](https://github.com/fmfn/BayesianOptimization) library for SMBO. +It estimates score map from previuos trials using gaussian process, +and chooses next candiate to try. +It's calssic but powerful method. + +You can modify behavior of *baye_opt* by giving some arguments. Here is those arguments. +- early_stop : Choice of early stopping methods. Currently ‘median_stop’ is supported. +- num_init_trial : How many trials to use to init SMBO. +- kappa : Kappa vlaue for ucb used in bayesian optimization. +- kappa_decay : Multiply kappa by kappa_decay every trials. +- kappa_decay_delay : Kappa isn’t multiplied to kappa_decay from first trials to kappa_decay_delay trials. + +If *early_stop* is set, some trials which is unlikely to get good score stopped in the middle. +It can save time by skipping inefficient trial. +Shortcoming of early stop is that it could skip trial which get lower score early but best score in the end. + +To estimate score map, Bayesian optimization needs some initial trial sampled randomly. +*num_init_trial* decides how many trials use random sampled hyper parameters. + +Bayesian optmization should balances between exploration and exploitation. +To be more sepecific, exploration means choosing unseen sample from search space +and exploitation means choosing sample which is likely going to provide good score. +If exploration overwhelms, bayesian optimization is almost same as random sampling, +In other case, Many good candiates is left unseen. So it's important to balance between them. +*kappa* is hyper parameter of bayesian optimization. +if higher kappa is higher, bayesian optimization explores more and vice versa. +you can modify this value to change balance. + +### ASHA + +If you have abundant GPU resouces, it must be better to run HPO parallely. +ASHA is good choice in that case. +ASHA runs multiple trials parallely, compares scores between them and terminates bed trials at specific iteration. +Left trials continue training, are compared again and some of them are terminated at next specific iteration and so on. +It's a like tournament. +Maybe you worry about that late bloomer is terminated eariler. +To avoid that case, ASHA run multiple tournaments some of which has very big interval between checkpoionts. + +There are some argument defining ASHA behavior like bayesian optimization. +- min_iterations : hpopt will run training at least ‘min_iterations’ times. +- reduction_factor : Used to set halving rate and amount. +- num_brackets : Number of brackets. Each bracket has a different halving rate, specified by the reduction factor. + +*min_iteration* is minimum iterations to compare scores literally. If you want that trial terminated after at least some iterations, you need to set this. + +Only 1/n trials can continue training every checkpoint. +*reduction_factor* means "n" used in previuos sentence. +That means if n is high, more of them terminates at checkpoint. It is recommended not to set it too high( > 4). +It could make HPO unstable. + +*num_brackets* means number of tournaments. More higher this value is, the more various checkpoint interval there are. + + +## Implementation Detail + +You need to implement loop to run each HPO trial after making hpopt class. +It's totally up to you how to implement the loop. +Although, I provide sample code expecting it could be simple guide to you. + +### Implement HPO loop + +Let't figure out how to implement loop code. +In this case, I assume that we use *baye_opt* as HPO algorithm. +Because the way of preceeding I'll expalain can be applied to all other environments +(one node with a GPU, multi node with multi GPU, etc.), +you are able to implement HPO with any environments. + +Please take a look at below code. + +``` python +import hpopt + +def run_hpo_trainer(train_config): + ... + +if __name__ == "__main__": + hpo = hpopt.create( + search_alg = "baye_opt", + ... + ) + + while True: + train_config = hpo.get_next_sample() + if train_config is None: + break + run_hpo_trainer(train_config) + + best_config = hpo.get_best_config() + hpo.print_results() +``` + +That's easy, right? +What you need to do is just getting *train_config* by hpo.get_next_sample() and train model with them. +I skipped how to implement *run_hpo_trainer* function now. +I'll explain in a little while. + +Do you remember that hpopt can adaptively decrease batch size if batch size is too big to GPU? +To enable this feature, you should add some code a little bit more. Take a look below code. + +``` python +import hpopt + +def run_hpo_trainer(train_config): + ... + +if __name__ == "__main__": + hpo = hpopt.create( + search_alg = "baye_opt", + batch_size_name = "bs", + ... + ) + + while True: + train_config = hpo.get_next_sample() + if train_config is None: + break + + try: + run_hpo_trainer(train_config) + # It can be different depending on your training framework. + except RuntimeError as err: + if str(err).startswith("CUDA out of memory"): + hpopt.reportOOM(train_config) + + best_config = hpo.get_best_config() + hpo.print_results() +``` + +You can see that *batch_size_name* is used to initiate hpopt class and +error handling syntax during invoking *run_hpo_trainer*. +If you report "cuda out of memory" situation to hpopt, +hpopt then decrease batch size search space automatically refering *batch_size_name*. + + +### Implment "run_hpo_trainer" function + +It's time to explain how to implement "run_hpo_trainer" function now. +Task of HPO is to optimize objective function which has input as hyper parameter and score as output. +So you need to make function which can get hyper parameters as input, run training with those hyper parameters and return score. +Do you remember "run_hpo_trainer" get **train_config** returned from *hpo.get_best_config()*? +You need to understand what *train_config* is first of all. + +**train_config** is dictionary which contains below keys. +- params : hyper parameters to optimize +- iterations : iterations for training +- subset_ratio : train dataset size ratio. + +Actually, there are others not written here. +I only introduce keys used in "run_hpo_trainer". +Now, what you need to do is to implement function which train models with these train configurations. +Before I show sample code, +please note that I assume that we use torch as framework, +and of course, you can change this code according to yours. +Let's see below code. + +``` python +import torch +import hpopt + + +def run_hpo_trainer(train_config): + total_epoch = train_config['iterations'] + lr = train_config['param']['lr'] + bs = train_config['param']['bs'] + subset_ratio = train_config['subset_ratio'] + + ... + train_size = int(subset_ratio * len(full_dataset)) + removed_data = len(full_dataset) - train_size + training_set, _ = torch.utils.data.random_split(full_dataset, [train_size, removed_data]) + training_loader = torch.utils.data.DataLoader(training_set, batch_size=bs) + ... + optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9) + ... + + def train_one_epoch(): + ... + + for epoch in range(total_epoch): + model.train(True) + score = train_one_epoch() + if hpopt.report(config=hp_config, score=score) == hpopt.Status.STOP: + break + ... + +if __name__ == "__main__": + hpo = hpopt.create(...) + + while True: + train_config = hpo.get_next_sample() + if train_config is None: + break + run_hpo_trainer(train_config) + + best_config = hpo.get_best_config() + hpo.print_results() +``` + +You can see that lr and bs are set and training set is splitted according to *subset_ratio*. +After preparation, model is trained with interations given from hpopt. +You shouldn't miss reporting score to hpopt after every epoch. +If hpopt determines that train doesn't need to proceed further, +*hpopt.report()* returns hpopt.Status.STOP. +Then you can just terminate train. + +That's it! you can now use hpopt freely according to your need. Please enjoy! diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..3d6b6e6 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,41 @@ +.. hpopt documentation master file, created by + sphinx-quickstart on Thu Jun 16 20:33:50 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to hpopt's documentation! +================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + contents/intro.md + contents/install.md + contents/getting_started.ipynb + contents/using.md + contents/apis.md + +.. toctree:: + :maxdepth: 2 + :caption: Flowcharts: + + contents/flowchart.md + contents/standard.md + contents/pause.md + +.. toctree:: + :maxdepth: 2 + :caption: HPO_OTE: + + contents/hpo_ote.md + contents/hpo_config.md + contents/cls_tutorial.md + contents/det_tutorial.md + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search`