notebooks/Grid Search Sample.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aff2abb3",
   "metadata": {},
   "source": [
    "## Grid search sample\n",
    "This notebook shows how CodeFlare pipelines can be used to perform grid search, where the standard parameters in a param grid can be input to the pipeline and one can call `grid_search_cv` to perform grid search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b182b659",
   "metadata": {},
   "outputs": [],
   "source": [
    "%config IPCompleter.use_jedi = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "da96167d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "03b2c2fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import numpy as np\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b3344db0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we will use a standard pipeline that is explained in more detail in a separate notebook/blog\n",
    "X_digits, y_digits = datasets.load_digits(return_X_y=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1000a8d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import codeflare.pipelines.Datamodel as dm\n",
    "import codeflare.pipelines.Runtime as rt\n",
    "\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1c6dab75",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-06-19 20:59:26,196\tINFO services.py:1269 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'node_ip_address': '9.211.53.245',\n",
       " 'raylet_ip_address': '9.211.53.245',\n",
       " 'redis_address': '9.211.53.245:6379',\n",
       " 'object_store_address': '/tmp/ray/session_2021-06-19_20-59-24_761561_86418/sockets/plasma_store',\n",
       " 'raylet_socket_name': '/tmp/ray/session_2021-06-19_20-59-24_761561_86418/sockets/raylet',\n",
       " 'webui_url': '127.0.0.1:8265',\n",
       " 'session_dir': '/tmp/ray/session_2021-06-19_20-59-24_761561_86418',\n",
       " 'metrics_export_port': 61314,\n",
       " 'node_id': 'd6148d041e7ad9fe60b016d54e9d91b6a0af574445694f2c2048f14d'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import ray\n",
    "ray.shutdown()\n",
    "\n",
    "ray.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0d0d5d4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA()\n",
    "# set the tolerance to a large value to make the example faster\n",
    "logistic = LogisticRegression(max_iter=10000, tol=0.1)\n",
    "\n",
    "pipeline = dm.Pipeline()\n",
    "node_pca = dm.EstimatorNode('pca', pca)\n",
    "node_logistic = dm.EstimatorNode('logistic', logistic)\n",
    "\n",
    "pipeline.add_edge(node_pca, node_logistic)\n",
    "\n",
    "# input to pipeline\n",
    "pipeline_input = dm.PipelineInput()\n",
    "pipeline_input.add_xy_arg(node_pca, dm.Xy(X_digits, y_digits))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5bfebb1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import codeflare.pipelines.utils as cf_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dfce8aa9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.47.2 (20210527.0053)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"81pt\" height=\"116pt\"\n",
       " viewBox=\"0.00 0.00 80.79 116.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 112)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-112 76.79,-112 76.79,4 -4,4\"/>\n",
       "<!-- pca -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>pca</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"36.4\" cy=\"-90\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"36.4\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">pca</text>\n",
       "</g>\n",
       "<!-- logistic -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>logistic</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"36.4\" cy=\"-18\" rx=\"36.29\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"36.4\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">logistic</text>\n",
       "</g>\n",
       "<!-- pca&#45;&gt;logistic -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>pca&#45;&gt;logistic</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M36.4,-71.7C36.4,-63.98 36.4,-54.71 36.4,-46.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"39.9,-46.1 36.4,-36.1 32.9,-46.1 39.9,-46.1\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fa2a9010150>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_param_graph = cf_utils.pipeline_to_graph(pipeline)\n",
    "non_param_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cf886598",
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {\n",
    "        'pca__n_components': [5, 15, 30, 45, 64],\n",
    "        'logistic__C': np.logspace(-4, 4, 4),\n",
    "    }\n",
    "\n",
    "pipeline_param = dm.PipelineParam.from_param_grid(param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "414f1f58",
   "metadata": {},
   "outputs": [],
   "source": [
    "parameterized_pipeline = pipeline.get_parameterized_pipeline(pipeline_param=pipeline_param)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f1bae14",
   "metadata": {},
   "source": [
    "### Expansion of parametric pipeline\n",
    "Given the `param_grid`, the parameterized pipeline is \"expanded\" to reflect the parallelism. One can see that we now have 9 nodes reflecting the combination of the parameter grid. Further, note that the k-fold cross validation also results in parallelism, which is data level parallelism and not reflected in the DAG itself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "05ce5b1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.47.2 (20210527.0053)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"462pt\" height=\"116pt\"\n",
       " viewBox=\"0.00 0.00 462.09 116.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 112)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-112 458.09,-112 458.09,4 -4,4\"/>\n",
       "<!-- pca__0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>pca__0</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"409.05\" cy=\"-90\" rx=\"36.29\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"409.05\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">pca__0</text>\n",
       "</g>\n",
       "<!-- logistic__0 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>logistic__0</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"286.05\" cy=\"-18\" rx=\"50.09\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"286.05\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">logistic__0</text>\n",
       "</g>\n",
       "<!-- pca__0&#45;&gt;logistic__0 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>pca__0&#45;&gt;logistic__0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M385.91,-75.83C367.48,-65.35 341.4,-50.5 320.51,-38.62\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"322.05,-35.46 311.62,-33.56 318.58,-41.55 322.05,-35.46\"/>\n",
       "</g>\n",
       "<!-- logistic__1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>logistic__1</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"404.05\" cy=\"-18\" rx=\"50.09\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"404.05\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">logistic__1</text>\n",
       "</g>\n",
       "<!-- pca__0&#45;&gt;logistic__1 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>pca__0&#45;&gt;logistic__1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M407.81,-71.7C407.26,-63.98 406.6,-54.71 405.98,-46.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"409.47,-45.83 405.27,-36.1 402.49,-46.33 409.47,-45.83\"/>\n",
       "</g>\n",
       "<!-- logistic__2 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>logistic__2</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"50.05\" cy=\"-18\" rx=\"50.09\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.05\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">logistic__2</text>\n",
       "</g>\n",
       "<!-- pca__0&#45;&gt;logistic__2 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>pca__0&#45;&gt;logistic__2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M381.62,-77.78C375.57,-75.61 369.15,-73.54 363.05,-72 252.5,-44.08 220.54,-59.84 109.05,-36 104.94,-35.12 100.7,-34.11 96.47,-33.03\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"97.23,-29.61 86.67,-30.41 95.42,-36.38 97.23,-29.61\"/>\n",
       "</g>\n",
       "<!-- logistic__3 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>logistic__3</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"168.05\" cy=\"-18\" rx=\"50.09\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"168.05\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">logistic__3</text>\n",
       "</g>\n",
       "<!-- pca__0&#45;&gt;logistic__3 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>pca__0&#45;&gt;logistic__3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M380.97,-78.14C375.09,-76 368.89,-73.85 363.05,-72 303.43,-53.14 287.31,-52.66 227.05,-36 223.28,-34.96 219.39,-33.87 215.47,-32.76\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.36,-29.38 205.78,-30.01 214.44,-36.11 216.36,-29.38\"/>\n",
       "</g>\n",
       "<!-- pca__1 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>pca__1</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"45.05\" cy=\"-90\" rx=\"36.29\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"45.05\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">pca__1</text>\n",
       "</g>\n",
       "<!-- pca__1&#45;&gt;logistic__0 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>pca__1&#45;&gt;logistic__0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M73.12,-78.14C79.01,-76 85.2,-73.85 91.05,-72 150.66,-53.14 166.78,-52.66 227.05,-36 230.81,-34.96 234.71,-33.87 238.62,-32.76\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"239.65,-36.11 248.31,-30.01 237.73,-29.38 239.65,-36.11\"/>\n",
       "</g>\n",
       "<!-- pca__1&#45;&gt;logistic__1 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>pca__1&#45;&gt;logistic__1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M72.47,-77.78C78.52,-75.61 84.94,-73.54 91.05,-72 201.59,-44.08 233.55,-59.84 345.05,-36 349.15,-35.12 353.39,-34.11 357.62,-33.03\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"358.67,-36.38 367.42,-30.41 356.86,-29.61 358.67,-36.38\"/>\n",
       "</g>\n",
       "<!-- pca__1&#45;&gt;logistic__2 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>pca__1&#45;&gt;logistic__2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.28,-71.7C46.83,-63.98 47.49,-54.71 48.11,-46.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"51.6,-46.33 48.82,-36.1 44.62,-45.83 51.6,-46.33\"/>\n",
       "</g>\n",
       "<!-- pca__1&#45;&gt;logistic__3 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>pca__1&#45;&gt;logistic__3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68.18,-75.83C86.61,-65.35 112.69,-50.5 133.58,-38.62\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"135.51,-41.55 142.47,-33.56 132.04,-35.46 135.51,-41.55\"/>\n",
       "</g>\n",
       "<!-- pca__2 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>pca__2</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"136.05\" cy=\"-90\" rx=\"36.29\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"136.05\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">pca__2</text>\n",
       "</g>\n",
       "<!-- pca__2&#45;&gt;logistic__0 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>pca__2&#45;&gt;logistic__0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M161.49,-77.13C185.04,-66.13 220.4,-49.64 247.43,-37.02\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"249.03,-40.14 256.61,-32.74 246.07,-33.79 249.03,-40.14\"/>\n",
       "</g>\n",
       "<!-- pca__2&#45;&gt;logistic__1 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>pca__2&#45;&gt;logistic__1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.57,-78.13C169.61,-75.95 176,-73.78 182.05,-72 253.23,-51.09 273.14,-54.28 345.05,-36 348.89,-35.02 352.86,-33.97 356.84,-32.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"357.99,-36.2 366.69,-30.15 356.12,-29.46 357.99,-36.2\"/>\n",
       "</g>\n",
       "<!-- pca__2&#45;&gt;logistic__2 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>pca__2&#45;&gt;logistic__2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M117.81,-74.15C106.05,-64.58 90.54,-51.96 77.33,-41.21\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.42,-38.39 69.45,-34.8 75,-43.82 79.42,-38.39\"/>\n",
       "</g>\n",
       "<!-- pca__2&#45;&gt;logistic__3 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>pca__2&#45;&gt;logistic__3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M143.79,-72.05C147.49,-63.97 151.99,-54.12 156.11,-45.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"159.4,-46.34 160.37,-35.79 153.03,-43.43 159.4,-46.34\"/>\n",
       "</g>\n",
       "<!-- pca__3 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>pca__3</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"227.05\" cy=\"-90\" rx=\"36.29\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"227.05\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">pca__3</text>\n",
       "</g>\n",
       "<!-- pca__3&#45;&gt;logistic__0 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>pca__3&#45;&gt;logistic__0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M240.43,-73.12C247.76,-64.42 257.01,-53.45 265.26,-43.66\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"268.12,-45.7 271.89,-35.8 262.77,-41.19 268.12,-45.7\"/>\n",
       "</g>\n",
       "<!-- pca__3&#45;&gt;logistic__1 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>pca__3&#45;&gt;logistic__1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M254.7,-78.06C283.46,-66.69 328.84,-48.74 362.03,-35.62\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"363.47,-38.81 371.48,-31.88 360.89,-32.3 363.47,-38.81\"/>\n",
       "</g>\n",
       "<!-- pca__3&#45;&gt;logistic__2 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>pca__3&#45;&gt;logistic__2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M199.39,-78.06C170.63,-66.69 125.25,-48.74 92.06,-35.62\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"93.2,-32.3 82.61,-31.88 90.63,-38.81 93.2,-32.3\"/>\n",
       "</g>\n",
       "<!-- pca__3&#45;&gt;logistic__3 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>pca__3&#45;&gt;logistic__3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M213.66,-73.12C206.33,-64.42 197.08,-53.45 188.83,-43.66\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"191.32,-41.19 182.2,-35.8 185.97,-45.7 191.32,-41.19\"/>\n",
       "</g>\n",
       "<!-- pca__4 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>pca__4</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"318.05\" cy=\"-90\" rx=\"36.29\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"318.05\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">pca__4</text>\n",
       "</g>\n",
       "<!-- pca__4&#45;&gt;logistic__0 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>pca__4&#45;&gt;logistic__0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M310.3,-72.05C306.6,-63.97 302.1,-54.12 297.98,-45.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"301.06,-43.43 293.72,-35.79 294.7,-46.34 301.06,-43.43\"/>\n",
       "</g>\n",
       "<!-- pca__4&#45;&gt;logistic__1 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>pca__4&#45;&gt;logistic__1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M336.28,-74.15C348.04,-64.58 363.55,-51.96 376.76,-41.21\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"379.09,-43.82 384.64,-34.8 374.67,-38.39 379.09,-43.82\"/>\n",
       "</g>\n",
       "<!-- pca__4&#45;&gt;logistic__2 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>pca__4&#45;&gt;logistic__2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.52,-78.13C284.48,-75.95 278.09,-73.78 272.05,-72 200.86,-51.09 180.95,-54.28 109.05,-36 105.2,-35.02 101.23,-33.97 97.25,-32.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"97.97,-29.46 87.4,-30.15 96.1,-36.2 97.97,-29.46\"/>\n",
       "</g>\n",
       "<!-- pca__4&#45;&gt;logistic__3 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>pca__4&#45;&gt;logistic__3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M292.6,-77.13C269.05,-66.13 233.69,-49.64 206.66,-37.02\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"208.02,-33.79 197.48,-32.74 205.06,-40.14 208.02,-33.79\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fa2a90193d0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_graph = cf_utils.pipeline_to_graph(parameterized_pipeline)\n",
    "param_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0af813c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "k = 5\n",
    "kf = KFold(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4020342e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.41 s, sys: 1.21 s, total: 2.63 s\n",
      "Wall time: 9.21 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "result = rt.grid_search_cv(kf, pipeline, pipeline_input, pipeline_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bcf25c0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9226679046734757\n",
      "pca__3{'copy': True, 'iterated_power': 'auto', 'n_components': 45, 'random_state': None, 'svd_solver': 'auto', 'tol': 0.0, 'whiten': False}=\r\n",
      "logistic__1{'C': 0.046415888336127774, 'class_weight': None, 'dual': False, 'fit_intercept': True, 'intercept_scaling': 1, 'l1_ratio': None, 'max_iter': 10000, 'multi_class': 'auto', 'n_jobs': None, 'penalty': 'l2', 'random_state': None, 'solver': 'lbfgs', 'tol': 0.1, 'verbose': 0, 'warm_start': False}=pca__3{'copy': True, 'iterated_power': 'auto', 'n_components': 45, 'random_state': None, 'svd_solver': 'auto', 'tol': 0.0, 'whiten': False} \r\n",
      "\n",
      "0.9260058805323429\n",
      "pca__4{'copy': True, 'iterated_power': 'auto', 'n_components': 64, 'random_state': None, 'svd_solver': 'auto', 'tol': 0.0, 'whiten': False}=\r\n",
      "logistic__1{'C': 0.046415888336127774, 'class_weight': None, 'dual': False, 'fit_intercept': True, 'intercept_scaling': 1, 'l1_ratio': None, 'max_iter': 10000, 'multi_class': 'auto', 'n_jobs': None, 'penalty': 'l2', 'random_state': None, 'solver': 'lbfgs', 'tol': 0.1, 'verbose': 0, 'warm_start': False}=pca__4{'copy': True, 'iterated_power': 'auto', 'n_components': 64, 'random_state': None, 'svd_solver': 'auto', 'tol': 0.0, 'whiten': False} \r\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import statistics\n",
    "\n",
    "# pick the best mean'\n",
    "best_pipeline = None\n",
    "best_mean_scores = 0.0\n",
    "\n",
    "for cv_pipeline, scores in result.items():\n",
    "    mean = statistics.mean(scores)\n",
    "    if mean > 0.92:\n",
    "        print(mean)\n",
    "        print(str(cv_pipeline))\n",
    "        \n",
    "    if mean > best_mean_scores:\n",
    "        best_pipeline = cv_pipeline\n",
    "        best_mean_scores = mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0aa813bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"pca__4{'copy': True, 'iterated_power': 'auto', 'n_components': 64, 'random_state': None, 'svd_solver': 'auto', 'tol': 0.0, 'whiten': False}=\\r\\nlogistic__1{'C': 0.046415888336127774, 'class_weight': None, 'dual': False, 'fit_intercept': True, 'intercept_scaling': 1, 'l1_ratio': None, 'max_iter': 10000, 'multi_class': 'auto', 'n_jobs': None, 'penalty': 'l2', 'random_state': None, 'solver': 'lbfgs', 'tol': 0.1, 'verbose': 0, 'warm_start': False}=pca__4{'copy': True, 'iterated_power': 'auto', 'n_components': 64, 'random_state': None, 'svd_solver': 'auto', 'tol': 0.0, 'whiten': False} \\r\\n\""
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(best_pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d1b0b7f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9260058805323429"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_mean_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7db8d362",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Currently, we are in process of making more parity of the CF pipeline grid search API to provide similar\n",
    "# methods as that of SKLearn Gridsearch APIs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e1694ca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "877e3f63",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "# Define a pipeline to search for the best combination of PCA truncation\n",
    "# and classifier regularization.\n",
    "pca = PCA()\n",
    "# set the tolerance to a large value to make the example faster\n",
    "logistic = LogisticRegression(max_iter=10000, tol=0.1)\n",
    "pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)])\n",
    "\n",
    "X_digits, y_digits = datasets.load_digits(return_X_y=True)\n",
    "\n",
    "# Parameters of pipelines can be set using ‘__’ separated parameter names:\n",
    "param_grid = {\n",
    "    'pca__n_components': [5, 15, 30, 45, 64],\n",
    "    'logistic__C': np.logspace(-4, 4, 4),\n",
    "}\n",
    "search = GridSearchCV(pipe, param_grid, n_jobs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "195c0d15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter (CV score=0.920):\n",
      "{'logistic__C': 0.046415888336127774, 'pca__n_components': 45}\n",
      "CPU times: user 1min 31s, sys: 46.2 s, total: 2min 17s\n",
      "Wall time: 21.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "search.fit(X_digits, y_digits)\n",
    "print(\"Best parameter (CV score=%0.3f):\" % search.best_score_)\n",
    "print(search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c568665a",
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}