GAN(gan=rgan,data=best(xing,stress),canon=stress,share=6,embed=10).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wZtYFEhHoBGp"
   },
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "KJPfzpxLQI5O"
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from deepgd import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "z4eBEyB1oaJe"
   },
   "source": [
    "# Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "Q_-MLFhpNvxF"
   },
   "outputs": [],
   "source": [
    "cuda_idx = 0\n",
    "canonicalization = Canonicalization(normalize=Standardization(norm_ord=1),\n",
    "                                    scale=False)\n",
    "config = StaticConfig({\n",
    "    \"name\": 'GAN(gan=rgan,data=best(xing,stress),canon=stress,share=6,embed=10)',\n",
    "    \"uid\": None,\n",
    "    \"link\": None,\n",
    "    \"generator\": {\n",
    "        \"params\": {\n",
    "            \"num_blocks\": 9,\n",
    "            \"normalize\": canonicalization\n",
    "        },\n",
    "        \"pretrained\": {\n",
    "            \"name\": None,\n",
    "            \"epoch\": -1,\n",
    "        },\n",
    "        \"optim\": torch.optim.AdamW,\n",
    "        \"lr\" : {\n",
    "            \"initial\": 1e-3,\n",
    "            \"decay\": 0.99,\n",
    "        },\n",
    "    },\n",
    "    \"discriminator\": {\n",
    "        \"params\": {\n",
    "            \"conv\": [2, 16, 16, 16],\n",
    "            \"dense\": [2],\n",
    "            \"shared_depth\": 6,\n",
    "            \"enet_depth\": 10,\n",
    "            \"enet_width\": 64,\n",
    "            \"aggr\": \"add\",\n",
    "            \"normalize\": canonicalization\n",
    "        },\n",
    "        \"pretrained\": {\n",
    "            \"name\": None,\n",
    "            \"epoch\": -1,\n",
    "        },\n",
    "        \"optim\": torch.optim.AdamW,\n",
    "        \"lr\" : {\n",
    "            \"initial\": 1e-3,\n",
    "            \"decay\": 0.99,\n",
    "        },\n",
    "        \"noise\": {\n",
    "            \"std\": 0,\n",
    "            \"decay\": 0.95,\n",
    "        },\n",
    "        \"repeat\": 1,\n",
    "        \"complete\": True,\n",
    "        \"adaptive\": True\n",
    "    },\n",
    "    \"alternate\": \"epoch\",\n",
    "    \"batchsize\": 24,\n",
    "    \"epoch\": {\n",
    "        \"start\": -1,\n",
    "        \"end\": None,\n",
    "    },\n",
    "    \"log_interval\": 1,\n",
    "    \"test\": {\n",
    "        \"name\": \"test\",\n",
    "        \"epoch\": -1,\n",
    "    },\n",
    "    \"gan_flavor\": \"rgan\",\n",
    "    \"gp_weight\": 0,\n",
    "})\n",
    "data_config = StaticConfig({\n",
    "    \"sparse\": False,\n",
    "    \"pivot\": None,\n",
    "    \"init\": \"pmds\",\n",
    "    \"edge\": {\n",
    "        \"index\": \"full_edge_index\",\n",
    "        \"attr\": \"full_edge_attr\",\n",
    "    },\n",
    "})\n",
    "loss_fns = {\n",
    "    Stress(): 1\n",
    "}\n",
    "ctrler_params = {\n",
    "    \"tau\": 0.95,\n",
    "    \"beta\": 1,\n",
    "    \"exploit_rate\": 0.5,\n",
    "    \"warmup\": 2,\n",
    "}\n",
    "paths = StaticConfig({\n",
    "    \"root\": \"artifacts\",\n",
    "    \"checkpoints\": lambda: f\"{paths.root}/checkpoints/{config.name}\",\n",
    "    \"gen_pretrain\": lambda: f\"{paths.root}/checkpoints/{config.generator.pretrained.name}\",\n",
    "    \"dis_pretrain\": lambda: f\"{paths.root}/checkpoints/{config.discriminator.pretrained.name}\",\n",
    "    \"tensorboard\": lambda: f\"{paths.root}/tensorboards/{config.name}\",\n",
    "    \"visualization\": lambda: f\"{paths.root}/visualizations/{config.name}_{config.test.name}\",\n",
    "    \"log\": lambda: f\"{paths.root}/logs/{config.name}.log\",\n",
    "    \"metrics\": lambda suffix: f\"{paths.root}/metrics/{config.name}_{suffix}.pickle\",\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "mdMEAbH10Qaq"
   },
   "outputs": [],
   "source": [
    "if \" \" in config.name:\n",
    "    raise Exception(\"Space is not allowed in model name.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WynW4ZAdBhep"
   },
   "source": [
    "# Prepare"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DR6-vYtr_i_P"
   },
   "source": [
    "## Get log command"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9-QpinlslcTO",
    "outputId": "48ae3716-212c-4d46-9efb-b7b4070f73de"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cd /users/PAS0027/osu10203/deepgd && tail -n1000 -f 'artifacts/logs/GAN(gan=rgan,data=best(xing,stress),canon=stress,share=6,embed=10).log'\n"
     ]
    }
   ],
   "source": [
    "print(f\"cd {os.getcwd()} && tail -n1000 -f '{paths.log()}'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensorboard dev upload --logdir 'artifacts/tensorboards/GAN(gan=rgan,data=best(xing,stress),canon=stress,share=6,embed=10)'\n"
     ]
    }
   ],
   "source": [
    "print(f\"tensorboard dev upload --logdir '{paths.tensorboard()}'\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "innqkwvH_ydD"
   },
   "source": [
    "## Set globals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "Qjq7JlSZlkQR"
   },
   "outputs": [],
   "source": [
    "if cuda_idx is not None and torch.cuda.is_available():\n",
    "    device = f'cuda:{cuda_idx}'\n",
    "    pynvml.nvmlInit()\n",
    "    cuda = pynvml.nvmlDeviceGetHandleByIndex(cuda_idx)\n",
    "else:\n",
    "    device = 'cpu'\n",
    "    cuda =  None\n",
    "np.set_printoptions(precision=2)\n",
    "warnings.filterwarnings(\"ignore\", category=RuntimeWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0da286l_ApEL"
   },
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RFuSHiZJMU4x",
    "outputId": "54065609-aaea-4536-d79c-73bc73b3f576"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load from 'cache/G_list.pickle'\n",
      "Load from 'cache/generate_data_list(list,sparse=False,pivot_mode=None,init_mode=pmds,edge_index=full_edge_index,edge_attr=full_edge_attr,pmds_list=ndarray,gviz_list=ndarray,noisy_layout=True,device=cpu).pickle'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/PAS0027/osu10203/.conda/envs/deepgd/lib/python3.9/site-packages/torch_geometric/deprecation.py:13: UserWarning: 'data.DataLoader' is deprecated, use 'loader.DataLoader' instead\n",
      "  warnings.warn(out)\n"
     ]
    }
   ],
   "source": [
    "G_list = load_G_list(data_path='data/rome', index_file='data_index.txt', cache='G_list', cache_prefix='cache/')\n",
    "data_list = generate_data_list(G_list, \n",
    "                               sparse=data_config.sparse, \n",
    "                               pivot_mode=data_config.pivot,\n",
    "                               init_mode=data_config.init,\n",
    "                               edge_index=data_config.edge.index,\n",
    "                               edge_attr=data_config.edge.attr,\n",
    "                               pmds_list=np.load('layouts/rome/pmds.npy', allow_pickle=True),\n",
    "                               gviz_list=np.load('layouts/rome/gviz.npy', allow_pickle=True),\n",
    "                               noisy_layout=True,\n",
    "                               device='cpu', \n",
    "                               cache=True,\n",
    "                               cache_prefix='cache/')\n",
    "train_loader = LazyDeviceMappingDataLoader(data_list[:10000], batch_size=config.batchsize, shuffle=True, device=device)\n",
    "val_loader = LazyDeviceMappingDataLoader(data_list[11000:], batch_size=config.batchsize, shuffle=False, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "djd1pLt2OxZq"
   },
   "outputs": [],
   "source": [
    "def draw_layout(G, method, draw=True):\n",
    "    if method == 'fa2':\n",
    "        layout = get_fa2_layout(G)\n",
    "    else:\n",
    "        try:\n",
    "            fn = getattr(nx.drawing.layout, f'{method}_layout')\n",
    "            layout = fn(G)\n",
    "        except:\n",
    "            layout = nx.drawing.nx_agraph.graphviz_layout(G, prog=method)\n",
    "    if draw:\n",
    "        nx.draw(G, pos=layout)\n",
    "    return layout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "2vBBw77lR7Em"
   },
   "outputs": [],
   "source": [
    "methods = ['neato', 'dot', 'fdp', 'sfdp', 'twopi', 'circo', 'shell', 'spring', 'circular', 'spectral', 'kamada_kawai', 'fa2', 'pmds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "dx6VVA9XIc2O"
   },
   "outputs": [],
   "source": [
    "from functools import lru_cache\n",
    "\n",
    "@lru_cache(maxsize=None)\n",
    "def load_pos(method):\n",
    "    return np.load(f'layouts/rome/{method}.npy', allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "05a37aeeb20049dfa2f97287b3990556",
      "a75afae3adfd4810a0b0991a062535ee",
      "a24ae105ed124595992f7e2e5f909247",
      "01b4cabaf8004dd4b9269b4e6889c498",
      "f3c7c84f2b314bd8b97a7358deaf4fa1",
      "35e4a83e692b4a7bafb46fdf192b2586",
      "62847b5e2678459f845159eef6d4feab",
      "143e81f00313436dacfa246a57108944",
      "e05833f11803400b95ff22246cdc8f2b",
      "53aeede91e074378b5d7f626165655ee",
      "79a43768c1794a10a788a49e5883aa2c"
     ]
    },
    "id": "MSmP-jem9I0X",
    "outputId": "c698f0bf-e96e-4de9-aeda-0b046fc0a03f",
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# best_list = []\n",
    "# best_layout_list = []\n",
    "# for idx, (G, data) in enumerate(zip(tqdm(G_list), data_list)):\n",
    "#     xing, stress, layout = {}, {}, {}\n",
    "#     for m in methods:\n",
    "#         batch = Batch.from_data_list([data])\n",
    "#         pos = load_pos(m)\n",
    "#         p = CanonicalizationByStress()(torch.tensor(pos[idx]).float(), batch)\n",
    "#         x = Xing()(p, batch).item()\n",
    "#         s = Stress()(p, batch).item()\n",
    "#         xing[m] = x\n",
    "#         stress[m] = s\n",
    "#         layout[m] = p.numpy()\n",
    "#         # plt.figure()\n",
    "#         # graph_vis(G, pos[idx])\n",
    "#         # plt.title(f'{m} stress={s:.2f} xing={x}')\n",
    "#     best, *_ = sorted(methods, key=lambda m: (xing[m], stress[m]))\n",
    "#     best_list.append(best)\n",
    "#     best_layout_list.append(layout[best])\n",
    "#     print(f'{best}, xing={xing[best]}, stress={stress[best]:.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Mx2bZNugdTqa",
    "outputId": "cec05a67-18e8-492f-aaa1-057274a8341a"
   },
   "outputs": [],
   "source": [
    "# pickle.dump(best_list, open('layouts/rome/best[xing,stress].pkl', 'wb'))\n",
    "# np.save('layouts/rome/best[xing,stress].npy', best_layout_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "7e0ae84dd0f3458d893c81b029fe1729",
      "18bdc3bf80fb434abde68de5b72a8870",
      "2e3552ded9594635a51832387ae754a0",
      "54f4aa5080c6498080a63be2cdeefee5",
      "3937586d0879451da5e2138ca3bfd0a0",
      "a9df6e84ce8c499eb2b2cf984995d791",
      "7af6b718dd3d4e01b264377a2fa938e2",
      "a774c86e64004bd6951645f1c24abad4",
      "c86575117cd041f2837762f37d1f387b",
      "6e9907cefb1241688fbd248b8281ed9f",
      "9b8e5405d8264944acab57e81f2f5ad6"
     ]
    },
    "id": "A1-w1fspC53p",
    "outputId": "09c2d5c3-1eef-41eb-dcee-581af1294804"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d67ee36109a428da712b5d120b083cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11531 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "best_layout_list = np.load('layouts/rome/best[xing,stress].npy', allow_pickle=True)\n",
    "for data, layout in zip(tqdm(data_list), best_layout_list):\n",
    "    data.gt_pos = torch.tensor(layout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "y6EGh4U7exL_"
   },
   "outputs": [],
   "source": [
    "train_loader = LazyDeviceMappingDataLoader(data_list[:10000], batch_size=config.batchsize, shuffle=True, device=device)\n",
    "val_loader = LazyDeviceMappingDataLoader(data_list[11000:], batch_size=config.batchsize, shuffle=False, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 103,
     "referenced_widgets": [
      "38a33cf69aff48f4aabd4902a7ce48b1",
      "57e277bcb44c4e2fa2a12105d75153d1",
      "20ce017c46bd4a7c9b420270ac579405",
      "13f2df4fe0ce4af88b6e3bd1637065b0",
      "2867faa145ca4860a198d4d0bf830f89",
      "02838cf669174ed9afa41a1a8e680765",
      "7e9e3c031667455f8baca163d3a213e5",
      "4000135e7dfa46529fa6244fccc5cb57",
      "e523e9faac5b4ec38643c05ac5ba99f7",
      "e8e3ee9741e4480dae5003ccc19a7891",
      "9a569045221b43acb3ba2fbf116b2c74"
     ]
    },
    "id": "YEfRRUK0NiNm",
    "outputId": "93f53d05-7667-4cda-a38e-c3a953124f79"
   },
   "outputs": [],
   "source": [
    "# for m in methods:\n",
    "#     layouts = []\n",
    "#     for G in tqdm(G_list):\n",
    "#         layout = draw_layout(G, method=m, draw=False)\n",
    "#         layouts.append(np.array(list(layout.values())))\n",
    "#     np.save(f'layouts/rome/{m}.npy', layouts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 564
    },
    "id": "upK7AHYqPmC5",
    "outputId": "29a88e61-f55f-409c-9194-12c6d5762528"
   },
   "outputs": [],
   "source": [
    "# from collections import Counter\n",
    "\n",
    "# letter_counts = Counter(best_list)\n",
    "# df = pd.DataFrame.from_dict(letter_counts, orient='index')\n",
    "# df.plot(kind='bar', figsize=[12, 8])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f-ODE2k8BFV6"
   },
   "source": [
    "## Create folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "3ykUMINYBRog"
   },
   "outputs": [],
   "source": [
    "mkdirs(paths.checkpoints(), paths.tensorboard(), paths.visualization(), f\"{paths.root}/logs\", f\"{paths.root}/metrics\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BFq9k5nzBIh5"
   },
   "source": [
    "## Load checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "W5NbsKkobIfA"
   },
   "outputs": [],
   "source": [
    "class EdgeFeatureDiscriminator(nn.Module):\n",
    "    def __init__(self, \n",
    "                 conv, \n",
    "                 dense,\n",
    "                 shared_depth,\n",
    "                 enet_depth,\n",
    "                 enet_width,\n",
    "                 aggr='add', \n",
    "                 root_weight=True,\n",
    "                 normalize=None):\n",
    "        super().__init__()\n",
    "        self.enet = nn.Sequential(*[\n",
    "            DenseLayer(in_dim=in_d,\n",
    "                       out_dim=out_d,\n",
    "                       skip=nonlin,\n",
    "                       bn=nonlin,\n",
    "                       act=nonlin,\n",
    "                       dp=None)\n",
    "            for in_d, out_d, nonlin \n",
    "            in zip([self._get_feature_dim()] + [enet_width] * (shared_depth-1),\n",
    "                   [enet_width] * shared_depth,\n",
    "                   [True] * (shared_depth-1) + [False])     \n",
    "        ])\n",
    "        self.blocks = nn.ModuleList([\n",
    "            GNNLayer(nfeat_dims=(in_d, out_d),\n",
    "                     efeat_dim=enet_width,\n",
    "                     edge_net=EdgeNet(nfeat_dims=(in_d, out_d), \n",
    "                                      efeat_dim=enet_width, \n",
    "                                      depth=enet_depth, \n",
    "                                      width=enet_width),\n",
    "                     aggr=aggr,\n",
    "                     dense=False,\n",
    "                     skip=nonlin,\n",
    "                     bn=nonlin,\n",
    "                     act=nonlin,\n",
    "                     root_weight=root_weight) \n",
    "            for in_d, out_d, nonlin \n",
    "            in zip(conv[:-1], conv[1:], [True] * (len(conv)-2) + [False])     \n",
    "        ])\n",
    "        self.pool = (gnn.global_mean_pool if aggr == 'mean' \n",
    "                     else gnn.global_add_pool if aggr == 'add' \n",
    "                     else None)\n",
    "        self.dense = nn.Identity() if not dense else nn.Sequential(*[\n",
    "            DenseLayer(in_dim=in_d,\n",
    "                       out_dim=out_d,\n",
    "                       skip=nonlin,\n",
    "                       bn=nonlin,\n",
    "                       act=nonlin,\n",
    "                       dp=None)\n",
    "            for in_d, out_d, nonlin \n",
    "            in zip(conv[-1:] + dense[:-1], dense, [True] * (len(dense)-1) + [False])     \n",
    "        ])\n",
    "        self.normalize = normalize or IdentityTransformation()\n",
    "    \n",
    "    def forward(self, batch):\n",
    "        x = torch.ones_like(batch.pos)\n",
    "        e = self.enet(self._get_features(self._get_edge_info(batch, layout='pos')))\n",
    "        for block in self.blocks:\n",
    "            x = block(x, e, batch)\n",
    "        x = self.pool(x, batch.batch)\n",
    "        x = self.dense(x)\n",
    "        return x\n",
    "    \n",
    "    def _get_edge_info(self, batch, layout='gt_pos'):\n",
    "        pos = self.normalize(batch[layout].float(), batch)\n",
    "        src, dst = get_edges(pos, batch)\n",
    "        v, u = l2_normalize(dst - src, return_norm=True)\n",
    "        d = batch.edge_attr[:, :1]\n",
    "        return {\n",
    "            \"src\": src,\n",
    "            \"dst\": dst,\n",
    "            \"v\": v,\n",
    "            \"u\": u,\n",
    "            \"d\": d,\n",
    "        }\n",
    "\n",
    "    def _get_features(self, edges):\n",
    "        return torch.cat([edges['src'], edges['dst'], edges['d']], dim=1)\n",
    "        \n",
    "    def _get_feature_dim(self):\n",
    "        return self._get_features({\n",
    "            \"src\": torch.zeros(1, 2),\n",
    "            \"dst\": torch.zeros(1, 2),\n",
    "            \"v\": torch.zeros(1, 2),\n",
    "            \"u\": torch.zeros(1, 1),\n",
    "            \"d\": torch.zeros(1, 1),\n",
    "        }).shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "96ZActxlL0fJ"
   },
   "outputs": [],
   "source": [
    "class StressDiscriminator(nn.Module):\n",
    "    def __init__(self, normalize=CanonicalizationByStress(), **kwargs):\n",
    "        super().__init__()\n",
    "        self.dummy = nn.Parameter(torch.zeros(1))\n",
    "        self.normalize = normalize\n",
    "        self.stress = Stress(reduce=None)\n",
    "\n",
    "    def forward(self, batch):\n",
    "        return -self.stress(self.normalize(batch.pos, batch), batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "IJ6xCYBUuhcw"
   },
   "outputs": [],
   "source": [
    "def get_ckpt_epoch(folder, epoch):\n",
    "    if not os.path.isdir(folder):\n",
    "        os.mkdir(folder)\n",
    "    if epoch >= 0:\n",
    "        return epoch\n",
    "    ckpt_files = os.listdir(folder)\n",
    "    last_epoch = 0\n",
    "    if ckpt_files:\n",
    "        last_epoch = sorted(list(map(lambda x: int(re.search('(?<=epoch_)(\\d+)(?=\\.)', x).group(1)), ckpt_files)))[-1]\n",
    "    return last_epoch + epoch + 1\n",
    "\n",
    "def start_epoch():\n",
    "    return get_ckpt_epoch(paths.checkpoints(), config.epoch.start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JacDYXka9U01",
    "outputId": "ef4e31ab-2144-497c-8e4f-4d3740ce2d52"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading from artifacts/checkpoints/GAN(gan=rgan,data=best(xing,stress),canon=stress,share=6,embed=10)/gen_epoch_1011.pt...\n",
      "Loading from artifacts/checkpoints/GAN(gan=rgan,data=best(xing,stress),canon=stress,share=6,embed=10)/gen_optim_epoch_1011.pt...\n"
     ]
    }
   ],
   "source": [
    "generator = Generator(**config.generator.params[...]).to(device)\n",
    "generator_optimizer = config.generator.optim(generator.parameters(), lr=config.generator.lr.initial * config.generator.lr.decay ** start_epoch())\n",
    "generator_scheduler = torch.optim.lr_scheduler.ExponentialLR(generator_optimizer, gamma=config.generator.lr.decay)\n",
    "if start_epoch() != 0:\n",
    "    gen_ckpt_epoch = start_epoch()\n",
    "elif config.generator.pretrained.name is not None and config.generator.pretrained.epoch != 0:\n",
    "    gen_pretrained_epoch = get_ckpt_epoch(paths.gen_pretrain(), config.generator.pretrained.epoch)\n",
    "    gen_ckpt_epoch = gen_pretrained_epoch \n",
    "else:\n",
    "    gen_ckpt_epoch = None\n",
    "if gen_ckpt_epoch is not None:\n",
    "    # Load generator\n",
    "    gen_ckpt_file = f\"{paths.checkpoints()}/gen_epoch_{gen_ckpt_epoch}.pt\"\n",
    "    print(f\"Loading from {gen_ckpt_file}...\")\n",
    "    generator.load_state_dict(torch.load(gen_ckpt_file, map_location=torch.device(device)))\n",
    "    # Load generator optimizer\n",
    "    gen_optim_ckpt_file = f\"{paths.checkpoints()}/gen_optim_epoch_{gen_ckpt_epoch}.pt\"\n",
    "    print(f\"Loading from {gen_optim_ckpt_file}...\")\n",
    "    generator_optimizer.load_state_dict(torch.load(gen_optim_ckpt_file, map_location=torch.device(device)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "oWQhD7-88VXe",
    "outputId": "3ba01600-b546-4e45-a6bd-c7a65af1ae9e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading from artifacts/checkpoints/GAN(gan=rgan,data=best(xing,stress),canon=stress,share=6,embed=10)/dis_epoch_1011.pt...\n",
      "Loading from artifacts/checkpoints/GAN(gan=rgan,data=best(xing,stress),canon=stress,share=6,embed=10)/dis_optim_epoch_1011.pt...\n"
     ]
    }
   ],
   "source": [
    "discriminator = EdgeFeatureDiscriminator(**config.discriminator.params[...]).to(device)\n",
    "discriminator_optimizer = config.discriminator.optim(discriminator.parameters(), lr=config.discriminator.lr.initial * config.discriminator.lr.decay ** start_epoch())\n",
    "discriminator_scheduler = torch.optim.lr_scheduler.ExponentialLR(discriminator_optimizer, gamma=config.discriminator.lr.decay)\n",
    "if start_epoch() != 0:\n",
    "    dis_ckpt_epoch = start_epoch()\n",
    "elif config.discriminator.pretrained.name is not None and config.discriminator.pretrained.epoch != 0:\n",
    "    dis_pretrained_epoch = get_ckpt_epoch(paths.dis_pretrain(), config.discriminator.pretrained.epoch)\n",
    "    dis_ckpt_epoch = dis_pretrained_epoch # f\"{paths.dis_pretrain()}/dis_epoch_{dis_pretrained_epoch}.pt\"\n",
    "else:\n",
    "    dis_ckpt_epoch = None\n",
    "if dis_ckpt_epoch is not None:\n",
    "    # Load discriminator\n",
    "    dis_ckpt_file = f\"{paths.checkpoints()}/dis_epoch_{dis_ckpt_epoch}.pt\"\n",
    "    print(f\"Loading from {dis_ckpt_file}...\")\n",
    "    discriminator.load_state_dict(torch.load(dis_ckpt_file, map_location=torch.device(device)))\n",
    "    # Load discriminator optimizer\n",
    "    dis_optim_ckpt_file = f\"{paths.checkpoints()}/dis_optim_epoch_{dis_ckpt_epoch}.pt\"\n",
    "    print(f\"Loading from {dis_optim_ckpt_file}...\")\n",
    "    discriminator_optimizer.load_state_dict(torch.load(dis_optim_ckpt_file, map_location=torch.device(device)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "N39dDHraedM6"
   },
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "BdIJSDSJedM-"
   },
   "outputs": [],
   "source": [
    "stress_criterion = StressDiscriminator().to(device)\n",
    "val_criterion = Stress(reduce=None)\n",
    "xing_criterion = Xing(reduce=None)\n",
    "dis_convert = DiscriminatorDataConverter(complete_graph=config.discriminator.complete, normalize=config.discriminator.params.normalize)\n",
    "tensorboard = SummaryWriter(log_dir=paths.tensorboard())\n",
    "epoch = start_epoch() + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "OHRWj1h0edM-"
   },
   "outputs": [],
   "source": [
    "def gradient_penalty(interpolated, discriminator, weight=10):\n",
    "    interpolated.pos.requires_grad_()\n",
    "    prob_interpolated = discriminator(interpolated)\n",
    "    gradients = autograd.grad(outputs=prob_interpolated, \n",
    "                              inputs=interpolated.pos,\n",
    "                              grad_outputs=torch.ones_like(prob_interpolated),\n",
    "                              create_graph=True, \n",
    "                              retain_graph=True, \n",
    "                              allow_unused=True)[0]\n",
    "    gradients_norm = torch.sqrt(gnn.global_add_pool(gradients.square().sum(dim=1), batch.batch) + 1e-8)\n",
    "    return weight * ((gradients_norm - 1) ** 2).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "o5wJJN9AedM_"
   },
   "outputs": [],
   "source": [
    "def get_gp_loss(batch, fake_pos, weight):\n",
    "    if weight > 0:\n",
    "        interp = dis_convert(batch, fake_pos, random.random())\n",
    "        return gradient_penalty(interp, discriminator, weight).mean()\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "QwuA6sRoedM_"
   },
   "outputs": [],
   "source": [
    "def get_sgan_loss(batch, fake_pos, mode='discriminator'):\n",
    "    real = dis_convert(batch)\n",
    "    fake = dis_convert(batch, fake_pos)\n",
    "    pred = discriminator(merge_batch(real, fake)).view(2, -1).T\n",
    "    if mode == 'discriminator':\n",
    "        label = torch.zeros(pred.shape[0]).long()\n",
    "    elif mode == 'generator':\n",
    "        label = torch.ones(pred.shape[0]).long()\n",
    "    else:\n",
    "        raise Exception\n",
    "    return nn.CrossEntropyLoss()(pred, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "GuAP1YchedNA"
   },
   "outputs": [],
   "source": [
    "def get_rgan_loss(batch, fake_pos, mode='discriminator'):\n",
    "    real = dis_convert(batch)\n",
    "    fake = dis_convert(batch, fake_pos)\n",
    "    pred = discriminator(merge_batch(real, fake)).view(2, -1).T\n",
    "    real_pred, fake_pred = pred[:,0], pred[:,1]\n",
    "    if mode == 'discriminator':\n",
    "        losses = - F.logsigmoid(real_pred - fake_pred)\n",
    "    elif mode == 'generator':\n",
    "        losses = - F.logsigmoid(fake_pred - real_pred)\n",
    "    else:\n",
    "        raise Exception\n",
    "    return losses.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "KFllQ7BredNA"
   },
   "outputs": [],
   "source": [
    "def get_wgan_loss(batch, fake_pos, mode='discriminator'):\n",
    "    real = dis_convert(batch)\n",
    "    fake = dis_convert(batch, fake_pos)\n",
    "    pred = discriminator(merge_batch(real, fake)).view(2, -1).T\n",
    "    real_pred, fake_pred = pred[:,0], pred[:,1]\n",
    "    if mode == 'discriminator':\n",
    "        losses = fake_pred - real_pred \n",
    "    elif mode == 'generator':\n",
    "        losses = real_pred - fake_pred\n",
    "    else:\n",
    "        raise Exception\n",
    "    return losses.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "r5Kg_tYEedNA"
   },
   "outputs": [],
   "source": [
    "def get_ragan_loss(batch, fake_pos, mode='discriminator'):\n",
    "    real = dis_convert(batch)\n",
    "    fake = dis_convert(batch, fake_pos)\n",
    "    pred = discriminator(merge_batch(real, fake)).view(2, -1).T\n",
    "    real_pred, fake_pred = pred[:,0], pred[:,1]\n",
    "    if mode == 'discriminator':\n",
    "        losses = - F.logsigmoid(real_pred - fake_pred.mean()) - F.logsigmoid(real_pred.mean() - fake_pred)\n",
    "    elif mode == 'generator':\n",
    "        losses = - F.logsigmoid(fake_pred - real_pred.mean()) - F.logsigmoid(fake_pred.mean() - real_pred)\n",
    "    else:\n",
    "        raise Exception\n",
    "    return losses.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dgdv2_loss(batch, fake_pos, mode='discriminator'):\n",
    "    fake = dis_convert(batch, fake_pos)\n",
    "    pred = discriminator(fake)\n",
    "    if mode == 'discriminator':\n",
    "        gt = get_gt(batch, fake_pos)\n",
    "        loss = criterion(pred, gt)\n",
    "    elif mode == 'generator':\n",
    "        losses = pred.sum(dim=0)\n",
    "#         losses = torch.tensor(config.importance).to(device) * losses #/ losses.detach()\n",
    "        loss = losses.sum()\n",
    "    else:\n",
    "        raise Exception\n",
    "    return -loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "3IE_84f-edNB"
   },
   "outputs": [],
   "source": [
    "def get_gan_loss(batch, fake_pos, mode='discriminator'):\n",
    "    return {\"sgan\": get_sgan_loss,\n",
    "            \"wgan\": get_wgan_loss,\n",
    "            \"rgan\": get_rgan_loss,\n",
    "            \"ragan\": get_ragan_loss,\n",
    "            \"dgdv2\": get_dgdv2_loss}[config.gan_flavor](batch, fake_pos, mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/",
     "height": 80
    },
    "id": "2NjekXgkedNB",
    "outputId": "da85b2ec-c0de-459c-c0dd-7ba3519419fa",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d08394c4d9bf44368ccf2565f9b367f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Tab(children=(VBox(children=(Wrapper(), Hud())), HBox(children=(Output(),), layout=Layout(height='500px', over…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/slurmtmp.5826416/ipykernel_134086/801679965.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    127\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdiscriminator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrepeat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m             \u001b[0mtrain_dis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malternate\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'iteration'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m                 \u001b[0mtrain_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/slurmtmp.5826416/ipykernel_134086/801679965.py\u001b[0m in \u001b[0;36mtrain_dis\u001b[0;34m(batch, epoch)\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;31m# train discriminator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mdiscriminator_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0mdiscriminator_optimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;31m# gradient penalty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deepgd/lib/python3.9/site-packages/torch/optim/lr_scheduler.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m                 \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step_count\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m                 \u001b[0mwrapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0;31m# Note that the returned function here is no longer a bound method,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deepgd/lib/python3.9/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m                 \u001b[0mprofile_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Optimizer.step#{}.step\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofile_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deepgd/lib/python3.9/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deepgd/lib/python3.9/site-packages/torch/optim/adamw.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    108\u001b[0m                 \u001b[0mstate_steps\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'step'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 110\u001b[0;31m             F.adamw(params_with_grad,\n\u001b[0m\u001b[1;32m    111\u001b[0m                     \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m                     \u001b[0mexp_avgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deepgd/lib/python3.9/site-packages/torch/optim/_functional.py\u001b[0m in \u001b[0;36madamw\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps)\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m         \u001b[0mexp_avg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m         \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    133\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m             \u001b[0;31m# Maintains the maximum of all 2nd moment running avg. till now\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def train_dis(batch, epoch):\n",
    "    generator.requires_grad_(False)\n",
    "    discriminator.zero_grad()\n",
    "    generator_output = generator(batch)\n",
    "    if config.discriminator.noise.std > 0:\n",
    "        generator_output = generator_output + torch.randn_like(generator_output) * config.discriminator.noise.std * config.discriminator.noise.decay ** epoch\n",
    "    discriminator_loss = get_gan_loss(batch, generator_output, mode='discriminator')\n",
    "    \n",
    "    # train discriminator\n",
    "    discriminator_loss.backward()\n",
    "    discriminator_optimizer.step()\n",
    "\n",
    "    # gradient penalty\n",
    "    if config.gp_weight > 0:\n",
    "        discriminator.zero_grad()\n",
    "        gp_loss = get_gp_loss(batch, generator_output, config.gp_weight)\n",
    "        gp_loss.backward()\n",
    "        discriminator_optimizer.step()\n",
    "\n",
    "    hud['dis_loss'] = format(discriminator_loss.item(), '.2e')\n",
    "    pbar().update()\n",
    "\n",
    "def train_gen(batch, epoch):\n",
    "    generator.requires_grad_(True)\n",
    "    generator.zero_grad()\n",
    "    discriminator.zero_grad()\n",
    "    generator_output = generator(batch)\n",
    "    if config.discriminator.noise.std > 0:\n",
    "        generator_output = generator_output + torch.randn_like(generator_output) * config.discriminator.noise.std * config.discriminator.noise.decay ** epoch\n",
    "    generator_loss = get_gan_loss(batch, generator_output, mode='generator') \n",
    "    \n",
    "    #train generator\n",
    "    generator_loss.backward()\n",
    "    generator_optimizer.step()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        dis_batch = dis_convert(batch, generator_output)\n",
    "        stress = stress_criterion(dis_batch).mean()\n",
    "        critic = discriminator(dis_batch).mean()\n",
    "    hud.append({'gen_loss': format(generator_loss.item(), '.2e'),\n",
    "                'stress': format(stress.item(), '.2e'),\n",
    "                'critic': format(critic.item(), '.2e')})\n",
    "    pbar().update()\n",
    "\n",
    "def cuda_memsafe_map(fn, *iterables, summary=False):\n",
    "    total, failed = 0, 0\n",
    "    iterator = zip(*iterables)\n",
    "    items = None\n",
    "    while True:\n",
    "        try:\n",
    "            items = next(iterator)\n",
    "            yield fn(*items)\n",
    "        except StopIteration:\n",
    "            if summary:\n",
    "                print(f'Iteration finished. {failed} out of {total} failed!')\n",
    "            break\n",
    "        except RuntimeError:\n",
    "            print('CUDA memory overflow! Skip batch...')\n",
    "            del items\n",
    "            failed += 1\n",
    "        torch.cuda.empty_cache()\n",
    "        total += 1\n",
    "    \n",
    "def validate(model, data_loader, criterion=val_criterion):\n",
    "    def val_one_batch(batch):\n",
    "        batch = preprocess_batch(model, batch)\n",
    "        pred = CanonicalizationByStress()(model(batch), batch)\n",
    "        gt = CanonicalizationByStress()(batch.gt_pos, batch)\n",
    "        loss = criterion(pred, batch)\n",
    "        gt_loss = criterion(gt, batch)\n",
    "        spc = (loss - gt_loss) / torch.maximum(torch.maximum(loss, gt_loss), torch.ones_like(loss)*1e-5)\n",
    "        return loss.mean().item(), spc.mean().item()\n",
    "    loss_all, spc_all = zip(*cuda_memsafe_map(val_one_batch, data_loader))\n",
    "    return np.mean(loss_all), np.mean(spc_all)\n",
    "\n",
    "def log(msg):\n",
    "    msg = f\"[{epoch:03}] {msg}\"\n",
    "    print(msg, file=open(paths.log(), \"a\"))\n",
    "    with log_out: \n",
    "        print(msg)\n",
    "\n",
    "print(f\"{'='*10} {config.link} {'='*10}\", file=open(paths.log(), \"a\"))\n",
    "hud = Hud()\n",
    "pbar = Wrapper(tqdm, total=len(train_loader)*2, smoothing=0)\n",
    "plot_out = Output()\n",
    "log_out = Output()\n",
    "tabs = {\"status\": VBox([pbar, hud]), \n",
    "        \"plot\": HBox([plot_out], layout=Layout(height='500px', overflow_y='auto')),\n",
    "        \"log\": HBox([log_out], layout=Layout(height='500px', overflow_y='auto'))}\n",
    "tab_bar = Tab(children=list(tabs.values()))\n",
    "[tab_bar.set_title(i, name) for i, name in enumerate(tabs)]\n",
    "display(tab_bar)\n",
    "while True:\n",
    "    if epoch % config.log_interval == 0:\n",
    "        torch.save(generator.state_dict(), f\"{paths.checkpoints()}/gen_epoch_{epoch}.pt\")\n",
    "        torch.save(generator_optimizer.state_dict(), f\"{paths.checkpoints()}/gen_optim_epoch_{epoch}.pt\")\n",
    "        torch.save(discriminator.state_dict(), f\"{paths.checkpoints()}/dis_epoch_{epoch}.pt\")\n",
    "        torch.save(discriminator_optimizer.state_dict(), f\"{paths.checkpoints()}/dis_optim_epoch_{epoch}.pt\")\n",
    "        generator.eval()\n",
    "        with torch.no_grad():\n",
    "            val_stress, val_stress_spc = validate(model=generator, data_loader=val_loader)\n",
    "            val_xing, val_xing_spc = validate(model=generator, data_loader=val_loader, criterion=xing_criterion)\n",
    "            with plot_out:\n",
    "                fig = plt.figure()\n",
    "                graph_vis(G_list[11100], generator(make_batch(data_list[11100]).to(device)).cpu())\n",
    "                plt.show()\n",
    "        # tensorboard.add_scalars('loss', {'train': train_loss, \n",
    "        #                                  'validation': val_loss}, epoch)\n",
    "        # for i, fn in enumerate(loss_fns):\n",
    "        #     tensorboard.add_scalars(type(fn).__name__, {'train': train_loss_comp[i].item(), \n",
    "        #                                           'validation': val_loss_comp[i].item()}, epoch)\n",
    "        hud.append({\n",
    "            'val_stress': format(val_stress, '.2f'),\n",
    "            'val_stress_spc': format(val_stress_spc, '.2%'),\n",
    "            'val_xing': format(val_xing, '.2f'),\n",
    "            'val_xing_spc': format(val_xing_spc, '.2%'),\n",
    "        })\n",
    "        log(f\"stress={hud.data['val_stress']}({hud.data['val_stress_spc']}) xing={hud.data['val_xing']}({hud.data['val_xing_spc']})\")\n",
    "        \n",
    "    # handle.update(tab_bar)\n",
    "    pbar().reset()\n",
    "    pbar().set_description(desc=f\"[epoch {epoch}/{config.epoch.end}]\")\n",
    "    hud(title=f\"epoch {epoch}\")\n",
    "    generator.train()\n",
    "    discriminator.train()\n",
    "    # proper: proper layout\n",
    "    for _ in range(config.discriminator.repeat):\n",
    "        for batch in train_loader:\n",
    "            train_dis(batch, epoch)\n",
    "            if config.alternate == 'iteration':\n",
    "                train_gen(batch, epoch)\n",
    "\n",
    "    if config.alternate == 'epoch':\n",
    "        for batch in train_loader:\n",
    "            train_gen(batch, epoch)\n",
    "\n",
    "    discriminator_scheduler.step()\n",
    "    generator_scheduler.step()\n",
    "\n",
    "    if epoch == config.epoch.end:\n",
    "        break\n",
    "    epoch += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0qYK_X_hrq9o"
   },
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XEvmAtNjWhcY",
    "outputId": "9f4c86a4-a4b0-48f1-cb58-ab3679173184"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading from artifacts/checkpoints/GAN(gan=rgan,data=best(xing,stress),canon=stress,share=6,embed=10)/gen_epoch_342.pt...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_epoch = 342\n",
    "\n",
    "test_generator = Generator(**config.generator.params[...]).to(device)\n",
    "test_ckpt_epoch = get_ckpt_epoch(paths.checkpoints(), test_epoch)\n",
    "test_ckpt_file = f\"{paths.checkpoints()}/gen_epoch_{test_ckpt_epoch}.pt\"\n",
    "print(f\"Loading from {test_ckpt_file}...\")\n",
    "test_generator.load_state_dict(torch.load(test_ckpt_file, map_location=torch.device(device)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "9b7d18992dbf4a84931f916fbaaa37d8",
      "2c664d7e1e6d474fb87094fc0da270d3",
      "d375f5f82a4645d7b29c541596dc1c7f",
      "ecfa311bb06b4f6fa504598263ff69d3",
      "d1859990877149eab2b414b4e2a0e0c7",
      "d2be6733734844bca3e1db9b49eb1c1e",
      "4d98072bdf674792a13d77e85dd285d1",
      "f7ed92c383d741a0a1cef2536ff2db13",
      "2ba110f835c64ec4b82974996aed10af",
      "09397ceac9014af0aebca848d4d96d48",
      "fd0709f48d6c4527bc00b5f407f3c1b9"
     ]
    },
    "id": "ZubmSTcYrq9o",
    "outputId": "fc2f65a2-2085-43ac-b110-cbfe742318a8",
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "234b5c429e9b478692c72718a6a9d959",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rotate = RotateByPrincipalComponents()\n",
    "def test_callback(*, idx, pred, metrics):\n",
    "    # graph_vis(G_list[idx], pred, file_name=f\"{paths.visualization()}/{idx}_{metrics['stress']:.2f}_{metrics['resolution_score']:.2f}.png\")\n",
    "    pred = rotate(torch.tensor(pred), data_list[idx])\n",
    "    graph_vis(G_list[idx], pred)\n",
    "    plt.title(f\"[pred] idx: {idx}, stress: {metrics['stress']:.2f}({metrics['stress_spc']:.2%}), xing: {metrics['xing']:.2f}({metrics['xing_spc']:.2%})\")\n",
    "    plt.show()\n",
    "    gt_pos = rotate(data_list[idx].gt_pos, data_list[idx])\n",
    "    graph_vis(G_list[idx], gt_pos, node_color='orange')\n",
    "    plt.title(f\"[gt] idx: {idx}, stress: {metrics['gt_stress']:.2f}, xing: {metrics['gt_xing']:.2f}\")\n",
    "    plt.show()\n",
    "    \n",
    "test_metrics = test(model=test_generator, \n",
    "                    criteria_list=[], \n",
    "                    dataset=data_list, \n",
    "                    idx_range=range(10000, 11000), \n",
    "#                     callback=test_callback,\n",
    "                    gt_pos=None)\n",
    "pickle.dump(test_metrics, open(paths.metrics(\"test\"), \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 318
    },
    "id": "t2rFXIr_PsrJ",
    "outputId": "6a4205b1-4630-4de4-ec72-bdba3d3ea114"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stress: tensor(384.0231)\n",
      "stress_spc: tensor(0.1024, dtype=torch.float64)\n",
      "xing: tensor(26.0990)\n",
      "xing_spc: tensor(0.1407, dtype=torch.float64)\n",
      "l1_angle: tensor(80.6249)\n",
      "l1_angle_spc: tensor(0.0523, dtype=torch.float64)\n",
      "edge: tensor(0.1630)\n",
      "edge_spc: tensor(0.1257, dtype=torch.float64)\n",
      "ring: tensor(281.2041)\n",
      "ring_spc: tensor(0.0502, dtype=torch.float64)\n",
      "tsne: tensor(0.2224)\n",
      "tsne_spc: tensor(0.1489, dtype=torch.float64)\n",
      "reso_score: tensor(0.6015)\n",
      "min_angle: tensor(5.7770)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_cbbb2_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >stress</th>\n",
       "      <th class=\"col_heading level0 col1\" >stress_spc</th>\n",
       "      <th class=\"col_heading level0 col2\" >xing</th>\n",
       "      <th class=\"col_heading level0 col3\" >xing_spc</th>\n",
       "      <th class=\"col_heading level0 col4\" >l1_angle</th>\n",
       "      <th class=\"col_heading level0 col5\" >l1_angle_spc</th>\n",
       "      <th class=\"col_heading level0 col6\" >edge</th>\n",
       "      <th class=\"col_heading level0 col7\" >edge_spc</th>\n",
       "      <th class=\"col_heading level0 col8\" >ring</th>\n",
       "      <th class=\"col_heading level0 col9\" >ring_spc</th>\n",
       "      <th class=\"col_heading level0 col10\" >tsne</th>\n",
       "      <th class=\"col_heading level0 col11\" >tsne_spc</th>\n",
       "      <th class=\"col_heading level0 col12\" >reso_score</th>\n",
       "      <th class=\"col_heading level0 col13\" >min_angle</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_cbbb2_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_cbbb2_row0_col0\" class=\"data row0 col0\" >384.02</td>\n",
       "      <td id=\"T_cbbb2_row0_col1\" class=\"data row0 col1\" >10.24%</td>\n",
       "      <td id=\"T_cbbb2_row0_col2\" class=\"data row0 col2\" >26.10</td>\n",
       "      <td id=\"T_cbbb2_row0_col3\" class=\"data row0 col3\" >14.07%</td>\n",
       "      <td id=\"T_cbbb2_row0_col4\" class=\"data row0 col4\" >80.62</td>\n",
       "      <td id=\"T_cbbb2_row0_col5\" class=\"data row0 col5\" >5.23%</td>\n",
       "      <td id=\"T_cbbb2_row0_col6\" class=\"data row0 col6\" >0.16</td>\n",
       "      <td id=\"T_cbbb2_row0_col7\" class=\"data row0 col7\" >12.57%</td>\n",
       "      <td id=\"T_cbbb2_row0_col8\" class=\"data row0 col8\" >281.20</td>\n",
       "      <td id=\"T_cbbb2_row0_col9\" class=\"data row0 col9\" >5.02%</td>\n",
       "      <td id=\"T_cbbb2_row0_col10\" class=\"data row0 col10\" >0.22</td>\n",
       "      <td id=\"T_cbbb2_row0_col11\" class=\"data row0 col11\" >14.89%</td>\n",
       "      <td id=\"T_cbbb2_row0_col12\" class=\"data row0 col12\" >0.60</td>\n",
       "      <td id=\"T_cbbb2_row0_col13\" class=\"data row0 col13\" >5.78</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x2aae52b9e580>"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = test_metrics\n",
    "print('stress:', metrics['stress'].mean())\n",
    "print('stress_spc:', metrics['stress_spc'].mean())\n",
    "print('xing:', metrics['xing'].mean())\n",
    "print('xing_spc:', metrics['xing_spc'].mean())\n",
    "print('l1_angle:', metrics['l1_angle'].mean())\n",
    "print('l1_angle_spc:', metrics['l1_angle_spc'].mean())\n",
    "print('edge:', metrics['edge'].mean())\n",
    "print('edge_spc:', metrics['edge_spc'].mean())\n",
    "print('ring:', metrics['ring'].mean())\n",
    "print('ring_spc:', metrics['ring_spc'].mean())\n",
    "print('tsne:', metrics['tsne'].mean())\n",
    "print('tsne_spc:', metrics['tsne_spc'].mean())\n",
    "print('reso_score:', metrics['resolution_score'].mean())\n",
    "print('min_angle:', metrics['min_angle'].mean())\n",
    "columns = [\n",
    "    'stress',\n",
    "    'stress_spc',\n",
    "    'xing',\n",
    "    'xing_spc',\n",
    "    'l1_angle',\n",
    "    'l1_angle_spc',\n",
    "    'edge',\n",
    "    'edge_spc',\n",
    "    'ring',\n",
    "    'ring_spc',\n",
    "    'tsne',\n",
    "    'tsne_spc',\n",
    "    'reso_score',\n",
    "    'min_angle'\n",
    "]\n",
    "df = pd.DataFrame(map(lambda m: metrics[m].mean().item(), list(metrics.keys())[:-1])).set_axis(columns).T\n",
    "df.style.format({c: \"{:.2f}\" for c in columns if 'spc' not in c} | {c: \"{:.2%}\" for c in columns if 'spc' in c})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "id": "5e7vPr0M3mmG"
   },
   "outputs": [],
   "source": [
    "metrics = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = ['fa2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 433,
     "referenced_widgets": [
      "a0cf6e046add4e639e52db69656e0db9",
      "8e237731bb1241ac858581b25389c012",
      "0891547293ad4211a138a4d92e01bb0b",
      "34912c8e581e47a99a1c4829c38cb75c",
      "5aa034c99c94424e835a2ee954b829ed",
      "248f6393eff74c3eb8740ca6d698045b",
      "686687a29c7f4ec98c4d76ca2062b14b",
      "996a84d1c23b47e4a2af252c6b60f55d",
      "3507f883bb954412b21c1c5641763ed7",
      "9582a849d91e457cadd514c95d6e1b5b",
      "52bfe674c2c547e09b7400a32a96f703",
      "93d58b3a12394fe1aa6f0431fc7cb0db",
      "af1d7855c9d646b0bcc7d95c4cb147a6",
      "f820e2b898cd461da04a4f651d27273d",
      "c0a112cfaa3d4d0aa9530372de84d8fb",
      "fd03985495694fbf9feb3f31ef8d18ce",
      "e02a2b36aa4c4d6fb5d2ac6693c1a927",
      "06f388ff5e3a48828d465096fe69977b",
      "c0052778c9c940e88935593e9f94b83b",
      "99a5e77a192d4a989c4534237eb16730",
      "77c6da092a6647fc9e54f7e276d75616",
      "8c13c0e8b5934117a662a78296bcb1b6",
      "cdcc3bd4235247b48c468c0f6b7682cc",
      "a2c6fa6fef4549cc853fa93b4b18cf95",
      "15cc02a756314083801b5f6798a1948c",
      "24d1c67fe52f452ab0c767e3c1a818aa",
      "c117db25c6fd484a9d0c7f210ccb1ade",
      "b090b60b61084cf692b3066dfac7f0a8",
      "5eab9848059c491b9c6cbac88683f025",
      "e87ffbaa1f204b52b9226337a506b040",
      "cf52198a9e4b46c7b49537a79919c09e",
      "51501227bc064ccbab97c22a679a0750",
      "d1b794a997dc457888a90bf44faee9b4",
      "21a755b50e5047bea797eccf24b35a0c",
      "16e78c3823ec4faf9f9cd9095b218ff5",
      "4ccd332c6b734acd8eff236eaf568dfa",
      "bd60fd13748347e48629df78fa455765",
      "e9187e4ba80b4fafa0334dae2e39aded",
      "4c39ec7c92b4402e8eedcb27beabaab1",
      "59a8194269f943bba571f710ad856973",
      "8e41bc25b9f348b1b85b381547cd60d1",
      "34d0f1529e64437fb25e639b41096072",
      "7e832c7543df43668eb72c0aa6356349",
      "f94162c9c5fe4c2581e3b80beaeb820b",
      "0284946fccf34607b25e1d6d81d4f72a",
      "8e8d4243a3624ba58d760bdf601d5126",
      "b1f0932594fe44f4be4beb8880172871",
      "820bb554d139437b8f6a6997f9002261",
      "459e375d1fe4497cb4197ef88e303c6d",
      "5726814966da46bbba942a114de4b319",
      "448beea5445f4642809801da46aa7e7c",
      "3746c0f0daf04f51805b4c95db20063a",
      "2ff74d1c705348438a32da37910c3300",
      "52f03147b5ab4cb390f9802b7a17a508",
      "5428511d652a47348203e8f3cd2e076d",
      "7b58569bee2e44c5a61a81f9a32c4c01",
      "c5e05d74b9404333a8155304e3497ba2",
      "3ad4e058bde44d87abab780368d49118",
      "8e437049b5c74ff1beddad93e30b19aa",
      "0c61377f4cc34b27ae34bdd6d976d868",
      "9ce35f9a9a23436ea77604513b5c4659",
      "dafbc43ad79c4feda093aefe490fc4b3",
      "497160b6bf80469a89680a03a468c8bb",
      "3265725ecb914981b6f07f68b8a582df",
      "1f0ac20d99d04ffb9ee75613526d3de6",
      "dee0e5716560483b8e2b107566439d94",
      "ca99841a1abd4fbd94927c49aaa711cb",
      "163a44ff1be04583ae7bd62726c55204",
      "f3b727045a0b4b4fabd147ec9c843e14",
      "7c4b4fe859cf48ab99b7e2780a24a539",
      "d9700cffab66480cb81e3722faab4e8b",
      "2c49acc1e0a64aa4b8ec461b0596848e",
      "691817157f43412f9c36002167606b75",
      "d0ae688d71734b56bb348de54e841798",
      "07a364ebcdaa491dbcc2304e4a8dae96",
      "538b277307b1449eaa79a2a91b4741d3",
      "f7b7dd4cbaa5468687b988c5e416a154",
      "877be862a4f045ec8b221097626becb5",
      "1bf37012df0844338ac689d4f6cc8eab",
      "33fe06641be544d3aab2c25cb979416f",
      "68de0d9261fa4e6999967b6f196d149c",
      "7017c5ba1ffc467495aca2689e516f7d",
      "e0997460cc21428695a0c278c6691baf",
      "522443bef4904996b6361ea265c4c56f",
      "dfe044b67917466f971ba84a11956d5e",
      "70b2d0ba1c1d4102841fb963480de1c7",
      "7b158daacd754a40bc0a48fc9ea6fea2",
      "8904f61ff30741e488eacf0e49440f66",
      "685dee6f1e1f4c2983d2d0b8eb90ed00",
      "005155a4d62d48cbbaf3d69614b08d81",
      "d2779dfa620041e4b32a7731926a5982",
      "53048bf27cb24080b4cdf20379302223",
      "0b5bff7794a14a20964da70b87158c2c",
      "ec664e55ea16435590393ae1617cccea",
      "f10d62ef464a45a7935610d869c282e3",
      "5e7c09bc35db4a3a8b9847c6d8478883",
      "987ea331b617446facc81b08bab1f8c4",
      "446c5a687b0d43358ab036d43ce281c3",
      "d89e216f62ae4a9999da1a91dd5e48b9",
      "09d03e7e66f54cafacefbfeda5fd6e5c",
      "d58c680562a44bbd9252e7ec6eebc06c",
      "71ea2594fdd942acac1cc61055438544",
      "fecda4df301b437f9fbfb61a7b593dc0",
      "bcf6741d926248cb92acf627d6557d2b",
      "5bd9868b9bda4f68b500a43916e2d148",
      "4bc5e02035de44ad827c4f881fd75afc",
      "7d80033a955e4feb947ca9cea931d90b",
      "e975c796d80d498992a50f87a2b0105c",
      "028b2ff105904a19a9a3e8ebf66967bb",
      "10dbf2cbdb1943eda6a0acaf5da3fe85",
      "14d59c7acc5e4e07a289db44406a4851",
      "d9447df93d714907bc9c768eb4910352",
      "d4a9ace081bf488680a7e43e0eaed71d",
      "022a60cde19546bfa6ecfb4947615ad9",
      "10c01207d9ba4fe197dc80717fceecc0",
      "f4995936570d4ba1a65e9a66e410b505",
      "b4bf66303e61491385cd616076a11c04",
      "a24ab2bfe7a44263bf60936d6e53ec32",
      "59e2e8ecc9f84d57af7bf44f5bba5df7",
      "be2eb92aab8045a99fd19195c3546b60",
      "e6715db2b048434796e910952cb5eb62",
      "8de2335007ba4a6b9935fa0c514c9f6a",
      "493a4bbe8057437a8c8acf4d03f8dcb0",
      "20b03955acc74e238695717124705520",
      "a89f9507496b4cf8bdd63bdb9bff79c4",
      "9752a1f6c8b6488d9628b8e0c023f10f",
      "61a0bebc687341a7bb3352febc8d4986",
      "07e0ff9c82a444788c60edee70688ca7",
      "4427901f35e54cc48e08527bb402c22a",
      "bd9dafb825954091ba975d71ab8c89ea",
      "7ec0f07bfe51484da38c18dc66d77804",
      "03943afebc4d4b668b54c7a0fd014a37",
      "a7a6b38a76424d738146b6a5786cf2c8",
      "1d1aa4729784470584c0001c1a296008",
      "e0c68ac9b5b442eeb77e6d9ea3c1da42",
      "c858ef9fd9324ab0ab885d85e3c8582f",
      "627bc2899c994a0c96c5d43e89dc77ff",
      "0e2c2f21fa8041038151170992ecb848",
      "2d9bcbf7629a4671bf8755e790276eb7",
      "77a4edf6ab1848a6a6df20c1c07358c6",
      "acf2212d6bb44c729b63281a98377095",
      "0c7a72818da3454190bab7f30a29e249",
      "d865af915b8b4aaa8cec8d40f1f8351d"
     ]
    },
    "id": "s1UnzfID4Ai6",
    "outputId": "3e204e76-68fd-4111-f527-91f6e3b7a3ca"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51cd093c07f14267a23d3ceb61e8f861",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for m in methods:\n",
    "    metrics[m] = test(model=test_generator, \n",
    "                      criteria_list=[], \n",
    "                      dataset=data_list, \n",
    "                      idx_range=range(10000, 11000), \n",
    "                      callback=None,\n",
    "                      gt_pos=load_pos(m))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "id": "dQ7hdCtk5u6K"
   },
   "outputs": [],
   "source": [
    "mean_metrics = {key : list(map(lambda m: metrics[key][m].mean().item(), list(metrics[key].keys())[:-1])) for key in metrics}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 313
    },
    "id": "OGyaCmLXTuvA",
    "outputId": "6c42d9f3-9869-4504-bc9c-d0eedaecfd90"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_ac54c_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >stress</th>\n",
       "      <th class=\"col_heading level0 col1\" >stress_spc</th>\n",
       "      <th class=\"col_heading level0 col2\" >xing</th>\n",
       "      <th class=\"col_heading level0 col3\" >xing_spc</th>\n",
       "      <th class=\"col_heading level0 col4\" >l1_angle</th>\n",
       "      <th class=\"col_heading level0 col5\" >l1_angle_spc</th>\n",
       "      <th class=\"col_heading level0 col6\" >edge</th>\n",
       "      <th class=\"col_heading level0 col7\" >edge_spc</th>\n",
       "      <th class=\"col_heading level0 col8\" >ring</th>\n",
       "      <th class=\"col_heading level0 col9\" >ring_spc</th>\n",
       "      <th class=\"col_heading level0 col10\" >tsne</th>\n",
       "      <th class=\"col_heading level0 col11\" >tsne_spc</th>\n",
       "      <th class=\"col_heading level0 col12\" >reso_score</th>\n",
       "      <th class=\"col_heading level0 col13\" >min_angle</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_ac54c_level0_row0\" class=\"row_heading level0 row0\" >fa2</th>\n",
       "      <td id=\"T_ac54c_row0_col0\" class=\"data row0 col0\" >384.01</td>\n",
       "      <td id=\"T_ac54c_row0_col1\" class=\"data row0 col1\" >-4.41%</td>\n",
       "      <td id=\"T_ac54c_row0_col2\" class=\"data row0 col2\" >26.10</td>\n",
       "      <td id=\"T_ac54c_row0_col3\" class=\"data row0 col3\" >1.34%</td>\n",
       "      <td id=\"T_ac54c_row0_col4\" class=\"data row0 col4\" >80.63</td>\n",
       "      <td id=\"T_ac54c_row0_col5\" class=\"data row0 col5\" >8.22%</td>\n",
       "      <td id=\"T_ac54c_row0_col6\" class=\"data row0 col6\" >0.16</td>\n",
       "      <td id=\"T_ac54c_row0_col7\" class=\"data row0 col7\" >-16.94%</td>\n",
       "      <td id=\"T_ac54c_row0_col8\" class=\"data row0 col8\" >281.20</td>\n",
       "      <td id=\"T_ac54c_row0_col9\" class=\"data row0 col9\" >3.84%</td>\n",
       "      <td id=\"T_ac54c_row0_col10\" class=\"data row0 col10\" >0.22</td>\n",
       "      <td id=\"T_ac54c_row0_col11\" class=\"data row0 col11\" >11.97%</td>\n",
       "      <td id=\"T_ac54c_row0_col12\" class=\"data row0 col12\" >0.60</td>\n",
       "      <td id=\"T_ac54c_row0_col13\" class=\"data row0 col13\" >5.70</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x2aae528e3580>"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns = [\n",
    "    'stress',\n",
    "    'stress_spc',\n",
    "    'xing',\n",
    "    'xing_spc',\n",
    "    'l1_angle',\n",
    "    'l1_angle_spc',\n",
    "    'edge',\n",
    "    'edge_spc',\n",
    "    'ring',\n",
    "    'ring_spc',\n",
    "    'tsne',\n",
    "    'tsne_spc',\n",
    "    'reso_score',\n",
    "    'min_angle'\n",
    "]\n",
    "df = pd.DataFrame(mean_metrics).set_axis(columns).T\n",
    "df.style.format({c: \"{:.2f}\" for c in columns if 'spc' not in c} | {c: \"{:.2%}\" for c in columns if 'spc' in c})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L5S1HWnHOWIl"
   },
   "source": [
    "# Large Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_pefPcV1OZdD"
   },
   "outputs": [],
   "source": [
    "scalability = pd.read_csv(f\"/__artifacts__/data/scalability.csv\", index_col=\"index\")\n",
    "scalability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oMhWAsdvPnTX"
   },
   "outputs": [],
   "source": [
    "rescale = CanonicalizationByStress()\n",
    "stressfn = Stress()\n",
    "rotate = RotateByPrincipalComponents()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LR83OcAaPID5"
   },
   "outputs": [],
   "source": [
    "stress_list = []\n",
    "spc_list = []\n",
    "pmds_list = np.load(\"layouts/new_large_graph/pmds.npy\", allow_pickle=True)\n",
    "gviz_list = np.load(\"layouts/new_large_graph/gviz.npy\", allow_pickle=True)\n",
    "with torch.no_grad():\n",
    "    for idx, col in tqdm(scalability.iterrows(), total=len(scalability)):\n",
    "        # if idx not in [406, 516]: continue\n",
    "        torch.cuda.empty_cache()\n",
    "        G = load_mtx(col['file'])\n",
    "        G.remove_edges_from(nx.selfloop_edges(G))\n",
    "        data = generate_data_list(G, \n",
    "                                sparse=data_config.sparse, \n",
    "                                pivot_mode=data_config.pivot,\n",
    "                                init_mode=data_config.init,\n",
    "                                edge_index=data_config.edge.index,\n",
    "                                edge_attr=data_config.edge.attr,\n",
    "                                pmds_list=pmds_list[idx],\n",
    "                                gviz_list=gviz_list[idx],\n",
    "                                device=device)\n",
    "        batch = Batch.from_data_list([data]).to(device)\n",
    "        # generator.train()\n",
    "        # generator(batch)\n",
    "        generator.eval()\n",
    "        pred = generator(batch)\n",
    "        pos = rotate(rescale(pred, batch), batch)\n",
    "        gt = rotate(rescale(batch.gt_pos, batch), batch)\n",
    "        stress = stressfn(pos, batch).item()\n",
    "        gt_stress = stressfn(gt, batch).item()\n",
    "        spc = (stress - gt_stress) / np.maximum(stress, gt_stress)\n",
    "        stress_list.append(stress)\n",
    "        spc_list.append(spc)\n",
    "\n",
    "        np.save(f\"/__artifacts__/gan_result/data/scalability_{idx}.npy\", pos.cpu().numpy())\n",
    "        graph_attr = dict(node_size=1, \n",
    "                        with_labels=False, \n",
    "                        labels=dict(zip(list(G.nodes), map(lambda n: n if type(n) is int else n[1:], list(G.nodes)))),\n",
    "                        font_color=\"white\", \n",
    "                        font_weight=\"bold\",\n",
    "                        font_size=12,\n",
    "                        width=0.1)\n",
    "\n",
    "        # gt_pos = pickle.load(open(f\"/__artifacts__/data/scalability_{idx}_gt.pkl\", \"rb\"))\n",
    "\n",
    "        plt.figure(figsize=[12, 9])\n",
    "        nx.draw(G, pos=gt.cpu().numpy(), node_color='orange', **graph_attr)\n",
    "        plt.title(f\"neato: large_{idx}\")\n",
    "        plt.axis(\"equal\")\n",
    "        plt.savefig(f\"/__artifacts__/gan_result/output/{idx}_{col['name']}_{col['n']}_{spc}_nx.png\", dpi=300)\n",
    "        plt.show()\n",
    "\n",
    "        plt.figure(figsize=[12, 9])\n",
    "        graph_vis(G, pos.cpu().numpy(), **graph_attr)\n",
    "        plt.title(f\"dgd: large_{idx}, spc={spc:.2%}\")\n",
    "        plt.axis(\"equal\")\n",
    "        plt.savefig(f\"/__artifacts__/gan_result/output/{idx}_{col['name']}_{col['n']}_{spc}_dgd.png\", dpi=300)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gRXTewkYeuq1"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "w_p3OD0woiSM",
    "DR6-vYtr_i_P",
    "innqkwvH_ydD",
    "f-ODE2k8BFV6",
    "N39dDHraedM6"
   ],
   "machine_shape": "hm",
   "name": "GAN(gan=rgan,data=best(xing,stress),dis=deep,share=16,embed=0,gp=0).ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "DeepGD",
   "language": "python",
   "name": "deepgd"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0228dc7bab2d4a288f8a7f2beddab0a3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "059c9bafa3a94a2c93f6223f69a5f933": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_2116fda717bf408bb518e16eac9c7bc0",
        "IPY_MODEL_95c05b3ac7f5416fb6bef74db45259d4",
        "IPY_MODEL_4d00170f382e48398421d20dc0fc9965"
       ],
       "layout": "IPY_MODEL_cccfea4b781a45eab585f39b292c88bd"
      }
     },
     "06347c463ad94819872c26fc538296dc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "0808cd4b6c1e4414ac8500de802c207e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "0eb8123d944f4aa683435276b26d62f1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "0ff6bd3f11a84399ba8312c4950e52c2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "10639dfe0c434bd7969b805a0bfbf1ce": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_8e296eee8f644341ac1565598c71900a",
        "IPY_MODEL_d99c37034d0746a98f2671ad724f9571",
        "IPY_MODEL_dfbdd6ad54f24570a64ee54a6126a9d4"
       ],
       "layout": "IPY_MODEL_e1e51208a5384f3abdbec86a031580c3"
      }
     },
     "10aced0957dc4eb39a8641d6aed04fe0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "10fbaf3af71c434f93d68af9e3550f69": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "129838b3c32240b694c2e8c78b6e332d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "143d155d7f7a4c3f8ddd35b4ddfe3592": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "1483a5c298924831b6db53011faa8d27": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "1484e839ba184e0eb6e3367e8cf998b1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "1af122bce11f4b4da2dd68a008df67a7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "1d51984999ec4168ba9f315d1214057a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "1f19e70ed7cd4ef1a9e882cffb3288fc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_848a71993247481eb29d07dcf8e8379a",
       "style": "IPY_MODEL_4ba7942f053244c8b6db966f13324313",
       "value": " 1000/1000 [00:55&lt;00:00, 21.63it/s]"
      }
     },
     "1fef1f074b564c5298c67f155718c602": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "2091900774fa436c99cf0f5fd52fe005": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "20f18e15616f4463aba04f9fce849497": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "20f470852c6040ffac4b2a34366c70b5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "2116fda717bf408bb518e16eac9c7bc0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_84f17793733047ae8b954f31e96e4118",
       "style": "IPY_MODEL_8a546ce455534be99327b98cd50c2e8a",
       "value": "100%"
      }
     },
     "218d293b47e549a88cb13dc427574b7a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_ddd5b2ee74bb4a6692da4dc3417400f1",
       "style": "IPY_MODEL_9ed91398625f4317b104cdbb74a22c31",
       "value": " 11531/11531 [00:00&lt;00:00, 55068.37it/s]"
      }
     },
     "22cf8819cdfb4d09a2e4ce9a50164b22": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "234054af390a43c59bf0bc5a2909b59f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "234b5c429e9b478692c72718a6a9d959": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_4873bb2167d5414f85cde169d2c55d61",
        "IPY_MODEL_b8d1d48ee1e543e8a45ad15abe30fdc0",
        "IPY_MODEL_b912650293474863ac1374895ae45f85"
       ],
       "layout": "IPY_MODEL_e1c6ced375684dfa8ca3c30114171327"
      }
     },
     "23d5379d3463497aa035e213210879d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "241217b13efd43d0b5c003fbc62172f8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "26e9d51ebc9b48d2bc660074c809275e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "27cda17cccdb4770b041c56ecf1a0bb2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "27f3545e4ccf4214a185ac5834c19667": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_d06c6478add646979376ece7bc411390",
       "max": 1000,
       "style": "IPY_MODEL_87faf360ba6b4f4bad8e789b48b8e752",
       "value": 1000
      }
     },
     "285f7e1957fa41a0a9b962de482be2da": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_5d40710eda784a44b64f54457b34ee36",
        "IPY_MODEL_91381e1e1cf14ab8afcc637f66bb381d",
        "IPY_MODEL_c73e82a2a2994ac1ab9be9600dded97a"
       ],
       "layout": "IPY_MODEL_784534fde8574abebeb49b37b1f01a83"
      }
     },
     "289bcea5c6c944ddbfb66bb55db62c01": {
      "model_module": "@jupyter-widgets/output",
      "model_module_version": "1.0.0",
      "model_name": "OutputModel",
      "state": {
       "layout": "IPY_MODEL_764d97620573410fa6f5cbfff6f7b77a",
       "outputs": [
        {
         "data": {
          "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>val_stress</th>\n      <th>val_stress_spc</th>\n      <th>val_xing</th>\n      <th>val_xing_spc</th>\n      <th>dis_loss</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>epoch 1012</th>\n      <td>390.59</td>\n      <td>6.38%</td>\n      <td>29.60</td>\n      <td>12.68%</td>\n      <td>4.08e+00</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
          "text/plain": "           val_stress val_stress_spc val_xing val_xing_spc  dis_loss\nepoch 1012     390.59          6.38%    29.60       12.68%  4.08e+00"
         },
         "metadata": {},
         "output_type": "display_data"
        }
       ]
      }
     },
     "2a0b95a105bd4384b3b871e2ba09beaf": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "2db69973e1e949f9a99154ff584825af": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "2ecf24a9517e4d7da1c9f464ae1efc8f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_1d51984999ec4168ba9f315d1214057a",
       "style": "IPY_MODEL_6492d05e5dcb40a3941628facfd56304",
       "value": "  7%"
      }
     },
     "2f6ebee507784c6cb420c41acd67a650": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "danger",
       "layout": "IPY_MODEL_fddf70b473aa473bbe6d646af28296d0",
       "max": 1000,
       "style": "IPY_MODEL_7bfdbc45a40e4253b674b8de337350ac",
       "value": 18
      }
     },
     "309c72d808e04bb2bfee2d3c68056c03": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_737a77980c05413e8fa007d33637226f"
       ],
       "layout": "IPY_MODEL_58de6c5ea8a14c5d915b2e7f9492fb2c"
      }
     },
     "30d27e5abd184f7fa7b6f36e3b645231": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "314a2ea29250400a878379c89d2ba15b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_48df34f536444900b7c56d4d6216ca38",
        "IPY_MODEL_41e93466dad94cb3883750fe87803470",
        "IPY_MODEL_e6a92fa0c4a7463e992bba736a65b22e"
       ],
       "layout": "IPY_MODEL_d954d327e9c6434f87a447878c5cf326"
      }
     },
     "31de8ec6fd3342a7a3cceb3e3bde9cd0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_b6c01a48e6044a05932e8274c7dbec00",
        "IPY_MODEL_5b4ce59b48b5434e83cb7e4b2d25be0e",
        "IPY_MODEL_97be70f252884888945daeb04b49f9fa"
       ],
       "layout": "IPY_MODEL_bae4590dca6c4b048829773d80e43bd8"
      }
     },
     "3896f84e333c4703b629bbf949d1a8d3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "38af150016154c7d9724a5efd1da012f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "38f08c34d2a04ae19732e7815f5c31b0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "VBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_eaf71abdb8c249779654aa7969085f20",
        "IPY_MODEL_289bcea5c6c944ddbfb66bb55db62c01"
       ],
       "layout": "IPY_MODEL_1484e839ba184e0eb6e3367e8cf998b1"
      }
     },
     "3b418d48ac4446ba839406628fa6b857": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_6ccd1e2a90de44c5b81241410c6ad185",
       "style": "IPY_MODEL_e12220b1bbdd4d9290785e6ae6383d9b",
       "value": "100%"
      }
     },
     "3e1fff53360a48b6ba952ed9e086874d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "401a5ff689054c8a80d6de58255414c9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "410f61a7f7804a878a074c724235475a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_6bd7bc11e57245a9b159a230b08a4092",
        "IPY_MODEL_dd7c372c557e4bd18e1a3fc136a537b2",
        "IPY_MODEL_f72558fc51d3452c879324c4c32de9e0"
       ],
       "layout": "IPY_MODEL_a73100b864114380a4a82650df097b6e"
      }
     },
     "41e93466dad94cb3883750fe87803470": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_241217b13efd43d0b5c003fbc62172f8",
       "max": 1000,
       "style": "IPY_MODEL_56fa375fb72448708e17ce45f6ec4766",
       "value": 1000
      }
     },
     "4504ddd20a6748768692ad04644c4fa6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "46604d6cbd4448d99837b791c6716187": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "4873bb2167d5414f85cde169d2c55d61": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_92be276f7b264062bcd0028a81955351",
       "style": "IPY_MODEL_6505577795a947d49cf41fd5d124e66e",
       "value": "100%"
      }
     },
     "48df34f536444900b7c56d4d6216ca38": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_94de0a1f656647d4b0e418f18986b884",
       "style": "IPY_MODEL_bf89fe4bb5f5453cbee5b334ab9811f9",
       "value": "100%"
      }
     },
     "49fa3229e7f844e29004d342292b3ec1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "4a73fb2c304d4c4c9781dc5e8f23b420": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_4504ddd20a6748768692ad04644c4fa6",
       "style": "IPY_MODEL_234054af390a43c59bf0bc5a2909b59f",
       "value": "100%"
      }
     },
     "4ba7942f053244c8b6db966f13324313": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "4d00170f382e48398421d20dc0fc9965": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_94412363ab3a4442a553b3b733c4d1fd",
       "style": "IPY_MODEL_cdf4ee189d214faaa723d8e015e640ae",
       "value": " 1000/1000 [00:51&lt;00:00, 21.00it/s]"
      }
     },
     "5189d4fd5c5144a0acc27d0fac90d481": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "51cd093c07f14267a23d3ceb61e8f861": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_d18f769b33ec4a6b9b3ab1bb09b53dcc",
        "IPY_MODEL_878eb120c8bf4609b6642463120dd006",
        "IPY_MODEL_b77202ad145a4e8ab40f968b67614cf4"
       ],
       "layout": "IPY_MODEL_fa7104db81ce40e9b7ab030caf41d7f4"
      }
     },
     "539e393ab1494b77b22f1053f768e80d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "53d75d048b73425386f14c01aca445be": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_2db69973e1e949f9a99154ff584825af",
       "max": 1000,
       "style": "IPY_MODEL_7a55b52f0ce84e75be1ea6d2635a2b30",
       "value": 1000
      }
     },
     "560a778d49944d6fab06c9239f2bec2f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_c025778136d14a0194c130b225b0d9e1",
       "style": "IPY_MODEL_1483a5c298924831b6db53011faa8d27",
       "value": "[epoch 1012/None]:   4%"
      }
     },
     "56fa375fb72448708e17ce45f6ec4766": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "58065643dbd14e8280250ffe97304697": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "583cc75651df497fb383b6d0df1a505d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "height": "500px",
       "overflow_y": "auto"
      }
     },
     "58de6c5ea8a14c5d915b2e7f9492fb2c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "height": "500px",
       "overflow_y": "auto"
      }
     },
     "595d902b2f8d44d88b5548c044e5339c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_7eaf02fc112642ea9e7e334fafcdaa3f",
        "IPY_MODEL_cc321c5bff804e8facd7e6909cfbf3b5",
        "IPY_MODEL_8095ba388a5e49eb829cab2c332ae738"
       ],
       "layout": "IPY_MODEL_a32e5fd9fb634ef0b49d36b571ea6227"
      }
     },
     "5adfcb0a88dd417384bde9ae3181eea7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_9ebbdf83647e4922be302dca1916e6aa",
       "style": "IPY_MODEL_6b4d43e8e0ce4c68889d7383e4e5e23d",
       "value": " 1000/1000 [00:54&lt;00:00, 17.38it/s]"
      }
     },
     "5b4ce59b48b5434e83cb7e4b2d25be0e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_8bc149ffaec3457f9e6015b48dcb6bdb",
       "max": 1000,
       "style": "IPY_MODEL_6483bf504dbd4d52aba19d07f631a9a4",
       "value": 1000
      }
     },
     "5d40710eda784a44b64f54457b34ee36": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_46604d6cbd4448d99837b791c6716187",
       "style": "IPY_MODEL_adb3c70c29364b09a64b7965162d6d1d",
       "value": "100%"
      }
     },
     "5dad478049274bf0b486c76902eced3d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "600a0222e84140f8ae90b3322d949a36": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "61b6ca63547746f8bc89cbdc2b122ca4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "6483bf504dbd4d52aba19d07f631a9a4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "6492d05e5dcb40a3941628facfd56304": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "64935b866e5b40acadb85b0e23ffca30": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "6505577795a947d49cf41fd5d124e66e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "65bdaedd66014470b71efc967295d7f5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "68b8caa225b24dcdad8a3f6fd3b71f34": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "696c0d89ba974d9289f9e79c17545dbb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "6a88bd2bf9ab44f8b86494a7186b8430": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "6b4d43e8e0ce4c68889d7383e4e5e23d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "6b7ed8fc80054cbba5be8b169340c794": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_dedcf5a7ff774221a1410352449587ff",
       "style": "IPY_MODEL_eb6a200c5d2a4ee9be0ee148c3f7a08b",
       "value": " 1000/1000 [00:52&lt;00:00, 16.31it/s]"
      }
     },
     "6bd7bc11e57245a9b159a230b08a4092": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_ad1dc96fad694fb79e0ff7beed1b0747",
       "style": "IPY_MODEL_767b563dedb04fe89497fb697f6f76cd",
       "value": "100%"
      }
     },
     "6ccd1e2a90de44c5b81241410c6ad185": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "6dcbdeaa088a40288dc638edf97d1cb8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "6dfe25ea3d5b4376accd9fe15390f398": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_8c66a97394d14f389697d5c813c925ef",
       "style": "IPY_MODEL_30d27e5abd184f7fa7b6f36e3b645231",
       "value": " 32/834 [00:06&lt;02:48,  4.76it/s]"
      }
     },
     "6fa5e4fef86e4f849273c94f6606a39c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "72a1265b9a9a401db03a49c0d3f4706e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_20f18e15616f4463aba04f9fce849497",
       "max": 11531,
       "style": "IPY_MODEL_f85a03eec90041c09182ab7b7a5a579b",
       "value": 11531
      }
     },
     "7362e0392c5e42e88255a317e09184e6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "737a77980c05413e8fa007d33637226f": {
      "model_module": "@jupyter-widgets/output",
      "model_module_version": "1.0.0",
      "model_name": "OutputModel",
      "state": {
       "layout": "IPY_MODEL_7c878e5515c34e2ea12c0da254be1c55",
       "outputs": [
        {
         "name": "stdout",
         "output_type": "stream",
         "text": "[1012] stress=390.59(6.38%) xing=29.60(12.68%)\n"
        }
       ]
      }
     },
     "7543322ef98f4a6fa11c809cd14f733d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_c24f4b41c181487ba4d839d878535724",
        "IPY_MODEL_a30a758bf0984907b2a807cb95b210fe",
        "IPY_MODEL_9ce41cf6cd7f4dbe8314d9cf23e0381e"
       ],
       "layout": "IPY_MODEL_1fef1f074b564c5298c67f155718c602"
      }
     },
     "75cb8cf3a90248adbe80ed6e70d49542": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_49fa3229e7f844e29004d342292b3ec1",
       "style": "IPY_MODEL_bdacf72306a2458b95f4b9124d1fb647",
       "value": " 74/1000 [00:04&lt;00:53, 17.18it/s]"
      }
     },
     "764d97620573410fa6f5cbfff6f7b77a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "767b563dedb04fe89497fb697f6f76cd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "77266a918f2949ffafdca91696177407": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "784534fde8574abebeb49b37b1f01a83": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "791cd4944aec449b957dde8687a8f67d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_f070c9676b1348c187c8e9d780970361",
       "style": "IPY_MODEL_1af122bce11f4b4da2dd68a008df67a7",
       "value": "100%"
      }
     },
     "7a55b52f0ce84e75be1ea6d2635a2b30": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "7bd1ff46da314095ad1cec06fc200812": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_e5c9824d87354b4f9841a61145274e7f",
       "max": 1000,
       "style": "IPY_MODEL_f8f4986844d24daba7dad3fe822be00a",
       "value": 1000
      }
     },
     "7bfdbc45a40e4253b674b8de337350ac": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "7c878e5515c34e2ea12c0da254be1c55": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "7d67ee36109a428da712b5d120b083cf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_3b418d48ac4446ba839406628fa6b857",
        "IPY_MODEL_72a1265b9a9a401db03a49c0d3f4706e",
        "IPY_MODEL_218d293b47e549a88cb13dc427574b7a"
       ],
       "layout": "IPY_MODEL_696c0d89ba974d9289f9e79c17545dbb"
      }
     },
     "7d9121058e1d4934b74759aa8e5e1c76": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_0808cd4b6c1e4414ac8500de802c207e",
       "style": "IPY_MODEL_9649dce59a9f44cd97091b53b9d71a18",
       "value": " 1000/1000 [00:49&lt;00:00, 21.52it/s]"
      }
     },
     "7eaf02fc112642ea9e7e334fafcdaa3f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_3e1fff53360a48b6ba952ed9e086874d",
       "style": "IPY_MODEL_e045698360eb4b038335faf9fbcff093",
       "value": "100%"
      }
     },
     "7f596927186347ae85abf5e8b2e047dc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "danger",
       "layout": "IPY_MODEL_6fa5e4fef86e4f849273c94f6606a39c",
       "max": 1000,
       "style": "IPY_MODEL_fad37bf2f8c148ba961d1f8ba0578ea3",
       "value": 74
      }
     },
     "8095ba388a5e49eb829cab2c332ae738": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_2091900774fa436c99cf0f5fd52fe005",
       "style": "IPY_MODEL_58065643dbd14e8280250ffe97304697",
       "value": " 1000/1000 [00:47&lt;00:00, 21.04it/s]"
      }
     },
     "80e8dfdc7a674251976a74114968cc1e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "822504e19e1347ce8ebdac5db2539a28": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "8350d5aeec574a2391f5c8bac4d621bb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "848a71993247481eb29d07dcf8e8379a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "84f17793733047ae8b954f31e96e4118": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "878eb120c8bf4609b6642463120dd006": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_c8ecb05471264f6b80d4d5f75bebc47d",
       "max": 1000,
       "style": "IPY_MODEL_10fbaf3af71c434f93d68af9e3550f69",
       "value": 1000
      }
     },
     "87faf360ba6b4f4bad8e789b48b8e752": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "88e89604ed304ec9b07769d1562bc418": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "8a546ce455534be99327b98cd50c2e8a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "8afb7108c1284501a5971ed753bdf6d8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_94f726a9063a436e80b5e47222f82704",
        "IPY_MODEL_2f6ebee507784c6cb420c41acd67a650",
        "IPY_MODEL_f483dfebacc543aa969e0a477c7c72ee"
       ],
       "layout": "IPY_MODEL_22cf8819cdfb4d09a2e4ce9a50164b22"
      }
     },
     "8bc149ffaec3457f9e6015b48dcb6bdb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "8c66a97394d14f389697d5c813c925ef": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "8e296eee8f644341ac1565598c71900a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_5189d4fd5c5144a0acc27d0fac90d481",
       "style": "IPY_MODEL_06347c463ad94819872c26fc538296dc",
       "value": "100%"
      }
     },
     "91381e1e1cf14ab8afcc637f66bb381d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_80e8dfdc7a674251976a74114968cc1e",
       "max": 1000,
       "style": "IPY_MODEL_20f470852c6040ffac4b2a34366c70b5",
       "value": 1000
      }
     },
     "92be276f7b264062bcd0028a81955351": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "94412363ab3a4442a553b3b733c4d1fd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "94de0a1f656647d4b0e418f18986b884": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "94f726a9063a436e80b5e47222f82704": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_ba0cc45ff56941aa83129ce8704fee47",
       "style": "IPY_MODEL_38af150016154c7d9724a5efd1da012f",
       "value": "  2%"
      }
     },
     "951895a410a046c6813874883fd4a251": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_560a778d49944d6fab06c9239f2bec2f",
        "IPY_MODEL_fc53bb7fe4fa497d9306c8139ab2947b",
        "IPY_MODEL_6dfe25ea3d5b4376accd9fe15390f398"
       ],
       "layout": "IPY_MODEL_5dad478049274bf0b486c76902eced3d"
      }
     },
     "95c05b3ac7f5416fb6bef74db45259d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_cd7f7b9480a843e9b9c2d93d706f51a5",
       "max": 1000,
       "style": "IPY_MODEL_d3fc3b0d14e441e7b977cbf6dc0e40df",
       "value": 1000
      }
     },
     "9649dce59a9f44cd97091b53b9d71a18": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "97be70f252884888945daeb04b49f9fa": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_68b8caa225b24dcdad8a3f6fd3b71f34",
       "style": "IPY_MODEL_d343977c3c624872986f7e88ba4d41e3",
       "value": " 1000/1000 [00:54&lt;00:00, 17.39it/s]"
      }
     },
     "97d1eceb8fdd46b8a5f736ba6578fad9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "980764e902934257898b3422ce94fd81": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_791cd4944aec449b957dde8687a8f67d",
        "IPY_MODEL_53d75d048b73425386f14c01aca445be",
        "IPY_MODEL_7d9121058e1d4934b74759aa8e5e1c76"
       ],
       "layout": "IPY_MODEL_d415d599b2034312bcebbb3ef82d50bf"
      }
     },
     "99aa560d63fe4fd882dd2945cf30b16f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_b11bd6204dd147b78ebae5a27da8ccbc",
        "IPY_MODEL_7bd1ff46da314095ad1cec06fc200812",
        "IPY_MODEL_1f19e70ed7cd4ef1a9e882cffb3288fc"
       ],
       "layout": "IPY_MODEL_be83b0dbcd634e0ba46a9a9d5b6c5a56"
      }
     },
     "9ce41cf6cd7f4dbe8314d9cf23e0381e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_ce8911143c6e45e8a8bf7dd4734e67f4",
       "style": "IPY_MODEL_143d155d7f7a4c3f8ddd35b4ddfe3592",
       "value": " 1000/1000 [00:53&lt;00:00, 20.60it/s]"
      }
     },
     "9ebbdf83647e4922be302dca1916e6aa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "9ed91398625f4317b104cdbb74a22c31": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "a30a758bf0984907b2a807cb95b210fe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_0eb8123d944f4aa683435276b26d62f1",
       "max": 1000,
       "style": "IPY_MODEL_b669a4e4f6e54b47900b33ec1df858ff",
       "value": 1000
      }
     },
     "a32e5fd9fb634ef0b49d36b571ea6227": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "a6967ce3ae864b70b19aa2beae73da68": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "a73100b864114380a4a82650df097b6e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "a77df8e7558646009c28219286e9b7df": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "a7e0e21bec2949d7bcefdd5262aa7a5f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "a9093cfe030b4d3c99c00f70637b3593": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "aa17e3d3b7494263a41fe7828a067a49": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "acc14848fddf4f9389db0d8116481aac": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_ea2402741a1f4d6c9ba04ce894484a57"
       ],
       "layout": "IPY_MODEL_583cc75651df497fb383b6d0df1a505d"
      }
     },
     "ad1dc96fad694fb79e0ff7beed1b0747": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "adb3c70c29364b09a64b7965162d6d1d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "b11bd6204dd147b78ebae5a27da8ccbc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_61b6ca63547746f8bc89cbdc2b122ca4",
       "style": "IPY_MODEL_d928669ea82b4fccb16af270fe174f40",
       "value": "100%"
      }
     },
     "b12a8e5046f643e98e2689eadaf5c45d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "b4eb1879719443a2842bbff596049ca1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_aa17e3d3b7494263a41fe7828a067a49",
       "max": 1000,
       "style": "IPY_MODEL_6a88bd2bf9ab44f8b86494a7186b8430",
       "value": 1000
      }
     },
     "b669a4e4f6e54b47900b33ec1df858ff": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "b6c01a48e6044a05932e8274c7dbec00": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_27cda17cccdb4770b041c56ecf1a0bb2",
       "style": "IPY_MODEL_129838b3c32240b694c2e8c78b6e332d",
       "value": "100%"
      }
     },
     "b77202ad145a4e8ab40f968b67614cf4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_b12a8e5046f643e98e2689eadaf5c45d",
       "style": "IPY_MODEL_a7e0e21bec2949d7bcefdd5262aa7a5f",
       "value": " 1000/1000 [00:52&lt;00:00, 17.24it/s]"
      }
     },
     "b7a7e70ec1af4deb99c1e7ebbf5c641d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "b8d1d48ee1e543e8a45ad15abe30fdc0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_7362e0392c5e42e88255a317e09184e6",
       "max": 1000,
       "style": "IPY_MODEL_a77df8e7558646009c28219286e9b7df",
       "value": 1000
      }
     },
     "b912650293474863ac1374895ae45f85": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_f29c1356d97e4211b4b7ea1834167c28",
       "style": "IPY_MODEL_88e89604ed304ec9b07769d1562bc418",
       "value": " 1000/1000 [00:54&lt;00:00, 17.11it/s]"
      }
     },
     "ba0cc45ff56941aa83129ce8704fee47": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "ba81b18d39d54ca0ae8e11b52d201b41": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "bae4590dca6c4b048829773d80e43bd8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "bbac528b092442fd9f06b8e3ca8936a7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "bc94159865314ad887e85dbbac9e59c1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "bdacf72306a2458b95f4b9124d1fb647": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "be83b0dbcd634e0ba46a9a9d5b6c5a56": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "bf89fe4bb5f5453cbee5b334ab9811f9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "c025778136d14a0194c130b225b0d9e1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "c24f4b41c181487ba4d839d878535724": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_d486e8d0539d4f3b9de44edda4cd8635",
       "style": "IPY_MODEL_23d5379d3463497aa035e213210879d4",
       "value": "100%"
      }
     },
     "c268fb821b914bddb1469b9c3b12b334": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_e5d8a9ae49e84aa9a796bb603bceff90",
        "IPY_MODEL_b4eb1879719443a2842bbff596049ca1",
        "IPY_MODEL_5adfcb0a88dd417384bde9ae3181eea7"
       ],
       "layout": "IPY_MODEL_e5f319954a854953a5f2230855541c17"
      }
     },
     "c73e82a2a2994ac1ab9be9600dded97a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_26e9d51ebc9b48d2bc660074c809275e",
       "style": "IPY_MODEL_77266a918f2949ffafdca91696177407",
       "value": " 1000/1000 [00:49&lt;00:00, 21.20it/s]"
      }
     },
     "c8ecb05471264f6b80d4d5f75bebc47d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "c97c2c1230d94c5ebf2d4b9089a23f2b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "cc321c5bff804e8facd7e6909cfbf3b5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_0228dc7bab2d4a288f8a7f2beddab0a3",
       "max": 1000,
       "style": "IPY_MODEL_d2a619edb0724b649e9c01e50c0f59bf",
       "value": 1000
      }
     },
     "cccfea4b781a45eab585f39b292c88bd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "cd7f7b9480a843e9b9c2d93d706f51a5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "cdf4ee189d214faaa723d8e015e640ae": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "ce8911143c6e45e8a8bf7dd4734e67f4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d06c6478add646979376ece7bc411390": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d08394c4d9bf44368ccf2565f9b367f0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "TabModel",
      "state": {
       "_titles": {
        "0": "status",
        "1": "plot",
        "2": "log"
       },
       "children": [
        "IPY_MODEL_38f08c34d2a04ae19732e7815f5c31b0",
        "IPY_MODEL_acc14848fddf4f9389db0d8116481aac",
        "IPY_MODEL_309c72d808e04bb2bfee2d3c68056c03"
       ],
       "layout": "IPY_MODEL_822504e19e1347ce8ebdac5db2539a28"
      }
     },
     "d18f769b33ec4a6b9b3ab1bb09b53dcc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_bbac528b092442fd9f06b8e3ca8936a7",
       "style": "IPY_MODEL_6dcbdeaa088a40288dc638edf97d1cb8",
       "value": "100%"
      }
     },
     "d2a619edb0724b649e9c01e50c0f59bf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "d343977c3c624872986f7e88ba4d41e3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "d3fc3b0d14e441e7b977cbf6dc0e40df": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "d415d599b2034312bcebbb3ef82d50bf": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d486e8d0539d4f3b9de44edda4cd8635": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d48d61742cec44c59f9148f2bf742066": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_4a73fb2c304d4c4c9781dc5e8f23b420",
        "IPY_MODEL_27f3545e4ccf4214a185ac5834c19667",
        "IPY_MODEL_6b7ed8fc80054cbba5be8b169340c794"
       ],
       "layout": "IPY_MODEL_bc94159865314ad887e85dbbac9e59c1"
      }
     },
     "d928669ea82b4fccb16af270fe174f40": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "d954d327e9c6434f87a447878c5cf326": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d99c37034d0746a98f2671ad724f9571": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_64935b866e5b40acadb85b0e23ffca30",
       "max": 1000,
       "style": "IPY_MODEL_600a0222e84140f8ae90b3322d949a36",
       "value": 1000
      }
     },
     "dd7c372c557e4bd18e1a3fc136a537b2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_a6967ce3ae864b70b19aa2beae73da68",
       "max": 1000,
       "style": "IPY_MODEL_10aced0957dc4eb39a8641d6aed04fe0",
       "value": 1000
      }
     },
     "ddd5b2ee74bb4a6692da4dc3417400f1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "dedcf5a7ff774221a1410352449587ff": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "dfbdd6ad54f24570a64ee54a6126a9d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_ec1fb56a11ac4f109a924cc5e4d41ca5",
       "style": "IPY_MODEL_ba81b18d39d54ca0ae8e11b52d201b41",
       "value": " 1000/1000 [00:53&lt;00:00, 21.01it/s]"
      }
     },
     "e045698360eb4b038335faf9fbcff093": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "e12220b1bbdd4d9290785e6ae6383d9b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "e1c6ced375684dfa8ca3c30114171327": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "e1e51208a5384f3abdbec86a031580c3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "e4221a6462ee4dd4af7f2e534bc741fc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "e5c9824d87354b4f9841a61145274e7f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "e5d8a9ae49e84aa9a796bb603bceff90": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_8350d5aeec574a2391f5c8bac4d621bb",
       "style": "IPY_MODEL_65bdaedd66014470b71efc967295d7f5",
       "value": "100%"
      }
     },
     "e5f319954a854953a5f2230855541c17": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "e6a92fa0c4a7463e992bba736a65b22e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_0ff6bd3f11a84399ba8312c4950e52c2",
       "style": "IPY_MODEL_c97c2c1230d94c5ebf2d4b9089a23f2b",
       "value": " 1000/1000 [00:53&lt;00:00, 17.66it/s]"
      }
     },
     "ea2402741a1f4d6c9ba04ce894484a57": {
      "model_module": "@jupyter-widgets/output",
      "model_module_version": "1.0.0",
      "model_name": "OutputModel",
      "state": {
       "layout": "IPY_MODEL_539e393ab1494b77b22f1053f768e80d",
       "outputs": [
        {
         "data": {
          "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb4AAAEuCAYAAADx63eqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABKR0lEQVR4nO3deVyU1f4H8M9sLOJ1A0VEQMxdQUUUNVHccRcsN8wtMLNMb9rNyq0s/WXZNfdK09wyTTCXRMUlMTfcN8IVUBQRcmOf5fn94YVkmZlnYAZmmM/79ep2m3nmec4gzmfOec75HokgCAKIiIishLS8G0BERFSWGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRVGHxERGRV5OXdAKKSSEjLwA/Rt7Hjwn1k5KjgYCvH4FZ1EOZfHx6ODuXdPCIyYxJBEITybgSRIQ7HpWDSpnNQqjVQaf759ZVLJVDIpFgR4oOujWuVYwuJyJwx+MiiJKRlIPDbaGQp1VqPsVfIEDnFnz0/IioW7/GRRfkh+jaUao3OY5RqDVYfu1NGLSIiS8PgI4uy48L9AsObxVFpBEScTyqjFhGRpWHwkUXJyFGJOy5X3HFEZH0YfGRRHGzFTUR2sOGEZSIqHoOPLMrgVnUgl0p0HiOXShDU2rWMWkRElobBRxYlzL8+FDLdv7YKmRShnTzLqEVEZGkYfGRRPBwdsCLEB/YKWZGen1wqgb1ChhUhPlzKQERacR0fWaSEtAysPnYHW0/fQY5agsp2CgS1dkVoJ0+GHhHpxOAji7Z//34sXLgQUVFR5d0UIrIQHOoki+bi4oIHDx6UdzOIyIIw+Mii1a5dG8nJyeXdDCKyIBzqJIum0WhgZ2eH58+fw9bWtrybQ0QWgD0+smhSqRS1atXCw4cPy7spRGQhGHxk8Xifj4gMweAji8f7fERkCAYfWTz2+IjIEJzcQhYrIS0DP0Tfxi+nbkMJGSrbKjC4VR2E+dfnInYi0orBRxbpcFwKJm06B6VaU2B/PrlUAoVMihUhPujauFY5tpCIzBWDjyxOQloGAr+NRpZSrfUYe4UMkVP82fMjoiJ4j48szg/Rt6FUa3Qeo1RrsPrYnTJqERFZEgYfWZwdF+4XGN4sjkojIOJ8Uhm1iIgsCYOPLE5GjkrccbnijiMi68LgI4vjYCsXd5yNuOOIyLow+MjiDG5Vp8gmtIXJpRIEtXYtoxYRkSVh8JHFCfOvD4VM96+uQiZFaCfPMmoREVkSBh9ZHA9HB6wI8YG9QgZZoY6fXCqBvUKGFSE+XMpARMXiOj6yWAlpGfho3QGcfChAkNvAwUaOoNauCO3kydAjIq14958sloejA9xSTqDpvypj1qxZ5d0cIrIQHOokixYTE4O2bduWdzOIyIJwqJMslkajQfXq1XHr1i04OTmVd3OIyEKwx0cW68aNG6hRowZDj4gMwuAji3XmzBn4+vqWdzOIyMIw+Mhi8f4eEZUEg48sFnt8RFQSnNxCFkmlUqFatWpISkpC1apVy7s5RGRB2OMjixQbGwtXV1eGHhEZjMFHFon394iopBh8ZJF4f4+ISorBRxaJPT4iKilObiGLk5OTg+rVq+PRo0dwcGAxaiIyDHt8ZHEuX76MBg0aMPSIqEQYfGRxeH+PiEqDwUcWh/f3iKg0GHxkcdjjI6LS4OQWsiiZmZlwcnLC48ePYWtrW97NISILxB4fWZQLFy6gWbNmDD0iKjEGH1kU3t8jotLiUCeZvYS0DPwQfRs7LtxHerYSNlIBQ9vVQ5h/fXg4ckkDERmGwUdm7XBcCiZtOgelWgOV5p9fVblUAoVMihUhPujauFY5tpCILA2Dj8xWQloGAr+NRpZSrfUYe4UMkVP82fMjItF4j4/M1g/Rt6FUa3Qeo1RrsPrYnTJqERFVBAw+Mls7LtwvMLxZHJVGQMT5pDJqERFVBAw+MlsZOSpxx+WKO46ICGDwkRlzsJWLO85G3HFERACDj8zY4FZ1IJdKdB4jl0oQ1Nq1jFpERBUBg4/MVph/fShkun9FFTIpQjt5llGLiKgiYPCR2fJwdMCKEB/YKaQQ1AXv48mlEtgrZFgR4sOlDERkEAYfmbWujWth8ivPUP3va5BpciGBgMq2coxo547IKf5cvE5EBuMCdjJ7Q4YMQf/+/XHw4EH07t0bb7zxRnk3iYgsGHt8ZNbS09MRFRWFQYMGQSLRPdGFiEgMBh+ZtT179qBjx46oUaNGeTeFiCoIBh+Zta1bt+L111/P/2+OzBNRaTH4yGzlDXMOHjwYADjUSURGweAjs7Vnzx506NCBw5xEZFQMPjJb27ZtKzDMCXCok4hKj8FHZikjIwMHDhzIH+YEONRJRMbB4COzlDfM6ejoWOBx9viIqLQYfGSWihvmZI+PiIyBwUdmJyMjA/v37y8wzElEZCwMPjI7e/bsQfv27YsMcwIc6iSi0mPwkdkpbpgT4FAnERkHg4/MCoc5icjUGHxkVn7//Xf4+fnBycmp2Oc51ElEpcXgI7Oybds2DB06tNjnONRJRMbA4COzkZGRgX379nGYk4hMisFHZkPfMCfAoU4iKj3uwE5mY+jQoejZsyfCwsIKPJ6QloEfom/j5xO3oJbKUdlWgcGt6iDMvz48HB3KqbVEZKkYfFRu8gJtx4X7yMhRQZObidfb1sPkns3yA+1wXAombToHpVoDleafX1W5VAKFTIoVIT7o2rhWeb0FIrJADD4qF2ICrb6TAwK/jUaWUq31PPYKGSKn+LPnR0Si8R4flbmEtAxM2nQOWUp1gdADAJVGQJZSjUmbzuGbA3FQqjU6z6VUa7D62B1TNpeIKhgGH5W5H6Jviwq0PZeTiwRjYSqNgIjzScZsHhFVcPLybgBZnx0X7osKNLEyclWlbRIRWRH2+KjMZeQYN6gcbPj9jYjEY/BRmXOwFRdUCpkEcqnuai1yqQRBrV2N0SwishIMPipTgiCguUM6BLXuXp9cKkE/LxcoZLp/RRUyKUI7eRqziURUwTH4qMzEx8ejb9++uLp9Kez0DE8qZFL8u0cjrAjxgb1CVqTnJ5dKYK+QYUWID5cyEJFBGHxkcmq1Gv/973/h6+uLzp0743z0Aax6o62oQOvauBYip/hjRDt3QJkFCYDKtnKMaOeOyCn+XLxORAbjAnYyqUuXLiE0NBSVKlXC999/j0aNGuU/l5CWgdXH7iDifBIyclQQlFl4w78JQjt5FtuLc3V1xenTp+Hqynt6RFRyDD4qkcLlxhxs5QXqZ2ZlZWHevHlYvXo15s+fj/Hjx0Mq1T7AoNFoULt2bcTExMDDw6PYY6pVq4aEhARUrVrVVG+LiKwAg48Mpq/c2Nteciz7ZBJatmyJJUuWwMXFRdR5R40aBX9/f7z11ltFnhMEAQqFAtnZ2ZDLuXyBiEqOwUcGSUjL0Fs/U1DlYLavDG8OG2TQuTdu3Ijt27cjIiKiyHPZ2dmoVq0asrOzDW4zEdHLOLmFDCKm3JjCxhbxdq8YfO5evXrh0KFDyM3NLfJceno6KleubPA5iYgKY/CRQcSVG0OJ6mfWqlULDRs2xIkTJ4o8x+AjImNh8JFBxJYbK2n9zMDAQERGRhZ5nMFHRMbC4CODiC03VtL6mX369GHwEZFJMfjIIINb1TFp/Uw/Pz/Ex8fjwYMHBR7PyMiAgwMrtBBR6TH4yCBh/vVNWj9TLpejR48e2L9/f4HH2eMjImNh8JFBPBwdsCLEB3ZyaZFC08aqn1ncfT4GHxEZC4OPDNa1cS2E1X0E5+c3UNlWDonEuPUze/fujQMHDkCt/metIIOPiIyFJTCoRP7ctwPvDh6M0aN7G/3cdevWhYuLC2JiYtC+fXsADD4iMh72+Mhg2dnZOHjwIPr27WuyaxQe7mTwEZGxMPjIYIcPH4aXlxecnJxMdo3CyxrS09M5q5OIjIJDnVZG364KYuzatQsDBw40aTtfffVVxMbGIi0tDY6OjkhPT4enJ3daJ6LSY4/PihyOS0Hgt9HYEnMX6TkqCADSc1TYEnMXgd9G43Bcit5zCIKA3bt3Y8CAASZtq62tLbp06YIDBw4AeLGOj0OdRGQMDD4rkZCWgUmbziFLqS5Sa1OlEZClVGPSpnNISMvQeZ6LFy/CxsYGTZo0MWVzARS8z8d7fERkLAw+KyFmVwWlWoPVx+7oPGbnzp0YOHAgJBLd1VuMIS/4NBoNg4+IjIbBZyXE7aog6N1VYdeuXSYf5sxTv359VKlSBZcuXWLwEZHRMPishDF2Vbh//z5u3bqFTp06GatZeuX1+hh8RGQsDD4rYYxdFXbv3o3AwEAoFApjNUuvl4OPyxmIyBgYfFZCzK4KEkGDAS20lxsry2HOPAEBATh79iyePXvGHh8RGQWDz0qI2VVBImjwy2cTceTIEQAvZoLO3HEZLebug+dHe3Cx0WickzbUO/PTmCpVqoSOHTvi+fPnDD4iMgqJIAi6ZzxQhXE4LgWTNp2DUq0pMNFFLpVAIZNiRYgPMm6cxsSJE9E++E1cqdYeKrWg9djSFqMWa9GiRZg+fTrUajWkUn5XI6LSYfBZmYS0DKw+dgcR55OQkauCg40cQa1dEdrJM79yy+U7DzDou9PQSLTf77NXyBA5xb9U2w+JdebMGbRr1w5qtbpMllEQUcXG4KMiZu64jC0xd3Uuf5BLJRjRzh3zBrUweXvu378PNzc3XLlyBU2bNjX59YioYuO4ERVhrDV/xpJXrqzw5rRERCXB4KMijLHmz5jS09NRo0YNBh8RGQV3Z6AiHGzlSBcRfrrW/BlTeno6HD2a4KKiIZrPiURmrrrIrhLG2HWCiKwD7/FREeZyjy8vzH6JSYRS/b+2vDS5JW+G6cQu9bHqj9s6Z6uW1QxUIjJ/DD4qIiEtA4HfRiNLqdZ6jKlndeYtvchVqaEu5W9oWc5AJSLzx6FOAlB0g1q5RAA0GshksgLBI5MAamUOFo9oZ7IgeXkLJWPI23WiLGagEpH54+QWKnaDWqUggVQmAyQS2MmlkEiAyrZyjPTzgNe93Ti+/QejtkGtViM1NRVxcXGYt+04cpTGmzhTljNQicj8cajTypVkWPPevXto1aoVTp48iQYNGhQ5XqlUIi0tDWlpaUhNTS3wb23//+nTp6hatSocHR2hHPR/EOS2Rn2fEglwZ34/o56TiCwThzqtnKgNalVq/N+OGLxWT50fVG3atEGPHj3QqVOnImGWmZmJGjVqwNHREU5OTnB0dMz//87OzmjevHmBxxwdHVG9enXIZDIAgOdHe4z+PstqBioRmT9+Glg5UYvVBWBvbBriNi3PDysfHx9cvHgRNWrUwMiRIwsEXNWqVUtVWkzscoo8giDovJ5cKkFQa9cSt4eIKhYGn5UTu1hdYmOHQ4cOFXisd+/eGD16NBYsWGDUvfIGt6qjdzlFgbbpCVmFTIrQTp7GaBoRVQCc3GLlKtnKRB1X3FBhQEAA/P39MW/ePKO2ScwWSgAgkwJSjQq1Hp6GvUIKKQoGpVwqgb1ChhUhPlzKQET5GHxWLCkpCbgTA2h0LxvQNVT49ddfY/Xq1YiNjTVauzwcHbAixAf2CpnWzXPtFVKMbOeB/e8HwCk5Bi3v70GInwcq28oACBByM9FYnoqdb/tx8ToRFcDgs1L79++Hr68v+jewg72tjc5jdQ0Vuri4YNasWXjnnXdgzAnCXRvXQuQUfwxv6wZNTgYkEkCmyUXLSk/xx/QAxH7WB/MGtUAD56rYtm0bEq6eRe6JjbgyNxDxC/rj4NutIT23Df26+GHv3r1GaxcRWT4uZ7AyarUac+fOxY8//ohNmzYhICBA1Aa1unpNKpUKvr6++PDDDzFixAijtvfRo0do0qQJ0tLScODAAcyYMQNnz54tclxaWhr8/f0xfvx4TJ8+Pf/x33//HVOmTEHTpk3x3//+F6+88opR20dEloc9PiuSnJyMnj174vjx4zh37hwCAgIA/NO7GtHOHZVt5fmL1Ue0c0fkFH+9Q4VyuRwrVqzA9OnT8ezZM6O2+eHDh6hduzYAoFu3bkhJScHly5eLHOfo6Ih9+/Zh6dKl+Omnn/If79u3L65cuYKOHTvCz88Ps2bNQmZmplHbSEQWRiCrcPDgQcHFxUWYM2eOoFKpTHKNcePGCVOnTjXqOQ8cOCB069Yt/78/+ugjYdq0aVqPv3btmuDs7Czs2rWryHN3794Vhg8fLri7uwtbt24VNBqNUdtKRJaBQ51mxBRb66jVasyfPx8rVqzA+vXr0bNnTyO3+h+PHj1C8+bNERUVBW9vb6Occ+PGjdi7dy82bdoEAIiLi0OXLl1w9+5dKBSKYl9z6tQp9O/fHzt27MCrr75a5Pk//vgDkydPhpOTE5YuXYrmzZsbpa1EZBk41GkmiquXmZ6jwpaYuwj8NhqH41IMPmdKSgr69OmDqKgonD171qShBwA1a9bEZ599hkmTJkGj0V0NRqzk5OT8oU4AaNy4MerXr499+/ZpfY2fnx82btyI4OBgXLlypcjzXbp0wblz5xAcHIyuXbti6tSpePLkiVHaS0Tmj8FnBl7ejaDwom2VRkCWUo1Jm84hIS1D9DmPHj0KHx8f+Pr64uDBg6hTp46xm12ssLAw5OTkYP369UY5X+HgA4CxY8cWuI9XnN69e2Px4sXo06cP4uPjizwvl8vx7rvv4urVq8jMzETTpk3x448/Gi2wich8MfjMgKh6mf/bWkcfjUaDBQsWYOjQofjhhx8wf/58yOVlV6BHJpNh5cqVmDFjBh4/flzq8xUXfEOHDsX+/fuRlpam87UjRozABx98gN69e+PRo0fFHlOzZk18//332LVrF3744Qd06NABp0+fLnW7ich8MfjMgKh6mSK21klNTUX//v2xe/duxMTEoE+fPsZspmi+vr4ICgrCJ598UupzPXz4EM7OzgUeq1atGvr27YstW7boff17772H119/HX379sXz5891tvnPP//E22+/jUGDBiE0NBQpKYYPLxOR+WPwmQGx9TIzcrUfd/z4cfj4+KB58+Y4cuQI3NzcjNW8Evniiy8QHh6OM2fOlOo8xfX4gBfDnevWrRN1jnnz5qF169YICgpCTk6O1uOkUinGjh2Lv/76C1WqVEHz5s2xZMkSqFTG2xuQiMofZ3WagRZz94najaCyrRxX5vYu8JggCFi0aBG++uorrF69GgMGDDBVMw22du1arFy5EidOnMjfcshQNWvWxNWrV1GrVsG1hGq1Gu7u7ti/f7+oWZlqtRqDQt5ESg0vZNZqjowctd5Zs9euXcN7772Hhw8fYunSpfnrHvOYYhYuEZkeg88MzNxxWe9uBHKpBCPauWPeoBb5jz1+/Bhjx45FcnIyfvnlF9SrV68MWiueRqOBv78/Ro8ejbfeesvg1yuVSlSqVAnZ2dnFBueMGTOg0WiwcOFCved6UZ3mLLJylID0n3Ppq04jCALCw8Mxbdo0+Pn54euvv4abm1upq90QUfnhUKcZELMbQeF6madPn4aPjw/q16+P6Ohosws94MXQ4YoVKzBr1iytk0t0SUlJQc2aNbX2FseMGYONGzfqHIpMSMvAlC3nMG5dDLKUmgKhB+ifNSuRSDBkyBBcu3YNTZo0QevWrfHhvK9ehKgRZ+ESUdlh8JkBMbsRCBDwQ/RtxKemY8mSJejfvz8WLVqE//73v7Cx0V1kujy1bNkSI0eOxHufzMPMHZfRYu4+eH60By3m7sPMHZd1hoO2+3t5mjZtCnd3dxw4cKDY5/PWRu68+EBvO/XNmq1UqRI+/fRTxMTE4GASkJWTW6rzEVH5YfCZiZfrZdrJ//fH8tIodLZSgy0xiej2VRR+2HMcJ0+eRHBwcDm11jA9R0/B8SoB+Pl0okGL81+u06nNmDFjip3k8vLaSDFj+WJmzQKAp6cnsmt7A1LdS0TEno+Iyh6Dz4x4ODogtJPnPzuKF9pZXKUBNFIFVH5jIKvqXMwZzE9CWgam74iDRGELdaEE0jcsmJycXGQpQ2HDhw9HZGRkkTWDYtZGFqZr1myB44wwC5fI2BLSMgweVbFWDD4zI24xu2Axw2ilWZyvb6gTAKpXr47AwMAia/rErI0srJJCJuqDw8FWXEGA4natJzIFU5Q8rMgYfGbGWIvZzUVp3o+Y4AOKL2EmtleWR9CokaVUi/rgGNyqjtZ7sXl07VpPZEymKHlY0TH4zExFG0YrzfsRG3w9e/ZEYmIiYmNj8x8T2yvLI5FKoREg6oNDzCxcQa3EKF8Xg9pAVBLGLHloLRh8ZqaiDaOJfz9FlyyImdwCvCg4PWrUqAK9PjG9MgCQAJBJJZBKdB/78geHrlm4cqkEdnIpGqYcxZDeXYrdNJfImCraKFFZYPCZmYo2jCYqgDRqpJ2NxIQJE3DixAnk1VQQ2+MDXszu3LBhA9RqNQBxvTIAGNSqDmzlL3p7uhT+4Ciyaz0AITcTw9q4Yt/Uzti75mv85z//Qbdu3bBkyRKwTgSZSkUbJSoLDD4zE+ZfH/o+rwsvZjdnYgLI3tYGvy2cgvr162Ps2LFo1qwZFi5ciAcPHuid1ZmnefPmcHV1RVRUFIB/emV2cgkEdcG/8HKpBPYKGdaObYvFw1ojK1ct6hqFPzg8HB0wb1ALXJnbG3cW9INHzFL4Sm7Dw9EBEokEo0ePxsmTJ7F582b069cPDx8+FHUdsl6GzMzMyMjAsmXLICizRJ3bUkaJygKDz8zUrWYHh/M/Qy7RFOkpCWoV7OQvymFZSi1IfcOC9goZVoT4oF1TT8yYMQN//fUXVq9ejWvXruH58+cYNWoUIiIikJure8E4UHRNX9fGtdDqwe/Q3IiGrUQDieRFvdMR7dwROcU/v6SYsYaXx40bV2RN4SuvvILo6Gj4+PigdevW+P333/Of4/RzepnYmZkpKSmYPXs2PD09cejQIfRsWK1CjRKVBdbqNDPffvsttm/fjp+278GPxxMQcT4JGbkqONjI4fj8FprLkrFi4Wfl3UyDJaRlYPWxO9h+NhGZuWpUtrNBUGtXhHbyLBDieYWfw8/dQ0auGrYyATZJF5B67BeMHNgL48ePR4sWLYq9xoVbSQicshCObfogM1cNe4UUaWf3on7uHfy57zdIpcV/zytprdTCnj9/Djc3N8TFxeX3VF8uZJ2eo4KQmwVPySOEBvfCF5E3WOuTALz4PQn8NhpZSu2jD7ZyCfzSorDr57UYOnQopk2bhkaNGol6rb1Chsgp/hbzhdnUGHxm5NatW/Dz88OJEyfQsGHDIs/fvXsXLVu2xPXr1+Hk5FQOLSy9pKQk+Pr64sGDomXEdBV+lkmBjpprOPDTYri4uGD8+PEYMWIEqlWrVuC1Wbm5gOSfiTKCWgU7WwVWjfLVGiTG/OAYO3YsvL298f7772t9PxA0ACRFChSU5HpUtky1I4eYL1+CWoXG8lRsmNK/yC0AFk03DIPPTGg0GvTo0QN9+/bF9OnTtR43YcIE1KpVC59//nkZts54MjIyULNmTWRmZhZ4XGz47Hn3Vdw4fxw//vgj9u3bh759+6LfsLH4/KymVMGl/YMDUMhkoj84jhw5gsmTJ2PX4RPos+SYzjbpIqaHSWXLlOFSmq3J8uSNqrw8SlTcqAox+MzGd999hx9//BHHjx/XuXfd7du30bZtW9y6dSu/t2NJBEGAQqFAZmZmgeLaJRlu/Pvvv7F582YsPZGCbNc2kMi034MTEySFPzgkqly0qpqD/04cIPqDQ6PRoEGDBuj50WocjM82uHrMy3R9yFHZMtaogCAIePLkCe7evYvExETcvXsXd+/exWZNR50jAHkkEuDO/H4leg/0DwafGbh79y58fHxw5MgRUZuqjh07FvXr18fs2bPLoHXG5+TkhNjYWNSsWTP/sdJ84zXGt+XiREVFYdq0abhw4cI/9VNF+PTTT7EhqyVUEoXo1xSHH3LmQ+wXs9d96mBsC/v8QHs53PL+WyaTwc3NDe7u7nBzc4Obmxs2ZLVEjkb/XEN+GTIOzm8tZ4Ig4K233sKUKVNEhR4AfPzxx3j11VcxdepUVKlSxcQtNL6qVavi6dOnBYKvNGuRTLWOqVu3bsjMzMTJkyfRoUMH0a8bM2YMflx1BeKjsnicfm4+xC4S3/TnDfw64+P8QHNzc0O7du0wZMiQ/P+uWrVqkddmiwxWzsw0Dv7NKmcbNmzA/fv38eGHH4p+TaNGjdCrVy8sX74cH330kQlbZxpVq1bFkydPCjzmYCsX1WsrLgzEvlamUeLEiRNo3769qB6cVCrFxIkTsXLlSoOCLz09HVBmAzb2ol9TGD/kzEd2drao3y8AkNlVwo0bNwy+Rph/fWw/lwSVRvtQqiWt3zV3XMdXjh48eIDp06fjxx9/hEJh2LDYJ598gsWLFyMjw/LWfFWrVg1Pnz4t8FhpKtaIea1MAjRUPMG4cePQoEEDzJ49G3FxcXrbOnbsWOzcuRNpaWl6j71x4wZGjhyJ7t27o3b2XUDHh5g+/JArP1lZWTh8+DDmzp2LgIAAODk5QaLKEfXakvbSxa535SQV42DwlRNBEPDOO+8gLCwMPj4+Br++WbNm6Ny5M1atWmWC1plWcT0+MRVetIVBmH99SATdRXpt5DKsmvo6YmNjsXXrVqSnpyMgIABt27bF4sWLkZycXOzrHB0dMXDgwGI3u82TkJCA0NBQdOzYEc2bN8fNmzexbsYbENRKnW0CXgTyywS1CrZyCT/kylBmZiYOHjyI2bNno3PnzqhZsyY+/vhjZGdnY8aMGXjw4AFCXm1o8kXiRcrgaSm4QKXHyS3lZNu2bZgzZw7OnTsHOzu7Ep3j4sWL6NOnD27dugV7+5IPq5W1sWPHonPnzhg/fnyBx/Omi2fnKiFI/glBfdPFjx8/juB3ZqJqv+lQF9phQddr1Wo1Dh06hE2bNuG3335Du3btEBISgqCgIPzrX/8qcP6xY8fir7/+KrAI/sGDB5g/fz42b96Mt99+G9OmTUP16tXznx/09se4Wq0DIJUV26a5A5rhyv1nBaafe0oe4empcPyxZ7tBE2qsidi1dNqOC2njgnt/nccff/yBI0eO4MKFC/D29kZAQAACAgLQsWNHVK5cucg1uUi84mDwlYPU1FR4eXkhIiIC7du3L9W5Bg0ahB49emDy5MlGap3pTZ06FR4eHvj3v/9d5LmEtAz0eHcB5A06Ilst6F2LFB8fj44dO2L16tVo7telxOuYMjMzsWvXLmzcuBHR0dHo06cPRo0ahV69ekEul6NVq1ZYtGgRevTogbS0NHz55ZdYs2YNxo4diw8//BC1ahUN5GPHjuHNKTMwdNZKRFy4L6pNKpUKHTp0wMSJE/Hmm28a8FO1DmLX0mktHqBRQ1Ar4RT3G/q0dEeXLl3QsWNHODjoDysuEq84GHzlICQkBLVr18aiRYtKfa4zZ84gKCgIN2/ehK2trRFaZ3pz5swB8GLaf2GPHz+Gu7s7/v77b733PZ89e4ZXX30VoaGhmDJlSqnaVLh3oJBooEg6j9Rjv+D1Pl1hb2+P69evo1WrVli2bBmGDh2KmTNnwtVV+9CWIAho1KgRNm7cCD8/P9FtuXTpEnr06IGLFy/CxYV7+uUR2+ta9nozvL3lMnJ1jH6XtHfGReIVA4OvjO3cuRPvv/8+Ll26hEqVKhnlnH379sWgQYPw1ltvGeV8pvbNN98gMTERixcvLvLc7t27sXjx4vxdFrRRq9UYOHAg3NzcsHLlylINC+r6Ji+XAq8Ksfht1QKkpaWhadOmWLp0Kbp37y7q3B/O+wrH/7bH46oNDSpxNXPmTMTGxmL79u0lfl8VjZi1dNCoofr7HuQ16gJS7YUgWBnHunFySxl68uQJJk2ahNWrVxst9ABg1qxZWLBgAZRK/ZMpzEFxszrzREdHw9/fX+85pk+fjpycHCxdurRUoZeQlvGixqdSXezu69kqAVHZnujYaxB69uyJatWqYdSoUXonxQAvAvW33OZIVLjprLZfnJkzZ+Lq1asIDw8v8XuraMSspYNUBrmju87QA7gxq7XjOr4yNG3aNAwcOBABAQFGPW+HDh3g0bwNhn+9A3E5VYxaPNcUipvVmSc6OlpvHdLvvvsOv//+O06ePGnwMpDCfoi+DaVa94xQuY0tWg77N4Z4qDBw4EDEx8fj6NGj2LRpEz799NNiJ8XkBWq2SihSSk2lEaDSqDFp0zmtw212dnZYvXo1hg0bhq5duxaYNGMNBEHAvXv3cOnSJVy8eBGXLl1Cer1Rosp6ia0cwI1ZrRd7fGXkwIEDiIqKwpdffmn0cx+OS8GD1uNx9omNwT2L8qCtx5eZmYmLFy/qnPATFRWFOXPmYPfu3UYJAzG9CLUARJxPQqtWreDq6op9+/ahZ8+eWLduHZKSkjB+/Hhs27YNbm5uGDFiBPbs2YPv/ripN1CVag1WH7uj9flOnTph8ODB+OCDD0r03ixFRkYGTp8+jdWrV2Py5Mno0qULatSogXbt2mHp0qV48uQJ+vfvD3uF2I8rccnHyjjWi3/yZeD58+cICwvD999/X2CavDHk9Sxy1QCkhvcsykNeybLCTp06BW9vb63DwHFxcRg5ciS2bt1a7LZNJWFoubOJEydi1apVGDhwIACgUqVKGDZsGIYNG4bU1FRs27YN8+fPx732/4ZET+WWvOE2XfeZFixYgBYtWuDQoUPo1q2byHdlngRBQHx8PC5dulSgJ3fv3j00adIELVu2hLe3NwYPHgwvL68iM2X/ElnWq76TA26nZrD8F2nFyS1aGHPfrXfffRcZGRlYu3at0dtprE1Uy9KNGzcQGBiIW7duFXj8s88+Q3p6OhYuXFjkNWlpaWjfvj1mzJhRqmn+giDg8uXL2L59O8LDw/Gs1xy9AQX8Uxw4KysLbm5uiImJgaen9soqnh/tgZi/WGIKUe/ZswdTpkwx6oQoU3v+/DkuX76cH3KXLl3C5cuXUaVKFXh7e+eHnLe3Nxo1agS5XP93cLGzOteM8cWbP53hmjvSisFXDGOu1zl69ChGjBiBK1eumOQ+jal2JjClR48eoWnTpkhNTS3weM+ePfHee+9hwIABBR7Pzc1F79694evri6+++srg62k0Gpw+fRrh4eEIDw+HWq3GkCFDEBwcjD0plfFLzD2Dvji8//77sLW1xYIFC7S+xth/LiNHjkTdunWL/VJQnjQaDW7dulUg4C5duoTk5GQ0b948P9xatmwJLy8v1KhRo1TXK+06Pq65I4DBV0RpKjQU7iVWspEh89ofmPlae7w5bJBJ2mvMnkVZyc3NRaVKlaBUKvNnZKpUKtSoUQPx8fEFPhwFQUBYWBgePXqE8PBwnXsVvkylUiE6Ohrh4eGIiIhA1apVERwcjODgYLRq1Sr/uiX5846Li0Pnzp2RmJiode2k2Kn37WoosWZSL71D4CkpKfDy8sLvv/+ONm3a6Hn3pvHkyZMiAXflyhXUrFkzP+DyQu6VV14R/WdlKLFr6bjmjrRh8BVS0qFDrZUiBDXsbWxM9g3TEnt8AGBvb4/U1NT8ihkxMTEYP348Ll++XOC4RYsWYcOGDTh27FiRMlKF5eTkICoqCuHh4di5cyc8PDwwZMgQBAUFoUmTJlpfV5LeQffu3REWFobhw4cXe04xgaqQCPCM+xmnD+4pUClG20zVDRs2YNGiRYiJiSn1bFZdVCoVbt68mX8PLu+fv//+G15eXgVCzsvLq9htdojMGYOvkJIESXnW8bPEe3wA4OLigrNnz6JOnToAXgTc7du3sXz58vxjdu7cibfffhsnTpyAu7t7sedJT09HZGQkwsPDsXfvXnh5eSE4OBhBQUHw8PAQ3R5Dewe//vorli1bhiNHjmg9p9hAzZsUs3HjRty4cQOvv/46Ro0aVWT7JEEQ0H3QMNi16odEaW2jLFtJS0srMtkkNjYWLi4uBe7DeXt7w9PTs0CtUiJLxeArROzQIQQNOt37BQ0aNMC1Sl44+8QOah0vNFX4WGrx3CZNmiA8PBzNmjUDAAwePBjDhw/P70FdvHgRPXr0wO7du4uU+3r8+DF27dqF8PBwHD58GB06dEBwcDAGDRoEZ2fnMmm/UqmEh4cHoqKi8t9DcV4O1PTsXNgrZHjN111roN6+fRubN2/Gpk2bkJubi5CQEISEhKBx48Y4HJeCtzeeQVaOssDaQDH3rZRKJeLi4oqEXHp6eoEhSm9vb7Ro0UJv75rIkjH4ChHb47OXAzMapOLmzZvYnOsDtdRG72tMNdyorWchhQCNKhcrQtqgbyvxvR9TyrsPuunPGxDktqhsK8egVi74/t/DceHYAbi6uiI5ORl+fn5YuHAhhg0bBgBITk7Gb7/9hu3bt+PkyZPo3r07goOD0b9//3Jb3D1r1iw8e/YM3377rajjQ0JC0KtXL4wZM0bvsYIg4Pz589i4cSO2bNkC51ea43mn96DSsfQ27wuOnSq9wBDlxYsXcf36dbi7uxe5F+fu7s5dIMjqMPgKKcnQoTlMMNE2VHf79x8gPH+ETZs2lfsHnLaAlkkAtTIHa0M7ob37vxAQEIB+/fph9OjRiIiIQHh4OK5cuYK+ffsiODgYgYGBoqrpm1piYiJat26NxMREUe1ZsOBFvc+vv/7aoOuo1WqMX7kff9xTARIdQ40aNVR/HUFm9Loiw5TNmze3mKUQRKbG4CukJEOH5jzBJDMzE35+fnjvvfcQFhZWptd+mdifq+uldXh0JxYymQz37t3DoEGDEBwcjG7dupnl7hMDBw7E4MGDi+wtWJzdu3dj6dKl2Ldvn8HXEfs7ZiNR4/teVeDp6QlXV1dR6+OIrA3/VhTi4eiAFSE+eiclvHx/ZnCrOqJ6ieVRKaJSpUrYtm0b/P394efnB29v7zJvAyCuJmZmdg4uZDti6KuvYujQoejUqZPZf3BPnDgRc+bMERV8Xl5euHLlSomuI7bCTK5GilmzZiE+Ph4pKSlwcXFBvXr18v/x8PDI//9169Y1yexQYxZ/IDIF9vi0MGSWnyVMMNm4cSM+//xzxMTEGL1smhhieywONlJc/bRPGbTIOG6nPEe3SfNg3zQA2SpB54e8IAioVq0a7ty5Y/BC7pKMKuTm5uLu3btISEhAfHx8/j95/52cnAxnZ2etwejm5gYbG/33rl/GheNkCRh8RqLtL7xE0EACDdaM61Duf+FDQ0ORlZWFjRs3lvn9PnO4D2pseX/m2UolhJcmnej6kO/YsSMWLFiALl26GHQtUyxbUSqVuHfvntZgvH//PmrWrKk1GN3d3QsMP1vCF0AigMFnVMX1Evs1d8Lm2aH4afmici8ynHe/b8qUKQgNDTXptbKzs3H06FFERkZi7969yOw7z6CamObOkLqRv19+kD/sJ9Uo4fWvTHz79iCDPvzLI1RUKhWSkpK0BuO9e/fg6OiYH4QpHt1wG7Wh0bE7gjmuKSXrw+ArA7t3784vMlzesxFjY2PRuXNnHDp0CF5eXkY7ryAIuHHjBiIjIxEZGYljx47B29sbgYGBCAwMRMQ9G4NrYpozMT0w2f8+/yUSSZFRADsbhcHDfm9/tgx7nrnAxtYOqpdul5bXMKJarcb9+/fzg/DTq1WghP4yZfq+3PAeIZkag6+MhISEwNnZGd988015NwXr16/H/PnzEX4gGpvPJpf4AyY9PR2HDx/OD7vs7Oz8oOvRo0eB9XUVbRhM7D03XQx5v6mpqXBxccHkjz+Dg88As6w/KXo4GwL+nOyTX7XnZbxHSGWBwVdGUlNT4eXlhYiICJ0brZaVQW9/jMtV/CCVK0R/wAiCgKtXr+YPX54+fRpt27ZFnz59EBgYiBYtWui8d1iRPtREV/jRwZAebocOHXDv3j0kJiaW+3pMbcR+GZBpcvH4hzfh7u6e/7vTsWNH3H+WW6G+HJH5YuG9MuLk5ITFixdj/PjxyMnJKde2JKRlIK6mPzRSeZGhOpVGQJbyxea1CWkZePLkCX799VeEhobCzc0NAwYMwO3btzFlyhTcv38fhw4dwgcffAAvLy+9H8hdG9dC5BR/tKycDiizAEEDKLMxop07Iqf4W0zoAYCDbemXWeRtRKvPtm3bcOrUKezcudNsQw94saxHLtXdPrlUgpEdGyIlJQXLly+HXC7HtGnTUKtWLbw2cyVylLqDU9+u9URisMdXhgRBQFBQELy8vDBv3rxya4eY+1MSaFDp/nkkhn+FTp06ITAwEH369EHDhg1L9eF7+PBhDB8+HNHR0RgzZgzs7Oxw+PDhEp+vvIjadkgEfbNYs7Oz4eTkhIEDB2Lz5s2lupaplWY4++HDh+i85DRyNPq/i1vKBCgyX+a9OriCkUgkWLFiBVq1aoXXXnsNLVu2LJd27LhwX+8HtgAp1G5tkJKSAnt7/bMxCytugkJXz0rYNm8yfv75ZzRq1AgpKSmoW7duSd9GuQrzr4/t55Kg0mj/kBfDwUb3X8ExY8ZAKpVi7dq1pbpOWShJ8Yc8zs7OyBURegCQkVu6e6tEHOosY3Xq1MH//d//Yfz48VCpyucvsNgqIDkalCj0DselIPDbaGyJuYv0HBUEAOk5Kuy69jf+NXQBJK4tkJ6ejgcPHuDhw4cGn98c5H3I2ytkgFAw/ORSCWRSCfSM+umt5hMTE4Nff/0Va9euNctybcXJG84e0c4dlW3lkEhe9NDEDGeLHT7W92WBSB8OdZYDQRDQu3dvdOvWDTNmzCjz65uytqjY4a6ZbaRYsfBTnD17Fs+fPzfpxqqmdPPhU/iHzkENn97IVGryZ1n2aVEbb/50Rs/PQYrIKZ2L7QFpNBrUrVsX9erVw/Hjx035FsyGpe4tSZaHX53KgUQiwffffw9fX18EBQWhcePGZXp9U9YWFVOTU6nWYP2pJHTp0gXJycm4fft2mf8MjOVe7Hm4PjiGM58uLvKctmE/mQRQ5mRDeewnXOsAePQpWqLtk08+QWpqKi5evGjK5psVMcPHCpkUoZ08y7BVVBFxqLOc1KtXD3PmzMGbb74JjUZ3UBhbmH99KGS6/+hL+gEj5v6hSiPgurI6/P390bRpU8TGxhp8HXOxZ88e9OtX/OQUbcN+I/08YHd4Ed4fEYj33nsPAwYMwM2bN/Nfl5iYiK+//hpz5sxBzZo1y+qtlLuXh48Lzw6VSyWwV8i03iMkMgSDrxy98847AIDly5eX6XULfsAUfK60HzBi7x9qZAp07NgRTZo0wV9//WXwdczFnj170L9/f63Pezg6YN6gFrgytzfuzO+HK3N7Y96gFvDydEGNGjVw5coVdOrUCe3bt8fHH3+M9PR0DBgwAK6urvj444/L8J2Yh9LcIyQSi/f4yllcXBw6deqEmJgY1KtXr0yvnZCWgc9/PYGom08BhV2pqoDk5ubi999/x/vHBWhkIir6CwJGtfdA1eSzuHLiMH766acSvovyc/v27RcLr+/fh1Qq/jtkQloGJnzzC25pHKGGHA62cvRsWA13D6zDwd+24OnTpzh37hxatWplusYTWTHe4ytnjRs3xvTp0xEWFob9+/eX6QJlD0cH9Kz+N57Eb8O2bdtKdI6LFy9i3bp12LRpE5o0aQKfvpNx4blNgVqSxZJIsCXmLmSSmrBLUZbo2uVtz5496Nu3r0Ghl1e9JkdTK7+Y84sZr2mQuw+EutZ5VJfewOTJk7FkyRK0bt3aVM0nsloc6jQD06ZNw+PHj8tlrVZiYiLc3d0Nek1qaiqWLl0KHx8fDBgwAJUrV8bx48dx9OhRLArrC4VMf6Fi4MW9vhw18KT5EMSnppek+eVq9+7dWu/vFSchLQOTNp1DllJdZAcDlUZAtlKD6gM+QPSFv/DGG28gMDAQEydORGpqaoFzzNxxGS3m7oPnR3vQYu4+zNxxGQlpGUZ7X0QVHYPPDMjlcqxZswYzZszA/fv3y/TaYoNPpVJh9+7deO2119CgQQOcOnUKCxcuRHx8PObNm4cGDRoA0D1BQSupHEv2l2xn8vKSnp6O48ePo2fPnqJfI2bGq0xhi5/PPcSECRPw119/wcbGBs2aNcPy5ctx8NqDYtdHbom5i8Bvo3E4LqWU74rIOjD4zETLli0xceJETJo0CWV521Vf8F27dg0ffPAB3NzcMH/+fPTu3RsJCQnYuHEjevToUeww38sTFMSQyOT4/Vpaid9DeYiKioKfnx+qVKki+jViZrxqIMmv31m9enUsWbIEBw8exM+7DuDNtSeRpVTrra9KRLrxHp8Z+eSTT+Dj44OtW7di2LBhZXLN4oLv8ePH2LJlC9auXYukpCSMHj0ahw8fRpMmTUSfN28248aTCaJ2MchWm/8cq5fLsKVny6Fo9x5m7rgsahun58+fIz1HCejYpDVPenYuunTpAldXV7i6uqJOnTqo22MMktLkOn+WeQWcubibSDfO6jQzp06dwuDBg3H58mU4OTmZ/HpOTk6IjY1FjRo1EBUVhbVr1yIyMhK9e/fGuHHj0LNnT8hE3rMrjtgqMXJBiZv/N7jE1zE1Q7dUUqlUOHPmDPbv348DBw7gwoULcJr4EwS5/tJjlRRSLAuwxf3795GUlISkpCTstusqarYsCzgT6cfgM0PTpk1DcnIyNm3aZNLrZGRkwNHREf/+97+xYcMGuLi4YNy4cRg+fDhq1KhhlGuI3anc4cF5XFo70yjXNDaxZdi+D/LAlZNHcODAARw+fBhubm7o1asXevbsCX9/f8zff6vEJblEb/KqZ7cHIuJQp1maN28evL29sXv3bp2Lo0vq2bNn+OWXX7BixQqoVCoolUpERkaiRQvjD5GJK0MlwaM/twIwz+ATMyklKzsHIz9bjW5VUhEcHIwVK1agdu3aBY4pTUkuB1u5qJ4zCzgT6cfJLWaoUqVKWL16Nd5++208ffrUKOfUaDQ4ePAg3njjDbi7u2Pv3r0ICgpCly5d8PXXX5sk9ADdszylEGCvkGFlSBs8v38bT548MUkbSkvMpBTI5Kjashd++uknjBo1qkjoAf/8LCRqJaSF+m/6KuaI2eRVImgwoAUrmxDpw+AzUwEBAejXrx8++OCDUp3n9u3bmD17Njw9PTF9+nT4+vrixo0bCA8PR506deDh4WGkFmtXuAyVIGgg1yhROfniizJUTZzNunSZ2DJsYvaJc1anIjtiFuySzkKTkwkJxJXkElNfVSJo8OsXk3D69GlR7SWyVrzHZ8aePXuGFi1aYOGKH3FF5VxgU9fBreponU2Ynp6ev49bbGwsRo4cibFjxxYpgTV79mxIpVLMnTu3bN7Q/3h7e0MulyMxMRFnz56Fh4cH3njjDXTr1g3jxo0r07bo8vz5c6xfvx5f3akNKOz0Hi9mYsmECRPw+PFj7Ny5Exs3bsTrr78uuj3aJthAUEMKoEuT2jh+IwXZasBGosHQdvUwofMrLOpMVAh7fGasSpUqmDx/JWb88QxbYhJ1LloWBAFHjx7FuHHj4ObmhvDwcEyZMgX37t3D4sWLi637WJKqLcbg4+ODO3fuoFu3btiwYQMAoEmTJmazS0NsbCzeffddeHh44MiRI+hWv7LeYUYx2zg9evQIW7duxd69e9GuXTuDQg/QXsDZv4ET1GoVjl5/hByNBBKJBErIsOlkPHov/iP/d4RVX4heYI/PjImZTWgnl2CA/DLC138Pe3t7jBs3DiEhIcXeYyqse/fu+Oijj9CjRw9jNluvTz75BGfPnoUgCLhz5w7i4uIQERGBdevWYefOnWXaljx5lWmWLVuGK1euYMKECZgwYQLq1q0relZn5BT/Ar2rl9f9ZeSoIIcaypvHkRq9BddOHzXKlw4xbVNIBHwe5IW5u2JFL8cgqsg4BcyMiZpNmKPE6YzK2LJlC3x9fQ0qcl1ePb7atWvD3d0dv/76K2rUqIETJ06U2758jx49wurVq7Fq1Sq4urri3XffxZAhQ2Br+896u7xJKfrW8b0cesUNSyohg+DZHrUbdMStLDsY4ycvbuNfNT7cfgmQFB3gUWkEqDQvqr4UDm6iiopDnWZMzGxCiUyOtCoN0LZtW4NCT6PR4O7du3BzcyttMw1Wu3ZtPH78GGFhYahVqxbWrVuHBg0a4O7du8jOzi6TNpw+fRpjxoxBo0aNcPPmTUREROD48eMYOXJkgdDLY8g+cS8Xoy785yeRKaCCzGjlxUTNOJXKoa9iTF7VFyJrwOAzY8acTVjYo0ePUKVKFdjb2xv82tJydnZGcnIypk6dimvXrmHbtm1QqVSoV69egZ3IjS07Oxvr169Hu3btMGzYMLRo0QI3b97EmjVr4OPjo/f12jaVLdxLEtcLM07QiP0dgZ4vRSqNkF8jlKiiY/CZMQdbcSPRJVm0XF7DnMCLHl9ycjJcXFzw+uuvw9HREb/99pvJhjsTExPx8ccfw93dHZs3b8bs2bNx8+ZNfPDBB3B0dDT69cLPJ+nthRkraMT+johRki9QRJaIwWemNBoNXpGlQVDr/jASM5uwOImJieUyzAn8E3wAMH36dKSkpGDNmjVGDT5BEHDw4EEEBQWhdevWyMzMxLFjxxAZGYn+/fuXqv6oLofjUpCZq32iycuMETRiFraLxaovZC0YfGYoKSkJgYGBSDq4AXa2Cp3HaitxpU959vj+9a9/Qa1WIz09HY0bN0ZAQACOHz8OZ2fnUi9if/bsGZYtW4ZmzZph6tSpCAwMREJCAhYvXoxGjRoZ6R0UL+/enljGCBoxC9tlEkAu4m96eo6KSxzIKjD4zMy2bdvg4+ODTp064WTULqwa5VtsuS+5FIAqB70qJZRoJl55Bp9EIkHt2rXx8OFDAC+WN0gkEly/fr3EPb68tXf16tXD0aNH8d133+HSpUt46623ULlyZWM2X6vv/riJHKW4XlxJe+qF6SoJl1cGbX6QFxQie7jc2JasAcc2zMTTp08xefJknDx5Ert27UK7du0A/DObcPWxO4g4n4T0HCVkGiVGtG+Ifq/YYXj/HmhZS4Hx48cbdL3ExER07NjRFG9FlLzhzldeeQV+fn5o2LAhIiIi8OTJE2g0mmI3uC1MpVJh165dWL58ef7au8uXL8PVtfSBYgi1Wo2NGzdi82UHQCFuslBJe+rFKfw7kpGrgoONHEGtXRHayRMejg6oVcWu+KovxeASB6rouIC9jBRezPxy2bGEq2cxevRo9OnTB19//TUcHLR/0Ny4cQMBAQG4d+9efi8pICAA3377rUGVQNq2bYtly5bBz8/PGG9PtLyfw+bjN6GRKVDZVoHBrerAMSMR8zfuRaVmAZDa2Ossy5a39m7lypVwc3PLX3tnY6N/vzpj0mg0+PXXXzFnzhzUrFkTia9+KPq1a8e2LfMF4wlpGS99gRKxR6KWLZKILB2Drwzo2sRUUKuQfXAZvp87Bf366d9HTRAEeHh4YP/+/fk7ol+6dAk9e/bETz/9hMDAQFFtcnZ2xoULF+Di4lKyN1UC2n4OUgmgEQBBo4ZE+s+QXOGqIqdPn8ayZcuwa9cuBAcH45133hG1DMHYBEHA7t27MWvWLNjY2ODzzz9Hz5494fXpfpFbB8lw9VNxf06mInaDYG5sSxUR7/GZmK7FzCqNALVEhn8F/hst2geIOp9EIkH37t1x6NCh/Me8vb2xY8cOjB49GtHR0XrPkZ2djadPn8LZ2dmg91Iaun4Oef/5cugBL34+WUo1Jvx0Gj5demP48OHw9vY2aO2dMQmCgKioKHTo0AGffPIJPv30U5w6dQq9evWCRCLB4FZ1IIHu9XtyqQTBPnXLqMXamXKNKJG5Y/CZmJjFzCqNYNBi5u7du+PgwYMFHuvQoQM2b96MIUOG4Nw53TML7927B1dXV1H30YxFzM9BG6Vag+bBk3Hjxg1Mnz7dJGvv9Dl27Bi6du2Kd955B1OnTsWFCxcwaNCg/Go5WVlZuLp9id7lJ8a8t1caplwjSmTuGHwmJqaklKGLmbt164YjR45ArS64XqxHjx747rvv0K9fP52zI8tjRqeo0lraSGU4/1hhsrV3upw5cwZ9+vTBqFGjMGbMGFy9ehXDhw8v8KXhwYMHCAgIgJ0qHd+NaqtzhqW2jWbLmpj1f8aaeUpkbhh8JmaKIaU6deqgVq1auHDhQpHngoKC8OWXX6J3796Ij48v9vXlEXyiS2tpe30ZD7lduXIFwcHBGDRoEAYMGIDr169j3LhxkMsL9oDOnz8PPz8/DBgwAJs3b0Zv77qia3qWJzHr/8yld0pkbBzHMDEHW7nICQ+G/VHkDXe2adOmyHOjR4/Gs2fP0KNHD0RHRxeZwFKWwZeeno4LFy5AIVEjVyh5j00uqJCQkGDyHeNv3LiBuXPnIioqCv/5z3+wadMmrfVMw8PD8dZbb2HlypV47bXX8h/Pq+lpzrMhS7LjBFFFwR6fiZlqSKm4+3wve/fddzFu3Dj06tULf//9d4HnTFWu7Pnz54iOjsbixYvxxhtvoFmzZnB2dsb7778P56y7kKJkQ50yCeCqvA8fHx/0798fu3fvLjLMW1oJCQkIDQ1Fhw4d0LRpU9y8eRPTpk0rNvQEQcD8+fPx3nvvYe/evQVCz5IYsuMEUUXC5QwmVtJNTPV5/Pgx3N3dkZaWpnX9miAI+M9//oOjR49i7bZd2HzuIXZcuI/0bCXs5BK85ute7Do5MZ49e4bz58/j7Nmz+f/cvXsXXl5eaNOmDXx8fNCmTRs0b94cCoVC1M9Bm7yfT017CbZu3YpVq1bh/v37CAsLw5tvvok6deoYfM48Dx48wBdffIGff/4ZEydOxPTp01G9enWtx2dnZyM0NBRxcXHYsWNHmS+WJ6LSY/CVAV3r+Eqz+3Xbtm2xaNEidO7cWesxgiAg+N1ZuODgC5nCpkTXf/r0Kc6dO1cg5JKSkuDt7Y02bdrk/9O0aVMoFNpri+b9HLKValF9P0Gjgb2NDCtH+RZp3/nz5/Hdd99h69atCAgIwMSJE9GjRw/RM1VTU1Px5ZdfYs2aNRg3bhw+/PBD1Kql+8/g4cOHGDx4MNzc3LBu3TpUqlRJ1LWIyLww+MrIy1UziispVRIzZsyAra0tPv30U53XNaTH+fjx4/yQy/v3gwcP0LJlywIh16RJkyITPcRISMtAj//+AaVaxK+dRoXO6UexfvkirYc8f/4cmzdvxqpVq/Ds2TNMmDAB48aN0xpiT58+xaJFi7B8+XIMHToUM2fOFNVru3jxIgYOHIhx48Zh9uzZZboUhIiMi8Fnwfbv34/PPvsMx44d03rMzB2XsSXmrs6lBBJoUPPJdaTtX4GUlBS0atWqQMg1btzYqEsJPD/aI+5unyDg2fejRdXfFAQBMTExWLVqFSIiIhAYGIiJEyeic+fOkEgkyMjIwJIlS/DNN9+gX79+mDNnDjw9xc1Y/O233xAaGoqlS5di+PDhol5DROaLwWfBMjMzUatWLSQnJ2vdgUBsaSpbqQY7RjVAo0aNTL5eTmybNDmZCNb8CQBYvHix6PM/efIEGzZswKpVq6BWq9G0aVP8+eef6NatG+bOnZtf6i2PtjqqoZ088cua5ViyZAkiIiLyC4cTkWXjeI0Fq1SpEnx9fXH06FGtx4hdP5crSNG0adMyWSQuavNUQQ1p4hl06dIF69evz9/CSIxq1arhrbfewuTJk/H3338jJiYGWVlZsLW1xePHj/Hyd73DcSkI/DYaW2LuIj1HBQH/bM3T7auD2HDwHE6dOsXQI6pAGHwWLCEtA3adxuC9o0p4frSn2E1EzbE0ldjNU12eXsOZM2cwcuRIfPPNN6LOrVarsX79ejRp0gTh4eHYtWsX7t27hzt37sDLywujR49Gy5YtsWLFClxNeKizjqpGKoeq/Vio7bXP8iQiy8OhTguVN0MyV6XGy/NECs/UFHOPTwoBIX4emDfYqwxa/oK+ma7D3dJxcMMSZGRkICIiAj6de2Psl+sRGft3kW2dPBwdoNFosH37dsyePRuOjo744osv0KVLlyLX1Wg0OHz4MFatWoVj2a6wa94dgkR7CHNrHqKKh8FngQyZqQlA//o5VS46PD2MtUsWws7OztjN1UrXTNf05HgMHjwYjx49wrp9p/F+RCwEiRTCS4MUeSE5tqEKW76ZBZlMhs8//xy9e/fOLx6tS7PZe5Gp1F84m1vzEFUsDD4LJKYX93JPRV/v6pvgpli34D+4desWwsPDy7yOZ2EJaRn47o+b2HjsOqQ29oAE+N//FE+Vi5ltJHhz2CBRgZdH7OxSiQS4M1//XolEZBl4j88CGbrjw8ulqZCbBQkKlqbq08oDW7ZswYgRI9CuXbv8UmgJaRmYueMyWszdp/UeorHlTTbZejYJUttKL1JHV+gBkNvYIsG+gUGhB5jn/U8iMj32+CyQ6J4KBOwe5YlXXnkFdnZ20Gg0sLe3x9OnT7UOaR46dAghISEY8u5sHMzxNHq1GV1KU9asJMORhvaciahi4FdZCyR2xweJWokhQ4YgPj4ederUgYeHB2QyGdasWYPGjRujUaNGqFu3boEqJN26dUPEgWgMX38ZkBUNIJVGgEqjxqRN5wyqL5qVlYWHDx/i4cOHSE5OLvbfSS7+0Hi2h0Rm+K9lSbYtCvOvj+3nkqDSaA9abs1DVPEw+CzQ4FZ1xPVU2jfEvIV/QalUIj4+Hrt378aNGzdw6dIl/Prrr4iLi8PTp0/RoEGD/CBs3LgxorPqQK6wgUrHvA+lWoPv/7iJt3yrFwmw4kItOzsbzs7OqF27doF/N2nSBAEBAXB2dsbEqHRkKks2AFGS4UhuzUNknTjUaYEMrb+ZV5lkW0wCctRAZTtF/lKAGjYaXL9+HdevX0dcXByuX7+O465DALn+2Z2anAxotr5fJMwK/9vZ2RnVqlXTew9OdCmzQko7HGmKOqpEZL4YfBZK7I4PJdkZorxmO4otZVZYSbZ1IiLrxVmdFkrMJqIJaRk6K5NkKV/cqys8S7O8ZjuKKmVWiL1CxuFIIjIIe3wVWElnLZbXbEdDZ3U2cq6MH97wZegRkUHY46vADF3vl0dMLU1TzHbMm2xiK9f/a2mvkDH0iKhEGHwVmNidGQovBcgLIHuFrMjQo1wqMenwYtfGtbB/amd007JG0NTXJ6KKj0OdFZjYySLaFn+X92zH8r4+EVVMDL4KjJVJiIiK4lBnBVZe9+qIiMwZg68CK897dURE5opDnVaA98qIiP7B4CMiIqvCoU4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIqDD4iIrIq/w/2e84ahnLjPwAAAABJRU5ErkJggg==\n",
          "text/plain": "<Figure size 432x288 with 1 Axes>"
         },
         "metadata": {},
         "output_type": "display_data"
        }
       ]
      }
     },
     "eaf71abdb8c249779654aa7969085f20": {
      "model_module": "@jupyter-widgets/output",
      "model_module_version": "1.0.0",
      "model_name": "OutputModel",
      "state": {
       "layout": "IPY_MODEL_b7a7e70ec1af4deb99c1e7ebbf5c641d",
       "outputs": [
        {
         "data": {
          "application/vnd.jupyter.widget-view+json": {
           "model_id": "951895a410a046c6813874883fd4a251",
           "version_major": 2,
           "version_minor": 0
          },
          "text/plain": "  0%|          | 0/834 [00:00<?, ?it/s]"
         },
         "metadata": {},
         "output_type": "display_data"
        }
       ]
      }
     },
     "eb6a200c5d2a4ee9be0ee148c3f7a08b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "eb9d58970e414533adb7a904da21ee1b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "ec1fb56a11ac4f109a924cc5e4d41ca5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "ee15fcacae404d8c8f40effe5db70654": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_2ecf24a9517e4d7da1c9f464ae1efc8f",
        "IPY_MODEL_7f596927186347ae85abf5e8b2e047dc",
        "IPY_MODEL_75cb8cf3a90248adbe80ed6e70d49542"
       ],
       "layout": "IPY_MODEL_2a0b95a105bd4384b3b871e2ba09beaf"
      }
     },
     "f070c9676b1348c187c8e9d780970361": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "f29c1356d97e4211b4b7ea1834167c28": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "f483dfebacc543aa969e0a477c7c72ee": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_97d1eceb8fdd46b8a5f736ba6578fad9",
       "style": "IPY_MODEL_e4221a6462ee4dd4af7f2e534bc741fc",
       "value": " 18/1000 [00:03&lt;02:36,  6.27it/s]"
      }
     },
     "f72558fc51d3452c879324c4c32de9e0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_a9093cfe030b4d3c99c00f70637b3593",
       "style": "IPY_MODEL_3896f84e333c4703b629bbf949d1a8d3",
       "value": " 1000/1000 [00:53&lt;00:00, 21.51it/s]"
      }
     },
     "f85a03eec90041c09182ab7b7a5a579b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "f8f4986844d24daba7dad3fe822be00a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "fa7104db81ce40e9b7ab030caf41d7f4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "fad37bf2f8c148ba961d1f8ba0578ea3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "fc53bb7fe4fa497d9306c8139ab2947b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_eb9d58970e414533adb7a904da21ee1b",
       "max": 834,
       "style": "IPY_MODEL_401a5ff689054c8a80d6de58255414c9",
       "value": 32
      }
     },
     "fddf70b473aa473bbe6d646af28296d0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}