FF_Updated.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "_ZuM_z_9j_bs"
      },
      "outputs": [],
      "source": [
        "## Setup Imports\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm.auto import tqdm \n",
        "\n",
        "import jax\n",
        "import jax.numpy as jnp\n",
        "from jax import jit, vmap, value_and_grad\n",
        "import optax\n",
        "\n",
        "from typing import NamedTuple, Tuple, List\n",
        "import functools\n",
        "import copy\n",
        "\n",
        "rng = jax.random.PRNGKey(42)\n",
        "\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import Dataset, DataLoader"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9LNYxauHkBbg"
      },
      "source": [
        "####Data Loader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "Wko73BZij_ms"
      },
      "outputs": [],
      "source": [
        "class MNIST_Dataset(Dataset):\n",
        "    \"\"\"\n",
        "    A custom dataset class that returns a pair of positive and negative image\n",
        "    samples as defined in the paper. \n",
        "    \"\"\"\n",
        "    def __init__(self, train):\n",
        "        super().__init__()\n",
        "\n",
        "        self.train = train\n",
        "        self.base_dataset = MNIST(root='train_mnist', download=True, train=train)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.base_dataset)\n",
        "    \n",
        "    def __getitem__(self, index: int):\n",
        "        # This function encodes the label within the image to make the data samples. \n",
        "        # For this example, as MNIST has 10 labels, we take the top-left corner's first\n",
        "        # 10 pixels and encode one-hot representation of the label into these 10 pixels.\n",
        "\n",
        "        # Load the base image and label\n",
        "        img = self.base_dataset[index][0]\n",
        "        label = self.base_dataset[index][1]\n",
        "\n",
        "        # Flatten the image and set the first 10 bits as 0\n",
        "        img = np.ravel(np.array(img, dtype=np.float32))\n",
        "        img[:10] = 0\n",
        "        \n",
        "        if self.train:\n",
        "            # Create a positive sample by setting the current label index as 255\n",
        "            # For example, if the label is '5', we set the 5th pixel to 255\n",
        "            pos_img = img.copy()\n",
        "            pos_img[label] = img.max()\n",
        "\n",
        "            # Create a negative sample by setting a random index in [0 -> 10]\n",
        "            # which is not the correct label as 255\n",
        "            neg_img = img.copy()\n",
        "            random_label = np.random.choice(np.setdiff1d(list(range(0, 10)), [label]))\n",
        "            neg_img[random_label] = img.max()\n",
        "\n",
        "            return img, pos_img, neg_img, label\n",
        "        else:\n",
        "            return img, label"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "VB0txA-GkE5F"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "A custom dataloader collate function to create batches of positive and negative data.\n",
        "Returns as jax array instead of the default pytorch tensor\n",
        "\"\"\"\n",
        "def train_collate_fn(batch):\n",
        "    transposed_data = list(zip(*batch))\n",
        "    imgs = jnp.array(transposed_data[0])\n",
        "    pos_imgs = jnp.array(transposed_data[1])\n",
        "    neg_imgs = jnp.array(transposed_data[2])\n",
        "    true_labels = jnp.array(transposed_data[3])\n",
        "\n",
        "    return imgs, pos_imgs, neg_imgs, true_labels\n",
        "\n",
        "def test_collate_fn(batch):\n",
        "    transposed_data = list(zip(*batch))\n",
        "    imgs = jnp.array(transposed_data[0])\n",
        "    true_labels = jnp.array(transposed_data[1])\n",
        "\n",
        "    return imgs, true_labels\n",
        "\n",
        "mnist_train = MNIST_Dataset(train=True)\n",
        "mnist_test = MNIST_Dataset(train=False)\n",
        "\n",
        "# Load the whole dataset into memory for custom batching\n",
        "train_loader = DataLoader(mnist_train, len(mnist_train), shuffle=True, collate_fn=train_collate_fn, drop_last=True)\n",
        "test_loader = DataLoader(mnist_test, len(mnist_test), shuffle=False, collate_fn=test_collate_fn, drop_last=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R1kCu9312bO_",
        "outputId": "79533640-fd27-4576-e8f7-fb6159dc66e7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train xs shape:  (60000, 784)\n",
            "Test x, y shape:  (10000, 784) (10000,)\n"
          ]
        }
      ],
      "source": [
        "train_imgs, train_pos_xs, train_neg_xs, train_labels = next(iter(train_loader))\n",
        "test_xs, test_labels = next(iter(test_loader))\n",
        "\n",
        "print(\"Train xs shape: \", train_pos_xs.shape)\n",
        "print(\"Test x, y shape: \", test_xs.shape, test_labels.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 922
        },
        "id": "l3hRYE9AIuCX",
        "outputId": "8566b8b8-1458-41c3-bf84-d6487f177bf9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "4 Random training samples and labels\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAN4CAYAAADeOtzyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACTF0lEQVR4nOzde1yU5b7//zegDAgCIgiaKKikmaVFyzykqLm00qVW25Zmph20lpjbtNbOrDyUkrrT0tBV/gzTcrm3JZbtDsvIrR3Uymprmq40aLk0MA+AJ1Dh+v3h11lOcyM3x2GY1/PxuP/gM9fcc42+7+HDfRo/Y4wRAAAALsvf0xMAAADwBjRNAAAANtA0AQAA2EDTBAAAYANNEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIANNE1eYvr06fLz86vQc5cvXy4/Pz9lZ2dX7aQukZ2dLT8/Py1fvrzaXgMAAE+iaaoBu3bt0j333KMrrrhCDodDzZo104gRI7Rr1y5PTw2otItN+cUlKChIV155pcaPH6/c3Nwam0evXr2cc/D391dYWJjatm2rkSNHasOGDZVa9+LFi/mDAE4XMx8UFKSDBw+6Pd6rVy916NDBAzOT4uPjXbaDiIgIXXPNNRo7dqy2bdtWqXXPnj1b69atq5qJeimapmq2du1aXX/99crMzNR9992nxYsX64EHHtDGjRt1/fXXKyMjw9Z6nnrqKZ05c6ZCcxg5cqTOnDmjli1bVuj5gB0zZ87UypUr9fLLL6tbt25asmSJunbtqtOnT9fYHJo3b66VK1dqxYoVmjdvngYNGqQvvvhC/fr10x//+EedO3euQuulaYKVoqIiPf/8856ehptOnTo5t4PU1FT17t1b69evV5cuXTRp0qQKr5emSZJBtdm3b59p0KCBadeunTl8+LDLY7/++qtp166dCQkJMfv37y91HSdPnqzuaVaJrKwsI8mkp6d7eiqoYenp6UaS+eqrr1zqkyZNMpLMqlWramQeycnJ5uqrr3arnz9/3owbN85IMn/+858rtO6rr77aJCcnV3KGqCsuZr5Tp07G4XCYgwcPujxeWhZrQsuWLc2AAQPc6qdPnzZDhgwxkszixYsrtO6QkBAzatSoSs7Qu7GnqRrNmzdPp0+f1quvvqro6GiXx6KiovTKK6/o1KlTmjt3rqR/nbe0e/du3X333WrUqJFuuukml8cudebMGU2YMEFRUVFq2LChBg0apIMHD8rPz0/Tp093jrM6pyk+Pl4DBw7UZ599ps6dOysoKEitWrXSihUrXF7j2LFjeuyxx3TNNdcoNDRUYWFhuvXWW/V///d/VfgvhbqoT58+kqSsrCxJ0vnz5/Xss8+qdevWcjgcio+P15NPPqmioiKX53399dfq37+/oqKiFBwcrISEBN1///0VnkdAQIAWLlyo9u3b6+WXX1Z+fr7zsfT0dPXp00dNmjSRw+FQ+/bttWTJEpfnx8fHa9euXdq0aZPzsEevXr0ksX34uieffFLFxcW29za98cYbSkpKUnBwsCIjIzVs2DAdOHDAbVxaWppatWql4OBgde7cWZ9++ql69erlzF1FBAcHa+XKlYqMjNSsWbNkjHE+9p//+Z/q1q2bGjdurODgYCUlJemtt95yeb6fn59OnTql119/3bkdjB49WpL0888/a9y4cWrbtq2Cg4PVuHFjDR06tFrPo/UUmqZqtH79esXHx6tHjx6Wj/fs2VPx8fH6n//5H5f60KFDdfr0ac2ePVtjxowpdf2jR4/WokWLdNttt2nOnDkKDg7WgAEDbM9v3759+rd/+zf9/ve/1wsvvKBGjRpp9OjRLuda/fTTT1q3bp0GDhyo+fPn6/HHH9fOnTuVnJysQ4cO2X4t+J79+/dLkho3bixJevDBB/XMM8/o+uuv14IFC5ScnKzU1FQNGzbM+ZzDhw+rX79+ys7O1hNPPKFFixZpxIgR2rp1a6XmEhAQoOHDh+v06dP67LPPnPUlS5aoZcuWevLJJ/XCCy8oLi5O48aNU1pamnPMiy++qObNm6tdu3ZauXKlVq5cqalTp0pi+/B1CQkJuvfee7V06dIy/79nzZqle++9V4mJiZo/f74mTpyozMxM9ezZU3l5ec5xS5Ys0fjx49W8eXPNnTtXPXr00JAhQ/TPf/6z0vMNDQ3V7bffroMHD2r37t3O+ksvvaTrrrtOM2fO1OzZs1WvXj0NHTrU5XfTypUr5XA41KNHD+d28NBDD0mSvvrqK33xxRcaNmyYFi5cqIcffliZmZnq1atXjR6erxGe3tVVV+Xl5RlJZvDgwZcdN2jQICPJFBQUmGnTphlJZvjw4W7jLj520fbt240kM3HiRJdxo0ePNpLMtGnTnLWLu5KzsrKctZYtWxpJZvPmzc7a4cOHjcPhMJMnT3bWCgsLTXFxsctrZGVlGYfDYWbOnOlSE4fnfNLFfH388cfm119/NQcOHDCrV682jRs3NsHBweaf//yn+e6774wk8+CDD7o897HHHjOSzCeffGKMMSYjI8PyUJ8dZR0Subjul156yVk7ffq027j+/fubVq1audRKOzxnd/tA3XLpIen9+/ebevXqmQkTJjgf/20Ws7OzTUBAgJk1a5bLenbu3Gnq1avnrBcVFZnGjRub3/3ud+bcuXPOccuXLzeSbB0iLu3w3EULFiwwksw777zjrP12Ozh79qzp0KGD6dOnj0u9tMNzVtvRli1bjCSzYsWKMufsTdjTVE1OnDghSWrYsOFlx118vKCgwFl7+OGHy1z/hx9+KEkaN26cS/2RRx6xPcf27du77AWLjo5W27Zt9dNPPzlrDodD/v4XYlJcXKyjR48qNDRUbdu21TfffGP7tVD39e3bV9HR0YqLi9OwYcMUGhqqjIwMXXHFFXr//fclye0k1MmTJ0uS8y/aiIgISdJ7771X4ZO2SxMaGirpX9umdOGQxUX5+fk6cuSIkpOT9dNPP7kcxisN2wdatWqlkSNH6tVXX9Uvv/xiOWbt2rUqKSnRXXfdpSNHjjiX2NhYJSYmauPGjZIuHJo+evSoxowZo3r16jmfP2LECDVq1KhK5lvWdnD8+HHl5+erR48etjN86fPPnTuno0ePqk2bNoqIiKhz2wFNUzW52AxdGkwrVs1VQkJCmev/+eef5e/v7za2TZs2tufYokULt1qjRo10/Phx588lJSVasGCBEhMT5XA4FBUVpejoaO3YscPWLxX4jrS0NG3YsEEbN27U7t279dNPP6l///6S/pXX3+YzNjZWERER+vnnnyVJycnJuvPOOzVjxgxFRUVp8ODBSk9PdzvvqSJOnjwpyXVb+/zzz9W3b1+FhIQoIiJC0dHRevLJJyXJVr7ZPiBduLr5/PnzpZ7b9OOPP8oYo8TEREVHR7ssP/zwgw4fPixJzu3gt9tJvXr1FB8fXyVztdoO3nvvPXXp0kVBQUGKjIxUdHS0lixZYjvDZ86c0TPPPKO4uDiX7SAvL6/ObQf1yh6CiggPD1fTpk21Y8eOy47bsWOHrrjiCoWFhTlrl3bt1SkgIMCybi45QXD27Nl6+umndf/99+vZZ59VZGSk/P39NXHiRJWUlNTIPOEdOnfurBtuuOGyY8q6Qaufn5/eeustbd26VevXr9dHH32k+++/Xy+88IK2bt3q/Cu5Ir7//ntJ//qFtH//ft18881q166d5s+fr7i4OAUGBur999/XggULbOWb7QPShb1N99xzj1599VU98cQTbo+XlJTIz89PH3zwgeXnbmVyXV6/3Q4+/fRTDRo0SD179tTixYvVtGlT1a9fX+np6Vq1apWtdT7yyCNKT0/XxIkT1bVrV4WHh8vPz0/Dhg2rc9sBTVM1GjhwoJYuXarPPvvMeRXcpT799FNlZ2c7T6Yrj5YtW6qkpERZWVlKTEx01vft21epOf/WW2+9pd69e2vZsmUu9by8PEVFRVXpa6HuupjXH3/8UVdddZWznpubq7y8PLd7iHXp0kVdunTRrFmztGrVKo0YMUKrV6/Wgw8+WKHXLy4u1qpVq9SgQQPntrh+/XoVFRXp3XffddnrevFQyaVKa/bYPnDRU089pTfeeENz5sxxe6x169YyxighIUFXXnllqeu4uB3s27dPvXv3dtbPnz+v7OxsXXvttZWa48mTJ5WRkaG4uDjndvj2228rKChIH330kRwOh3Nsenq62/Mvtx2MGjVKL7zwgrNWWFjocoJ7XcHhuWr0+OOPKzg4WA899JCOHj3q8tixY8f08MMPq0GDBnr88cfLve6Lhz0WL17sUl+0aFHFJ2whICDAZc+TJK1Zs8byLrhAaW677TZJF65Eu9T8+fMlyXnV5/Hjx93y1qlTJ0mq8CG64uJiTZgwQT/88IMmTJjg3Kt78S/+S18vPz/f8pdFSEiI5S8Atg9c1Lp1a91zzz165ZVXlJOT4/LYHXfcoYCAAM2YMcMtL8YY5++HG264QY0bN9bSpUt1/vx555g333zT5bSJijhz5oxGjhypY8eOaerUqc4GKCAgQH5+fiouLnaOzc7OtryJZXm2g0WLFrmss65gT1M1SkxM1Ouvv64RI0bommuu0QMPPKCEhARlZ2dr2bJlOnLkiP7617+qdevW5V53UlKS7rzzTr344os6evSounTpok2bNunvf/+7pLIPg9g1cOBAzZw5U/fdd5+6deumnTt36s0331SrVq2qZP3wDR07dtSoUaP06quvKi8vT8nJyfryyy/1+uuva8iQIc6/ql9//XUtXrxYt99+u1q3bq0TJ05o6dKlCgsLczZel5Ofn6833nhDknT69Gnt27dPa9eu1f79+zVs2DA9++yzzrH9+vVTYGCg/vCHP+ihhx7SyZMntXTpUjVp0sTthN6kpCQtWbJEzz33nNq0aaMmTZqoT58+bB9wMXXqVK1cuVJ79+7V1Vdf7ay3bt1azz33nKZMmaLs7GwNGTJEDRs2VFZWljIyMjR27Fg99thjCgwM1PTp0/XII4+oT58+uuuuu5Sdna3ly5erdevWtj/XDx486NwOTp48qd27d2vNmjXKycnR5MmTXY5uDBgwQPPnz9ctt9yiu+++W4cPH1ZaWpratGnjdnpJUlKSPv74Y82fP1/NmjVTQkKCbrzxRg0cOFArV65UeHi42rdvry1btujjjz923m6kTvHUZXu+ZMeOHWb48OGmadOmpn79+iY2NtYMHz7c7Ny502XcxdsK/Prrr27r+O0tB4wx5tSpUyYlJcVERkaa0NBQM2TIELN3714jyTz//PPOcaXdcsDqstTk5GSXy1oLCwvN5MmTTdOmTU1wcLDp3r272bJli9s4bjngu0q7I/hvnTt3zsyYMcMkJCSY+vXrm7i4ODNlyhRTWFjoHPPNN9+Y4cOHmxYtWhiHw2GaNGliBg4caL7++usy55GcnGwkOZfQ0FCTmJho7rnnHvO3v/3N8jnvvvuuufbaa01QUJCJj483c+bMMa+99prb9pKTk2MGDBhgGjZs6HLpt93tA3XL5TI/atQoI8ny9hdvv/22uemmm0xISIgJCQkx7dq1MykpKWbv3r0u4xYuXGhatmxpHA6H6dy5s/n8889NUlKSueWWW8qc28XbyUgyfn5+JiwszFx99dVmzJgxZtu2bZbPWbZsmUlMTDQOh8O0a9fOpKenW/7O2bNnj+nZs6cJDg42kpy3Hzh+/Li57777TFRUlAkNDTX9+/c3e/bsMS1btqxzdxD3M+Y3+9Tg1b777jtdd911euONNzRixAhPTwcAUEklJSWKjo7WHXfcoaVLl3p6Oj6Nc5q8mNUX+L744ovy9/dXz549PTAjAEBlFBYWup0ftGLFCh07dqxSX6OCqsE5TV5s7ty52r59u3r37q169erpgw8+0AcffKCxY8cqLi7O09MDAJTT1q1b9eijj2ro0KFq3LixvvnmGy1btkwdOnTQ0KFDPT09n8fhOS+2YcMGzZgxQ7t379bJkyfVokULjRw5UlOnTnW5mywAwDtkZ2drwoQJ+vLLL3Xs2DFFRkbqtttu0/PPP68mTZp4eno+j6YJAADABs5pAgAAsIGmCQAAwIZqO/ElLS1N8+bNU05Ojjp27KhFixapc+fOZT6vpKREhw4dUsOGDavsBo2AXcYYnThxQs2aNXN+e315VTT7EvmH51RF9iU+++GdbOe/Om7+tHr1ahMYGGhee+01s2vXLjNmzBgTERFhcnNzy3zugQMHXG5Qx8LiieXAgQM1nn3yz1Iblopmv7L5J/sstWEpK//V0jR17tzZpKSkOH8uLi42zZo1M6mpqWU+Ny8vz+P/aCwseXl5NZ79iuQ/Pz/fcvH0vx+L9y4VzX5l8+/Nn/1sg3VnKSv/VX547uzZs9q+fbumTJnirPn7+6tv377asmWL2/iioiKXL+I8ceJEVU8JKLeKHB4ob/alyuf/4pfPAlWloofGfPmzn+2w7igr/1V+IviRI0dUXFysmJgYl3pMTIzbNz9LUmpqqsLDw50LN2WEtypv9iXyj7qDz374Ao9fPTdlyhTl5+c7lwMHDnh6SkCNIf/wVWQf3qjKD89FRUUpICBAubm5LvXc3FzFxsa6jXc4HHI4HFU9DaDGlTf7UuXzz1VGqC18+bOf7dB3VPmepsDAQCUlJSkzM9NZKykpUWZmprp27VrVLwfUGmQfvoz8wydU+DKJy1i9erVxOBxm+fLlZvfu3Wbs2LEmIiLC5OTklPlcrjpgqQ1Lfn5+jWef/LPUhqWi2a9s/sk+S21Yysp/tTRNxhizaNEi06JFCxMYGGg6d+5stm7daut5bDgstWGpzC+Oimaf/LPUhqUy2a9M/sk+S21Yysp/rfvC3oKCAoWHh3t6GvBx+fn5HrmMmPzD08g+fFlZ+ff41XMAAADegKYJAADABpomAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwgaYJAADABpomAAAAG2iaAAAAbKBpAgAAsIGmCQAAwIZ6np4Aak7Dhg3daqGhoZZjBwwYYFmPjo62rM+fP9+tVlRUVI7ZAdWL/MNXkf2qw54mAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAZOBPdi8fHxlvX/+I//sKx37drVrdahQ4cqmUvTpk3dahMmTKiSdQNWyD98Fdn3HPY0AQAA2EDTBAAAYANNEwAAgA00TQAAADbQNAEAANjgZ4wxnp7EpQoKChQeHu7paXhMu3btLOsTJ050q40YMcJybHBwsGXdz8/PrXbgwAHLsSdOnLCsX3XVVZb1I0eOuNV69eplOXbPnj2W9dokPz9fYWFhNf665J/8exrZ9wyyXzuUlX/2NAEAANhA0wQAAGADTRMAAIANNE0AAAA20DQBAADYwHfPVbPSrgaZM2eOZf2Pf/yjZb1hw4aVnsuPP/7oVuvfv7/l2Pr161vWS7v6ISoqylYNvoX8w1eR/bqJPU0AAAA20DQBAADYQNMEAABgA00TAACADZwIXs1uv/12y/qDDz5Yba+5f/9+y/rvf/97t1ppt9Jv06ZNlc4Jvon8w1eR/bqJPU0AAAA20DQBAADYQNMEAABgA00TAACADTRNAAAANpT76rnNmzdr3rx52r59u3755RdlZGRoyJAhzseNMZo2bZqWLl2qvLw8de/eXUuWLFFiYmJVzttrDB06tErWk52d7Vb76quvLMf+x3/8h2W9tKslrFx11VW2x/oKsl9+5L/uIP/lQ/brpnLvaTp16pQ6duyotLQ0y8fnzp2rhQsX6i9/+Yu2bdumkJAQ9e/fX4WFhZWeLOBJZB++jPwDFdjTdOutt+rWW2+1fMwYoxdffFFPPfWUBg8eLElasWKFYmJitG7dOg0bNqxyswU8iOzDl5F/oIrPacrKylJOTo769u3rrIWHh+vGG2/Uli1bLJ9TVFSkgoIClwXwNhXJvkT+UTfw2Q9fUaVNU05OjiQpJibGpR4TE+N87LdSU1MVHh7uXOLi4qpySkCNqEj2JfKPuoHPfvgKj189N2XKFOXn5zuX8pywBng78g9fRfbhjar0u+diY2MlSbm5uWratKmznpubq06dOlk+x+FwyOFwVOU0apUxY8ZY1seOHWtZ/9vf/mZZ37dvn1vt8OHDFZ9YGX77FyMuryLZl8j/b5F/78RnvzuyXzdV6Z6mhIQExcbGKjMz01krKCjQtm3b1LVr16p8KaBWIfvwZeQfvqLce5pOnjzp0vlmZWXpu+++U2RkpFq0aKGJEyfqueeeU2JiohISEvT000+rWbNmLvfzALwR2YcvI/9ABZqmr7/+Wr1793b+PGnSJEnSqFGjtHz5cv35z3/WqVOnNHbsWOXl5emmm27Shx9+qKCgoKqbNeABZB++jPwDFWiaevXqJWNMqY/7+flp5syZmjlzZqUmBtQ2ZB++jPwDteDqOQAAAG9QpVfPwd2hQ4cs69OnT6/ZiZQTJ2+iKpB/+CqyXzexpwkAAMAGmiYAAAAbaJoAAABsoGkCAACwgRPBvdiECRMs6yEhIZVe9zXXXFOu8V988YVbrbRvNweqAvmHryL7nsOeJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbuHrOQxo0aGBZb9++vWV92rRpbrXbbrutXK/p7+/eI5eUlJRrHaV9NcB9993nVisuLi7XuuE7yD98Fdn3buxpAgAAsIGmCQAAwAaaJgAAABtomgAAAGygaQIAALCBq+eqUP369d1q1113neXYt99+27LetGlTy/qZM2fcaqVdzVDa9/7ccsstbrXSruQoTb161pG544473GovvfSS5dizZ8+W6zXhHci/K/LvO8i+q7qcffY0AQAA2EDTBAAAYANNEwAAgA00TQAAADb4GWOMpydxqYKCAoWHh3t6GpcVGBhoWbc62W7t2rXlWveMGTMs65988olb7fPPP7ccGxkZaXsdHTp0KMfsymfEiBGW9XXr1lnWi4qKqm0u5ZWfn6+wsLAaf13yT/49jeyXjuzb463Zl8rOP3uaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGzg6rnLsLo1viTNnDnTsv7444/bXvcHH3xgWR85cqRlPS8vz60WHR1tOfb999+3rF9//fVutdJuaz937lzLemlXXAwePNiybuXjjz+2rM+ZM8etdvz4cdvrlaTvvvuuXONLwxVE5N+KL+Sf7JN9K76QfYmr5wAAAKoETRMAAIANNE0AAAA20DQBAADYQNMEAABgQz1PT6C2CAgIcKs9++yzlmMfe+wxy/qpU6fcak888YTl2NWrV1vWra6UkKQbbrjBrfbyyy9bjr3uuuss6z/++KNb7U9/+pPl2I0bN1rWS7uqoFu3bm610r5/aNCgQZb1DRs2WNatHDhwwLKekJBgex34F/Lvivz7DrLviuxfHnuaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGzgu+f+H6srCRYtWmQ59vTp05b1sWPHutX+9re/WY698cYbLev33XefZf3WW291qwUHB1uOLe37kdLT091qpV2JUJ2GDx9uWb/77rttr+PRRx+1rO/bt69Cc/otX/v+LfJfc2p7/sk+2a8utT37Et89BwAAUCVomgAAAGygaQIAALCBpgkAAMCGcp0InpqaqrVr12rPnj0KDg5Wt27dNGfOHLVt29Y5prCwUJMnT9bq1atVVFSk/v37a/HixYqJibH1Gp46GfCXX35xq0VHR1uOLSoqsqzv2bPHrRYSEmI5tk2bNuWYnbXp06db1lNTUy3rxcXFlX5NX2F1MiD5v4D8121kn+z7sio9EXzTpk1KSUnR1q1btWHDBp07d079+vVz+d6dRx99VOvXr9eaNWu0adMmHTp0SHfccUfF3wFQS5B/+CqyD1xQri/s/fDDD11+Xr58uZo0aaLt27erZ8+eys/P17Jly7Rq1Sr16dNH0oVLHa+66ipt3bpVXbp0qbqZAzWM/MNXkX3ggkqd05Sfny9JioyMlCRt375d586dU9++fZ1j2rVrpxYtWmjLli2W6ygqKlJBQYHLAngD8g9fRfbhqyrcNJWUlGjixInq3r27OnToIEnKyclRYGCgIiIiXMbGxMQoJyfHcj2pqakKDw93LnFxcRWdElBjyD98FdmHL6tw05SSkqLvv/9eq1evrtQEpkyZovz8fOfiibuUAuVF/uGryD58WbnOabpo/Pjxeu+997R582Y1b97cWY+NjdXZs2eVl5fn8hdHbm6uYmNjLdflcDjkcDgqMo0qZfXXUGlXUJQ2344dO9p+vffff9+yvnnzZsv6unXr3GrZ2dmWY7lSonqRf/Lvq8g+2fd15drTZIzR+PHjlZGRoU8++UQJCQkujyclJal+/frKzMx01vbu3at//OMf6tq1a9XMGPAQ8g9fRfaBC8q1pyklJUWrVq3SO++8o4YNGzo79PDwcAUHBys8PFwPPPCAJk2apMjISIWFhemRRx5R165duXoCXo/8w1eRfeCCcjVNS5YskST16tXLpZ6enq7Ro0dLkhYsWCB/f3/deeedLjc4A7wd+YevIvvABeVqmuzcPDwoKEhpaWlKS0ur8KSA2oj8w1eRfeACvnsOAADAhgpdPVcX9ezZ0602ZMgQy7HXX3+9Zf3w4cNutddee81y7PHjxy3rZ8+eLWWGQPUh//BVZB/lwZ4mAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAY/Y+da0hpUUFCg8PBwT08DPi4/P19hYWE1/rrkH55G9uHLyso/e5oAAABsoGkCAACwgaYJAADABpomAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwgaYJAADABpomAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwgaYJAADAhlrXNBljPD0FwGM5JP/wNLIPX1ZWDmtd03TixAlPTwHwWA7JPzyN7MOXlZVDP1PL2vuSkhIdOnRIDRs21IkTJxQXF6cDBw4oLCzM01OrFgUFBbzHWsQYoxMnTqhZs2by96/5vynIf93jLe+R7Ncsb8lFZXjTe7Sb/3o1OCdb/P391bx5c0mSn5+fJCksLKzW/4NXFu+x9ggPD/fYa5P/ussb3iPZr3m8x9rDTv5r3eE5AACA2oimCQAAwIZa3TQ5HA5NmzZNDofD01OpNrxHlMYX/t14j7DiC/9mvEfvVOtOBAcAAKiNavWeJgAAgNqCpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwoVY3TWlpaYqPj1dQUJBuvPFGffnll56eUoVt3rxZf/jDH9SsWTP5+flp3bp1Lo8bY/TMM8+oadOmCg4OVt++ffXjjz96ZrIVkJqaqt/97ndq2LChmjRpoiFDhmjv3r0uYwoLC5WSkqLGjRsrNDRUd955p3Jzcz0049qN7HtP9iXyX9XIv/fk39eyX2ubpv/6r//SpEmTNG3aNH3zzTfq2LGj+vfvr8OHD3t6ahVy6tQpdezYUWlpaZaPz507VwsXLtRf/vIXbdu2TSEhIerfv78KCwtreKYVs2nTJqWkpGjr1q3asGGDzp07p379+unUqVPOMY8++qjWr1+vNWvWaNOmTTp06JDuuOMOD866diL73pV9ifxXJfLvXfn3ueybWqpz584mJSXF+XNxcbFp1qyZSU1N9eCsqoYkk5GR4fy5pKTExMbGmnnz5jlreXl5xuFwmL/+9a8emGHlHT582EgymzZtMsZceD/169c3a9ascY754YcfjCSzZcsWT02zViL73p19Y8h/ZZB/785/Xc9+rdzTdPbsWW3fvl19+/Z11vz9/dW3b19t2bLFgzOrHllZWcrJyXF5v+Hh4brxxhu99v3m5+dLkiIjIyVJ27dv17lz51zeY7t27dSiRQuvfY/Vgex7f/Yl8l9R5N/781/Xs18rm6YjR46ouLhYMTExLvWYmBjl5OR4aFbV5+J7qivvt6SkRBMnTlT37t3VoUMHSRfeY2BgoCIiIlzGeut7rC5kX86fvfX9kv+KI/9y/uyN79cXsl/P0xNA3ZOSkqLvv/9en332maenAtQ48g9f5QvZr5V7mqKiohQQEOB2dn1ubq5iY2M9NKvqc/E91YX3O378eL333nvauHGjmjdv7qzHxsbq7NmzysvLcxnvje+xOpF9OX/2xvdL/iuH/Mv5s7e9X1/Jfq1smgIDA5WUlKTMzExnraSkRJmZmeratasHZ1Y9EhISFBsb6/J+CwoKtG3bNq95v8YYjR8/XhkZGfrkk0+UkJDg8nhSUpLq16/v8h737t2rf/zjH17zHmsC2fe+7Evkv6qQf+/Lv89l38Mnopdq9erVxuFwmOXLl5vdu3ebsWPHmoiICJOTk+PpqVXIiRMnzLfffmu+/fZbI8nMnz/ffPvtt+bnn382xhjz/PPPm4iICPPOO++YHTt2mMGDB5uEhARz5swZD8/cnj/96U8mPDzc/O///q/55ZdfnMvp06edYx5++GHTokUL88knn5ivv/7adO3a1XTt2tWDs66dyL53Zd8Y8l+VyL935d/Xsl9rmyZjjFm0aJFp0aKFCQwMNJ07dzZbt2719JQqbOPGjUaS2zJq1ChjzIVLT59++mkTExNjHA6Hufnmm83evXs9O+lysHpvkkx6erpzzJkzZ8y4ceNMo0aNTIMGDcztt99ufvnlF89NuhYj+96TfWPIf1Uj/96Tf1/Lvp8xxlTvviwAAADvVyvPaQIAAKhtaJoAAABsoGkCAACwgaYJAADABpomAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwgaYJAADABpomAAAAG2iavMT06dPl5+dXoecuX75cfn5+ys7OrtpJXSI7O1t+fn5avnx5tb0GAACeRNNUA3bt2qV77rlHV1xxhRwOh5o1a6YRI0Zo165dnp4aUGkXm/KLS1BQkK688kqNHz9eubm5NTaPXr16Oefg7++vsLAwtW3bViNHjtSGDRsqte7FixfzBwGcLmY+KChIBw8edHu8V69e6tChgwdmJsXHx7tsBxEREbrmmms0duxYbdu2rVLrnj17ttatW1c1E/VSNE3VbO3atbr++uuVmZmp++67T4sXL9YDDzygjRs36vrrr1dGRoat9Tz11FM6c+ZMheYwcuRInTlzRi1btqzQ8wE7Zs6cqZUrV+rll19Wt27dtGTJEnXt2lWnT5+usTk0b95cK1eu1IoVKzRv3jwNGjRIX3zxhfr166c//vGPOnfuXIXWS9MEK0VFRXr++ec9PQ03nTp1cm4Hqamp6t27t9avX68uXbpo0qRJFV4vTZMkg2qzb98+06BBA9OuXTtz+PBhl8d+/fVX065dOxMSEmL2799f6jpOnjxZ3dOsEllZWUaSSU9P9/RUUMPS09ONJPPVV1+51CdNmmQkmVWrVtXIPJKTk83VV1/tVj9//rwZN26ckWT+/Oc/V2jdV199tUlOTq7kDFFXXMx8p06djMPhMAcPHnR5vLQs1oSWLVuaAQMGuNVPnz5thgwZYiSZxYsXV2jdISEhZtSoUZWcoXdjT1M1mjdvnk6fPq1XX31V0dHRLo9FRUXplVde0alTpzR37lxJ/zpvaffu3br77rvVqFEj3XTTTS6PXerMmTOaMGGCoqKi1LBhQw0aNEgHDx6Un5+fpk+f7hxndU5TfHy8Bg4cqM8++0ydO3dWUFCQWrVqpRUrVri8xrFjx/TYY4/pmmuuUWhoqMLCwnTrrbfq//7v/6rwXwp1UZ8+fSRJWVlZkqTz58/r2WefVevWreVwOBQfH68nn3xSRUVFLs/7+uuv1b9/f0VFRSk4OFgJCQm6//77KzyPgIAALVy4UO3bt9fLL7+s/Px852Pp6enq06ePmjRpIofDofbt22vJkiUuz4+Pj9euXbu0adMm52GPXr16SWL78HVPPvmkiouLbe9teuONN5SUlKTg4GBFRkZq2LBhOnDggNu4tLQ0tWrVSsHBwercubM+/fRT9erVy5m7iggODtbKlSsVGRmpWbNmyRjjfOw///M/1a1bNzVu3FjBwcFKSkrSW2+95fJ8Pz8/nTp1Sq+//rpzOxg9erQk6eeff9a4cePUtm1bBQcHq3Hjxho6dGi1nkfrKTRN1Wj9+vWKj49Xjx49LB/v2bOn4uPj9T//8z8u9aFDh+r06dOaPXu2xowZU+r6R48erUWLFum2227TnDlzFBwcrAEDBtie3759+/Rv//Zv+v3vf68XXnhBjRo10ujRo13Otfrpp5+0bt06DRw4UPPnz9fjjz+unTt3Kjk5WYcOHbL9WvA9+/fvlyQ1btxYkvTggw/qmWee0fXXX68FCxYoOTlZqampGjZsmPM5hw8fVr9+/ZSdna0nnnhCixYt0ogRI7R169ZKzSUgIEDDhw/X6dOn9dlnnznrS5YsUcuWLfXkk0/qhRdeUFxcnMaNG6e0tDTnmBdffFHNmzdXu3bttHLlSq1cuVJTp06VxPbh6xISEnTvvfdq6dKlZf5/z5o1S/fee68SExM1f/58TZw4UZmZmerZs6fy8vKc45YsWaLx48erefPmmjt3rnr06KEhQ4bon//8Z6XnGxoaqttvv10HDx7U7t27nfWXXnpJ1113nWbOnKnZs2erXr16Gjp0qMvvppUrV8rhcKhHjx7O7eChhx6SJH311Vf64osvNGzYMC1cuFAPP/ywMjMz1atXrxo9PF8jPL2rq67Ky8szkszgwYMvO27QoEFGkikoKDDTpk0zkszw4cPdxl187KLt27cbSWbixIku40aPHm0kmWnTpjlrF3clZ2VlOWstW7Y0kszmzZudtcOHDxuHw2EmT57srBUWFpri4mKX18jKyjIOh8PMnDnTpSYOz/mki/n6+OOPza+//moOHDhgVq9ebRo3bmyCg4PNP//5T/Pdd98ZSebBBx90ee5jjz1mJJlPPvnEGGNMRkaG5aE+O8o6JHJx3S+99JKzdvr0abdx/fv3N61atXKplXZ4zu72gbrl0kPS+/fvN/Xq1TMTJkxwPv7bLGZnZ5uAgAAza9Ysl/Xs3LnT1KtXz1kvKioyjRs3Nr/73e/MuXPnnOOWL19uJNk6RFza4bmLFixYYCSZd955x1n77XZw9uxZ06FDB9OnTx+XemmH56y2oy1bthhJZsWKFWXO2Zuwp6manDhxQpLUsGHDy467+HhBQYGz9vDDD5e5/g8//FCSNG7cOJf6I488YnuO7du3d9kLFh0drbZt2+qnn35y1hwOh/z9L8SkuLhYR48eVWhoqNq2batvvvnG9muh7uvbt6+io6MVFxenYcOGKTQ0VBkZGbriiiv0/vvvS5LbSaiTJ0+WJOdftBEREZKk9957r8InbZcmNDRU0r+2TenCIYuL8vPzdeTIESUnJ+unn35yOYxXGrYPtGrVSiNHjtSrr76qX375xXLM2rVrVVJSorvuuktHjhxxLrGxsUpMTNTGjRslXTg0ffToUY0ZM0b16tVzPn/EiBFq1KhRlcy3rO3g+PHjys/PV48ePWxn+NLnnzt3TkePHlWbNm0UERFR57YDmqZqcrEZujSYVqyaq4SEhDLX//PPP8vf399tbJs2bWzPsUWLFm61Ro0a6fjx486fS0pKtGDBAiUmJsrhcCgqKkrR0dHasWOHrV8q8B1paWnasGGDNm7cqN27d+unn35S//79Jf0rr7/NZ2xsrCIiIvTzzz9LkpKTk3XnnXdqxowZioqK0uDBg5Wenu523lNFnDx5UpLrtvb555+rb9++CgkJUUREhKKjo/Xkk09Kkq18s31AunB18/nz50s9t+nHH3+UMUaJiYmKjo52WX744QcdPnxYkpzbwW+3k3r16ik+Pr5K5mq1Hbz33nvq0qWLgoKCFBkZqejoaC1ZssR2hs+cOaNnnnlGcXFxLttBXl5endsO6pU9BBURHh6upk2baseOHZcdt2PHDl1xxRUKCwtz1i7t2qtTQECAZd1ccoLg7Nmz9fTTT+v+++/Xs88+q8jISPn7+2vixIkqKSmpkXnCO3Tu3Fk33HDDZceUdYNWPz8/vfXWW9q6davWr1+vjz76SPfff79eeOEFbd261flXckV8//33kv71C2n//v26+eab1a5dO82fP19xcXEKDAzU+++/rwULFtjKN9sHpAt7m+655x69+uqreuKJJ9weLykpkZ+fnz744APLz93K5Lq8frsdfPrppxo0aJB69uypxYsXq2nTpqpfv77S09O1atUqW+t85JFHlJ6erokTJ6pr164KDw+Xn5+fhg0bVue2A5qmajRw4EAtXbpUn332mfMquEt9+umnys7Odp5MVx4tW7ZUSUmJsrKylJiY6Kzv27evUnP+rbfeeku9e/fWsmXLXOp5eXmKioqq0tdC3XUxrz/++KOuuuoqZz03N1d5eXlu9xDr0qWLunTpolmzZmnVqlUaMWKEVq9erQcffLBCr19cXKxVq1apQYMGzm1x/fr1Kioq0rvvvuuy1/XioZJLldbssX3goqeeekpvvPGG5syZ4/ZY69atZYxRQkKCrrzyylLXcXE72Ldvn3r37u2snz9/XtnZ2br22msrNceTJ08qIyNDcXFxzu3w7bffVlBQkD766CM5HA7n2PT0dLfnX247GDVqlF544QVnrbCw0OUE97qCw3PV6PHHH1dwcLAeeughHT161OWxY8eO6eGHH1aDBg30+OOPl3vdFw97LF682KW+aNGiik/YQkBAgMueJ0las2aN5V1wgdLcdtttki5ciXap+fPnS5Lzqs/jx4+75a1Tp06SVOFDdMXFxZowYYJ++OEHTZgwwblX9+Jf/Je+Xn5+vuUvi5CQEMtfAGwfuKh169a655579MorrygnJ8flsTvuuEMBAQGaMWOGW16MMc7fDzfccIMaN26spUuX6vz5884xb775pstpExVx5swZjRw5UseOHdPUqVOdDVBAQID8/PxUXFzsHJudnW15E8vybAeLFi1yWWddwZ6mapSYmKjXX39dI0aM0DXXXKMHHnhACQkJys7O1rJly3TkyBH99a9/VevWrcu97qSkJN1555168cUXdfToUXXp0kWbNm3S3//+d0llHwaxa+DAgZo5c6buu+8+devWTTt37tSbb76pVq1aVcn64Rs6duyoUaNG6dVXX1VeXp6Sk5P15Zdf6vXXX9eQIUOcf1W//vrrWrx4sW6//Xa1bt1aJ06c0NKlSxUWFuZsvC4nPz9fb7zxhiTp9OnT2rdvn9auXav9+/dr2LBhevbZZ51j+/Xrp8DAQP3hD3/QQw89pJMnT2rp0qVq0qSJ2wm9SUlJWrJkiZ577jm1adNGTZo0UZ8+fdg+4GLq1KlauXKl9u7dq6uvvtpZb926tZ577jlNmTJF2dnZGjJkiBo2bKisrCxlZGRo7NixeuyxxxQYGKjp06frkUceUZ8+fXTXXXcpOztby5cvV+vWrW1/rh88eNC5HZw8eVK7d+/WmjVrlJOTo8mTJ7sc3RgwYIDmz5+vW265RXfffbcOHz6stLQ0tWnTxu30kqSkJH388ceaP3++mjVrpoSEBN14440aOHCgVq5cqfDwcLVv315btmzRxx9/7LzdSJ3iqcv2fMmOHTvM8OHDTdOmTU39+vVNbGysGT58uNm5c6fLuIu3Ffj111/d1vHbWw4YY8ypU6dMSkqKiYyMNKGhoWbIkCFm7969RpJ5/vnnneNKu+WA1WWpycnJLpe1FhYWmsmTJ5umTZua4OBg0717d7Nlyxa3cdxywHeVdkfw3zp37pyZMWOGSUhIMPXr1zdxcXFmypQpprCw0Dnmm2++McOHDzctWrQwDofDNGnSxAwcONB8/fXXZc4jOTnZSHIuoaGhJjEx0dxzzz3mb3/7m+Vz3n33XXPttdeaoKAgEx8fb+bMmWNee+01t+0lJyfHDBgwwDRs2NDl0m+72wfqlstlftSoUUaS5e0v3n77bXPTTTeZkJAQExISYtq1a2dSUlLM3r17XcYtXLjQtGzZ0jgcDtO5c2fz+eefm6SkJHPLLbeUObeLt5ORZPz8/ExYWJi5+uqrzZgxY8y2bdssn7Ns2TKTmJhoHA6HadeunUlPT7f8nbNnzx7Ts2dPExwcbCQ5bz9w/Phxc99995moqCgTGhpq+vfvb/bs2WNatmxZ5+4g7mfMb/apwat99913uu666/TGG29oxIgRnp4OAKCSSkpKFB0drTvuuENLly719HR8Guc0eTGrL/B98cUX5e/vr549e3pgRgCAyigsLHQ7P2jFihU6duxYpb5GBVWDc5q82Ny5c7V9+3b17t1b9erV0wcffKAPPvhAY8eOVVxcnKenBwAop61bt+rRRx/V0KFD1bhxY33zzTdatmyZOnTooKFDh3p6ej6Pw3NebMOGDZoxY4Z2796tkydPqkWLFho5cqSmTp3qcjdZAIB3yM7O1oQJE/Tll1/q2LFjioyM1G233abnn39eTZo08fT0fB5NEwAAgA2c0wQAAGADTRMAAIAN1XbiS1pamubNm6ecnBx17NhRixYtUufOnct8XklJiQ4dOqSGDRtW2Q0aAbuMMTpx4oSaNWvm/Pb68qpo9iXyD8+piuxLfPbDO9nOf3Xc/Gn16tUmMDDQvPbaa2bXrl1mzJgxJiIiwuTm5pb53AMHDrjcoI6FxRPLgQMHajz75J+lNiwVzX5l80/2WWrDUlb+q6Vp6ty5s0lJSXH+XFxcbJo1a2ZSU1PLfG5eXp7H/9FYWPLy8mo8+1WZ//z8fMvF0/+uLLV/qWj2K5v/uvbZzzbonUtZ+a/yw3Nnz57V9u3bNWXKFGfN399fffv21ZYtW9zGFxUVuXwR54kTJ6p6SkC5VeTwQHmzL1Vf/i9+KS1QXhU9NMZnvyu2Qe9UVv6r/ETwI0eOqLi4WDExMS71mJgYt29+lqTU1FSFh4c7F27KCG9V3uxL5B91B5/98AUev3puypQpys/Pdy4HDhzw9JSAGkP+4avIPrxRlR+ei4qKUkBAgHJzc13qubm5io2NdRvvcDjkcDiqehpAjStv9qXqyz9XH6Gm8dnvim2wbqryPU2BgYFKSkpSZmams1ZSUqLMzEx17dq1ql8OqDXIPnwZ+YdPqPBlEpexevVq43A4zPLly83u3bvN2LFjTUREhMnJySnzuVxdwFIblvz8/BrPPvlnqQ1LRbNf2fyTfZbasJSV/2ppmowxZtGiRaZFixYmMDDQdO7c2WzdutXW89hwWGrDUplfHBXNPvlnqQ1LZbJfmfyTfZbasJSV/1r3hb0FBQUKDw/39DTg4/Lz8z1yyTD5h6eRffiysvLv8avnAAAAvAFNEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIANNE0AAAA20DQBAADYQNMEAABgA00TAACADTRNAAAANtA0AQAA2EDTBAAAYANNEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIANNE0AAAA20DQBAADYQNMEAABgA00TAACADTRNAAAANtA0AQAA2EDTBAAAYANNEwAAgA00TQAAADbQNAEAANhA0wQAAGBDPU9PoLYICQlxqwUFBVmOHThwoGW9U6dOVTmlGvHSSy9Z1rOzs2t2IvAo8u+K/PsOsu+K7F8ee5oAAABsoGkCAACwgaYJAADABpomAAAAG2iaAAAAbKizV88NHz7csn7TTTdZ1rt37+5Wu+aaa6p0TrXRbbfdZlnv0aOHZf3w4cPVOR1UEfJvD/mve8i+PWS/YtjTBAAAYANNEwAAgA00TQAAADbQNAEAANjgZ4wxnp7EpQoKChQeHl7p9ZT2tkpKSmzXDxw4UK7X/PTTT91qv/76q+XYH374oVzrLo8OHTpY1idMmGB7HY899phlfcGCBRWak7fJz89XWFhYjb8u+a888l85ZJ/sW/GF7Etl5589TQAAADbQNAEAANhA0wQAAGADTRMAAIANNE0AAAA2lPtrVDZv3qx58+Zp+/bt+uWXX5SRkaEhQ4Y4HzfGaNq0aVq6dKny8vLUvXt3LVmyRImJiVU57zL9/e9/t6wXFRVZ1p977jm32n//939X6ZyqWlxcnGW9Z8+elV53dnZ2pddR13hL9iXyX1nk35235J/sVw7Zv7xy72k6deqUOnbsqLS0NMvH586dq4ULF+ovf/mLtm3bppCQEPXv31+FhYWVnizgSWQfvoz8AxXY03Trrbfq1ltvtXzMGKMXX3xRTz31lAYPHixJWrFihWJiYrRu3ToNGzascrMFPIjsw5eRf6CKz2nKyspSTk6O+vbt66yFh4frxhtv1JYtWyyfU1RUpIKCApcF8DYVyb5E/lE38NkPX1GlTVNOTo4kKSYmxqUeExPjfOy3UlNTFR4e7lxKO1YL1GYVyb5E/lE38NkPX+Hxq+emTJmi/Px851Le29cD3oz8w1eRfXijcp/TdDmxsbGSpNzcXDVt2tRZz83NVadOnSyf43A45HA4qnIakqS2bdtW+To9KT4+3q22Zs0ay7HXX3+97fW+8847lvWPP/7Y9jpQsexL5N8u8l+78dlffch+7VKle5oSEhIUGxurzMxMZ62goEDbtm1T165dq/KlgFqF7MOXkX/4inLvaTp58qT27dvn/DkrK0vfffedIiMj1aJFC02cOFHPPfecEhMTlZCQoKefflrNmjVzuZ8H4I3IPnwZ+Qcq0DR9/fXX6t27t/PnSZMmSZJGjRql5cuX689//rNOnTqlsWPHKi8vTzfddJM+/PBDBQUFVd2sAQ8g+/Bl5B+oQNPUq1cvGWNKfdzPz08zZ87UzJkzKzUxoLYh+/Bl5B+oBVfPAQAAeIMqvXoO9jVo0MCyfunN4S716quvutWio6MrPY+nn37asn7ixIlKrxsoDfmHryL73o09TQAAADbQNAEAANhA0wQAAGADTRMAAIANnAjuIdOnT7esT548uUbnkZaWZlkv78mA27dvd6stX77ccmx2dna51o26h/zDV5F978aeJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbuHrOQ9q0aePpKUiSevToUSXrue2229xqV111leXYu+++27JeXFxcJXNB7Uf+3ZF/30D23XlT9tnTBAAAYANNEwAAgA00TQAAADbQNAEAANhA0wQAAGCDnzHGeHoSlyooKFB4eLinp1Ht2rdvb1mPjIystteMiYlxq40cOdJybHp6umW9ZcuWlvU5c+a41QIDAy3HfvHFF5b13r17u9XOnz9vOba65efnKywsrMZfl/yT/0t5Iv9kv3qRfXe1JftS2flnTxMAAIANNE0AAAA20DQBAADYQNMEAABgA1+j4iG7d++utnV3797dsv7II4+41e69917Lsf/4xz/K9ZqfffaZW+2VV16xHNutWzfL+pVXXulWq85/J3gO+XdH/n0D2XfnTdlnTxMAAIANNE0AAAA20DQBAADYQNMEAABgA00TAACADVw958W6dOliWX/++ect648//rhbrbxXSpTmm2++cau9+eablmOvv/56y/qGDRvcaldccUXlJoY6i/zDV5F9z2FPEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIANXD3nxR577DHLenBwsGV979691TkdN19++aVl/dy5c5b12NjY6pwO6hjyD19F9j2HPU0AAAA20DQBAADYQNMEAABgA00TAACADTRNAAAANnD1nBeLioqyrF933XWW9b/+9a9utdmzZ1uO3bx5c7nmMnToULfaoEGDLMfWr1+/XOsGrJB/+Cqy7znsaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsKNeJ4KmpqVq7dq327Nmj4OBgdevWTXPmzFHbtm2dYwoLCzV58mStXr1aRUVF6t+/vxYvXqyYmJgqn7yv++677yzrPXr0sKz//ve/d6t1797dcuyRI0fKNZcrrrjCrRYQEFCudTzwwAPlGl/TyH/tQv5rDtmvXci+55RrT9OmTZuUkpKirVu3asOGDTp37pz69eunU6dOOcc8+uijWr9+vdasWaNNmzbp0KFDuuOOO6p84kBNI//wVWQfuKBce5o+/PBDl5+XL1+uJk2aaPv27erZs6fy8/O1bNkyrVq1Sn369JEkpaen66qrrtLWrVvVpUuXqps5UMPIP3wV2QcuqNQ5Tfn5+ZKkyMhISdL27dt17tw59e3b1zmmXbt2atGihbZs2WK5jqKiIhUUFLgsgDcg//BVZB++qsJNU0lJiSZOnKju3burQ4cOkqScnBwFBgYqIiLCZWxMTIxycnIs15Oamqrw8HDnEhcXV9EpATWG/MNXkX34sgo3TSkpKfr++++1evXqSk1gypQpys/Pdy4HDhyo1PqAmkD+4avIPnxZhb5GZfz48Xrvvfe0efNmNW/e3FmPjY3V2bNnlZeX5/IXR25urmJjYy3X5XA45HA4KjINn/cf//EflvWgoCDL+pgxY9xqDRo0sBzbokWLik+sDEuXLrWsr1y5stpesyqR/9qB/Nc8sl87kH3PKdeeJmOMxo8fr4yMDH3yySdKSEhweTwpKUn169dXZmams7Z371794x//UNeuXatmxoCHkH/4KrIPXFCuPU0pKSlatWqV3nnnHTVs2NB5rDo8PFzBwcEKDw/XAw88oEmTJikyMlJhYWF65JFH1LVrV66egNcj//BVZB+4oFxN05IlSyRJvXr1cqmnp6dr9OjRkqQFCxbI399fd955p8sNzgBvR/7hq8g+cEG5miZjTJljgoKClJaWprS0tApPCqiNyD98FdkHLuC75wAAAGzwM3b+hKhBBQUFCg8P9/Q0vFpgYKBlPTQ01K320EMPWY6Nioqq9Dy+/PJLy/p///d/W9ZrUxTz8/MVFhZW469L/iuP/FcO2fdeZL/yyso/e5oAAABsoGkCAACwgaYJAADABpomAAAAGzgRHLDAybDwVWQfvowTwQEAAKoATRMAAIANNE0AAAA20DQBAADYQNMEAABgA00TAACADTRNAAAANtA0AQAA2EDTBAAAYANNEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIANNE0AAAA20DQBAADYQNMEAABgA00TAACADTRNAAAANtA0AQAA2EDTBAAAYANNEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIANNE0AAAA21LqmyRjj6SkAHssh+YenkX34srJyWOuaphMnTnh6CoDHckj+4WlkH76srBz6mVrW3peUlOjQoUNq2LChTpw4obi4OB04cEBhYWGenlq1KCgo4D3WIsYYnThxQs2aNZO/f83/TUH+6x5veY9kv2Z5Sy4qw5veo93816vBOdni7++v5s2bS5L8/PwkSWFhYbX+H7yyeI+1R3h4uMdem/zXXd7wHsl+zeM91h528l/rDs8BAADURjRNAAAANtTqpsnhcGjatGlyOByenkq14T2iNL7w78Z7hBVf+DfjPXqnWnciOAAAQG1Uq/c0AQAA1BY0TQAAADbQNAEAANhA0wQAAGADTRMAAIANtbppSktLU3x8vIKCgnTjjTfqyy+/9PSUKmzz5s36wx/+oGbNmsnPz0/r1q1zedwYo2eeeUZNmzZVcHCw+vbtqx9//NEzk62A1NRU/e53v1PDhg3VpEkTDRkyRHv37nUZU1hYqJSUFDVu3FihoaG68847lZub66EZ125k33uyL5H/qkb+vSf/vpb9Wts0/dd//ZcmTZqkadOm6ZtvvlHHjh3Vv39/HT582NNTq5BTp06pY8eOSktLs3x87ty5Wrhwof7yl79o27ZtCgkJUf/+/VVYWFjDM62YTZs2KSUlRVu3btWGDRt07tw59evXT6dOnXKOefTRR7V+/XqtWbNGmzZt0qFDh3THHXd4cNa1E9n3ruxL5L8qkX/vyr/PZd/UUp07dzYpKSnOn4uLi02zZs1MamqqB2dVNSSZjIwM588lJSUmNjbWzJs3z1nLy8szDofD/PWvf/XADCvv8OHDRpLZtGmTMebC+6lfv75Zs2aNc8wPP/xgJJktW7Z4apq1Etn37uwbQ/4rg/x7d/7revZr5Z6ms2fPavv27erbt6+z5u/vr759+2rLli0enFn1yMrKUk5Ojsv7DQ8P14033ui17zc/P1+SFBkZKUnavn27zp075/Ie27VrpxYtWnjte6wOZN/7sy+R/4oi/96f/7qe/VrZNB05ckTFxcWKiYlxqcfExCgnJ8dDs6o+F99TXXm/JSUlmjhxorp3764OHTpIuvAeAwMDFRER4TLWW99jdSH7cv7sre+X/Fcc+ZfzZ298v76Q/XqengDqnpSUFH3//ff67LPPPD0VoMaRf/gqX8h+rdzTFBUVpYCAALez63NzcxUbG+uhWVWfi++pLrzf8ePH67333tPGjRvVvHlzZz02NlZnz55VXl6ey3hvfI/ViezL+bM3vl/yXznkX86fve39+kr2a2XTFBgYqKSkJGVmZjprJSUlyszMVNeuXT04s+qRkJCg2NhYl/dbUFCgbdu2ec37NcZo/PjxysjI0CeffKKEhASXx5OSklS/fn2X97h371794x//8Jr3WBPIvvdlXyL/VYX8e1/+fS77Hj4RvVSrV682DofDLF++3OzevduMHTvWREREmJycHE9PrUJOnDhhvv32W/Ptt98aSWb+/Pnm22+/NT///LMxxpjnn3/eREREmHfeecfs2LHDDB482CQkJJgzZ854eOb2/OlPfzLh4eHmf//3f80vv/ziXE6fPu0c8/DDD5sWLVqYTz75xHz99dema9eupmvXrh6cde1E9r0r+8aQ/6pE/r0r/76W/VrbNBljzKJFi0yLFi1MYGCg6dy5s9m6daunp1RhGzduNJLcllGjRhljLlx6+vTTT5uYmBjjcDjMzTffbPbu3evZSZeD1XuTZNLT051jzpw5Y8aNG2caNWpkGjRoYG6//Xbzyy+/eG7StRjZ957sG0P+qxr59578+1r2/Ywxpnr3ZQEAAHi/WnlOEwAAQG1D0wQAAGADTRMAAIANNE0AAAA20DQBAADYQNMEAABgA00TAACADTRNAAAANtA0AQAA2EDTBAAAYANNEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIANNE0AAAA20DQBAADYQNPkJaZPny4/P78KPXf58uXy8/NTdnZ21U7qEtnZ2fLz89Py5cur7TUAAPAkmqYasGvXLt1zzz264oor5HA41KxZM40YMUK7du3y9NSASrvYlF9cgoKCdOWVV2r8+PHKzc2tsXn06tXLOQd/f3+FhYWpbdu2GjlypDZs2FCpdS9evJg/COB0MfNBQUE6ePCg2+O9evVShw4dPDAzKT4+3mU7iIiI0DXXXKOxY8dq27ZtlVr37NmztW7duqqZqJeiaapma9eu1fXXX6/MzEzdd999Wrx4sR544AFt3LhR119/vTIyMmyt56mnntKZM2cqNIeRI0fqzJkzatmyZYWeD9gxc+ZMrVy5Ui+//LK6deumJUuWqGvXrjp9+nSNzaF58+ZauXKlVqxYoXnz5mnQoEH64osv1K9fP/3xj3/UuXPnKrRemiZYKSoq0vPPP+/pabjp1KmTcztITU1V7969tX79enXp0kWTJk2q8HppmiQZVJt9+/aZBg0amHbt2pnDhw+7PPbrr7+adu3amZCQELN///5S13Hy5MnqnmaVyMrKMpJMenq6p6eCGpaenm4kma+++sqlPmnSJCPJrFq1qkbmkZycbK6++mq3+vnz5824ceOMJPPnP/+5Quu++uqrTXJyciVniLriYuY7depkHA6HOXjwoMvjpWWxJrRs2dIMGDDArX769GkzZMgQI8ksXry4QusOCQkxo0aNquQMvRt7mqrRvHnzdPr0ab366quKjo52eSwqKkqvvPKKTp06pblz50r613lLu3fv1t13361GjRrppptucnnsUmfOnNGECRMUFRWlhg0batCgQTp48KD8/Pw0ffp05zirc5ri4+M1cOBAffbZZ+rcubOCgoLUqlUrrVixwuU1jh07pscee0zXXHONQkNDFRYWpltvvVX/93//V4X/UqiL+vTpI0nKysqSJJ0/f17PPvusWrduLYfDofj4eD355JMqKipyed7XX3+t/v37KyoqSsHBwUpISND9999f4XkEBARo4cKFat++vV5++WXl5+c7H0tPT1efPn3UpEkTORwOtW/fXkuWLHF5fnx8vHbt2qVNmzY5D3v06tVLEtuHr3vyySdVXFxse2/TG2+8oaSkJAUHBysyMlLDhg3TgQMH3MalpaWpVatWCg4OVufOnfXpp5+qV69eztxVRHBwsFauXKnIyEjNmjVLxhjnY//5n/+pbt26qXHjxgoODlZSUpLeeustl+f7+fnp1KlTev31153bwejRoyVJP//8s8aNG6e2bdsqODhYjRs31tChQ6v1PFpPoWmqRuvXr1d8fLx69Ohh+XjPnj0VHx+v//mf/3GpDx06VKdPn9bs2bM1ZsyYUtc/evRoLVq0SLfddpvmzJmj4OBgDRgwwPb89u3bp3/7t3/T73//e73wwgtq1KiRRo8e7XKu1U8//aR169Zp4MCBmj9/vh5//HHt3LlTycnJOnTokO3Xgu/Zv3+/JKlx48aSpAcffFDPPPOMrr/+ei1YsEDJyclKTU3VsGHDnM85fPiw+vXrp+zsbD3xxBNatGiRRowYoa1bt1ZqLgEBARo+fLhOnz6tzz77zFlfsmSJWrZsqSeffFIvvPCC4uLiNG7cOKWlpTnHvPjii2revLnatWunlStXauXKlZo6daoktg9fl5CQoHvvvVdLly4t8/971qxZuvfee5WYmKj58+dr4sSJyszMVM+ePZWXl+cct2TJEo0fP17NmzfX3Llz1aNHDw0ZMkT//Oc/Kz3f0NBQ3X777Tp48KB2797trL/00ku67rrrNHPmTM2ePVv16tXT0KFDXX43rVy5Ug6HQz169HBuBw899JAk6auvvtIXX3yhYcOGaeHChXr44YeVmZmpXr161ejh+Rrh6V1ddVVeXp6RZAYPHnzZcYMGDTKSTEFBgZk2bZqRZIYPH+427uJjF23fvt1IMhMnTnQZN3r0aCPJTJs2zVm7uCs5KyvLWWvZsqWRZDZv3uysHT582DgcDjN58mRnrbCw0BQXF7u8RlZWlnE4HGbmzJkuNXF4ziddzNfHH39sfv31V3PgwAGzevVq07hxYxMcHGz++c9/mu+++85IMg8++KDLcx977DEjyXzyySfGGGMyMjIsD/XZUdYhkYvrfumll5y106dPu43r37+/adWqlUuttMNzdrcP1C2XHpLev3+/qVevnpkwYYLz8d9mMTs72wQEBJhZs2a5rGfnzp2mXr16znpRUZFp3Lix+d3vfmfOnTvnHLd8+XIjydYh4tIOz120YMECI8m88847ztpvt4OzZ8+aDh06mD59+rjUSzs8Z7UdbdmyxUgyK1asKHPO3oQ9TdXkxIkTkqSGDRtedtzFxwsKCpy1hx9+uMz1f/jhh5KkcePGudQfeeQR23Ns3769y16w6OhotW3bVj/99JOz5nA45O9/ISbFxcU6evSoQkND1bZtW33zzTe2Xwt1X9++fRUdHa24uDgNGzZMoaGhysjI0BVXXKH3339fktxOQp08ebIkOf+ijYiIkCS99957FT5puzShoaGS/rVtShcOWVyUn5+vI0eOKDk5WT/99JPLYbzSsH2gVatWGjlypF599VX98ssvlmPWrl2rkpIS3XXXXTpy5IhziY2NVWJiojZu3CjpwqHpo0ePasyYMapXr57z+SNGjFCjRo2qZL5lbQfHjx9Xfn6+evToYTvDlz7/3LlzOnr0qNq0aaOIiIg6tx3QNFWTi83QpcG0YtVcJSQklLn+n3/+Wf7+/m5j27RpY3uOLVq0cKs1atRIx48fd/5cUlKiBQsWKDExUQ6HQ1FRUYqOjtaOHTts/VKB70hLS9OGDRu0ceNG7d69Wz/99JP69+8v6V95/W0+Y2NjFRERoZ9//lmSlJycrDvvvFMzZsxQVFSUBg8erPT0dLfzniri5MmTkly3tc8//1x9+/ZVSEiIIiIiFB0drSeffFKSbOWb7QPShaubz58/X+q5TT/++KOMMUpMTFR0dLTL8sMPP+jw4cOS5NwOfrud1KtXT/Hx8VUyV6vt4L333lOXLl0UFBSkyMhIRUdHa8mSJbYzfObMGT3zzDOKi4tz2Q7y8vLq3HZQr+whqIjw8HA1bdpUO3bsuOy4HTt26IorrlBYWJizdmnXXp0CAgIs6+aSEwRnz56tp59+Wvfff7+effZZRUZGyt/fXxMnTlRJSUmNzBPeoXPnzrrhhhsuO6asG7T6+fnprbfe0tatW7V+/Xp99NFHuv/++/XCCy9o69atzr+SK+L777+X9K9fSPv379fNN9+sdu3aaf78+YqLi1NgYKDef/99LViwwFa+2T4gXdjbdM899+jVV1/VE0884fZ4SUmJ/Pz89MEHH1h+7lYm1+X12+3g008/1aBBg9SzZ08tXrxYTZs2Vf369ZWenq5Vq1bZWucjjzyi9PR0TZw4UV27dlV4eLj8/Pw0bNiwOrcd0DRVo4EDB2rp0qX67LPPnFfBXerTTz9Vdna282S68mjZsqVKSkqUlZWlxMREZ33fvn2VmvNvvfXWW+rdu7eWLVvmUs/Ly1NUVFSVvhbqrot5/fHHH3XVVVc567m5ucrLy3O7h1iXLl3UpUsXzZo1S6tWrdKIESO0evVqPfjggxV6/eLiYq1atUoNGjRwbovr169XUVGR3n33XZe9rhcPlVyqtGaP7QMXPfXUU3rjjTc0Z84ct8dat24tY4wSEhJ05ZVXlrqOi9vBvn371Lt3b2f9/Pnzys7O1rXXXlupOZ48eVIZGRmKi4tzbodvv/22goKC9NFHH8nhcDjHpqenuz3/ctvBqFGj9MILLzhrhYWFLie41xUcnqtGjz/+uIKDg/XQQw/p6NGjLo8dO3ZMDz/8sBo0aKDHH3+83Ou+eNhj8eLFLvVFixZVfMIWAgICXPY8SdKaNWss74ILlOa2226TdOFKtEvNnz9fkpxXfR4/ftwtb506dZKkCh+iKy4u1oQJE/TDDz9owoQJzr26F//iv/T18vPzLX9ZhISEWP4CYPvARa1bt9Y999yjV155RTk5OS6P3XHHHQoICNCMGTPc8mKMcf5+uOGGG9S4cWMtXbpU58+fd4558803XU6bqIgzZ85o5MiROnbsmKZOnepsgAICAuTn56fi4mLn2OzsbMubWJZnO1i0aJHLOusK9jRVo8TERL3++usaMWKErrnmGj3wwANKSEhQdna2li1bpiNHjuivf/2rWrduXe51JyUl6c4779SLL76oo0ePqkuXLtq0aZP+/ve/Syr7MIhdAwcO1MyZM3XfffepW7du2rlzp9588021atWqStYP39CxY0eNGjVKr776qvLy8pScnKwvv/xSr7/+uoYMGeL8q/r111/X4sWLdfvtt6t169Y6ceKEli5dqrCwMGfjdTn5+fl64403JEmnT5/Wvn37tHbtWu3fv1/Dhg3Ts88+6xzbr18/BQYG6g9/+IMeeughnTx5UkuXLlWTJk3cTuhNSkrSkiVL9Nxzz6lNmzZq0qSJ+vTpw/YBF1OnTtXKlSu1d+9eXX311c5669at9dxzz2nKlCnKzs7WkCFD1LBhQ2VlZSkjI0Njx47VY489psDAQE2fPl2PPPKI+vTpo7vuukvZ2dlavny5Wrdubftz/eDBg87t4OTJk9q9e7fWrFmjnJwcTZ482eXoxoABAzR//nzdcsstuvvuu3X48GGlpaWpTZs2bqeXJCUl6eOPP9b8+fPVrFkzJSQk6MYbb9TAgQO1cuVKhYeHq3379tqyZYs+/vhj5+1G6hRPXbbnS3bs2GGGDx9umjZtaurXr29iY2PN8OHDzc6dO13GXbytwK+//uq2jt/ecsAYY06dOmVSUlJMZGSkCQ0NNUOGDDF79+41kszzzz/vHFfaLQesLktNTk52uay1sLDQTJ482TRt2tQEBweb7t27my1btriN45YDvqu0O4L/1rlz58yMGTNMQkKCqV+/vomLizNTpkwxhYWFzjHffPONGT58uGnRooVxOBymSZMmZuDAgebrr78ucx7JyclGknMJDQ01iYmJ5p577jF/+9vfLJ/z7rvvmmuvvdYEBQWZ+Ph4M2fOHPPaa6+5bS85OTlmwIABpmHDhi6XftvdPlC3XC7zo0aNMpIsb3/x9ttvm5tuusmEhISYkJAQ065dO5OSkmL27t3rMm7hwoWmZcuWxuFwmM6dO5vPP//cJCUlmVtuuaXMuV28nYwk4+fnZ8LCwszVV19txowZY7Zt22b5nGXLlpnExETjcDhMu3btTHp6uuXvnD179piePXua4OBgI8l5+4Hjx4+b++67z0RFRZnQ0FDTv39/s2fPHtOyZcs6dwdxP2N+s08NXu27777TddddpzfeeEMjRozw9HQAAJVUUlKi6Oho3XHHHVq6dKmnp+PTOKfJi1l9ge+LL74of39/9ezZ0wMzAgBURmFhodv5QStWrNCxY8cq9TUqqBqc0+TF5s6dq+3bt6t3796qV6+ePvjgA33wwQcaO3as4uLiPD09AEA5bd26VY8++qiGDh2qxo0b65tvvtGyZcvUoUMHDR061NPT83kcnvNiGzZs0IwZM7R7926dPHlSLVq00MiRIzV16lSXu8kCALxDdna2JkyYoC+//FLHjh1TZGSkbrvtNj3//PNq0qSJp6fn82iaAAAAbOCcJgAAABtomgAAAGyothNf0tLSNG/ePOXk5Khjx45atGiROnfuXObzSkpKdOjQITVs2LDKbtAI2GWM0YkTJ9SsWTPnt9eXV0WzL5F/eE5VZF/isx/eyXb+q+PmT6tXrzaBgYHmtddeM7t27TJjxowxERERJjc3t8znHjhwwOUGdSwsnlgOHDhQ49kn/yy1Yalo9iubf7LPUhuWsvJfLU1T586dTUpKivPn4uJi06xZM5Oamlrmc/Py8jz+j8bCkpeXV+PZ95b85+fnWy6enhdL1SwVzX5l8+8N2S/PwnbinUtZ+a/yw3Nnz57V9u3bNWXKFGfN399fffv21ZYtW9zGFxUVuXwR54kTJ6p6SkC5VeTwQHmzL3ln/i9+4S3qpooeGuOz3xXbiXcqK/9VfiL4kSNHVFxcrJiYGJd6TEyM2zc/S1JqaqrCw8OdCzdlhLcqb/Yl8o+6g89++AKPXz03ZcoU5efnO5cDBw54ekpAjSH/8FVkH96oyg/PRUVFKSAgQLm5uS713NxcxcbGuo13OBxyOBxVPQ2gxpU3+5J35p8rm2CFz35XbCd1U5XvaQoMDFRSUpIyMzOdtZKSEmVmZqpr165V/XJArUH24cvIP3xChS+TuIzVq1cbh8Nhli9fbnbv3m3Gjh1rIiIiTE5OTpnP5eoCltqw5Ofn13j2yT9LbVgqmv3K5p/ss9SGpaz8V0vTZIwxixYtMi1atDCBgYGmc+fOZuvWrbaex4bDUhuWyvziqGj2yT9LbVgqk/3K5J/ss9SGpaz817ov7C0oKFB4eLinpwEfl5+f75FLhsk/PI3sw5eVlX+PXz0HAADgDWiaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwgaYJAADABpomAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwgaYJAADABpomAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABvqeXoCqJ3i4+Mt6/fdd59l/amnnnKrffrpp5ZjBw8ebFnPz8+3NzmgmpF/+Cqyf3nsaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwwc8YYzw9iUsVFBQoPDzc09PwCtHR0Zb1W2+91bI+depUt5qfn5/l2ODgYMt606ZNbc6u9HWvWLHCsl7a1RmekJ+fr7CwsBp/XfJvH/mvHmS/9iP71aes/LOnCQAAwAaaJgAAABtomgAAAGygaQIAALCBr1HxkMDAQMt6ixYtLOsLFy50qzVu3NhybFJSku15lHbCXnVeH/D2229X27rhHcg/fBXZ927saQIAALCBpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwodxXz23evFnz5s3T9u3b9csvvygjI0NDhgxxPm6M0bRp07R06VLl5eWpe/fuWrJkiRITE6ty3l6jtCsUJk2aZFl/7rnnbK+nln0Djm0//PCDp6dQIWS//Mi/O/LvG8i+O2/N/qXKvafp1KlT6tixo9LS0iwfnzt3rhYuXKi//OUv2rZtm0JCQtS/f38VFhZWerKAJ5F9+DLyD1RgT9Ott95a6pcCGmP04osv6qmnntLgwYMlXfiCvpiYGK1bt07Dhg2r3GwBDyL78GXkH6jic5qysrKUk5Ojvn37Omvh4eG68cYbtWXLFsvnFBUVqaCgwGUBvE1Fsi+Rf9QNfPbDV1Rp05STkyNJiomJcanHxMQ4H/ut1NRUhYeHO5e4uLiqnBJQIyqSfYn8o27gsx++wuNXz02ZMkX5+fnO5cCBA56eElBjyD98FdmHN6rS756LjY2VJOXm5qpp06bOem5urjp16mT5HIfDIYfDUZXTqHbJycmW9WuvvdatNnXqVMuxUVFRVTqnS7377ruW9Z9//tmttnz5csuxH330kWW9PPNetGiRZX3//v221+EtKpJ9ifxXB/Jf8/jsJ/uXqsvZr9I9TQkJCYqNjVVmZqazVlBQoG3btqlr165V+VJArUL24cvIP3xFufc0nTx5Uvv27XP+nJWVpe+++06RkZFq0aKFJk6cqOeee06JiYlKSEjQ008/rWbNmrnczwPwRmQfvoz8AxVomr7++mv17t3b+fPFG3WNGjVKy5cv15///GedOnVKY8eOVV5enm666SZ9+OGHCgoKqrpZAx5A9uHLyD9QgaapV69el70bqZ+fn2bOnKmZM2dWamJAbUP24cvIP1ALrp4DAADwBlV69VxdU9rdb9944w3Lenh4eKVfc/fu3ZZ1q6sfSvsen+nTp1vWT58+7VbLzc21HNu4ceNSZmjNat6lfZcSvAP5t4/81y1k3z5fyz57mgAAAGygaQIAALCBpgkAAMAGmiYAAAAbOBH8Mkq7vLa0+o8//uhWK+2EuCNHjljW9+zZY1m3OhmwvKKjo91qpd0av7T3ePbsWcv63Llz3WqlvUd4B/Lvjvz7BrLvjuxfwJ4mAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABu4eu4yPvzwQ8v6HXfcYVnfvHlzdU7HNqsrJSTpo48+sr2O0m7pb3WlhFT61wvAe5F/d+TfN5B9d2T/AvY0AQAA2EDTBAAAYANNEwAAgA00TQAAADbQNAEAANjgZ0r7ohkPKSgoUHh4uKen4dXef/99y3r//v3dajt37rQc27dvX8t6Xf5OoUvl5+crLCysxl+X/Fce+a8csu+9yH7llZV/9jQBAADYQNMEAABgA00TAACADTRNAAAANvA1Kl7sT3/6k2X9pptusqxbnfPv6yf9wXuRf/gqsu857GkCAACwgaYJAADABpomAAAAG2iaAAAAbKBpAgAAsIGr57zE4MGD3Wpz5syxHNugQQPL+q+//upW40oJeAPyD19F9msX9jQBAADYQNMEAABgA00TAACADTRNAAAANtA0AQAA2MDVc15i4sSJbrXyXCkhSf3796/KKQE1hvzDV5H92oU9TQAAADbQNAEAANhA0wQAAGADTRMAAIANNE0AAAA2cPWch0RERFjWMzIyLOu9evVyq2VlZVmOve222yzre/bssTU3oLqRf/gqsu/d2NMEAABgA00TAACADTRNAAAANtA0AQAA2FCuE8FTU1O1du1a7dmzR8HBwerWrZvmzJmjtm3bOscUFhZq8uTJWr16tYqKitS/f38tXrxYMTExVT55b9azZ0/Leo8ePSzrJSUlbrW33nrLciwn/VUP8l91yL93IftVh+x7t3Ltadq0aZNSUlK0detWbdiwQefOnVO/fv106tQp55hHH31U69ev15o1a7Rp0yYdOnRId9xxR5VPHKhp5B++iuwDF5RrT9OHH37o8vPy5cvVpEkTbd++XT179lR+fr6WLVumVatWqU+fPpKk9PR0XXXVVdq6dau6dOlSdTMHahj5h68i+8AFlTqnKT8/X5IUGRkpSdq+fbvOnTunvn37Ose0a9dOLVq00JYtWyzXUVRUpIKCApcF8AbkH76K7MNXVbhpKikp0cSJE9W9e3d16NBBkpSTk6PAwEC3m3fFxMQoJyfHcj2pqakKDw93LnFxcRWdElBjyD98FdmHL6tw05SSkqLvv/9eq1evrtQEpkyZovz8fOdy4MCBSq0PqAnkH76K7MOXVehrVMaPH6/33ntPmzdvVvPmzZ312NhYnT17Vnl5eS5/ceTm5io2NtZyXQ6HQw6HoyLT8Ar/3//3/1nWBw4cWK71vPvuu2616dOnV2RKqCTybx/5r1vIvn1kv24q154mY4zGjx+vjIwMffLJJ0pISHB5PCkpSfXr11dmZqaztnfvXv3jH/9Q165dq2bGgIeQf/gqsg9cUK49TSkpKVq1apXeeecdNWzY0HmsOjw8XMHBwQoPD9cDDzygSZMmKTIyUmFhYXrkkUfUtWtXrp6A1yP/8FVkH7igXE3TkiVLJLl/63J6erpGjx4tSVqwYIH8/f115513utzgDPB25B++iuwDF5SraTLGlDkmKChIaWlpSktLq/CkgNqI/MNXkX3gAr57DgAAwIYKXT0Ha8nJyW6122+/3XJseHh4uda9YMECt9rp06fLtQ6gOpF/+Cqy7zvY0wQAAGADTRMAAIANNE0AAAA20DQBAADY4GfsXEtagwoKCsp9olxN8/Pzs6yfP3/e9jr27dtnWW/btm2F5oSqlZ+fr7CwsBp/XfJP/j2N7JeO7Nd9ZeWfPU0AAAA20DQBAADYQNMEAABgA00TAACADTRNAAAANvA1KhXw1FNPWdatLkT89ddfLcdOmDChSucE1BTyD19F9sGeJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbuHruMq644grL+ujRo22vY8yYMZb1jz76qCJTAmoM+YevIvsoDXuaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGzg6rnLCA4Otqy3bNnSsr5p0ya32qefflqlcwJqCvmHryL7KA17mgAAAGygaQIAALCBpgkAAMAGmiYAAAAbOBH8Mvbt22dZr1ePfzbUfeQfvorsozTsaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsqHVNkzHG01MAPJZD8g9PI/vwZWXlsNY1TSdOnPD0FACP5ZD8w9PIPnxZWTn0M7WsvS8pKdGhQ4fUsGFDnThxQnFxcTpw4IDCwsI8PbVqUVBQwHusRYwxOnHihJo1ayZ//5r/m4L81z3e8h7Jfs3yllxUhje9R7v5r3U3nfD391fz5s0lSX5+fpKksLCwWv8PXlm8x9ojPDzcY69N/usub3iPZL/m8R5rDzv5r3WH5wAAAGojmiYAAAAbanXT5HA4NG3aNDkcDk9PpdrwHlEaX/h34z3Cii/8m/EevVOtOxEcAACgNqrVe5oAAABqC5omAAAAG2iaAAAAbKBpAgAAsIGmCQAAwIZa3TSlpaUpPj5eQUFBuvHGG/Xll196ekoVtnnzZv3hD39Qs2bN5Ofnp3Xr1rk8bozRM888o6ZNmyo4OFh9+/bVjz/+6JnJVkBqaqp+97vfqWHDhmrSpImGDBmivXv3uowpLCxUSkqKGjdurNDQUN15553Kzc310IxrN7LvPdmXyH9VI//ek39fy36tbZr+67/+S5MmTdK0adP0zTffqGPHjurfv78OHz7s6alVyKlTp9SxY0elpaVZPj537lwtXLhQf/nLX7Rt2zaFhISof//+KiwsrOGZVsymTZuUkpKirVu3asOGDTp37pz69eunU6dOOcc8+uijWr9+vdasWaNNmzbp0KFDuuOOOzw469qJ7HtX9iXyX5XIv3fl3+eyb2qpzp07m5SUFOfPxcXFplmzZiY1NdWDs6oakkxGRobz55KSEhMbG2vmzZvnrOXl5RmHw2H++te/emCGlXf48GEjyWzatMkYc+H91K9f36xZs8Y55ocffjCSzJYtWzw1zVqJ7Ht39o0h/5VB/r07/3U9+7VyT9PZs2e1fft29e3b11nz9/dX3759tWXLFg/OrHpkZWUpJyfH5f2Gh4frxhtv9Nr3m5+fL0mKjIyUJG3fvl3nzp1zeY/t2rVTixYtvPY9Vgey7/3Zl8h/RZF/789/Xc9+rWyajhw5ouLiYsXExLjUY2JilJOT46FZVZ+L76muvN+SkhJNnDhR3bt3V4cOHSRdeI+BgYGKiIhwGeut77G6kH05f/bW90v+K478y/mzN75fX8h+PU9PAHVPSkqKvv/+e3322WeengpQ48g/fJUvZL9W7mmKiopSQECA29n1ubm5io2N9dCsqs/F91QX3u/48eP13nvvaePGjWrevLmzHhsbq7NnzyovL89lvDe+x+pE9uX82RvfL/mvHPIv58/e9n59Jfu1smkKDAxUUlKSMjMznbWSkhJlZmaqa9euHpxZ9UhISFBsbKzL+y0oKNC2bdu85v0aYzR+/HhlZGTok08+UUJCgsvjSUlJql+/vst73Lt3r/7xj394zXusCWTf+7Ivkf+qQv69L/8+l30Pn4heqtWrVxuHw2GWL19udu/ebcaOHWsiIiJMTk6Op6dWISdOnDDffvut+fbbb40kM3/+fPPtt9+an3/+2RhjzPPPP28iIiLMO++8Y3bs2GEGDx5sEhISzJkzZzw8c3v+9Kc/mfDwcPO///u/5pdffnEup0+fdo55+OGHTYsWLcwnn3xivv76a9O1a1fTtWtXD866diL73pV9Y8h/VSL/3pV/X8t+rW2ajDFm0aJFpkWLFiYwMNB07tzZbN261dNTqrCNGzcaSW7LqFGjjDEXLj19+umnTUxMjHE4HObmm282e/fu9eyky8HqvUky6enpzjFnzpwx48aNM40aNTINGjQwt99+u/nll188N+lajOx7T/aNIf9Vjfx7T/59Lft+xhhTvfuyAAAAvF+tPKcJAACgtqFpAgAAsIGmCQAAwAaaJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwgaYJAADABpomAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGygafIS06dPl5+fX4Weu3z5cvn5+Sk7O7tqJ3WJ7Oxs+fn5afny5dX2GgAAeBJNUw3YtWuX7rnnHl1xxRVyOBxq1qyZRowYoV27dnl6akClXWzKLy5BQUG68sorNX78eOXm5tbYPHr16uWcg7+/v8LCwtS2bVuNHDlSGzZsqNS6Fy9ezB8EcLqY+aCgIB08eNDt8V69eqlDhw4emJkUHx/vsh1ERETommuu0dixY7Vt27ZKrXv27Nlat25d1UzUS9E0VbO1a9fq+uuvV2Zmpu677z4tXrxYDzzwgDZu3Kjrr79eGRkZttbz1FNP6cyZMxWaw8iRI3XmzBm1bNmyQs8H7Jg5c6ZWrlypl19+Wd26ddOSJUvUtWtXnT59usbm0Lx5c61cuVIrVqzQvHnzNGjQIH3xxRfq16+f/vjHP+rcuXMVWi9NE6wUFRXp+eef9/Q03HTq1Mm5HaSmpqp3795av369unTpokmTJlV4vTRNkgyqzb59+0yDBg1Mu3btzOHDh10e+/XXX027du1MSEiI2b9/f6nrOHnyZHVPs0pkZWUZSSY9Pd3TU0ENS09PN5LMV1995VKfNGmSkWRWrVpVI/NITk42V199tVv9/PnzZty4cUaS+fOf/1yhdV999dUmOTm5kjNEXXEx8506dTIOh8McPHjQ5fHSslgTWrZsaQYMGOBWP336tBkyZIiRZBYvXlyhdYeEhJhRo0ZVcobejT1N1WjevHk6ffq0Xn31VUVHR7s8FhUVpVdeeUWnTp3S3LlzJf3rvKXdu3fr7rvvVqNGjXTTTTe5PHapM2fOaMKECYqKilLDhg01aNAgHTx4UH5+fpo+fbpznNU5TfHx8Ro4cKA+++wzde7cWUFBQWrVqpVWrFjh8hrHjh3TY489pmuuuUahoaEKCwvTrbfeqv/7v/+rwn8p1EV9+vSRJGVlZUmSzp8/r2effVatW7eWw+FQfHy8nnzySRUVFbk87+uvv1b//v0VFRWl4OBgJSQk6P7776/wPAICArRw4UK1b99eL7/8svLz852Ppaenq0+fPmrSpIkcDofat2+vJUuWuDw/Pj5eu3bt0qZNm5yHPXr16iWJ7cPXPfnkkyouLra9t+mNN95QUlKSgoODFRkZqWHDhunAgQNu49LS0tSqVSsFBwerc+fO+vTTT9WrVy9n7ioiODhYK1euVGRkpGbNmiVjjPOx//zP/1S3bt3UuHFjBQcHKykpSW+99ZbL8/38/HTq1Cm9/vrrzu1g9OjRkqSff/5Z48aNU9u2bRUcHKzGjRtr6NCh1XoerafQNFWj9evXKz4+Xj169LB8vGfPnoqPj9f//M//uNSHDh2q06dPa/bs2RozZkyp6x89erQWLVqk2267TXPmzFFwcLAGDBhge3779u3Tv/3bv+n3v/+9XnjhBTVq1EijR492Odfqp59+0rp16zRw4EDNnz9fjz/+uHbu3Knk5GQdOnTI9mvB9+zfv1+S1LhxY0nSgw8+qGeeeUbXX3+9FixYoOTkZKWmpmrYsGHO5xw+fFj9+vVTdna2nnjiCS1atEgjRozQ1q1bKzWXgIAADR8+XKdPn9Znn33mrC9ZskQtW7bUk08+qRdeeEFxcXEaN26c0tLSnGNefPFFNW/eXO3atdPKlSu1cuVKTZ06VRLbh69LSEjQvffeq6VLl5b5/z1r1izde++9SkxM1Pz58zVx4kRlZmaqZ8+eysvLc45bsmSJxo8fr+bNm2vu3Lnq0aOHhgwZon/+85+Vnm9oaKhuv/12HTx4ULt373bWX3rpJV133XWaOXOmZs+erXr16mno0KEuv5tWrlwph8OhHj16OLeDhx56SJL01Vdf6YsvvtCwYcO0cOFCPfzww8rMzFSvXr1q9PB8jfD0rq66Ki8vz0gygwcPvuy4QYMGGUmmoKDATJs2zUgyw4cPdxt38bGLtm/fbiSZiRMnuowbPXq0kWSmTZvmrF3clZyVleWstWzZ0kgymzdvdtYOHz5sHA6HmTx5srNWWFhoiouLXV4jKyvLOBwOM3PmTJeaODznky7m6+OPPza//vqrOXDggFm9erVp3LixCQ4ONv/85z/Nd999ZySZBx980OW5jz32mJFkPvnkE2OMMRkZGZaH+uwo65DIxXW/9NJLztrp06fdxvXv39+0atXKpVba4Tm72wfqlksPSe/fv9/Uq1fPTJgwwfn4b7OYnZ1tAgICzKxZs1zWs3PnTlOvXj1nvaioyDRu3Nj87ne/M+fOnXOOW758uZFk6xBxaYfnLlqwYIGRZN555x1n7bfbwdmzZ02HDh1Mnz59XOqlHZ6z2o62bNliJJkVK1aUOWdvwp6manLixAlJUsOGDS877uLjBQUFztrDDz9c5vo//PBDSdK4ceNc6o888ojtObZv395lL1h0dLTatm2rn376yVlzOBzy978Qk+LiYh09elShoaFq27atvvnmG9uvhbqvb9++io6OVlxcnIYNG6bQ0FBlZGToiiuu0Pvvvy9JbiehTp48WZKcf9FGRERIkt57770Kn7RdmtDQUEn/2jalC4csLsrPz9eRI0eUnJysn376yeUwXmnYPtCqVSuNHDlSr776qn755RfLMWvXrlVJSYnuuusuHTlyxLnExsYqMTFRGzdulHTh0PTRo0c1ZswY1atXz/n8ESNGqFGjRlUy37K2g+PHjys/P189evSwneFLn3/u3DkdPXpUbdq0UURERJ3bDmiaqsnFZujSYFqxaq4SEhLKXP/PP/8sf39/t7Ft2rSxPccWLVq41Ro1aqTjx487fy4pKdGCBQuUmJgoh8OhqKgoRUdHa8eOHbZ+qcB3pKWlacOGDdq4caN2796tn376Sf3795f0r7z+Np+xsbGKiIjQzz//LElKTk7WnXfeqRkzZigqKkqDBw9Wenq623lPFXHy5ElJrtva559/rr59+yokJEQRERGKjo7Wk08+KUm28s32AenC1c3nz58v9dymH3/8UcYYJSYmKjo62mX54YcfdPjwYUlybge/3U7q1aun+Pj4Kpmr1Xbw3nvvqUuXLgoKClJkZKSio6O1ZMkS2xk+c+aMnnnmGcXFxblsB3l5eXVuO6hX9hBURHh4uJo2baodO3ZcdtyOHTt0xRVXKCwszFm7tGuvTgEBAZZ1c8kJgrNnz9bTTz+t+++/X88++6wiIyPl7++viRMnqqSkpEbmCe/QuXNn3XDDDZcdU9YNWv38/PTWW29p69atWr9+vT766CPdf//9euGFF7R161bnX8kV8f3330v61y+k/fv36+abb1a7du00f/58xcXFKTAwUO+//74WLFhgK99sH5Au7G2655579Oqrr+qJJ55we7ykpER+fn764IMPLD93K5Pr8vrtdvDpp59q0KBB6tmzpxYvXqymTZuqfv36Sk9P16pVq2yt85FHHlF6eromTpyorl27Kjw8XH5+fho2bFid2w5omqrRwIEDtXTpUn322WfOq+Au9emnnyo7O9t5Ml15tGzZUiUlJcrKylJiYqKzvm/fvkrN+bfeeust9e7dW8uWLXOp5+XlKSoqqkpfC3XXxbz++OOPuuqqq5z13Nxc5eXlud1DrEuXLurSpYtmzZqlVatWacSIEVq9erUefPDBCr1+cXGxVq1apQYNGji3xfXr16uoqEjvvvuuy17Xi4dKLlVas8f2gYueeuopvfHGG5ozZ47bY61bt5YxRgkJCbryyitLXcfF7WDfvn3q3bu3s37+/HllZ2fr2muvrdQcT548qYyMDMXFxTm3w7fffltBQUH66KOP5HA4nGPT09Pdnn+57WDUqFF64YUXnLXCwkKXE9zrCg7PVaPHH39cwcHBeuihh3T06FGXx44dO6aHH35YDRo00OOPP17udV887LF48WKX+qJFiyo+YQsBAQEue54kac2aNZZ3wQVKc9ttt0m6cCXapebPny9Jzqs+jx8/7pa3Tp06SVKFD9EVFxdrwoQJ+uGHHzRhwgTnXt2Lf/Ff+nr5+fmWvyxCQkIsfwGwfeCi1q1b65577tErr7yinJwcl8fuuOMOBQQEaMaMGW55McY4fz/ccMMNaty4sZYuXarz5887x7z55psup01UxJkzZzRy5EgdO3ZMU6dOdTZAAQEB8vPzU3FxsXNsdna25U0sy7MdLFq0yGWddQV7mqpRYmKiXn/9dY0YMULXXHONHnjgASUkJCg7O1vLli3TkSNH9Ne//lWtW7cu97qTkpJ055136sUXX9TRo0fVpUsXbdq0SX//+98llX0YxK6BAwdq5syZuu+++9StWzft3LlTb775plq1alUl64dv6Nixo0aNGqVXX31VeXl5Sk5O1pdffqnXX39dQ4YMcf5V/frrr2vx4sW6/fbb1bp1a504cUJLly5VWFiYs/G6nPz8fL3xxhuSpNOnT2vfvn1au3at9u/fr2HDhunZZ591ju3Xr58CAwP1hz/8QQ899JBOnjyppUuXqkmTJm4n9CYlJWnJkiV67rnn1KZNGzVp0kR9+vRh+4CLqVOnauXKldq7d6+uvvpqZ71169Z67rnnNGXKFGVnZ2vIkCFq2LChsrKylJGRobFjx+qxxx5TYGCgpk+frkceeUR9+vTRXXfdpezsbC1fvlytW7e2/bl+8OBB53Zw8uRJ7d69W2vWrFFOTo4mT57scnRjwIABmj9/vm655RbdfffdOnz4sNLS0tSmTRu300uSkpL08ccfa/78+WrWrJkSEhJ04403auDAgVq5cqXCw8PVvn17bdmyRR9//LHzdiN1iqcu2/MlO3bsMMOHDzdNmzY19evXN7GxsWb48OFm586dLuMu3lbg119/dVvHb285YIwxp06dMikpKSYyMtKEhoaaIUOGmL179xpJ5vnnn3eOK+2WA1aXpSYnJ7tc1lpYWGgmT55smjZtaoKDg0337t3Nli1b3MZxywHfVdodwX/r3LlzZsaMGSYhIcHUr1/fxMXFmSlTppjCwkLnmG+++cYMHz7ctGjRwjgcDtOkSRMzcOBA8/XXX5c5j+TkZCPJuYSGhprExERzzz33mL/97W+Wz3n33XfNtddea4KCgkx8fLyZM2eOee2119y2l5ycHDNgwADTsGFDl0u/7W4fqFsul/lRo0YZSZa3v3j77bfNTTfdZEJCQkxISIhp166dSUlJMXv37nUZt3DhQtOyZUvjcDhM586dzeeff26SkpLMLbfcUubcLt5ORpLx8/MzYWFh5uqrrzZjxowx27Zts3zOsmXLTGJionE4HKZdu3YmPT3d8nfOnj17TM+ePU1wcLCR5Lz9wPHjx819991noqKiTGhoqOnfv7/Zs2ePadmyZZ27g7ifMb/Zpwav9t133+m6667TG2+8oREjRnh6OgCASiopKVF0dLTuuOMOLV261NPT8Wmc0+TFrL7A98UXX5S/v7969uzpgRkBACqjsLDQ7fygFStW6NixY5X6GhVUDc5p8mJz587V9u3b1bt3b9WrV08ffPCBPvjgA40dO1ZxcXGenh4AoJy2bt2qRx99VEOHDlXjxo31zTffaNmyZerQoYOGDh3q6en5PA7PebENGzZoxowZ2r17t06ePKkWLVpo5MiRmjp1qsvdZAEA3iE7O1sTJkzQl19+qWPHjikyMlK33Xabnn/+eTVp0sTT0/N5NE0AAAA2cE4TAACADTRNAAAANlTbiS9paWmaN2+ecnJy1LFjRy1atEidO3cu83klJSU6dOiQGjZsWGU3aATsMsboxIkTatasmfPb68urotmXyD88pyqyL/HZD+9kO//VcfOn1atXm8DAQPPaa6+ZXbt2mTFjxpiIiAiTm5tb5nMPHDjgcoM6FhZPLAcOHKjx7JN/ltqwVDT7lc0/2WepDUtZ+a+Wpqlz584mJSXF+XNxcbFp1qyZSU1NLfO5eXl5Hv9HY2HJy8ur8exfmv8DBw6Y/Px8l8XT/yYsvrFUNPuVzX91f/b/dntiu2KxWsrKf5Ufnjt79qy2b9+uKVOmOGv+/v7q27evtmzZ4ja+qKjI5Ys4T5w4UdVTAsqtIocHypt9qfT8h4WFOb9YFqhJFT00Vts/+9meYEdZ+a/yE8GPHDmi4uJixcTEuNRjYmLcvvlZklJTUxUeHu5cuCkjvFV5sy+Rf9QdfPbDF3j86rkpU6YoPz/fuRw4cMDTUwJqDPmHryL78EZVfnguKipKAQEBys3Ndann5uYqNjbWbbzD4ZDD4ajqaQA1rrzZl0rPf3h4eLXMEagutf2znyvyUBWqfE9TYGCgkpKSlJmZ6ayVlJQoMzNTXbt2reqXA2oNsg9fRv7hEyp8mcRlrF692jgcDrN8+XKze/duM3bsWBMREWFycnLKfC5XM7DUhiU/P7/Gs0/+WWrDUtHsVzb/ZJ+lNixl5b9amiZjjFm0aJFp0aKFCQwMNJ07dzZbt2619Tw2HJbasFTmF0dFs0/+WWrDUpnsVyb/ZJ+lNixl5b/WfWFvQUEB53PA4/Lz8z1yiTL5h6eRffiysvLv8avnAAAAvAFNEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIANNE0AAAA20DQBAADYQNMEAABgA00TAACADTRNAAAANtA0AQAA2EDTBAAAYANNEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIAN9Tw9AdSchg0butWKioosx549e7a6pwPUKPIPX0X2qw57mgAAAGygaQIAALCBpgkAAMAGmiYAAAAbOBHcSzRo0MCtdvPNN1uO7du3r2V9/PjxbrVNmzZZjp01a5ZlPTMzs7QpAtWG/MNXkf3ahT1NAAAANtA0AQAA2EDTBAAAYANNEwAAgA00TQAAADb4GWOMpydxqYKCAoWHh3t6GtUuJCTEst6nTx/L+mOPPeZW6969e7le08/Pz61W2n9/QUGBZT0jI8Oy/t///d9utZ07d1qOPXToUGlTrDXy8/MVFhZW469L/sm/p5H96kX2a7ey8s+eJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbuHquCll9R1Bp3wU0efJky3p5r4ooj/JcQVEVfvnlF8v6smXLLOvr1q1zq+3atcty7Llz5yo8Lzu4gqj8yL8rb80/2S8/su/KW7MvcfUcAABAlaBpAgAAsIGmCQAAwAaaJgAAABs4EfwyAgMDLeu33HKLZd3qdvfdunWr0jlVhtXJgPv27bMcW9qJfO3atbOsR0VFVXxilzFv3jzL+lNPPWVZLy4urpLX5WRY8m/FF/JP9sm+FV/IvsSJ4AAAAFWCpgkAAMAGmiYAAAAbaJoAAABsoGkCAACwoV55n7B582bNmzdP27dv1y+//KKMjAwNGTLE+bgxRtOmTdPSpUuVl5en7t27a8mSJUpMTKzKeVe5evXc/yleeOEFy7F/+tOfqns61eLdd991q6WkpFiOPXTokGW9tCsorP5N7r77bsuxjRo1Km2Kbh5//HHL+s8//2xZ/8tf/mJ73eVVV7Mvkf/fIv/u6mr+yb4rsn955d7TdOrUKXXs2FFpaWmWj8+dO1cLFy7UX/7yF23btk0hISHq37+/CgsLKz1ZwJPIPnwZ+QcqsKfp1ltv1a233mr5mDFGL774op566ikNHjxYkrRixQrFxMRo3bp1GjZsWOVmC3gQ2YcvI/9AFZ/TlJWVpZycHJdvdw4PD9eNN96oLVu2WD6nqKhIBQUFLgvgbSqSfYn8o27gsx++okqbppycHElSTEyMSz0mJsb52G+lpqYqPDzcucTFxVXllIAaUZHsS+QfdQOf/fAVHr96bsqUKcrPz3cuBw4c8PSUgBpD/uGryD68UbnPabqc2NhYSVJubq6aNm3qrOfm5qpTp06Wz3E4HHI4HFU5jQr5+uuv3WodOnTwwEyszZkzx622Zs2acq3ju+++q/Q89uzZY1n/93//d7fa+++/bzn2zTfftKxHRETYnsczzzxjWa+pKyh+qyLZl8i/XeTfVV3IP9m3h+y78nT2q3RPU0JCgmJjY5WZmemsFRQUaNu2beratWtVvhRQq5B9+DLyD19R7j1NJ0+edPl25KysLH333XeKjIxUixYtNHHiRD333HNKTExUQkKCnn76aTVr1szlfh6ANyL78GXkH6hA0/T111+rd+/ezp8nTZokSRo1apSWL1+uP//5zzp16pTGjh2rvLw83XTTTfrwww8VFBRUdbMGPIDsw5eRf6ACTVOvXr1kjCn1cT8/P82cOVMzZ86s1MSA2obsw5eRf6AWXD0HAADgDfzM5f508ICCggKFh4fX+OsWFxe71T7++GPLsfn5+Zb1O+64o9LzeOeddyzrVt/vc/jw4Uq/nieMHTvWsr548eJKr9vqe6QqIj8/X2FhYVWyrvIg/+S/Mqoi/2Sf7FeX2p59qez8s6cJAADABpomAAAAG2iaAAAAbKBpAgAAsIETwf+fi18DcKnHH3/ccmxpJ7MFBwfbfr1PPvnEsj5gwADL+rlz52yvu7Yr7b4t33//vVstPj6+XOvmRPCKIf81p7bnn+yT/epS27MvcSI4AABAlaBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGyomtPN64Brr73WrTZu3DjLsfXr17e93v/6r/+yrE+bNs2yXpeulCjNrbfeallv2bKl7XV88cUXVTUdiPzXJPJfu5D9mlMXss+eJgAAABtomgAAAGygaQIAALCBpgkAAMAGmiYAAAAbuHru//noo4/caiUlJeVah9X45cuXW47dt29fudbtjdq2bWtZX7x4sWXd399+D5+ZmVmhOcEa+a965N87kP2qV5ezz54mAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAZOBP9/rE7kM8ZYji0uLrasz5492622YcOGyk3MSwQGBrrVxo8fbzk2Ojrasm71771582bLsVb/1qg48l855N97kf3K8bXss6cJAADABpomAAAAG2iaAAAAbKBpAgAAsIGmCQAAwAY/U9plAh5SUFCg8PDwGn9dq6siSvunKe02+O3atavSOdVGDofDsj537ly3WkpKSqVf79Zbb7WsV/eVKfn5+QoLC6vW17BC/ms3X8g/2Sf7Vnwh+1LZ+WdPEwAAgA00TQAAADbQNAEAANhA0wQAAGADTRMAAIANfPdcBZR2FUFERIRbLS8vr3onU02aNm1qWV++fLll/eabb670a/77v/+7W+3TTz+t9HpRtci/O/LvG8i+O1/LPnuaAAAAbKBpAgAAsIGmCQAAwAaaJgAAABtomgAAAGzg6rn/57//+7/dakOHDrUcGxcXZ1l/88033WojR460HHvs2LFyzK76/OlPf7Ks33XXXZb1Hj162F738ePHLetvvfWWZT09Pd2tVlhYaPv1UHHk3xX59x1k3xXZvzz2NAEAANhA0wQAAGADTRMAAIANNE0AAAA2+BljjN3BqampWrt2rfbs2aPg4GB169ZNc+bMUdu2bZ1jCgsLNXnyZK1evVpFRUXq37+/Fi9erJiYGFuvUVBQoPDw8PK/k0pq06aNW+2jjz6yHNuyZUvb6929e7dlvbQT5axOiJOk3Nxct1rXrl0tx3br1s2yHhgY6FZLSEiwHFvarfRLY/V+0tLSLMdOnz69XOv2hPz8fIWFhbnUyP8F5N9dXco/2Sf75VGXsi9Z5/9S5drTtGnTJqWkpGjr1q3asGGDzp07p379+unUqVPOMY8++qjWr1+vNWv+//buJSSq94/j+Ef76XTTMYuczCR3FkIL6SJCREmtui8jCoKoxsDaRVSbyC5QUEhtolZdcGFRiyi0jEKLTIgopCAqsDFaeMEuSvP8F9LwnxzpmXHGc47n/YJZnMfD9H3ks/g0njOnUa2treru7taWLVtS3wHgEuQffkX2gRFJfeXAvXv34o6vXr2quXPnqqOjQytXrlRfX58uX76sa9euafXq1ZJG2vOiRYvU3t6uFStWpG9yYIKRf/gV2QdGjOuapr6+PklSYWGhJKmjo0PDw8OqqamJnVNeXq7S0lK1tbUlfI9fv36pv78/7gV4AfmHX5F9+FXKpSkajaqurk7V1dWqqKiQJEUiEeXm5qqgoCDu3KKiIkUikYTvU19fr2AwGHuN9eVhgJuQf/gV2YefpVyawuGwXr9+rRs3boxrgEOHDqmvry/2+vz587jeD5gI5B9+RfbhZyk9RqW2tlZ3797V48ePVVJSElsPhUIaGhpSb29v3P84enp6FAqFEr5XIBBQIBBIZYy0ev/+/ai148ePJzz38OHDCdcXLlw4am3x4sVJzVFdXZ3U+cnIysoatZbEzZOSpN7e3oTr27ZtG7V2//79pN7bK8g/+f+bX/JP9sn+3/yS/T+S+qTJGKPa2lo1NTWppaVl1C2LlZWVysnJUXNzc2ytq6tLnz59GvMWScAryD/8iuwDI5L6pCkcDuvatWu6ffu28vLyYn+rDgaDmjZtmoLBoHbt2qWDBw+qsLBQ+fn52r9/v6qqqrh7Ap5H/uFXZB8YkVRpunjxoiRp1apVcetXrlzRzp07JUnnzp1Tdna2tm7dGvcFZ4DXkX/4FdkHRiRVmmz+Bjp16lQ1NDSM+Y2ggFeRf/gV2QdG8Ow5AAAAC0k9e24iOPX8oWSUl5cnXD9x4sSotQ0bNmR6HGuJ7qBob29PeO7Tp08Trl+4cCHh+mS7Xfhfzx/KFPKfOeTfDtkfG9kfbTJlX0rzs+cAAAD8itIEAABggdIEAABggdIEAABggQvB0ygnJ2fU2p+ngP9t+/btCdfnz5+f1pn+36lTp0atjfVk8e/fv2dsDi/gYtjkkf/Jgewnj+xPHlwIDgAAkAaUJgAAAAuUJgAAAAuUJgAAAAuUJgAAAAvcPQckwB1E8CuyDz/j7jkAAIA0oDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYoDQBAABYcF1pMsY4PQLgWA7JP5xG9uFn/8qh60rTwMCA0yMAjuWQ/MNpZB9+9q8cZhmX1ftoNKru7m7l5eVpYGBACxYs0OfPn5Wfn+/0aBnR39/PHl3EGKOBgQEVFxcrO3vi/09B/icfr+yR7E8sr+RiPLy0R9v8/zeBM1nJzs5WSUmJJCkrK0uSlJ+f7/pf+HixR/cIBoOO/dvkf/Lywh7J/sRjj+5hk3/X/XkOAADAjShNAAAAFlxdmgKBgI4dO6ZAIOD0KBnDHjEWP/ze2CMS8cPvjD16k+suBAcAAHAjV3/SBAAA4BaUJgAAAAuUJgAAAAuUJgAAAAuuLk0NDQ1auHChpk6dquXLl+v58+dOj5Syx48fa/369SouLlZWVpZu3boV93NjjI4ePap58+Zp2rRpqqmp0bt375wZNgX19fVaunSp8vLyNHfuXG3atEldXV1x5/z8+VPhcFizZ8/WzJkztXXrVvX09Dg0sbuRfe9kXyL/6Ub+vZN/v2XftaXp5s2bOnjwoI4dO6aXL19qyZIlWrdunb5+/er0aCkZHBzUkiVL1NDQkPDnp0+f1vnz53Xp0iU9e/ZMM2bM0Lp16/Tz588JnjQ1ra2tCofDam9v14MHDzQ8PKy1a9dqcHAwds6BAwd0584dNTY2qrW1Vd3d3dqyZYuDU7sT2fdW9iXyn07k31v59132jUstW7bMhMPh2PHv379NcXGxqa+vd3Cq9JBkmpqaYsfRaNSEQiFz5syZ2Fpvb68JBALm+vXrDkw4fl+/fjWSTGtrqzFmZD85OTmmsbExds7bt2+NJNPW1ubUmK5E9r2dfWPI/3iQf2/nf7Jn35WfNA0NDamjo0M1NTWxtezsbNXU1Kitrc3ByTLjw4cPikQicfsNBoNavny5Z/fb19cnSSosLJQkdXR0aHh4OG6P5eXlKi0t9eweM4Hsez/7EvlPFfn3fv4ne/ZdWZq+ffum379/q6ioKG69qKhIkUjEoaky58+eJst+o9Go6urqVF1drYqKCkkje8zNzVVBQUHcuV7dY6aQfcWOvbpf8p868q/YsRf364fs/+f0AJh8wuGwXr9+rSdPnjg9CjDhyD/8yg/Zd+UnTXPmzNGUKVNGXV3f09OjUCjk0FSZ82dPk2G/tbW1unv3rh4+fKiSkpLYeigU0tDQkHp7e+PO9+IeM4nsK3bsxf2S//Eh/4ode22/fsm+K0tTbm6uKisr1dzcHFuLRqNqbm5WVVWVg5NlRllZmUKhUNx++/v79ezZM8/s1xij2tpaNTU1qaWlRWVlZXE/r6ysVE5OTtweu7q69OnTJ8/scSKQfe9lXyL/6UL+vZd/32Xf4QvRx3Tjxg0TCATM1atXzZs3b8zu3btNQUGBiUQiTo+WkoGBAdPZ2Wk6OzuNJHP27FnT2dlpPn78aIwx5uTJk6agoMDcvn3bvHr1ymzcuNGUlZWZHz9+ODy5nb1795pgMGgePXpkvnz5Ent9//49ds6ePXtMaWmpaWlpMS9evDBVVVWmqqrKwandiex7K/vGkP90Iv/eyr/fsu/a0mSMMRcuXDClpaUmNzfXLFu2zLS3tzs9UsoePnxoJI167dixwxgzcuvpkSNHTFFRkQkEAmbNmjWmq6vL2aGTkGhvksyVK1di5/z48cPs27fPzJo1y0yfPt1s3rzZfPnyxbmhXYzseyf7xpD/dCP/3sm/37KfZYwxmf0sCwAAwPtceU0TAACA21CaAAAALFCaAAAALFCaAAAALFCaAAAALFCaAAAALFCaAAAALFCaAAAALFCaAAAALFCaAAAALFCaAAAALFCaAAAALPwP/6zVkcc0ChQAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 700x1100 with 12 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "print(\"4 Random training samples and labels\")\n",
        "indxs = [5, 30, 79, 114]\n",
        "\n",
        "plt.figure(figsize=(7, 11))\n",
        "\n",
        "for i, idx in enumerate(indxs):\n",
        "    i *= 3\n",
        "    img, pos_img, neg_img, label = mnist_train[idx]\n",
        "\n",
        "    plt.subplot(4, 3, i + 1)\n",
        "    plt.imshow(img.reshape((28, 28)), cmap=\"gray\")\n",
        "    plt.title(\"Original\")\n",
        "\n",
        "    plt.subplot(4, 3, i + 2)\n",
        "    plt.imshow(pos_img.reshape((28, 28)), cmap=\"gray\")\n",
        "    plt.title(\"Pos Data\")\n",
        "\n",
        "    plt.subplot(4, 3, i + 3)\n",
        "    plt.imshow(neg_img.reshape((28, 28)), cmap=\"gray\")\n",
        "    plt.title(\"Neg Data\")\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_nEN8FMOkNaT"
      },
      "source": [
        "####Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "LXv5snx3kMYZ"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Here we define a custon Linear layer for the model. Each layer contains its\n",
        "own weights, biases, and also the optimizer state. Since each layer is trained\n",
        "locally without any backward pass, the optimizer does loacl optimizer for each\n",
        "layer independently.\n",
        "\"\"\"\n",
        "\n",
        "class LayerState(NamedTuple):\n",
        "    params: Tuple[jnp.ndarray, jnp.ndarray]\n",
        "    opt_state: Tuple\n",
        "\n",
        "\n",
        "def init_layer_state(in_dim, out_dim, PRNGkey, opt, scale=1e-3) -> LayerState:\n",
        "    \"\"\"\n",
        "    Initialize the layer weights, biases, and optimizer state\n",
        "    \"\"\"\n",
        "    initializer = jax.nn.initializers.glorot_uniform()\n",
        "\n",
        "    weights_key, bias_key = jax.random.split(PRNGkey)\n",
        "    weights = initializer(weights_key, (out_dim, in_dim), jnp.float32) * scale\n",
        "    biases = jax.random.normal(bias_key, (out_dim, ), jnp.float32) * scale\n",
        "    opt_state = opt.init((weights, biases))\n",
        "\n",
        "    return LayerState((weights, biases), opt_state)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "-vl7UUCukU4K"
      },
      "outputs": [],
      "source": [
        "class Layer:\n",
        "    \"\"\"\n",
        "    Stateless layer namespace that contains the functions for training layers.\n",
        "    \"\"\"\n",
        "\n",
        "    def forward(params: Tuple, x: jnp.ndarray):\n",
        "        \"\"\"\n",
        "        params: Tuple of (weights, biases) of the current layer\n",
        "        x: input array\n",
        "\n",
        "        This function performs a forward pass for a single flattened input.\n",
        "        The input is first normalized, and linearly transformed using the\n",
        "        layer weights and bias. Finally, it is non-liniearized using ReLU activation\n",
        "        \"\"\" \n",
        "        x /= (jnp.sqrt(jnp.mean(x ** 2, axis=-1, keepdims=True)) + 1e-6)\n",
        "\n",
        "        w, b = params\n",
        "        return jax.nn.relu(jnp.dot(w, x) + b)\n",
        "\n",
        "    \n",
        "    ## Batchify the forward function\n",
        "    @functools.partial(jax.vmap, in_axes=(None, 0)) # Batch dim 0\n",
        "    def b_forward(params: Tuple, xs: jnp.ndarray):\n",
        "        return Layer.forward(params, xs)\n",
        "    \n",
        "\n",
        "    def goodness(params: Tuple, x: jnp.ndarray):\n",
        "        \"\"\"\n",
        "        Calculates the sum squared goodness for a single input as \n",
        "        mentioned in the paper. It first gets the activation of the current layer\n",
        "        for the input, and then takes the L2 norm of the activations\n",
        "        \"\"\"\n",
        "\n",
        "        h = Layer.forward(params, x)  \n",
        "        return jnp.power(h, 2).sum(), h\n",
        "\n",
        "    ## Batchify the goodness function\n",
        "    @functools.partial(jax.vmap, in_axes=(None, 0)) # Batch dim 0\n",
        "    def b_goodness(params: Tuple, xs: jnp.ndarray):\n",
        "        return Layer.goodness(params, xs)\n",
        "\n",
        "    \"\"\"\n",
        "    The main Forward-Forward algorithm is below. We first get the goodness value\n",
        "    for all positive and negative samples respectively.\n",
        "    The custom loss function finds the distance between the Mean-squared\n",
        "    result and the threshold value which we set as the length of the input vector.\n",
        "    The loss is calculated as a mean across the entire batch combining both the \n",
        "    positive and negative data.\n",
        "    \"\"\"\n",
        "    def forward_forward(params: Tuple, pos_xs: jnp.ndarray, neg_xs: jnp.ndarray):\n",
        "        \"\"\"\n",
        "        pos_xs : batch of positive inputs - (batch_size, 784)\n",
        "        neg_xs : batch of negative inputs - (batch_size, 784)\n",
        "        \"\"\"\n",
        "\n",
        "        pos_goodness, _ = Layer.b_goodness(params, pos_xs)\n",
        "        neg_goodness, _ = Layer.b_goodness(params, neg_xs)\n",
        "\n",
        "        pos_logits = pos_goodness - pos_xs.shape[-1]\n",
        "        neg_logits = neg_goodness - neg_xs.shape[-1]\n",
        "        \n",
        "        loss = -jax.nn.log_sigmoid(jnp.concatenate([pos_logits, -neg_logits])).mean()\n",
        "        \n",
        "        return loss"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sqE9zjOlnVDK"
      },
      "source": [
        "####Utils"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "WdjUZ054nMx6"
      },
      "outputs": [],
      "source": [
        "# Utility Functions\n",
        "\n",
        "def plot_image(inputs, idx=0):\n",
        "    print(\"Embedded Label:\",  np.argmax(inputs[idx][:10]))\n",
        "    im = inputs[idx].reshape(28, 28)\n",
        "    plt.figure(figsize = (4, 4))\n",
        "    plt.imshow(im, cmap=\"gray\")\n",
        "    plt.show()\n",
        "\n",
        "def batch(arr, batch_size):\n",
        "  for i in range(0, len(arr), batch_size):\n",
        "    yield arr[i: i+batch_size]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "wIz1Evle2YPY"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "We implement hard-negative mining over the number of episodes.\n",
        "At the end of episodes we replace the original negative samples using hard-mined\n",
        "samples. For this, we evaluate each image using the current trained model and \n",
        "set the negative label as the second most probable label predicted by the model\n",
        "for that image. This allows the model to better differentiate between positive \n",
        "and negative samples which was mentioned in the paper as an extension.\n",
        "\"\"\"\n",
        "def get_negative_label(params, x):\n",
        "    goodnesses = jnp.zeros(10)\n",
        "    label = jnp.argmax(jnp.split(x, [10])[0])\n",
        "    for query_label in range(10):\n",
        "        img = jnp.array(x.copy())\n",
        "        img = img.at[query_label].set(img.max())\n",
        "        for layer_params in params:\n",
        "            g, img = Layer.goodness(layer_params, img)\n",
        "            goodnesses = goodnesses.at[query_label].add(g)\n",
        "            \n",
        "    # get negative label that gives second best goodness\n",
        "    label_top2 = jax.lax.top_k(goodnesses, 2)[1]\n",
        "    \n",
        "    neg_label = label_top2[1]\n",
        "    neg_img = jnp.array(x.copy())\n",
        "    neg_img = neg_img.at[label].set(0)  # zero the positive label\n",
        "    neg_img = neg_img.at[neg_label].set(neg_img.max())\n",
        "    return neg_img\n",
        "\n",
        "def get_hard_negs(params, xs):\n",
        "    b_get_negative_label = vmap(get_negative_label, in_axes=(None, 0))\n",
        "    neg_images = b_get_negative_label(params, xs)\n",
        "\n",
        "    return neg_images"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "R5SwqCNq1u3D"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Function to predict the label for a single input image. It follows the\n",
        "method outlined in the paper which queries over all the labels\n",
        "and calculates the sum of goodness for all the layers. The label that \n",
        "gives the highest goodness is the predicted output.\n",
        "\"\"\"\n",
        "def predict_one_sample(params, x):\n",
        "    goodnesses = jnp.zeros(10)\n",
        "    for query_label in range(10):\n",
        "        img = jnp.array(x.copy())\n",
        "        img = img.at[query_label].set(img.max())\n",
        "        for layer_params in params:\n",
        "            g, img = Layer.goodness(layer_params, img)\n",
        "            goodnesses = goodnesses.at[query_label].add(g)\n",
        "\n",
        "    return jnp.argmax(goodnesses)\n",
        "\n",
        "\"\"\"\n",
        "Calculates accuracy over the whole dataset\n",
        "\"\"\"\n",
        "def accuracy(params, xs, ys):\n",
        "    b_predict = vmap(predict_one_sample, in_axes=(None, 0))\n",
        "    y_hats = b_predict(params, xs)\n",
        "\n",
        "    if ys.shape[-1] == 10:\n",
        "        labels = jnp.argmax(ys, axis=-1)\n",
        "    else:\n",
        "        labels = ys\n",
        "\n",
        "    return jnp.mean(labels == y_hats)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vwp9LKIBn3sz"
      },
      "source": [
        "####Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "Y0C92m2zna86"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "This function trains a single layer for the specified number of epochs.\n",
        "For each step, it takes a batch of positive and negative imagesand passes them\n",
        "through the layer's forward_forward function that returns the loss for that batch\n",
        "of data. The optimizer takes the gradients over the loss and updates the parameters\n",
        "of the layer. This is not counted as backpropagation because the gradients are not\n",
        "passed through any other layer, and the updates happen locally.\n",
        "\"\"\"\n",
        "def train_layer(\n",
        "        opt: optax.GradientTransformation,\n",
        "        num_epochs: int,\n",
        "        batch_size: int,\n",
        "        initial_state: LayerState,\n",
        "        layer_idx: int,\n",
        "        pos_xs: np.ndarray,\n",
        "        neg_xs: np.ndarray):\n",
        "    \n",
        "    @jit\n",
        "    def step(state: LayerState, pos_xs, neg_xs):\n",
        "        # Get the loss and gradients for the current batch of data\n",
        "        # and update the layer parameters\n",
        "        loss, grads = value_and_grad(Layer.forward_forward)(state.params, pos_xs, neg_xs)\n",
        "        updates, opt_state = opt.update(grads, state.opt_state)\n",
        "        params = optax.apply_updates(state.params, updates)\n",
        "        return loss, LayerState(params, opt_state)\n",
        "\n",
        "    # Collects the loss over each epoch\n",
        "    loss_hist = []\n",
        "\n",
        "    # Gets a batch of positive and negative data\n",
        "    pos_batches = list(batch(pos_xs, batch_size))\n",
        "    neg_batches = list(batch(neg_xs, batch_size))\n",
        "\n",
        "    state = initial_state\n",
        "\n",
        "    pbar = tqdm(range(num_epochs), desc=f\"Training Layer {layer_idx}\")\n",
        "    for epoch in pbar:\n",
        "        loss_sum = 0.\n",
        "        for pos_batch, neg_batch in zip(pos_batches, neg_batches):\n",
        "            loss, state = step(state, pos_batch, neg_batch)\n",
        "            loss_sum += loss\n",
        "        loss_hist.append(float(loss_sum))\n",
        "        pbar.set_postfix({'loss': loss_sum})\n",
        "    \n",
        "    # Return the updated and trained layer state\n",
        "    return state, loss_hist"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "CdcC_oy5sNx-"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Trains all the layers of the model for a fixed number of episodes.\n",
        "Each layer is trained for num_epochs, and the whole model is trained over \n",
        "num_episodes. Hard-negative mining is used in-between episodes to improve\n",
        "generalization. \n",
        "\"\"\"\n",
        "def train_net(\n",
        "    opt: optax.GradientTransformation,\n",
        "    num_epochs: int,\n",
        "    batch_size: int,\n",
        "    initial_states: List[LayerState],\n",
        "    pos_xs: np.ndarray,\n",
        "    neg_xs: np.ndarray,\n",
        "    num_episodes: int):\n",
        "\n",
        "    # Collects the loss history over all episodes\n",
        "    loss_all = []\n",
        "\n",
        "    pos_in = copy.deepcopy(pos_xs)\n",
        "    neg_in = copy.deepcopy(neg_xs)\n",
        "\n",
        "    for i in range(num_episodes):\n",
        "        print(f\"Start training episode {i+1}.\")\n",
        "\n",
        "        final_states = []\n",
        "        loss_hists = []\n",
        "        for layer_idx, layer_state in enumerate(initial_states):\n",
        "            final_state, loss_hist = train_layer(opt, num_epochs, batch_size, layer_state, layer_idx, pos_xs, neg_xs)\n",
        "            final_states.append(final_state)\n",
        "            loss_hists.append(loss_hist)\n",
        "            pos_xs = Layer.b_forward(final_state.params, pos_xs)\n",
        "            neg_xs = Layer.b_forward(final_state.params, neg_xs)\n",
        "        \n",
        "        loss_all.append(loss_hists)\n",
        "        \n",
        "        # Evaluate at the end of episode\n",
        "        train_acc = accuracy([s.params for s in final_states], train_imgs, train_labels)\n",
        "        print(\"Train Accuracy: \", train_acc)\n",
        "        test_acc = accuracy([s.params for s in final_states], test_xs, test_labels)\n",
        "        print(\"Test Accuracy: \", test_acc)\n",
        "\n",
        "        # Re-assign parameters for next episode\n",
        "        initial_states = final_states\n",
        "\n",
        "        # Re-assign positive data as previous\n",
        "        # Update negative data with hard-mining\n",
        "        pos_xs = pos_in\n",
        "        neg_xs = get_hard_negs([s.params for s in final_states], train_imgs)\n",
        "\n",
        "    # Returns the final trained states of all layers\n",
        "    return final_states, loss_all"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "16Hb6TPL3hPq"
      },
      "source": [
        "####Run"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 461,
          "referenced_widgets": [
            "4c3358d0c30b4e97b014e10cf90add37",
            "ce8d4e7a40fa49a59d05e55f2ac1106b",
            "324ee7c3f499452b8ccd1f45fa982d89",
            "7c77f93a5ac8436eaf2dc52186da85aa",
            "8556556b60a147caae19c5b982468a25",
            "bc5e338f90d049089556f7d5d99a9574",
            "8841c567c4ba4882a862e449daaae763",
            "57cae0eec5c240b4957e82faa8823a0a",
            "93862f217bac47289ebe0efa59f21edd",
            "27d54db2f0e442beb29315b04f16304f",
            "a9b02b0b86984515af3fcd4803c9d8af",
            "c2c9fc55e2e94dc4b01cd5b734cc5dda",
            "f47fbd388a664f709feb685697f60b84",
            "b963143aa51d4a58b05b3f0a1bd09d3a",
            "d5ff6d42aaf14629875d66bdbabbde6a",
            "4f8f4d68c05041e78c8f9a20fde72685",
            "9d02b79dbce148acaa903e01607b0618",
            "9033fe7c667e4b728d9753889b3fe86e",
            "c99151acec604436b05bc6704eb261b5",
            "db4ce281dbcb4bb7958cc92d55f64b66",
            "5aa7c39a37b743389775767b65fd1784",
            "3431975a9aae4608b06b67d8441d5c8d",
            "464e51c5c0f64cbf8910c7c8a0fd601e",
            "32a32c9100f14fb096ab7b369e62ffe1",
            "ca7ed09443144209897b6214ce72f8e9",
            "ec769a83862e4d8eaf12c97418fc9238",
            "3ef6d0bd9bc244279adbf6883283a9a1",
            "3bf083af0b6749e0b19b1f30d4f3a51d",
            "4e871d790a204e159ed834a29d64bda1",
            "3147b88f0fc54e3e927f60f9d142f506",
            "039ca27f587749b6882a35805f647516",
            "b7cb7bfd1c6d4d129ea501a7a23c27fc",
            "ecf557718b84483c99cab5e213125426",
            "bd69e362c71943bd88f431e198bb0a05",
            "33c10ca24b1d4ab7b659de0408b61c48",
            "a70c55ff27cf48caa343c6e3ed2a6ca0",
            "ad27cb5b68fe4704ad3d67a45ae608c7",
            "5b4f7300e1a7463199876c01cd626249",
            "e9d1f4b17de349b6b26edd555ab1fa26",
            "f5f304fafb30429995c300b6c594de11",
            "f3deb3ededbf42e18523ea03f3e59b52",
            "ae810c25a6b74b93bda10538fb7581fe",
            "57d156afccc849078cff7b8fa4714363",
            "43c90325fd144c61b72acfc0530c564d",
            "8b129fdd6cbf4773b4a43ce5fd2250e2",
            "02bf0e6c4290487794c3f7e3bfec4b1a",
            "f84149369e06474dacccd22256aabcc6",
            "4d66d5e91977481d8f44842893a7de23",
            "af8e19652a7e45e5ac679ee8c59544ee",
            "5cef791fcff64958beb4c96744c82912",
            "30eb52e35c6d4fe9b83ce9b8b610ff1d",
            "b3a790e54e5e4aa18cbbcb8dc82f2de1",
            "f0dcd51dc3584bd1ab721b7abda51e98",
            "700f9aef759d4ff88a6529769dad630a",
            "dcf761f22c8b428a990794498da9628d",
            "7175c5eacfcb473a8407471d01f9a24e",
            "8459970fd69345e7adf165d46b253a3f",
            "68038f99b99c4181a483a6ba38506080",
            "19495359382e481797c15589994a3530",
            "250acf8e90df42d0a6203cd2b67a78c5",
            "d362014d564b4f9b8f42b3a71077a962",
            "59f233cef02544619c519795f506a3c3",
            "0fd30554bc464ac0a6205cac72672b12",
            "962884a27f1b45a3aa4c06184703f0ee",
            "d5718f1265c64d73961f726fecc765e9",
            "faf02fba0bc74207ba3ea3276f262f48",
            "162beda3fd084f6797476574a10316a2",
            "138d8fb04da741b3be19bf33717b8563",
            "d3b490a013b74ef78df729bfe363b4d9",
            "bfe43079e654472d85060895961d31b0",
            "5625688b08b14f4598abe887f05c0487",
            "ee669bb6587e40228f7b837711ff2f1e",
            "2fa79a374f11499cb9b0945527489602",
            "7168897ea4b6490aa9aafb55aa0263d9",
            "12d68c888845433f8febce89c396a66d",
            "27ea5630f79e451b808b54f85cf6a143",
            "0ccf8dde9b074dd7b4f6de0fe68087a6",
            "d059863fe9f9441e989d23afa78fa411",
            "3a1818934a3a40348f8479c591b49884",
            "34a6667c6a8d4318843bcd76437b39c5",
            "469bb22b0eb84a288464943765cf9691",
            "12ed1d98467c4eca9d923e5b2ac6d282",
            "9c150a43608d49508562e8d6b30bb735",
            "402cdf4dd9a94a6aa553951be24680ee",
            "5ae06ddbf5e14ded89d532c568a9ddfc",
            "557b76964fe34f1092fbd631c565e978",
            "e4e5975c2b3245de951342c7027888be",
            "b91bb77a815a45f0861917fca8efad58",
            "8ebfe7a5fa3f44a3ab142731a7bed1d7",
            "0c5065fc6e414f9cbe957ce80a40d7e4",
            "ed936491a42c4d2d8a1850040deca36a",
            "b73d6f8b428e48179951e63eabf335d4",
            "8fa0a27a45444dc6b7034c16f20c0ed4",
            "9a5ec28fb7dc4a7b8dfa4b81285c1cba",
            "ef8acb5e2c4641c38f2b8e477a04c8a7",
            "c8462a70dcf74cbf82b6f6ed4add9192",
            "facc4a192b364f939a3ed144afc601cf",
            "c1c0544556534410ae14aa5de18344ac",
            "1ad06a79653f481c982d26a94d986f50"
          ]
        },
        "id": "QMWZPBWz3u37",
        "outputId": "6fbd0a6d-1f24-4fad-b372-f562bfb5049b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Start training episode 1.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4c3358d0c30b4e97b014e10cf90add37",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training Layer 0:   0%|          | 0/60 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c2c9fc55e2e94dc4b01cd5b734cc5dda",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training Layer 1:   0%|          | 0/60 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "464e51c5c0f64cbf8910c7c8a0fd601e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training Layer 2:   0%|          | 0/60 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Accuracy:  0.95491666\n",
            "Test Accuracy:  0.9517\n",
            "Start training episode 2.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bd69e362c71943bd88f431e198bb0a05",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training Layer 0:   0%|          | 0/60 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8b129fdd6cbf4773b4a43ce5fd2250e2",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training Layer 1:   0%|          | 0/60 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7175c5eacfcb473a8407471d01f9a24e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training Layer 2:   0%|          | 0/60 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Accuracy:  0.9710167\n",
            "Test Accuracy:  0.96709996\n",
            "Start training episode 3.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "162beda3fd084f6797476574a10316a2",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training Layer 0:   0%|          | 0/60 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d059863fe9f9441e989d23afa78fa411",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training Layer 1:   0%|          | 0/60 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8ebfe7a5fa3f44a3ab142731a7bed1d7",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training Layer 2:   0%|          | 0/60 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Accuracy:  0.97643334\n",
            "Test Accuracy:  0.97239995\n"
          ]
        }
      ],
      "source": [
        "lr = 0.001\n",
        "rng = jax.random.PRNGKey(39)\n",
        "opt = optax.sgd(lr)\n",
        "n_epochs = 60\n",
        "n_episodes = 3\n",
        "batch_size = 100\n",
        "\n",
        "states = [\n",
        "    init_layer_state(784, 512, rng, opt, scale=1),\n",
        "    init_layer_state(512, 512, rng, opt, scale=1),\n",
        "    init_layer_state(512, 512, rng, opt, scale=1),\n",
        "]\n",
        "\n",
        "states, loss_hists = train_net(opt, n_epochs, batch_size, states, train_pos_xs, train_neg_xs, n_episodes)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "## Conclusion\n",
        "\n",
        "With the help of the JAX, this example has shown how the Forward-Forward algorithm works.\n",
        "With a 2000-unit, 4-hidden-layer, fully-connected network run over 60 epochs, Geoffrey Hinton \n",
        "reported test accuracy error results of 1.36% in the paper. He also noted that backpropagation \n",
        "only needs 20 epochs to reach a comparable level of performance. A second run with the learning \n",
        "rate doubled and 40 training epochs results in a marginally higher error rate of 1.46%.\n",
        "With 3 layers of 512 hidden units, the present example produces a 1.8% test accuracy error \n",
        "after 5 episodes and 60 epochs. The findings can be improved to match the claims of the paper \n",
        "with proper tuning of the Learning Rate, model architecture (number of units in \"Dense\" layers, \n",
        "kernel activations, initializations, regularization, etc.), and other factors.\n",
        "\"\"\""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "02bf0e6c4290487794c3f7e3bfec4b1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5cef791fcff64958beb4c96744c82912",
            "placeholder": "​",
            "style": "IPY_MODEL_30eb52e35c6d4fe9b83ce9b8b610ff1d",
            "value": "Training Layer 1: 100%"
          }
        },
        "039ca27f587749b6882a35805f647516": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0c5065fc6e414f9cbe957ce80a40d7e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9a5ec28fb7dc4a7b8dfa4b81285c1cba",
            "placeholder": "​",
            "style": "IPY_MODEL_ef8acb5e2c4641c38f2b8e477a04c8a7",
            "value": "Training Layer 2: 100%"
          }
        },
        "0ccf8dde9b074dd7b4f6de0fe68087a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0fd30554bc464ac0a6205cac72672b12": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "12d68c888845433f8febce89c396a66d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "12ed1d98467c4eca9d923e5b2ac6d282": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "138d8fb04da741b3be19bf33717b8563": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ee669bb6587e40228f7b837711ff2f1e",
            "placeholder": "​",
            "style": "IPY_MODEL_2fa79a374f11499cb9b0945527489602",
            "value": "Training Layer 0: 100%"
          }
        },
        "162beda3fd084f6797476574a10316a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_138d8fb04da741b3be19bf33717b8563",
              "IPY_MODEL_d3b490a013b74ef78df729bfe363b4d9",
              "IPY_MODEL_bfe43079e654472d85060895961d31b0"
            ],
            "layout": "IPY_MODEL_5625688b08b14f4598abe887f05c0487"
          }
        },
        "19495359382e481797c15589994a3530": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d5718f1265c64d73961f726fecc765e9",
            "placeholder": "​",
            "style": "IPY_MODEL_faf02fba0bc74207ba3ea3276f262f48",
            "value": " 60/60 [00:11&lt;00:00,  4.90it/s, loss=85.242935]"
          }
        },
        "1ad06a79653f481c982d26a94d986f50": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "250acf8e90df42d0a6203cd2b67a78c5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27d54db2f0e442beb29315b04f16304f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27ea5630f79e451b808b54f85cf6a143": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2fa79a374f11499cb9b0945527489602": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "30eb52e35c6d4fe9b83ce9b8b610ff1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3147b88f0fc54e3e927f60f9d142f506": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "324ee7c3f499452b8ccd1f45fa982d89": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_57cae0eec5c240b4957e82faa8823a0a",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_93862f217bac47289ebe0efa59f21edd",
            "value": 60
          }
        },
        "32a32c9100f14fb096ab7b369e62ffe1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3bf083af0b6749e0b19b1f30d4f3a51d",
            "placeholder": "​",
            "style": "IPY_MODEL_4e871d790a204e159ed834a29d64bda1",
            "value": "Training Layer 2: 100%"
          }
        },
        "33c10ca24b1d4ab7b659de0408b61c48": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9d1f4b17de349b6b26edd555ab1fa26",
            "placeholder": "​",
            "style": "IPY_MODEL_f5f304fafb30429995c300b6c594de11",
            "value": "Training Layer 0: 100%"
          }
        },
        "3431975a9aae4608b06b67d8441d5c8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "34a6667c6a8d4318843bcd76437b39c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ae06ddbf5e14ded89d532c568a9ddfc",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_557b76964fe34f1092fbd631c565e978",
            "value": 60
          }
        },
        "3a1818934a3a40348f8479c591b49884": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c150a43608d49508562e8d6b30bb735",
            "placeholder": "​",
            "style": "IPY_MODEL_402cdf4dd9a94a6aa553951be24680ee",
            "value": "Training Layer 1: 100%"
          }
        },
        "3bf083af0b6749e0b19b1f30d4f3a51d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3ef6d0bd9bc244279adbf6883283a9a1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "402cdf4dd9a94a6aa553951be24680ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "43c90325fd144c61b72acfc0530c564d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "464e51c5c0f64cbf8910c7c8a0fd601e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_32a32c9100f14fb096ab7b369e62ffe1",
              "IPY_MODEL_ca7ed09443144209897b6214ce72f8e9",
              "IPY_MODEL_ec769a83862e4d8eaf12c97418fc9238"
            ],
            "layout": "IPY_MODEL_3ef6d0bd9bc244279adbf6883283a9a1"
          }
        },
        "469bb22b0eb84a288464943765cf9691": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4e5975c2b3245de951342c7027888be",
            "placeholder": "​",
            "style": "IPY_MODEL_b91bb77a815a45f0861917fca8efad58",
            "value": " 60/60 [00:11&lt;00:00,  3.03it/s, loss=22.82753]"
          }
        },
        "4c3358d0c30b4e97b014e10cf90add37": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ce8d4e7a40fa49a59d05e55f2ac1106b",
              "IPY_MODEL_324ee7c3f499452b8ccd1f45fa982d89",
              "IPY_MODEL_7c77f93a5ac8436eaf2dc52186da85aa"
            ],
            "layout": "IPY_MODEL_8556556b60a147caae19c5b982468a25"
          }
        },
        "4d66d5e91977481d8f44842893a7de23": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_700f9aef759d4ff88a6529769dad630a",
            "placeholder": "​",
            "style": "IPY_MODEL_dcf761f22c8b428a990794498da9628d",
            "value": " 60/60 [00:11&lt;00:00,  5.92it/s, loss=57.226974]"
          }
        },
        "4e871d790a204e159ed834a29d64bda1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4f8f4d68c05041e78c8f9a20fde72685": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "557b76964fe34f1092fbd631c565e978": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5625688b08b14f4598abe887f05c0487": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "57cae0eec5c240b4957e82faa8823a0a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "57d156afccc849078cff7b8fa4714363": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "59f233cef02544619c519795f506a3c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5aa7c39a37b743389775767b65fd1784": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ae06ddbf5e14ded89d532c568a9ddfc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b4f7300e1a7463199876c01cd626249": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5cef791fcff64958beb4c96744c82912": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68038f99b99c4181a483a6ba38506080": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0fd30554bc464ac0a6205cac72672b12",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_962884a27f1b45a3aa4c06184703f0ee",
            "value": 60
          }
        },
        "700f9aef759d4ff88a6529769dad630a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7168897ea4b6490aa9aafb55aa0263d9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7175c5eacfcb473a8407471d01f9a24e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8459970fd69345e7adf165d46b253a3f",
              "IPY_MODEL_68038f99b99c4181a483a6ba38506080",
              "IPY_MODEL_19495359382e481797c15589994a3530"
            ],
            "layout": "IPY_MODEL_250acf8e90df42d0a6203cd2b67a78c5"
          }
        },
        "7c77f93a5ac8436eaf2dc52186da85aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27d54db2f0e442beb29315b04f16304f",
            "placeholder": "​",
            "style": "IPY_MODEL_a9b02b0b86984515af3fcd4803c9d8af",
            "value": " 60/60 [00:11&lt;00:00,  6.11it/s, loss=379.6463]"
          }
        },
        "8459970fd69345e7adf165d46b253a3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d362014d564b4f9b8f42b3a71077a962",
            "placeholder": "​",
            "style": "IPY_MODEL_59f233cef02544619c519795f506a3c3",
            "value": "Training Layer 2: 100%"
          }
        },
        "8556556b60a147caae19c5b982468a25": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8841c567c4ba4882a862e449daaae763": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8b129fdd6cbf4773b4a43ce5fd2250e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_02bf0e6c4290487794c3f7e3bfec4b1a",
              "IPY_MODEL_f84149369e06474dacccd22256aabcc6",
              "IPY_MODEL_4d66d5e91977481d8f44842893a7de23"
            ],
            "layout": "IPY_MODEL_af8e19652a7e45e5ac679ee8c59544ee"
          }
        },
        "8ebfe7a5fa3f44a3ab142731a7bed1d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0c5065fc6e414f9cbe957ce80a40d7e4",
              "IPY_MODEL_ed936491a42c4d2d8a1850040deca36a",
              "IPY_MODEL_b73d6f8b428e48179951e63eabf335d4"
            ],
            "layout": "IPY_MODEL_8fa0a27a45444dc6b7034c16f20c0ed4"
          }
        },
        "8fa0a27a45444dc6b7034c16f20c0ed4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9033fe7c667e4b728d9753889b3fe86e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "93862f217bac47289ebe0efa59f21edd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "962884a27f1b45a3aa4c06184703f0ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9a5ec28fb7dc4a7b8dfa4b81285c1cba": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c150a43608d49508562e8d6b30bb735": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9d02b79dbce148acaa903e01607b0618": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a70c55ff27cf48caa343c6e3ed2a6ca0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f3deb3ededbf42e18523ea03f3e59b52",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ae810c25a6b74b93bda10538fb7581fe",
            "value": 60
          }
        },
        "a9b02b0b86984515af3fcd4803c9d8af": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ad27cb5b68fe4704ad3d67a45ae608c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_57d156afccc849078cff7b8fa4714363",
            "placeholder": "​",
            "style": "IPY_MODEL_43c90325fd144c61b72acfc0530c564d",
            "value": " 60/60 [00:10&lt;00:00,  5.99it/s, loss=455.1321]"
          }
        },
        "ae810c25a6b74b93bda10538fb7581fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "af8e19652a7e45e5ac679ee8c59544ee": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b3a790e54e5e4aa18cbbcb8dc82f2de1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b73d6f8b428e48179951e63eabf335d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c1c0544556534410ae14aa5de18344ac",
            "placeholder": "​",
            "style": "IPY_MODEL_1ad06a79653f481c982d26a94d986f50",
            "value": " 60/60 [00:11&lt;00:00,  4.35it/s, loss=35.32199]"
          }
        },
        "b7cb7bfd1c6d4d129ea501a7a23c27fc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b91bb77a815a45f0861917fca8efad58": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b963143aa51d4a58b05b3f0a1bd09d3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c99151acec604436b05bc6704eb261b5",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_db4ce281dbcb4bb7958cc92d55f64b66",
            "value": 60
          }
        },
        "bc5e338f90d049089556f7d5d99a9574": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd69e362c71943bd88f431e198bb0a05": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_33c10ca24b1d4ab7b659de0408b61c48",
              "IPY_MODEL_a70c55ff27cf48caa343c6e3ed2a6ca0",
              "IPY_MODEL_ad27cb5b68fe4704ad3d67a45ae608c7"
            ],
            "layout": "IPY_MODEL_5b4f7300e1a7463199876c01cd626249"
          }
        },
        "bfe43079e654472d85060895961d31b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27ea5630f79e451b808b54f85cf6a143",
            "placeholder": "​",
            "style": "IPY_MODEL_0ccf8dde9b074dd7b4f6de0fe68087a6",
            "value": " 60/60 [00:11&lt;00:00,  5.27it/s, loss=295.02963]"
          }
        },
        "c1c0544556534410ae14aa5de18344ac": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c2c9fc55e2e94dc4b01cd5b734cc5dda": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f47fbd388a664f709feb685697f60b84",
              "IPY_MODEL_b963143aa51d4a58b05b3f0a1bd09d3a",
              "IPY_MODEL_d5ff6d42aaf14629875d66bdbabbde6a"
            ],
            "layout": "IPY_MODEL_4f8f4d68c05041e78c8f9a20fde72685"
          }
        },
        "c8462a70dcf74cbf82b6f6ed4add9192": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c99151acec604436b05bc6704eb261b5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ca7ed09443144209897b6214ce72f8e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3147b88f0fc54e3e927f60f9d142f506",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_039ca27f587749b6882a35805f647516",
            "value": 60
          }
        },
        "ce8d4e7a40fa49a59d05e55f2ac1106b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc5e338f90d049089556f7d5d99a9574",
            "placeholder": "​",
            "style": "IPY_MODEL_8841c567c4ba4882a862e449daaae763",
            "value": "Training Layer 0: 100%"
          }
        },
        "d059863fe9f9441e989d23afa78fa411": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3a1818934a3a40348f8479c591b49884",
              "IPY_MODEL_34a6667c6a8d4318843bcd76437b39c5",
              "IPY_MODEL_469bb22b0eb84a288464943765cf9691"
            ],
            "layout": "IPY_MODEL_12ed1d98467c4eca9d923e5b2ac6d282"
          }
        },
        "d362014d564b4f9b8f42b3a71077a962": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3b490a013b74ef78df729bfe363b4d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7168897ea4b6490aa9aafb55aa0263d9",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_12d68c888845433f8febce89c396a66d",
            "value": 60
          }
        },
        "d5718f1265c64d73961f726fecc765e9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5ff6d42aaf14629875d66bdbabbde6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5aa7c39a37b743389775767b65fd1784",
            "placeholder": "​",
            "style": "IPY_MODEL_3431975a9aae4608b06b67d8441d5c8d",
            "value": " 60/60 [00:11&lt;00:00,  5.76it/s, loss=32.615166]"
          }
        },
        "db4ce281dbcb4bb7958cc92d55f64b66": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dcf761f22c8b428a990794498da9628d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e4e5975c2b3245de951342c7027888be": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9d1f4b17de349b6b26edd555ab1fa26": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec769a83862e4d8eaf12c97418fc9238": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b7cb7bfd1c6d4d129ea501a7a23c27fc",
            "placeholder": "​",
            "style": "IPY_MODEL_ecf557718b84483c99cab5e213125426",
            "value": " 60/60 [00:11&lt;00:00,  5.85it/s, loss=45.579185]"
          }
        },
        "ecf557718b84483c99cab5e213125426": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ed936491a42c4d2d8a1850040deca36a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c8462a70dcf74cbf82b6f6ed4add9192",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_facc4a192b364f939a3ed144afc601cf",
            "value": 60
          }
        },
        "ee669bb6587e40228f7b837711ff2f1e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef8acb5e2c4641c38f2b8e477a04c8a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f0dcd51dc3584bd1ab721b7abda51e98": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f3deb3ededbf42e18523ea03f3e59b52": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f47fbd388a664f709feb685697f60b84": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d02b79dbce148acaa903e01607b0618",
            "placeholder": "​",
            "style": "IPY_MODEL_9033fe7c667e4b728d9753889b3fe86e",
            "value": "Training Layer 1: 100%"
          }
        },
        "f5f304fafb30429995c300b6c594de11": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f84149369e06474dacccd22256aabcc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b3a790e54e5e4aa18cbbcb8dc82f2de1",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f0dcd51dc3584bd1ab721b7abda51e98",
            "value": 60
          }
        },
        "facc4a192b364f939a3ed144afc601cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "faf02fba0bc74207ba3ea3276f262f48": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}